Lots more changes
This commit is contained in:
@@ -12,10 +12,12 @@ import (
|
||||
"scratchbox/internal/storage"
|
||||
)
|
||||
|
||||
var testStorageKey = []byte("0123456789abcdef0123456789abcdef")
|
||||
|
||||
func TestNewWorker(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
store, err := storage.NewFilesystemStore(filepath.Join(t.TempDir(), "data"))
|
||||
store, err := storage.NewFilesystemStore(filepath.Join(t.TempDir(), "data"), testStorageKey)
|
||||
if err != nil {
|
||||
t.Fatalf("NewFilesystemStore() error = %v", err)
|
||||
}
|
||||
@@ -29,7 +31,7 @@ func TestNewWorker(t *testing.T) {
|
||||
func TestWorkerStartRemovesExpiredAndLogs(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
store, err := storage.NewFilesystemStore(filepath.Join(t.TempDir(), "data"))
|
||||
store, err := storage.NewFilesystemStore(filepath.Join(t.TempDir(), "data"), testStorageKey)
|
||||
if err != nil {
|
||||
t.Fatalf("NewFilesystemStore() error = %v", err)
|
||||
}
|
||||
@@ -53,7 +55,7 @@ func TestWorkerStartRemovesExpiredAndLogs(t *testing.T) {
|
||||
func TestWorkerStartWithNoExpiredEntries(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
store, err := storage.NewFilesystemStore(filepath.Join(t.TempDir(), "data"))
|
||||
store, err := storage.NewFilesystemStore(filepath.Join(t.TempDir(), "data"), testStorageKey)
|
||||
if err != nil {
|
||||
t.Fatalf("NewFilesystemStore() error = %v", err)
|
||||
}
|
||||
|
||||
+163
-38
@@ -1,6 +1,8 @@
|
||||
package config
|
||||
|
||||
import (
|
||||
"crypto/rand"
|
||||
"encoding/base64"
|
||||
"errors"
|
||||
"fmt"
|
||||
"net/netip"
|
||||
@@ -14,23 +16,34 @@ import (
|
||||
)
|
||||
|
||||
const (
|
||||
defaultListenAddr = ":8080"
|
||||
defaultReadHeaderTimeout = "5s"
|
||||
defaultReadTimeout = "30s"
|
||||
defaultWriteTimeout = "30s"
|
||||
defaultIdleTimeout = "120s"
|
||||
defaultMaxHeaderBytes = 1 << 20
|
||||
defaultMaxUploadSize = "100MiB"
|
||||
defaultTTL = "1h"
|
||||
defaultRawCacheMaxSize = "200MiB"
|
||||
defaultRawCacheMaxEntries = 32
|
||||
defaultDataDir = "./data"
|
||||
defaultCleanupInterval = "1m"
|
||||
defaultRateLimitEnabled = true
|
||||
defaultRequestsPerMinute = 30
|
||||
defaultRateLimitBurst = 10
|
||||
maxDefaultTTLLimit = 24 * time.Hour
|
||||
minCleanupInterval = 10 * time.Second
|
||||
defaultListenAddr = ":8080"
|
||||
defaultReadHeaderTimeout = "5s"
|
||||
defaultReadTimeout = "30s"
|
||||
defaultWriteTimeout = "30s"
|
||||
defaultIdleTimeout = "120s"
|
||||
defaultMaxHeaderBytes = 1 << 20
|
||||
defaultAccessLogEnabled = true
|
||||
defaultAccessLogMaxSize = "100MiB"
|
||||
defaultAccessLogMaxBackups = 5
|
||||
defaultMaxUploadSize = "100MiB"
|
||||
defaultTTL = "15m"
|
||||
defaultRawCacheMaxSize = "200MiB"
|
||||
defaultRawCacheMaxEntries = 32
|
||||
defaultDataDir = "./data"
|
||||
defaultCleanupInterval = "1m"
|
||||
defaultStorageEncryptionKey = "vVPlqMfKa8OtD3x0RKp4rVCJ4QoScf5mWdgfg4f+5GU="
|
||||
defaultEncryptionKeyFile = "config.key"
|
||||
defaultUIRateLimitEnabled = false
|
||||
defaultAPIReadRateLimitEnabled = false
|
||||
defaultAPIWriteRateLimitEnabled = true
|
||||
defaultUIRequestsPerMinute = 360
|
||||
defaultUIRateLimitBurst = 120
|
||||
defaultAPIReadRequestsPerMinute = 300
|
||||
defaultAPIReadRateLimitBurst = 100
|
||||
defaultAPIWriteRequestsPerMinute = 30
|
||||
defaultAPIWriteRateLimitBurst = 10
|
||||
maxDefaultTTLLimit = 24 * time.Hour
|
||||
minCleanupInterval = 10 * time.Second
|
||||
)
|
||||
|
||||
type Config struct {
|
||||
@@ -41,16 +54,25 @@ type Config struct {
|
||||
}
|
||||
|
||||
type ServerConfig struct {
|
||||
ListenAddr string `yaml:"listen_addr" comment:"Address to bind the HTTP server to."`
|
||||
ReadHeaderTimeout string `yaml:"read_header_timeout" comment:"Maximum duration for reading request headers (for slowloris protection). Must be > 0."`
|
||||
ReadTimeout string `yaml:"read_timeout" comment:"Maximum duration for reading the full request, including body. Must be > 0."`
|
||||
WriteTimeout string `yaml:"write_timeout" comment:"Maximum duration before timing out writes of a response. Must be > 0."`
|
||||
IdleTimeout string `yaml:"idle_timeout" comment:"Maximum amount of time to wait for the next request when keep-alives are enabled. Must be > 0."`
|
||||
MaxHeaderBytes int `yaml:"max_header_bytes" comment:"Maximum size of request headers in bytes. Must be > 0."`
|
||||
ReadHeaderTimeoutDur time.Duration `yaml:"-"`
|
||||
ReadTimeoutDur time.Duration `yaml:"-"`
|
||||
WriteTimeoutDur time.Duration `yaml:"-"`
|
||||
IdleTimeoutDur time.Duration `yaml:"-"`
|
||||
ListenAddr string `yaml:"listen_addr" comment:"Address to bind the HTTP server to."`
|
||||
ReadHeaderTimeout string `yaml:"read_header_timeout" comment:"Maximum duration for reading request headers (for slowloris protection). Must be > 0."`
|
||||
ReadTimeout string `yaml:"read_timeout" comment:"Maximum duration for reading the full request, including body. Must be > 0."`
|
||||
WriteTimeout string `yaml:"write_timeout" comment:"Maximum duration before timing out writes of a response. Must be > 0."`
|
||||
IdleTimeout string `yaml:"idle_timeout" comment:"Maximum amount of time to wait for the next request when keep-alives are enabled. Must be > 0."`
|
||||
MaxHeaderBytes int `yaml:"max_header_bytes" comment:"Maximum size of request headers in bytes. Must be > 0."`
|
||||
AccessLog AccessLogConfig `yaml:"access_log" comment:"Access log settings for all HTTP requests."`
|
||||
ReadHeaderTimeoutDur time.Duration `yaml:"-"`
|
||||
ReadTimeoutDur time.Duration `yaml:"-"`
|
||||
WriteTimeoutDur time.Duration `yaml:"-"`
|
||||
IdleTimeoutDur time.Duration `yaml:"-"`
|
||||
}
|
||||
|
||||
type AccessLogConfig struct {
|
||||
Enabled bool `yaml:"enabled" comment:"Enable HTTP access logs."`
|
||||
FilePath string `yaml:"file_path" comment:"Optional file path for access logs. When set, logs are tee'd to stdout and this file."`
|
||||
MaxSize string `yaml:"max_size" comment:"Maximum size for access log file before rotation. Supports B, KB, MB, GB and KiB, MiB, GiB."`
|
||||
MaxBackups int `yaml:"max_backups" comment:"Number of rotated access log files to keep."`
|
||||
MaxSizeBytes int64 `yaml:"-"`
|
||||
}
|
||||
|
||||
type LimitsConfig struct {
|
||||
@@ -66,14 +88,18 @@ type LimitsConfig struct {
|
||||
type StorageConfig struct {
|
||||
DataDir string `yaml:"data_dir" comment:"Directory where scratch files and metadata are stored."`
|
||||
CleanupInterval string `yaml:"cleanup_interval" comment:"How often expired scratches are deleted (minimum 10s)."`
|
||||
EncryptionKeyFile string `yaml:"encryption_key_file" comment:"Path to a file containing a base64-encoded 32-byte AES key for at-rest encryption. Relative paths are resolved from the config file directory."`
|
||||
CleanupIntervalParsed time.Duration `yaml:"-"`
|
||||
EncryptionKeyBytes []byte `yaml:"-"`
|
||||
}
|
||||
|
||||
type SecurityConfig struct {
|
||||
AllowedIPs []string `yaml:"allowed_ips" comment:"Allowlist for POST /api/scratch as IPs or CIDRs (examples: 127.0.0.1, 192.168.1.0/24). Keep 127.0.0.1 for local access; add trusted ranges as needed. Empty means unrestricted."`
|
||||
TrustProxyHeaders bool `yaml:"trust_proxy_headers" comment:"Honor X-Forwarded-For / X-Real-IP only when request source IP matches security.trusted_proxy_ips."`
|
||||
TrustedProxyIPs []string `yaml:"trusted_proxy_ips" comment:"IPs/CIDRs of reverse proxies allowed to supply forwarded client IP headers. Required when trust_proxy_headers is true."`
|
||||
RateLimit RateLimitConfig `yaml:"rate_limit" comment:"Per-client token bucket on write and scratch read routes."`
|
||||
RateLimitUI RateLimitConfig `yaml:"rate_limit_ui" comment:"Per-client token bucket on UI routes (/, /u, /s/{id})."`
|
||||
RateLimitAPIRead RateLimitConfig `yaml:"rate_limit_api_read" comment:"Per-client token bucket on API read routes (GET /api/config, GET /api/scratch/{id}, GET /api/raw/{id})."`
|
||||
RateLimitAPIWrite RateLimitConfig `yaml:"rate_limit_api_write" comment:"Per-client token bucket on API write route (POST /api/scratch)."`
|
||||
AllowedPrefixes []netip.Prefix `yaml:"-"`
|
||||
TrustedProxyPrefixes []netip.Prefix `yaml:"-"`
|
||||
}
|
||||
@@ -95,6 +121,9 @@ func Load(path string) (Config, error) {
|
||||
return Config{}, fmt.Errorf("parse config yaml: %w", err)
|
||||
}
|
||||
}
|
||||
if err := cfg.loadEncryptionKey(path); err != nil {
|
||||
return Config{}, err
|
||||
}
|
||||
|
||||
if err := cfg.Validate(); err != nil {
|
||||
return Config{}, err
|
||||
@@ -115,6 +144,11 @@ func defaults() Config {
|
||||
WriteTimeout: defaultWriteTimeout,
|
||||
IdleTimeout: defaultIdleTimeout,
|
||||
MaxHeaderBytes: defaultMaxHeaderBytes,
|
||||
AccessLog: AccessLogConfig{
|
||||
Enabled: defaultAccessLogEnabled,
|
||||
MaxSize: defaultAccessLogMaxSize,
|
||||
MaxBackups: defaultAccessLogMaxBackups,
|
||||
},
|
||||
},
|
||||
Limits: LimitsConfig{
|
||||
MaxUploadSize: defaultMaxUploadSize,
|
||||
@@ -123,17 +157,29 @@ func defaults() Config {
|
||||
RawCacheMaxEntries: defaultRawCacheMaxEntries,
|
||||
},
|
||||
Storage: StorageConfig{
|
||||
DataDir: defaultDataDir,
|
||||
CleanupInterval: defaultCleanupInterval,
|
||||
DataDir: defaultDataDir,
|
||||
CleanupInterval: defaultCleanupInterval,
|
||||
EncryptionKeyFile: defaultEncryptionKeyFile,
|
||||
EncryptionKeyBytes: mustDecodeDefaultEncryptionKey(),
|
||||
},
|
||||
Security: SecurityConfig{
|
||||
AllowedIPs: []string{
|
||||
"127.0.0.1",
|
||||
},
|
||||
RateLimit: RateLimitConfig{
|
||||
Enabled: defaultRateLimitEnabled,
|
||||
RequestsPerMinute: defaultRequestsPerMinute,
|
||||
Burst: defaultRateLimitBurst,
|
||||
RateLimitUI: RateLimitConfig{
|
||||
Enabled: defaultUIRateLimitEnabled,
|
||||
RequestsPerMinute: defaultUIRequestsPerMinute,
|
||||
Burst: defaultUIRateLimitBurst,
|
||||
},
|
||||
RateLimitAPIRead: RateLimitConfig{
|
||||
Enabled: defaultAPIReadRateLimitEnabled,
|
||||
RequestsPerMinute: defaultAPIReadRequestsPerMinute,
|
||||
Burst: defaultAPIReadRateLimitBurst,
|
||||
},
|
||||
RateLimitAPIWrite: RateLimitConfig{
|
||||
Enabled: defaultAPIWriteRateLimitEnabled,
|
||||
RequestsPerMinute: defaultAPIWriteRequestsPerMinute,
|
||||
Burst: defaultAPIWriteRateLimitBurst,
|
||||
},
|
||||
},
|
||||
}
|
||||
@@ -182,6 +228,14 @@ func (c *Config) Validate() error {
|
||||
if c.Server.MaxHeaderBytes <= 0 {
|
||||
return errors.New("server.max_header_bytes must be > 0")
|
||||
}
|
||||
accessLogMaxSizeBytes, err := parseHumanSize(c.Server.AccessLog.MaxSize)
|
||||
if err != nil {
|
||||
return fmt.Errorf("server.access_log.max_size invalid: %w", err)
|
||||
}
|
||||
c.Server.AccessLog.MaxSizeBytes = accessLogMaxSizeBytes
|
||||
if c.Server.AccessLog.MaxBackups <= 0 {
|
||||
return errors.New("server.access_log.max_backups must be > 0")
|
||||
}
|
||||
|
||||
sizeBytes, err := parseHumanSize(c.Limits.MaxUploadSize)
|
||||
if err != nil {
|
||||
@@ -222,6 +276,12 @@ func (c *Config) Validate() error {
|
||||
if err := ensureWritableDir(c.Storage.DataDir); err != nil {
|
||||
return fmt.Errorf("storage.data_dir not writable: %w", err)
|
||||
}
|
||||
if strings.TrimSpace(c.Storage.EncryptionKeyFile) == "" {
|
||||
return errors.New("storage.encryption_key_file is required")
|
||||
}
|
||||
if len(c.Storage.EncryptionKeyBytes) != 32 {
|
||||
return fmt.Errorf("storage.encryption_key_file must decode to 32 bytes, got %d", len(c.Storage.EncryptionKeyBytes))
|
||||
}
|
||||
|
||||
prefixes, err := parseAllowedPrefixes(c.Security.AllowedIPs)
|
||||
if err != nil {
|
||||
@@ -237,11 +297,23 @@ func (c *Config) Validate() error {
|
||||
return errors.New("security.trusted_proxy_ips is required when security.trust_proxy_headers is true")
|
||||
}
|
||||
|
||||
if c.Security.RateLimit.RequestsPerMinute <= 0 {
|
||||
return errors.New("security.rate_limit.requests_per_minute must be > 0")
|
||||
if c.Security.RateLimitUI.RequestsPerMinute <= 0 {
|
||||
return errors.New("security.rate_limit_ui.requests_per_minute must be > 0")
|
||||
}
|
||||
if c.Security.RateLimit.Burst <= 0 {
|
||||
return errors.New("security.rate_limit.burst must be > 0")
|
||||
if c.Security.RateLimitUI.Burst <= 0 {
|
||||
return errors.New("security.rate_limit_ui.burst must be > 0")
|
||||
}
|
||||
if c.Security.RateLimitAPIRead.RequestsPerMinute <= 0 {
|
||||
return errors.New("security.rate_limit_api_read.requests_per_minute must be > 0")
|
||||
}
|
||||
if c.Security.RateLimitAPIRead.Burst <= 0 {
|
||||
return errors.New("security.rate_limit_api_read.burst must be > 0")
|
||||
}
|
||||
if c.Security.RateLimitAPIWrite.RequestsPerMinute <= 0 {
|
||||
return errors.New("security.rate_limit_api_write.requests_per_minute must be > 0")
|
||||
}
|
||||
if c.Security.RateLimitAPIWrite.Burst <= 0 {
|
||||
return errors.New("security.rate_limit_api_write.burst must be > 0")
|
||||
}
|
||||
|
||||
return nil
|
||||
@@ -338,3 +410,56 @@ func parseHumanSize(raw string) (int64, error) {
|
||||
}
|
||||
return size, nil
|
||||
}
|
||||
|
||||
func GenerateEncryptionKey() (string, error) {
|
||||
raw := make([]byte, 32)
|
||||
if _, err := rand.Read(raw); err != nil {
|
||||
return "", fmt.Errorf("generate encryption key bytes: %w", err)
|
||||
}
|
||||
return base64.StdEncoding.EncodeToString(raw), nil
|
||||
}
|
||||
|
||||
func (c *Config) loadEncryptionKey(configPath string) error {
|
||||
if configPath == "" {
|
||||
return nil
|
||||
}
|
||||
keyPath := strings.TrimSpace(c.Storage.EncryptionKeyFile)
|
||||
if keyPath == "" {
|
||||
return errors.New("storage.encryption_key_file is required")
|
||||
}
|
||||
if !filepath.IsAbs(keyPath) {
|
||||
keyPath = filepath.Join(filepath.Dir(configPath), keyPath)
|
||||
}
|
||||
keyRaw, err := os.ReadFile(keyPath)
|
||||
if err != nil {
|
||||
return fmt.Errorf("read storage.encryption_key_file %q: %w", keyPath, err)
|
||||
}
|
||||
encKey, err := decodeEncryptionKey(strings.TrimSpace(string(keyRaw)))
|
||||
if err != nil {
|
||||
return fmt.Errorf("storage.encryption_key_file invalid: %w", err)
|
||||
}
|
||||
c.Storage.EncryptionKeyBytes = encKey
|
||||
return nil
|
||||
}
|
||||
|
||||
func decodeEncryptionKey(raw string) ([]byte, error) {
|
||||
if raw == "" {
|
||||
return nil, errors.New("key is empty")
|
||||
}
|
||||
encKey, err := base64.StdEncoding.DecodeString(raw)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("invalid base64: %w", err)
|
||||
}
|
||||
if len(encKey) != 32 {
|
||||
return nil, fmt.Errorf("must decode to 32 bytes, got %d", len(encKey))
|
||||
}
|
||||
return encKey, nil
|
||||
}
|
||||
|
||||
func mustDecodeDefaultEncryptionKey() []byte {
|
||||
encKey, err := decodeEncryptionKey(defaultStorageEncryptionKey)
|
||||
if err != nil {
|
||||
panic(fmt.Sprintf("invalid built-in encryption key: %v", err))
|
||||
}
|
||||
return encKey
|
||||
}
|
||||
|
||||
+165
-11
@@ -1,6 +1,7 @@
|
||||
package config
|
||||
|
||||
import (
|
||||
"encoding/base64"
|
||||
"net/netip"
|
||||
"os"
|
||||
"path/filepath"
|
||||
@@ -37,8 +38,17 @@ func TestValidateAppliesDefaultsAndParsedFields(t *testing.T) {
|
||||
if cfg.Server.MaxHeaderBytes != 1<<20 {
|
||||
t.Fatalf("MaxHeaderBytes = %d, want %d", cfg.Server.MaxHeaderBytes, 1<<20)
|
||||
}
|
||||
if cfg.Limits.DefaultTTLDuration != time.Hour {
|
||||
t.Fatalf("DefaultTTLDuration = %s, want 1h", cfg.Limits.DefaultTTLDuration)
|
||||
if !cfg.Server.AccessLog.Enabled {
|
||||
t.Fatalf("AccessLog.Enabled = %v, want true", cfg.Server.AccessLog.Enabled)
|
||||
}
|
||||
if cfg.Server.AccessLog.MaxSizeBytes != 100*1024*1024 {
|
||||
t.Fatalf("AccessLog.MaxSizeBytes = %d, want %d", cfg.Server.AccessLog.MaxSizeBytes, int64(100*1024*1024))
|
||||
}
|
||||
if cfg.Server.AccessLog.MaxBackups != 5 {
|
||||
t.Fatalf("AccessLog.MaxBackups = %d, want 5", cfg.Server.AccessLog.MaxBackups)
|
||||
}
|
||||
if cfg.Limits.DefaultTTLDuration != 15*time.Minute {
|
||||
t.Fatalf("DefaultTTLDuration = %s, want 15m", cfg.Limits.DefaultTTLDuration)
|
||||
}
|
||||
if cfg.Limits.RawCacheMaxBytes != 200*1024*1024 {
|
||||
t.Fatalf("RawCacheMaxBytes = %d, want %d", cfg.Limits.RawCacheMaxBytes, int64(200*1024*1024))
|
||||
@@ -49,6 +59,9 @@ func TestValidateAppliesDefaultsAndParsedFields(t *testing.T) {
|
||||
if cfg.Storage.CleanupIntervalParsed != time.Minute {
|
||||
t.Fatalf("CleanupIntervalParsed = %s, want 1m", cfg.Storage.CleanupIntervalParsed)
|
||||
}
|
||||
if len(cfg.Storage.EncryptionKeyBytes) != 32 {
|
||||
t.Fatalf("EncryptionKeyBytes length = %d, want 32", len(cfg.Storage.EncryptionKeyBytes))
|
||||
}
|
||||
if len(cfg.Security.AllowedPrefixes) != 1 {
|
||||
t.Fatalf("AllowedPrefixes len = %d, want 1", len(cfg.Security.AllowedPrefixes))
|
||||
}
|
||||
@@ -63,6 +76,7 @@ func TestLoadParsesConfiguredFields(t *testing.T) {
|
||||
tmp := t.TempDir()
|
||||
dataDir := filepath.Join(tmp, "store")
|
||||
configPath := filepath.Join(tmp, "config.yaml")
|
||||
keyPath := filepath.Join(tmp, "scratchbox.key")
|
||||
content := `
|
||||
server:
|
||||
listen_addr: "127.0.0.1:9090"
|
||||
@@ -71,6 +85,11 @@ server:
|
||||
write_timeout: "35s"
|
||||
idle_timeout: "90s"
|
||||
max_header_bytes: 2097152
|
||||
access_log:
|
||||
enabled: true
|
||||
file_path: "` + filepath.Join(tmp, "access.log") + `"
|
||||
max_size: "2MiB"
|
||||
max_backups: 7
|
||||
limits:
|
||||
max_upload_size: "2MiB"
|
||||
default_ttl: "2h"
|
||||
@@ -79,6 +98,7 @@ limits:
|
||||
storage:
|
||||
data_dir: "` + dataDir + `"
|
||||
cleanup_interval: "45s"
|
||||
encryption_key_file: "` + filepath.Base(keyPath) + `"
|
||||
security:
|
||||
allowed_ips:
|
||||
- "127.0.0.1"
|
||||
@@ -86,7 +106,15 @@ security:
|
||||
trust_proxy_headers: true
|
||||
trusted_proxy_ips:
|
||||
- "10.0.0.0/8"
|
||||
rate_limit:
|
||||
rate_limit_ui:
|
||||
enabled: true
|
||||
requests_per_minute: 12
|
||||
burst: 4
|
||||
rate_limit_api_read:
|
||||
enabled: true
|
||||
requests_per_minute: 12
|
||||
burst: 4
|
||||
rate_limit_api_write:
|
||||
enabled: true
|
||||
requests_per_minute: 12
|
||||
burst: 4
|
||||
@@ -94,6 +122,9 @@ security:
|
||||
if err := os.WriteFile(configPath, []byte(content), 0o644); err != nil {
|
||||
t.Fatalf("WriteFile() error = %v", err)
|
||||
}
|
||||
if err := os.WriteFile(keyPath, []byte("AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=\n"), 0o600); err != nil {
|
||||
t.Fatalf("WriteFile(key) error = %v", err)
|
||||
}
|
||||
|
||||
cfg, err := Load(configPath)
|
||||
if err != nil {
|
||||
@@ -118,6 +149,15 @@ security:
|
||||
if cfg.Server.MaxHeaderBytes != 2*1024*1024 {
|
||||
t.Fatalf("MaxHeaderBytes = %d, want %d", cfg.Server.MaxHeaderBytes, 2*1024*1024)
|
||||
}
|
||||
if !cfg.Server.AccessLog.Enabled {
|
||||
t.Fatalf("AccessLog.Enabled = false, want true")
|
||||
}
|
||||
if cfg.Server.AccessLog.MaxSizeBytes != 2*1024*1024 {
|
||||
t.Fatalf("AccessLog.MaxSizeBytes = %d, want %d", cfg.Server.AccessLog.MaxSizeBytes, int64(2*1024*1024))
|
||||
}
|
||||
if cfg.Server.AccessLog.MaxBackups != 7 {
|
||||
t.Fatalf("AccessLog.MaxBackups = %d, want 7", cfg.Server.AccessLog.MaxBackups)
|
||||
}
|
||||
if cfg.Limits.MaxUploadSizeBytes != 2*1024*1024 {
|
||||
t.Fatalf("MaxUploadSizeBytes = %d, want %d", cfg.Limits.MaxUploadSizeBytes, int64(2*1024*1024))
|
||||
}
|
||||
@@ -133,11 +173,23 @@ security:
|
||||
if cfg.Storage.CleanupIntervalParsed != 45*time.Second {
|
||||
t.Fatalf("CleanupIntervalParsed = %s, want 45s", cfg.Storage.CleanupIntervalParsed)
|
||||
}
|
||||
if len(cfg.Storage.EncryptionKeyBytes) != 32 {
|
||||
t.Fatalf("EncryptionKeyBytes length = %d, want 32", len(cfg.Storage.EncryptionKeyBytes))
|
||||
}
|
||||
if cfg.Storage.EncryptionKeyBytes[0] != 0 {
|
||||
t.Fatalf("EncryptionKeyBytes[0] = %d, want 0", cfg.Storage.EncryptionKeyBytes[0])
|
||||
}
|
||||
if !cfg.Security.TrustProxyHeaders {
|
||||
t.Fatalf("TrustProxyHeaders = false, want true")
|
||||
}
|
||||
if cfg.Security.RateLimit.RequestsPerMinute != 12 || cfg.Security.RateLimit.Burst != 4 {
|
||||
t.Fatalf("unexpected rate_limit values: %+v", cfg.Security.RateLimit)
|
||||
if cfg.Security.RateLimitUI.RequestsPerMinute != 12 || cfg.Security.RateLimitUI.Burst != 4 {
|
||||
t.Fatalf("unexpected rate_limit_ui values: %+v", cfg.Security.RateLimitUI)
|
||||
}
|
||||
if cfg.Security.RateLimitAPIRead.RequestsPerMinute != 12 || cfg.Security.RateLimitAPIRead.Burst != 4 {
|
||||
t.Fatalf("unexpected rate_limit_api_read values: %+v", cfg.Security.RateLimitAPIRead)
|
||||
}
|
||||
if cfg.Security.RateLimitAPIWrite.RequestsPerMinute != 12 || cfg.Security.RateLimitAPIWrite.Burst != 4 {
|
||||
t.Fatalf("unexpected rate_limit_api_write values: %+v", cfg.Security.RateLimitAPIWrite)
|
||||
}
|
||||
if len(cfg.Security.AllowedPrefixes) != 2 {
|
||||
t.Fatalf("AllowedPrefixes len = %d, want 2", len(cfg.Security.AllowedPrefixes))
|
||||
@@ -204,6 +256,20 @@ func TestValidateRejectsInvalidValues(t *testing.T) {
|
||||
},
|
||||
wantErr: "server.max_header_bytes",
|
||||
},
|
||||
{
|
||||
name: "invalid access log max size",
|
||||
mutate: func(cfg *Config) {
|
||||
cfg.Server.AccessLog.MaxSize = "oops"
|
||||
},
|
||||
wantErr: "server.access_log.max_size",
|
||||
},
|
||||
{
|
||||
name: "invalid access log max backups",
|
||||
mutate: func(cfg *Config) {
|
||||
cfg.Server.AccessLog.MaxBackups = 0
|
||||
},
|
||||
wantErr: "server.access_log.max_backups",
|
||||
},
|
||||
{
|
||||
name: "invalid ttl parse",
|
||||
mutate: func(cfg *Config) {
|
||||
@@ -255,11 +321,25 @@ func TestValidateRejectsInvalidValues(t *testing.T) {
|
||||
wantErr: "security.trusted_proxy_ips is required",
|
||||
},
|
||||
{
|
||||
name: "invalid rate limit burst",
|
||||
name: "invalid ui rate limit burst",
|
||||
mutate: func(cfg *Config) {
|
||||
cfg.Security.RateLimit.Burst = 0
|
||||
cfg.Security.RateLimitUI.Burst = 0
|
||||
},
|
||||
wantErr: "security.rate_limit.burst",
|
||||
wantErr: "security.rate_limit_ui.burst",
|
||||
},
|
||||
{
|
||||
name: "invalid api read rate limit burst",
|
||||
mutate: func(cfg *Config) {
|
||||
cfg.Security.RateLimitAPIRead.Burst = 0
|
||||
},
|
||||
wantErr: "security.rate_limit_api_read.burst",
|
||||
},
|
||||
{
|
||||
name: "invalid api write rate limit burst",
|
||||
mutate: func(cfg *Config) {
|
||||
cfg.Security.RateLimitAPIWrite.Burst = 0
|
||||
},
|
||||
wantErr: "security.rate_limit_api_write.burst",
|
||||
},
|
||||
{
|
||||
name: "data dir required",
|
||||
@@ -283,11 +363,39 @@ func TestValidateRejectsInvalidValues(t *testing.T) {
|
||||
wantErr: "storage.cleanup_interval",
|
||||
},
|
||||
{
|
||||
name: "invalid rate limit rpm",
|
||||
name: "encryption key file required",
|
||||
mutate: func(cfg *Config) {
|
||||
cfg.Security.RateLimit.RequestsPerMinute = 0
|
||||
cfg.Storage.EncryptionKeyFile = " "
|
||||
},
|
||||
wantErr: "security.rate_limit.requests_per_minute",
|
||||
wantErr: "storage.encryption_key_file is required",
|
||||
},
|
||||
{
|
||||
name: "encryption key bytes invalid length",
|
||||
mutate: func(cfg *Config) {
|
||||
cfg.Storage.EncryptionKeyBytes = []byte("short")
|
||||
},
|
||||
wantErr: "storage.encryption_key_file must decode to 32 bytes",
|
||||
},
|
||||
{
|
||||
name: "invalid ui rate limit rpm",
|
||||
mutate: func(cfg *Config) {
|
||||
cfg.Security.RateLimitUI.RequestsPerMinute = 0
|
||||
},
|
||||
wantErr: "security.rate_limit_ui.requests_per_minute",
|
||||
},
|
||||
{
|
||||
name: "invalid api read rate limit rpm",
|
||||
mutate: func(cfg *Config) {
|
||||
cfg.Security.RateLimitAPIRead.RequestsPerMinute = 0
|
||||
},
|
||||
wantErr: "security.rate_limit_api_read.requests_per_minute",
|
||||
},
|
||||
{
|
||||
name: "invalid api write rate limit rpm",
|
||||
mutate: func(cfg *Config) {
|
||||
cfg.Security.RateLimitAPIWrite.RequestsPerMinute = 0
|
||||
},
|
||||
wantErr: "security.rate_limit_api_write.requests_per_minute",
|
||||
},
|
||||
}
|
||||
|
||||
@@ -328,6 +436,29 @@ func TestLoadErrors(t *testing.T) {
|
||||
if err == nil || !strings.Contains(err.Error(), "parse config yaml") {
|
||||
t.Fatalf("Load() error = %v, want yaml parse error", err)
|
||||
}
|
||||
|
||||
validConfigPath := filepath.Join(tmp, "valid.yaml")
|
||||
validConfig := `
|
||||
storage:
|
||||
data_dir: "` + filepath.Join(tmp, "store") + `"
|
||||
cleanup_interval: "10s"
|
||||
`
|
||||
if err := os.WriteFile(validConfigPath, []byte(validConfig), 0o644); err != nil {
|
||||
t.Fatalf("WriteFile(valid config) error = %v", err)
|
||||
}
|
||||
_, err = Load(validConfigPath)
|
||||
if err == nil || !strings.Contains(err.Error(), "read storage.encryption_key_file") {
|
||||
t.Fatalf("Load() error = %v, want missing key file error", err)
|
||||
}
|
||||
|
||||
keyPath := filepath.Join(tmp, "config.key")
|
||||
if err := os.WriteFile(keyPath, []byte("not-base64"), 0o600); err != nil {
|
||||
t.Fatalf("WriteFile(invalid key) error = %v", err)
|
||||
}
|
||||
_, err = Load(validConfigPath)
|
||||
if err == nil || !strings.Contains(err.Error(), "storage.encryption_key_file invalid: invalid base64") {
|
||||
t.Fatalf("Load() error = %v, want invalid key base64 error", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseAllowedPrefixesTrimsAndParses(t *testing.T) {
|
||||
@@ -385,3 +516,26 @@ func TestParseHumanSizeVariants(t *testing.T) {
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestGenerateEncryptionKey(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
k1, err := GenerateEncryptionKey()
|
||||
if err != nil {
|
||||
t.Fatalf("GenerateEncryptionKey() error = %v", err)
|
||||
}
|
||||
k2, err := GenerateEncryptionKey()
|
||||
if err != nil {
|
||||
t.Fatalf("GenerateEncryptionKey() second error = %v", err)
|
||||
}
|
||||
if k1 == k2 {
|
||||
t.Fatalf("GenerateEncryptionKey() produced duplicate keys")
|
||||
}
|
||||
b1, err := base64.StdEncoding.DecodeString(k1)
|
||||
if err != nil {
|
||||
t.Fatalf("DecodeString(k1) error = %v", err)
|
||||
}
|
||||
if len(b1) != 32 {
|
||||
t.Fatalf("decoded key length = %d, want 32", len(b1))
|
||||
}
|
||||
}
|
||||
|
||||
+44
-26
@@ -18,46 +18,63 @@ import (
|
||||
)
|
||||
|
||||
type Server struct {
|
||||
cfg config.Config
|
||||
store *storage.FilesystemStore
|
||||
logger *log.Logger
|
||||
rawCache *rawContentCache
|
||||
cfg config.Config
|
||||
store *storage.FilesystemStore
|
||||
logger *log.Logger
|
||||
accessLogger *log.Logger
|
||||
rawCache *rawContentCache
|
||||
}
|
||||
|
||||
func NewServer(cfg config.Config, store *storage.FilesystemStore, logger *log.Logger) (*Server, error) {
|
||||
func NewServer(cfg config.Config, store *storage.FilesystemStore, logger *log.Logger, accessLogger *log.Logger) (*Server, error) {
|
||||
return &Server{
|
||||
cfg: cfg,
|
||||
store: store,
|
||||
logger: logger,
|
||||
rawCache: newRawContentCache(cfg.Limits.RawCacheMaxEntries, cfg.Limits.RawCacheMaxBytes),
|
||||
cfg: cfg,
|
||||
store: store,
|
||||
logger: logger,
|
||||
accessLogger: accessLogger,
|
||||
rawCache: newRawContentCache(cfg.Limits.RawCacheMaxEntries, cfg.Limits.RawCacheMaxBytes),
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (s *Server) Routes() http.Handler {
|
||||
mux := http.NewServeMux()
|
||||
mux.Handle("GET /{$}", http.HandlerFunc(s.serveSPA))
|
||||
mux.Handle("GET /u", http.HandlerFunc(s.serveSPA))
|
||||
mux.Handle("GET /api/config", http.HandlerFunc(s.getUIConfig))
|
||||
uiMiddlewares := make([]func(http.Handler) http.Handler, 0, 1)
|
||||
apiReadMiddlewares := make([]func(http.Handler) http.Handler, 0, 1)
|
||||
apiWriteMiddlewares := make([]func(http.Handler) http.Handler, 0, 1)
|
||||
if s.cfg.Security.RateLimitUI.Enabled {
|
||||
uiLimiter := NewRateLimiter(s.cfg.Security.RateLimitUI.RequestsPerMinute, s.cfg.Security.RateLimitUI.Burst)
|
||||
uiMiddlewares = append(uiMiddlewares, RateLimitMiddleware(uiLimiter, s.cfg.Security.TrustProxyHeaders, s.cfg.Security.TrustedProxyPrefixes))
|
||||
}
|
||||
if s.cfg.Security.RateLimitAPIRead.Enabled {
|
||||
apiReadLimiter := NewRateLimiter(s.cfg.Security.RateLimitAPIRead.RequestsPerMinute, s.cfg.Security.RateLimitAPIRead.Burst)
|
||||
apiReadMiddlewares = append(apiReadMiddlewares, RateLimitMiddleware(apiReadLimiter, s.cfg.Security.TrustProxyHeaders, s.cfg.Security.TrustedProxyPrefixes))
|
||||
}
|
||||
if s.cfg.Security.RateLimitAPIWrite.Enabled {
|
||||
apiWriteLimiter := NewRateLimiter(s.cfg.Security.RateLimitAPIWrite.RequestsPerMinute, s.cfg.Security.RateLimitAPIWrite.Burst)
|
||||
apiWriteMiddlewares = append(apiWriteMiddlewares, RateLimitMiddleware(apiWriteLimiter, s.cfg.Security.TrustProxyHeaders, s.cfg.Security.TrustedProxyPrefixes))
|
||||
}
|
||||
|
||||
mux.Handle("GET /{$}", Chain(http.HandlerFunc(s.serveSPA), uiMiddlewares...))
|
||||
mux.Handle("GET /u", Chain(http.HandlerFunc(s.serveSPA), uiMiddlewares...))
|
||||
mux.Handle("GET /s/{id}", Chain(http.HandlerFunc(s.scratchPage), uiMiddlewares...))
|
||||
|
||||
mux.Handle("GET /api/config", Chain(http.HandlerFunc(s.getUIConfig), apiReadMiddlewares...))
|
||||
mux.Handle("GET /api/scratch/{id}", Chain(http.HandlerFunc(s.getScratch), apiReadMiddlewares...))
|
||||
mux.Handle("GET /api/raw/{id}", Chain(http.HandlerFunc(s.rawScratch), apiReadMiddlewares...))
|
||||
|
||||
writeHandler := http.HandlerFunc(s.createScratch)
|
||||
middlewares := make([]func(http.Handler) http.Handler, 0, 2)
|
||||
middlewares = append(middlewares, AllowlistMiddleware(s.cfg.Security.AllowedPrefixes, s.cfg.Security.TrustProxyHeaders, s.cfg.Security.TrustedProxyPrefixes, s.logger))
|
||||
readMiddlewares := make([]func(http.Handler) http.Handler, 0, 1)
|
||||
if s.cfg.Security.RateLimit.Enabled {
|
||||
writeLimiter := NewRateLimiter(s.cfg.Security.RateLimit.RequestsPerMinute, s.cfg.Security.RateLimit.Burst)
|
||||
readLimiter := NewRateLimiter(s.cfg.Security.RateLimit.RequestsPerMinute, s.cfg.Security.RateLimit.Burst)
|
||||
middlewares = append(middlewares, RateLimitMiddleware(writeLimiter, s.cfg.Security.TrustProxyHeaders, s.cfg.Security.TrustedProxyPrefixes))
|
||||
readMiddlewares = append(readMiddlewares, RateLimitMiddleware(readLimiter, s.cfg.Security.TrustProxyHeaders, s.cfg.Security.TrustedProxyPrefixes))
|
||||
}
|
||||
mux.Handle("POST /api/scratch", Chain(writeHandler, middlewares...))
|
||||
mux.Handle("GET /api/scratch/{id}", Chain(http.HandlerFunc(s.getScratch), readMiddlewares...))
|
||||
mux.Handle("GET /s/{id}", Chain(http.HandlerFunc(s.scratchPage), readMiddlewares...))
|
||||
mux.Handle("GET /api/raw/{id}", Chain(http.HandlerFunc(s.rawScratch), readMiddlewares...))
|
||||
writeMiddlewares := make([]func(http.Handler) http.Handler, 0, 2)
|
||||
writeMiddlewares = append(writeMiddlewares, AllowlistMiddleware(s.cfg.Security.AllowedPrefixes, s.cfg.Security.TrustProxyHeaders, s.cfg.Security.TrustedProxyPrefixes, s.logger))
|
||||
writeMiddlewares = append(writeMiddlewares, apiWriteMiddlewares...)
|
||||
mux.Handle("POST /api/scratch", Chain(writeHandler, writeMiddlewares...))
|
||||
|
||||
mux.Handle("GET /assets/", http.FileServerFS(webassets.StaticFS()))
|
||||
mux.Handle("GET /static/", http.StripPrefix("/static/", http.FileServerFS(webassets.StaticFS())))
|
||||
mux.Handle("GET /{path...}", http.HandlerFunc(s.notFoundPage))
|
||||
return mux
|
||||
return AccessLogMiddleware(
|
||||
s.accessLogger,
|
||||
s.cfg.Security.TrustProxyHeaders,
|
||||
s.cfg.Security.TrustedProxyPrefixes,
|
||||
)(mux)
|
||||
}
|
||||
|
||||
func (s *Server) serveSPA(w http.ResponseWriter, r *http.Request) {
|
||||
@@ -68,6 +85,7 @@ func (s *Server) getUIConfig(w http.ResponseWriter, r *http.Request) {
|
||||
payload := map[string]any{
|
||||
"max_upload_size_bytes": s.cfg.Limits.MaxUploadSizeBytes,
|
||||
"upload_allowed": s.isUploadAllowedForRequest(r),
|
||||
"default_ttl": s.cfg.Limits.DefaultTTL,
|
||||
}
|
||||
writeJSON(w, http.StatusOK, payload)
|
||||
}
|
||||
|
||||
@@ -162,7 +162,7 @@ func TestReadRoutesAreRateLimitedWhenEnabled(t *testing.T) {
|
||||
|
||||
createReq := httptest.NewRequest(http.MethodPost, "/api/scratch", bytes.NewBufferString("hello scratch"))
|
||||
createReq.Header.Set("Content-Type", "text/plain; charset=utf-8")
|
||||
createReq.RemoteAddr = "127.0.0.1:9111"
|
||||
createReq.RemoteAddr = "127.0.0.2:9111"
|
||||
createRec := httptest.NewRecorder()
|
||||
handler.ServeHTTP(createRec, createReq)
|
||||
if createRec.Code != http.StatusCreated {
|
||||
@@ -1041,7 +1041,7 @@ func newTestHandlerAndStore(t *testing.T, opts testServerOptions) (http.Handler,
|
||||
t.Helper()
|
||||
|
||||
dataDir := filepath.Join(t.TempDir(), "data")
|
||||
store, err := storage.NewFilesystemStore(dataDir)
|
||||
store, err := storage.NewFilesystemStore(dataDir, testStorageKey)
|
||||
if err != nil {
|
||||
t.Fatalf("NewFilesystemStore() error = %v", err)
|
||||
}
|
||||
@@ -1060,7 +1060,17 @@ func newTestHandlerAndStore(t *testing.T, opts testServerOptions) (http.Handler,
|
||||
DataDir: dataDir,
|
||||
},
|
||||
Security: config.SecurityConfig{
|
||||
RateLimit: config.RateLimitConfig{
|
||||
RateLimitUI: config.RateLimitConfig{
|
||||
Enabled: opts.rateLimitEnable,
|
||||
RequestsPerMinute: 120,
|
||||
Burst: 1,
|
||||
},
|
||||
RateLimitAPIRead: config.RateLimitConfig{
|
||||
Enabled: opts.rateLimitEnable,
|
||||
RequestsPerMinute: 120,
|
||||
Burst: 1,
|
||||
},
|
||||
RateLimitAPIWrite: config.RateLimitConfig{
|
||||
Enabled: opts.rateLimitEnable,
|
||||
RequestsPerMinute: 120,
|
||||
Burst: 1,
|
||||
|
||||
@@ -15,6 +15,8 @@ import (
|
||||
"scratchbox/internal/storage"
|
||||
)
|
||||
|
||||
var testStorageKey = []byte("0123456789abcdef0123456789abcdef")
|
||||
|
||||
func TestMultipartFileCount(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
@@ -49,18 +51,20 @@ func TestWriteCreateErrorBranches(t *testing.T) {
|
||||
|
||||
func TestNewServerInitializes(t *testing.T) {
|
||||
tmp := t.TempDir()
|
||||
store, err := storage.NewFilesystemStore(filepath.Join(tmp, "data"))
|
||||
store, err := storage.NewFilesystemStore(filepath.Join(tmp, "data"), testStorageKey)
|
||||
if err != nil {
|
||||
t.Fatalf("NewFilesystemStore() error = %v", err)
|
||||
}
|
||||
cfg := config.Config{
|
||||
Limits: config.LimitsConfig{MaxUploadSizeBytes: 1024},
|
||||
Security: config.SecurityConfig{
|
||||
RateLimit: config.RateLimitConfig{Enabled: false},
|
||||
RateLimitUI: config.RateLimitConfig{Enabled: false, RequestsPerMinute: 30, Burst: 10},
|
||||
RateLimitAPIRead: config.RateLimitConfig{Enabled: false, RequestsPerMinute: 30, Burst: 10},
|
||||
RateLimitAPIWrite: config.RateLimitConfig{Enabled: false, RequestsPerMinute: 30, Burst: 10},
|
||||
},
|
||||
}
|
||||
|
||||
srv, err := NewServer(cfg, store, log.New(io.Discard, "", 0))
|
||||
srv, err := NewServer(cfg, store, log.New(io.Discard, "", 0), log.New(io.Discard, "", 0))
|
||||
if err != nil {
|
||||
t.Fatalf("NewServer() error = %v", err)
|
||||
}
|
||||
@@ -118,17 +122,19 @@ func TestGetUIConfigUploadDenied(t *testing.T) {
|
||||
|
||||
func TestWriteCreateErrorBranchesWithConfig(t *testing.T) {
|
||||
tmp := t.TempDir()
|
||||
store, err := storage.NewFilesystemStore(filepath.Join(tmp, "data"))
|
||||
store, err := storage.NewFilesystemStore(filepath.Join(tmp, "data"), testStorageKey)
|
||||
if err != nil {
|
||||
t.Fatalf("NewFilesystemStore() error = %v", err)
|
||||
}
|
||||
cfg := config.Config{
|
||||
Limits: config.LimitsConfig{MaxUploadSizeBytes: 1024},
|
||||
Security: config.SecurityConfig{
|
||||
RateLimit: config.RateLimitConfig{Enabled: false},
|
||||
RateLimitUI: config.RateLimitConfig{Enabled: false, RequestsPerMinute: 30, Burst: 10},
|
||||
RateLimitAPIRead: config.RateLimitConfig{Enabled: false, RequestsPerMinute: 30, Burst: 10},
|
||||
RateLimitAPIWrite: config.RateLimitConfig{Enabled: false, RequestsPerMinute: 30, Burst: 10},
|
||||
},
|
||||
}
|
||||
if _, err := NewServer(cfg, store, log.New(io.Discard, "", 0)); err != nil {
|
||||
if _, err := NewServer(cfg, store, log.New(io.Discard, "", 0), log.New(io.Discard, "", 0)); err != nil {
|
||||
t.Fatalf("NewServer() error = %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -107,6 +107,53 @@ func RateLimitMiddleware(limiter *RateLimiter, trustProxyHeaders bool, trustedPr
|
||||
}
|
||||
}
|
||||
|
||||
func AccessLogMiddleware(logger *log.Logger, trustProxyHeaders bool, trustedProxies []netip.Prefix) func(http.Handler) http.Handler {
|
||||
return func(next http.Handler) http.Handler {
|
||||
if logger == nil {
|
||||
return next
|
||||
}
|
||||
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
if !shouldLogAccessRequest(r.Method, r.URL.Path) {
|
||||
next.ServeHTTP(w, r)
|
||||
return
|
||||
}
|
||||
|
||||
start := time.Now()
|
||||
recorder := &accessLogResponseWriter{ResponseWriter: w}
|
||||
next.ServeHTTP(recorder, r)
|
||||
|
||||
status := recorder.status
|
||||
if status == 0 {
|
||||
status = http.StatusOK
|
||||
}
|
||||
clientIP := "-"
|
||||
if ip, ok := extractClientIP(r, trustProxyHeaders, trustedProxies); ok {
|
||||
clientIP = ip.String()
|
||||
}
|
||||
logger.Printf(
|
||||
"method=%s path=%s status=%d bytes=%d duration_ms=%d ip=%s ua=%q",
|
||||
r.Method,
|
||||
r.URL.RequestURI(),
|
||||
status,
|
||||
recorder.bytes,
|
||||
time.Since(start).Milliseconds(),
|
||||
clientIP,
|
||||
r.UserAgent(),
|
||||
)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func shouldLogAccessRequest(method string, path string) bool {
|
||||
if method == http.MethodPost && path == "/api/scratch" {
|
||||
return true
|
||||
}
|
||||
if method == http.MethodGet && strings.HasPrefix(path, "/api/raw/") && len(path) > len("/api/raw/") {
|
||||
return true
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func Chain(handler http.Handler, middlewares ...func(http.Handler) http.Handler) http.Handler {
|
||||
wrapped := handler
|
||||
for i := len(middlewares) - 1; i >= 0; i-- {
|
||||
@@ -115,6 +162,26 @@ func Chain(handler http.Handler, middlewares ...func(http.Handler) http.Handler)
|
||||
return wrapped
|
||||
}
|
||||
|
||||
type accessLogResponseWriter struct {
|
||||
http.ResponseWriter
|
||||
status int
|
||||
bytes int
|
||||
}
|
||||
|
||||
func (w *accessLogResponseWriter) WriteHeader(statusCode int) {
|
||||
w.status = statusCode
|
||||
w.ResponseWriter.WriteHeader(statusCode)
|
||||
}
|
||||
|
||||
func (w *accessLogResponseWriter) Write(p []byte) (int, error) {
|
||||
if w.status == 0 {
|
||||
w.status = http.StatusOK
|
||||
}
|
||||
n, err := w.ResponseWriter.Write(p)
|
||||
w.bytes += n
|
||||
return n, err
|
||||
}
|
||||
|
||||
func extractClientIP(r *http.Request, trustProxyHeaders bool, trustedProxies []netip.Prefix) (netip.Addr, bool) {
|
||||
remoteIP, ok := remoteIPFromRequest(r)
|
||||
if !ok {
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
package httpapi
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/json"
|
||||
"io"
|
||||
"log"
|
||||
@@ -228,3 +229,95 @@ func TestExtractClientIPIgnoresHeadersFromUntrustedProxy(t *testing.T) {
|
||||
t.Fatalf("extractClientIP() = (%v,%v), want (203.0.113.10,true)", ip, ok)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAccessLogMiddlewareWritesStructuredLog(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
var buf bytes.Buffer
|
||||
logger := log.New(&buf, "", 0)
|
||||
handler := AccessLogMiddleware(logger, false, nil)(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
|
||||
w.WriteHeader(http.StatusCreated)
|
||||
_, _ = w.Write([]byte("ok"))
|
||||
}))
|
||||
|
||||
req := httptest.NewRequest(http.MethodPost, "/api/scratch?ttl=1h", nil)
|
||||
req.RemoteAddr = "198.51.100.12:4444"
|
||||
req.Header.Set("User-Agent", "middleware-test")
|
||||
rec := httptest.NewRecorder()
|
||||
handler.ServeHTTP(rec, req)
|
||||
|
||||
got := buf.String()
|
||||
if !strings.Contains(got, "method=POST") {
|
||||
t.Fatalf("missing method field in log: %q", got)
|
||||
}
|
||||
if !strings.Contains(got, "path=/api/scratch?ttl=1h") {
|
||||
t.Fatalf("missing path field in log: %q", got)
|
||||
}
|
||||
if !strings.Contains(got, "status=201") {
|
||||
t.Fatalf("missing status field in log: %q", got)
|
||||
}
|
||||
if !strings.Contains(got, "bytes=2") {
|
||||
t.Fatalf("missing bytes field in log: %q", got)
|
||||
}
|
||||
if !strings.Contains(got, "ip=198.51.100.12") {
|
||||
t.Fatalf("missing ip field in log: %q", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAccessLogMiddlewareSkipsConfigAndStaticPaths(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
var buf bytes.Buffer
|
||||
logger := log.New(&buf, "", 0)
|
||||
handler := AccessLogMiddleware(logger, false, nil)(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
|
||||
w.WriteHeader(http.StatusOK)
|
||||
}))
|
||||
|
||||
reqConfig := httptest.NewRequest(http.MethodGet, "/api/config", nil)
|
||||
reqConfig.RemoteAddr = "198.51.100.12:4444"
|
||||
handler.ServeHTTP(httptest.NewRecorder(), reqConfig)
|
||||
|
||||
reqStatic := httptest.NewRequest(http.MethodGet, "/static/app.js", nil)
|
||||
reqStatic.RemoteAddr = "198.51.100.12:4444"
|
||||
handler.ServeHTTP(httptest.NewRecorder(), reqStatic)
|
||||
|
||||
reqAsset := httptest.NewRequest(http.MethodGet, "/assets/logo.svg", nil)
|
||||
reqAsset.RemoteAddr = "198.51.100.12:4444"
|
||||
handler.ServeHTTP(httptest.NewRecorder(), reqAsset)
|
||||
|
||||
reqScratchView := httptest.NewRequest(http.MethodGet, "/s/abc123", nil)
|
||||
reqScratchView.RemoteAddr = "198.51.100.12:4444"
|
||||
handler.ServeHTTP(httptest.NewRecorder(), reqScratchView)
|
||||
|
||||
reqUpload := httptest.NewRequest(http.MethodGet, "/u", nil)
|
||||
reqUpload.RemoteAddr = "198.51.100.12:4444"
|
||||
handler.ServeHTTP(httptest.NewRecorder(), reqUpload)
|
||||
|
||||
if buf.Len() != 0 {
|
||||
t.Fatalf("expected no access logs for skipped paths, got %q", buf.String())
|
||||
}
|
||||
}
|
||||
|
||||
func TestAccessLogMiddlewareOnlyAllowsConfiguredRoutes(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
var buf bytes.Buffer
|
||||
logger := log.New(&buf, "", 0)
|
||||
handler := AccessLogMiddleware(logger, false, nil)(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
|
||||
w.WriteHeader(http.StatusOK)
|
||||
}))
|
||||
|
||||
disallowed := httptest.NewRequest(http.MethodGet, "/api/scratch/some-id", nil)
|
||||
disallowed.RemoteAddr = "198.51.100.12:4444"
|
||||
handler.ServeHTTP(httptest.NewRecorder(), disallowed)
|
||||
if buf.Len() != 0 {
|
||||
t.Fatalf("expected no access log for disallowed route, got %q", buf.String())
|
||||
}
|
||||
|
||||
allowed := httptest.NewRequest(http.MethodGet, "/api/raw/some-id", nil)
|
||||
allowed.RemoteAddr = "198.51.100.12:4444"
|
||||
handler.ServeHTTP(httptest.NewRecorder(), allowed)
|
||||
if !strings.Contains(buf.String(), "method=GET path=/api/raw/some-id") {
|
||||
t.Fatalf("expected access log for allowed route, got %q", buf.String())
|
||||
}
|
||||
}
|
||||
|
||||
+388
-134
@@ -3,7 +3,11 @@ package storage
|
||||
import (
|
||||
"compress/gzip"
|
||||
"context"
|
||||
"crypto/aes"
|
||||
"crypto/cipher"
|
||||
"crypto/rand"
|
||||
"crypto/sha256"
|
||||
"encoding/binary"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
@@ -17,14 +21,16 @@ import (
|
||||
)
|
||||
|
||||
const (
|
||||
indexFilename = "metadata.json"
|
||||
filesDirName = "scratches"
|
||||
idBytes = 12
|
||||
maxCreateIDAttempts = 100
|
||||
indexFilename = "metadata"
|
||||
filesDirName = "scratches"
|
||||
encryptedChunkSize = 64 * 1024
|
||||
)
|
||||
|
||||
var ErrNotFound = errors.New("scratch not found")
|
||||
|
||||
var encryptedFileMagic = [4]byte{'S', 'B', 'X', '1'}
|
||||
var encryptedIndexMagic = [4]byte{'S', 'M', 'D', '1'}
|
||||
|
||||
type Metadata struct {
|
||||
ID string `json:"id"`
|
||||
FilePath string `json:"file_path"`
|
||||
@@ -35,35 +41,63 @@ type Metadata struct {
|
||||
Size int64 `json:"size"`
|
||||
}
|
||||
|
||||
type FilesystemStore struct {
|
||||
dataDir string
|
||||
filesDir string
|
||||
index string
|
||||
idGenerator func() (string, error)
|
||||
|
||||
mu sync.RWMutex
|
||||
metadata map[string]Metadata
|
||||
reservedIDs map[string]struct{}
|
||||
type indexDocument struct {
|
||||
DefaultTTL string `json:"default_ttl,omitempty"`
|
||||
Scratches map[string]Metadata `json:"scratches"`
|
||||
}
|
||||
|
||||
func NewFilesystemStore(dataDir string) (*FilesystemStore, error) {
|
||||
type FilesystemStore struct {
|
||||
dataDir string
|
||||
filesDir string
|
||||
index string
|
||||
defaultTTL time.Duration
|
||||
fileCipher cipher.AEAD
|
||||
|
||||
mu sync.RWMutex
|
||||
metadata map[string]Metadata
|
||||
}
|
||||
|
||||
func NewFilesystemStore(dataDir string, encryptionKey []byte) (*FilesystemStore, error) {
|
||||
return newFilesystemStore(dataDir, 0, encryptionKey)
|
||||
}
|
||||
|
||||
func NewFilesystemStoreWithDefaultTTL(dataDir string, defaultTTL time.Duration, encryptionKey []byte) (*FilesystemStore, error) {
|
||||
return newFilesystemStore(dataDir, defaultTTL, encryptionKey)
|
||||
}
|
||||
|
||||
func newFilesystemStore(dataDir string, defaultTTL time.Duration, encryptionKey []byte) (*FilesystemStore, error) {
|
||||
filesDir := filepath.Join(dataDir, filesDirName)
|
||||
if err := os.MkdirAll(filesDir, 0o755); err != nil {
|
||||
return nil, fmt.Errorf("create files dir: %w", err)
|
||||
}
|
||||
|
||||
if len(encryptionKey) != 32 {
|
||||
return nil, fmt.Errorf("storage encryption key must be 32 bytes, got %d", len(encryptionKey))
|
||||
}
|
||||
block, err := aes.NewCipher(encryptionKey)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("init storage cipher: %w", err)
|
||||
}
|
||||
fileCipher, err := cipher.NewGCM(block)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("init storage aead: %w", err)
|
||||
}
|
||||
|
||||
store := &FilesystemStore{
|
||||
dataDir: dataDir,
|
||||
filesDir: filesDir,
|
||||
index: filepath.Join(dataDir, indexFilename),
|
||||
idGenerator: generateID,
|
||||
metadata: map[string]Metadata{},
|
||||
reservedIDs: map[string]struct{}{},
|
||||
dataDir: dataDir,
|
||||
filesDir: filesDir,
|
||||
index: filepath.Join(dataDir, indexFilename),
|
||||
fileCipher: fileCipher,
|
||||
metadata: map[string]Metadata{},
|
||||
}
|
||||
if err := store.loadIndex(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if err := store.reconcileOnStartup(time.Now()); err != nil {
|
||||
now := time.Now().UTC()
|
||||
if err := store.applyDefaultTTLOnStartup(defaultTTL); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if err := store.reconcileOnStartup(now); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return store, nil
|
||||
@@ -74,107 +108,84 @@ func (s *FilesystemStore) Create(ctx context.Context, body io.Reader, contentTyp
|
||||
}
|
||||
|
||||
func (s *FilesystemStore) CreateWithOriginalName(ctx context.Context, body io.Reader, contentType string, originalName string, ttl time.Duration) (Metadata, error) {
|
||||
for attempt := 0; attempt < maxCreateIDAttempts; attempt++ {
|
||||
id, err := s.idGenerator()
|
||||
if err != nil {
|
||||
return Metadata{}, fmt.Errorf("generate id: %w", err)
|
||||
}
|
||||
finalPath := filepath.Join(s.filesDir, id)
|
||||
|
||||
s.mu.RLock()
|
||||
_, exists := s.metadata[id]
|
||||
_, reserved := s.reservedIDs[id]
|
||||
s.mu.RUnlock()
|
||||
if exists || reserved {
|
||||
continue
|
||||
}
|
||||
if _, err := os.Stat(finalPath); err == nil {
|
||||
continue
|
||||
} else if !errors.Is(err, os.ErrNotExist) {
|
||||
return Metadata{}, fmt.Errorf("stat candidate scratch path: %w", err)
|
||||
}
|
||||
|
||||
s.mu.Lock()
|
||||
if _, exists := s.metadata[id]; exists {
|
||||
s.mu.Unlock()
|
||||
continue
|
||||
}
|
||||
if _, reserved := s.reservedIDs[id]; reserved {
|
||||
s.mu.Unlock()
|
||||
continue
|
||||
}
|
||||
// Reserve the ID before consuming the request body to avoid
|
||||
// concurrent writers choosing the same ID.
|
||||
s.reservedIDs[id] = struct{}{}
|
||||
s.mu.Unlock()
|
||||
|
||||
tempFile, err := os.CreateTemp(s.filesDir, id+".tmp-*")
|
||||
if err != nil {
|
||||
s.mu.Lock()
|
||||
delete(s.reservedIDs, id)
|
||||
s.mu.Unlock()
|
||||
return Metadata{}, fmt.Errorf("create temp file: %w", err)
|
||||
}
|
||||
|
||||
gzipWriter := gzip.NewWriter(tempFile)
|
||||
size, copyErr := io.Copy(gzipWriter, body)
|
||||
gzipCloseErr := gzipWriter.Close()
|
||||
closeErr := tempFile.Close()
|
||||
if copyErr != nil {
|
||||
_ = os.Remove(tempFile.Name())
|
||||
s.mu.Lock()
|
||||
delete(s.reservedIDs, id)
|
||||
s.mu.Unlock()
|
||||
return Metadata{}, fmt.Errorf("write scratch content: %w", copyErr)
|
||||
}
|
||||
if gzipCloseErr != nil {
|
||||
_ = os.Remove(tempFile.Name())
|
||||
s.mu.Lock()
|
||||
delete(s.reservedIDs, id)
|
||||
s.mu.Unlock()
|
||||
return Metadata{}, fmt.Errorf("close gzip writer: %w", gzipCloseErr)
|
||||
}
|
||||
if closeErr != nil {
|
||||
_ = os.Remove(tempFile.Name())
|
||||
s.mu.Lock()
|
||||
delete(s.reservedIDs, id)
|
||||
s.mu.Unlock()
|
||||
return Metadata{}, fmt.Errorf("close temp file: %w", closeErr)
|
||||
}
|
||||
|
||||
if err := os.Rename(tempFile.Name(), finalPath); err != nil {
|
||||
_ = os.Remove(tempFile.Name())
|
||||
s.mu.Lock()
|
||||
delete(s.reservedIDs, id)
|
||||
s.mu.Unlock()
|
||||
return Metadata{}, fmt.Errorf("atomically move scratch file: %w", err)
|
||||
}
|
||||
|
||||
now := time.Now().UTC()
|
||||
meta := Metadata{
|
||||
ID: id,
|
||||
FilePath: finalPath,
|
||||
CreatedAt: now,
|
||||
ExpiresAt: now.Add(ttl),
|
||||
ContentType: contentType,
|
||||
OriginalName: normalizeOriginalName(originalName),
|
||||
Size: size,
|
||||
}
|
||||
|
||||
s.mu.Lock()
|
||||
delete(s.reservedIDs, id)
|
||||
s.metadata[id] = meta
|
||||
if err := s.persistLocked(); err != nil {
|
||||
delete(s.metadata, id)
|
||||
s.mu.Unlock()
|
||||
_ = os.Remove(finalPath)
|
||||
return Metadata{}, err
|
||||
}
|
||||
s.mu.Unlock()
|
||||
return meta, nil
|
||||
tempFile, err := os.CreateTemp(s.filesDir, "scratch.tmp-*")
|
||||
if err != nil {
|
||||
return Metadata{}, fmt.Errorf("create temp file: %w", err)
|
||||
}
|
||||
|
||||
return Metadata{}, fmt.Errorf("failed to allocate unique scratch id after %d attempts", maxCreateIDAttempts)
|
||||
blobHash := sha256.New()
|
||||
blobSink := io.MultiWriter(tempFile, blobHash)
|
||||
scratchWriter := io.Writer(blobSink)
|
||||
encryptedWriter, err := newEncryptedFileWriter(blobSink, s.fileCipher)
|
||||
if err != nil {
|
||||
_ = tempFile.Close()
|
||||
_ = os.Remove(tempFile.Name())
|
||||
return Metadata{}, fmt.Errorf("init encrypted scratch writer: %w", err)
|
||||
}
|
||||
scratchWriter = encryptedWriter
|
||||
|
||||
gzipWriter := gzip.NewWriter(scratchWriter)
|
||||
size, copyErr := io.Copy(gzipWriter, body)
|
||||
gzipCloseErr := gzipWriter.Close()
|
||||
encryptCloseErr := encryptedWriter.Close()
|
||||
closeErr := tempFile.Close()
|
||||
if copyErr != nil {
|
||||
_ = os.Remove(tempFile.Name())
|
||||
return Metadata{}, fmt.Errorf("write scratch content: %w", copyErr)
|
||||
}
|
||||
if gzipCloseErr != nil {
|
||||
_ = os.Remove(tempFile.Name())
|
||||
return Metadata{}, fmt.Errorf("close gzip writer: %w", gzipCloseErr)
|
||||
}
|
||||
if encryptCloseErr != nil {
|
||||
_ = os.Remove(tempFile.Name())
|
||||
return Metadata{}, fmt.Errorf("close encrypted writer: %w", encryptCloseErr)
|
||||
}
|
||||
if closeErr != nil {
|
||||
_ = os.Remove(tempFile.Name())
|
||||
return Metadata{}, fmt.Errorf("close temp file: %w", closeErr)
|
||||
}
|
||||
|
||||
id := hex.EncodeToString(blobHash.Sum(nil))
|
||||
finalPath := filepath.Join(s.filesDir, id)
|
||||
if _, statErr := os.Stat(finalPath); statErr == nil {
|
||||
_ = os.Remove(tempFile.Name())
|
||||
return Metadata{}, errors.New("scratch id collision for content hash")
|
||||
} else if !errors.Is(statErr, os.ErrNotExist) {
|
||||
_ = os.Remove(tempFile.Name())
|
||||
return Metadata{}, fmt.Errorf("stat candidate scratch path: %w", statErr)
|
||||
}
|
||||
if err := os.Rename(tempFile.Name(), finalPath); err != nil {
|
||||
_ = os.Remove(tempFile.Name())
|
||||
return Metadata{}, fmt.Errorf("atomically move scratch file: %w", err)
|
||||
}
|
||||
|
||||
now := time.Now().UTC()
|
||||
meta := Metadata{
|
||||
ID: id,
|
||||
FilePath: finalPath,
|
||||
CreatedAt: now,
|
||||
ExpiresAt: now.Add(ttl),
|
||||
ContentType: contentType,
|
||||
OriginalName: normalizeOriginalName(originalName),
|
||||
Size: size,
|
||||
}
|
||||
|
||||
s.mu.Lock()
|
||||
if _, exists := s.metadata[id]; exists {
|
||||
s.mu.Unlock()
|
||||
_ = os.Remove(finalPath)
|
||||
return Metadata{}, errors.New("scratch id collision in metadata index")
|
||||
}
|
||||
s.metadata[id] = meta
|
||||
if err := s.persistLocked(); err != nil {
|
||||
delete(s.metadata, id)
|
||||
s.mu.Unlock()
|
||||
_ = os.Remove(finalPath)
|
||||
return Metadata{}, err
|
||||
}
|
||||
s.mu.Unlock()
|
||||
return meta, nil
|
||||
}
|
||||
|
||||
func normalizeOriginalName(originalName string) string {
|
||||
@@ -212,7 +223,12 @@ func (s *FilesystemStore) Open(id string) (io.ReadCloser, Metadata, error) {
|
||||
}
|
||||
return nil, Metadata{}, fmt.Errorf("open scratch file: %w", err)
|
||||
}
|
||||
gzipReader, err := gzip.NewReader(file)
|
||||
decryptedReader, decryptErr := newEncryptedFileReader(file, s.fileCipher)
|
||||
if decryptErr != nil {
|
||||
_ = file.Close()
|
||||
return nil, Metadata{}, fmt.Errorf("create encrypted reader: %w", decryptErr)
|
||||
}
|
||||
gzipReader, err := gzip.NewReader(decryptedReader)
|
||||
if err != nil {
|
||||
_ = file.Close()
|
||||
return nil, Metadata{}, fmt.Errorf("create gzip reader: %w", err)
|
||||
@@ -286,22 +302,46 @@ func (s *FilesystemStore) loadIndex() error {
|
||||
}
|
||||
return fmt.Errorf("read metadata index: %w", err)
|
||||
}
|
||||
plaintext, err := decryptIndexPayload(b, s.fileCipher)
|
||||
if err != nil {
|
||||
return fmt.Errorf("decrypt metadata index: %w", err)
|
||||
}
|
||||
|
||||
loaded := map[string]Metadata{}
|
||||
if err := json.Unmarshal(b, &loaded); err != nil {
|
||||
doc := indexDocument{}
|
||||
if err := json.Unmarshal(plaintext, &doc); err != nil {
|
||||
return fmt.Errorf("decode metadata index: %w", err)
|
||||
}
|
||||
s.metadata = loaded
|
||||
if doc.Scratches == nil {
|
||||
doc.Scratches = map[string]Metadata{}
|
||||
}
|
||||
s.metadata = doc.Scratches
|
||||
if ttlRaw := strings.TrimSpace(doc.DefaultTTL); ttlRaw != "" {
|
||||
ttl, parseErr := time.ParseDuration(ttlRaw)
|
||||
if parseErr != nil {
|
||||
return fmt.Errorf("decode metadata index default ttl: %w", parseErr)
|
||||
}
|
||||
s.defaultTTL = ttl
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *FilesystemStore) persistLocked() error {
|
||||
tmp := s.index + ".tmp"
|
||||
payload, err := json.MarshalIndent(s.metadata, "", " ")
|
||||
doc := indexDocument{
|
||||
Scratches: s.metadata,
|
||||
}
|
||||
if s.defaultTTL > 0 {
|
||||
doc.DefaultTTL = s.defaultTTL.String()
|
||||
}
|
||||
payload, err := json.MarshalIndent(doc, "", " ")
|
||||
if err != nil {
|
||||
return fmt.Errorf("encode metadata index: %w", err)
|
||||
}
|
||||
if err := os.WriteFile(tmp, payload, 0o644); err != nil {
|
||||
encryptedPayload, err := encryptIndexPayload(payload, s.fileCipher)
|
||||
if err != nil {
|
||||
return fmt.Errorf("encrypt metadata index: %w", err)
|
||||
}
|
||||
if err := os.WriteFile(tmp, encryptedPayload, 0o644); err != nil {
|
||||
return fmt.Errorf("write metadata temp file: %w", err)
|
||||
}
|
||||
if err := os.Rename(tmp, s.index); err != nil {
|
||||
@@ -310,6 +350,28 @@ func (s *FilesystemStore) persistLocked() error {
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *FilesystemStore) applyDefaultTTLOnStartup(defaultTTL time.Duration) error {
|
||||
if defaultTTL <= 0 {
|
||||
return nil
|
||||
}
|
||||
|
||||
if s.defaultTTL <= 0 {
|
||||
s.defaultTTL = defaultTTL
|
||||
return s.persistLocked()
|
||||
}
|
||||
if s.defaultTTL == defaultTTL {
|
||||
return nil
|
||||
}
|
||||
|
||||
delta := defaultTTL - s.defaultTTL
|
||||
for id, meta := range s.metadata {
|
||||
meta.ExpiresAt = meta.ExpiresAt.Add(delta)
|
||||
s.metadata[id] = meta
|
||||
}
|
||||
s.defaultTTL = defaultTTL
|
||||
return s.persistLocked()
|
||||
}
|
||||
|
||||
func (s *FilesystemStore) reconcileOnStartup(now time.Time) error {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
@@ -337,14 +399,6 @@ func (s *FilesystemStore) reconcileOnStartup(now time.Time) error {
|
||||
return s.persistLocked()
|
||||
}
|
||||
|
||||
func generateID() (string, error) {
|
||||
b := make([]byte, idBytes)
|
||||
if _, err := rand.Read(b); err != nil {
|
||||
return "", err
|
||||
}
|
||||
return hex.EncodeToString(b), nil
|
||||
}
|
||||
|
||||
type combinedReadCloser struct {
|
||||
reader io.Reader
|
||||
closers []io.Closer
|
||||
@@ -363,3 +417,203 @@ func (c *combinedReadCloser) Close() error {
|
||||
}
|
||||
return firstErr
|
||||
}
|
||||
|
||||
type encryptedFileWriter struct {
|
||||
dst io.Writer
|
||||
aead cipher.AEAD
|
||||
noncePrefix [4]byte
|
||||
counter uint64
|
||||
pending []byte
|
||||
}
|
||||
|
||||
func newEncryptedFileWriter(dst io.Writer, aead cipher.AEAD) (*encryptedFileWriter, error) {
|
||||
writer := &encryptedFileWriter{
|
||||
dst: dst,
|
||||
aead: aead,
|
||||
pending: make([]byte, 0, encryptedChunkSize),
|
||||
}
|
||||
if _, err := rand.Read(writer.noncePrefix[:]); err != nil {
|
||||
return nil, fmt.Errorf("generate nonce prefix: %w", err)
|
||||
}
|
||||
header := make([]byte, 0, len(encryptedFileMagic)+len(writer.noncePrefix))
|
||||
header = append(header, encryptedFileMagic[:]...)
|
||||
header = append(header, writer.noncePrefix[:]...)
|
||||
if _, err := writer.dst.Write(header); err != nil {
|
||||
return nil, fmt.Errorf("write encrypted file header: %w", err)
|
||||
}
|
||||
return writer, nil
|
||||
}
|
||||
|
||||
func (w *encryptedFileWriter) Write(p []byte) (int, error) {
|
||||
written := len(p)
|
||||
for len(p) > 0 {
|
||||
space := encryptedChunkSize - len(w.pending)
|
||||
if space > len(p) {
|
||||
space = len(p)
|
||||
}
|
||||
w.pending = append(w.pending, p[:space]...)
|
||||
p = p[space:]
|
||||
if len(w.pending) == encryptedChunkSize {
|
||||
if err := w.flushChunk(w.pending); err != nil {
|
||||
return written - len(p), err
|
||||
}
|
||||
w.pending = w.pending[:0]
|
||||
}
|
||||
}
|
||||
return written, nil
|
||||
}
|
||||
|
||||
func (w *encryptedFileWriter) Close() error {
|
||||
if len(w.pending) == 0 {
|
||||
return nil
|
||||
}
|
||||
if err := w.flushChunk(w.pending); err != nil {
|
||||
return err
|
||||
}
|
||||
w.pending = w.pending[:0]
|
||||
return nil
|
||||
}
|
||||
|
||||
func (w *encryptedFileWriter) flushChunk(plaintext []byte) error {
|
||||
nonce := make([]byte, w.aead.NonceSize())
|
||||
copy(nonce, w.noncePrefix[:])
|
||||
binary.BigEndian.PutUint64(nonce[len(w.noncePrefix):], w.counter)
|
||||
ciphertext := w.aead.Seal(nil, nonce, plaintext, nil)
|
||||
|
||||
var sizeBuf [4]byte
|
||||
binary.BigEndian.PutUint32(sizeBuf[:], uint32(len(ciphertext)))
|
||||
if _, err := w.dst.Write(sizeBuf[:]); err != nil {
|
||||
return fmt.Errorf("write encrypted chunk size: %w", err)
|
||||
}
|
||||
if _, err := w.dst.Write(ciphertext); err != nil {
|
||||
return fmt.Errorf("write encrypted chunk payload: %w", err)
|
||||
}
|
||||
w.counter++
|
||||
return nil
|
||||
}
|
||||
|
||||
type encryptedFileReader struct {
|
||||
src io.Reader
|
||||
aead cipher.AEAD
|
||||
noncePrefix [4]byte
|
||||
counter uint64
|
||||
buf []byte
|
||||
offset int
|
||||
eof bool
|
||||
}
|
||||
|
||||
func newEncryptedFileReader(src io.Reader, aead cipher.AEAD) (*encryptedFileReader, error) {
|
||||
reader := &encryptedFileReader{
|
||||
src: src,
|
||||
aead: aead,
|
||||
}
|
||||
header := make([]byte, len(encryptedFileMagic)+len(reader.noncePrefix))
|
||||
if _, err := io.ReadFull(src, header); err != nil {
|
||||
return nil, fmt.Errorf("read encrypted file header: %w", err)
|
||||
}
|
||||
if !equalBytes(header[:len(encryptedFileMagic)], encryptedFileMagic[:]) {
|
||||
return nil, errors.New("invalid encrypted file header")
|
||||
}
|
||||
copy(reader.noncePrefix[:], header[len(encryptedFileMagic):])
|
||||
return reader, nil
|
||||
}
|
||||
|
||||
func (r *encryptedFileReader) Read(p []byte) (int, error) {
|
||||
for r.offset >= len(r.buf) {
|
||||
if r.eof {
|
||||
return 0, io.EOF
|
||||
}
|
||||
if err := r.loadNextChunk(); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
}
|
||||
n := copy(p, r.buf[r.offset:])
|
||||
r.offset += n
|
||||
return n, nil
|
||||
}
|
||||
|
||||
func (r *encryptedFileReader) loadNextChunk() error {
|
||||
var sizeBuf [4]byte
|
||||
_, err := io.ReadFull(r.src, sizeBuf[:])
|
||||
if err != nil {
|
||||
if errors.Is(err, io.EOF) {
|
||||
r.eof = true
|
||||
return io.EOF
|
||||
}
|
||||
if errors.Is(err, io.ErrUnexpectedEOF) {
|
||||
return errors.New("truncated encrypted chunk size")
|
||||
}
|
||||
return fmt.Errorf("read encrypted chunk size: %w", err)
|
||||
}
|
||||
chunkSize := binary.BigEndian.Uint32(sizeBuf[:])
|
||||
if chunkSize == 0 {
|
||||
return errors.New("invalid empty encrypted chunk")
|
||||
}
|
||||
maxChunkSize := uint32(encryptedChunkSize + r.aead.Overhead())
|
||||
if chunkSize > maxChunkSize {
|
||||
return fmt.Errorf("encrypted chunk too large: %d", chunkSize)
|
||||
}
|
||||
ciphertext := make([]byte, chunkSize)
|
||||
if _, err := io.ReadFull(r.src, ciphertext); err != nil {
|
||||
return fmt.Errorf("read encrypted chunk payload: %w", err)
|
||||
}
|
||||
nonce := make([]byte, r.aead.NonceSize())
|
||||
copy(nonce, r.noncePrefix[:])
|
||||
binary.BigEndian.PutUint64(nonce[len(r.noncePrefix):], r.counter)
|
||||
plaintext, err := r.aead.Open(nil, nonce, ciphertext, nil)
|
||||
if err != nil {
|
||||
return fmt.Errorf("decrypt chunk: %w", err)
|
||||
}
|
||||
r.counter++
|
||||
r.buf = plaintext
|
||||
r.offset = 0
|
||||
return nil
|
||||
}
|
||||
|
||||
func equalBytes(a, b []byte) bool {
|
||||
if len(a) != len(b) {
|
||||
return false
|
||||
}
|
||||
for i := range a {
|
||||
if a[i] != b[i] {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
func encryptIndexPayload(plaintext []byte, aead cipher.AEAD) ([]byte, error) {
|
||||
nonce := make([]byte, aead.NonceSize())
|
||||
if _, err := rand.Read(nonce); err != nil {
|
||||
return nil, fmt.Errorf("generate index nonce: %w", err)
|
||||
}
|
||||
ciphertext := aead.Seal(nil, nonce, plaintext, nil)
|
||||
out := make([]byte, 0, len(encryptedIndexMagic)+len(nonce)+len(ciphertext))
|
||||
out = append(out, encryptedIndexMagic[:]...)
|
||||
out = append(out, nonce...)
|
||||
out = append(out, ciphertext...)
|
||||
return out, nil
|
||||
}
|
||||
|
||||
func decryptIndexPayload(raw []byte, aead cipher.AEAD) ([]byte, error) {
|
||||
headerLen := len(encryptedIndexMagic)
|
||||
nonceSize := aead.NonceSize()
|
||||
if len(raw) < headerLen+nonceSize {
|
||||
return nil, errors.New("encrypted index is too short")
|
||||
}
|
||||
if !equalBytes(raw[:headerLen], encryptedIndexMagic[:]) {
|
||||
return nil, errors.New("invalid encrypted index header")
|
||||
}
|
||||
nonceStart := headerLen
|
||||
nonceEnd := nonceStart + nonceSize
|
||||
nonce := raw[nonceStart:nonceEnd]
|
||||
ciphertext := raw[nonceEnd:]
|
||||
if len(ciphertext) == 0 {
|
||||
return nil, errors.New("encrypted index ciphertext is empty")
|
||||
}
|
||||
plaintext, err := aead.Open(nil, nonce, ciphertext, nil)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("decrypt index payload: %w", err)
|
||||
}
|
||||
return plaintext, nil
|
||||
}
|
||||
|
||||
+212
-278
@@ -3,6 +3,11 @@ package storage
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"crypto/aes"
|
||||
"crypto/cipher"
|
||||
"crypto/rand"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
@@ -15,10 +20,25 @@ import (
|
||||
"time"
|
||||
)
|
||||
|
||||
var testEncryptionKey = []byte("0123456789abcdef0123456789abcdef")
|
||||
|
||||
func mustTestAEAD(t *testing.T) cipher.AEAD {
|
||||
t.Helper()
|
||||
block, err := aes.NewCipher(testEncryptionKey)
|
||||
if err != nil {
|
||||
t.Fatalf("aes.NewCipher() error = %v", err)
|
||||
}
|
||||
aead, err := cipher.NewGCM(block)
|
||||
if err != nil {
|
||||
t.Fatalf("cipher.NewGCM() error = %v", err)
|
||||
}
|
||||
return aead
|
||||
}
|
||||
|
||||
func TestCreateStoresGzipAndOpenIsTransparent(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
store, err := NewFilesystemStore(filepath.Join(t.TempDir(), "data"))
|
||||
store, err := NewFilesystemStore(filepath.Join(t.TempDir(), "data"), testEncryptionKey)
|
||||
if err != nil {
|
||||
t.Fatalf("NewFilesystemStore() error = %v", err)
|
||||
}
|
||||
@@ -36,8 +56,8 @@ func TestCreateStoresGzipAndOpenIsTransparent(t *testing.T) {
|
||||
if err != nil {
|
||||
t.Fatalf("ReadFile() error = %v", err)
|
||||
}
|
||||
if len(onDisk) < 2 || onDisk[0] != 0x1f || onDisk[1] != 0x8b {
|
||||
t.Fatalf("stored file does not have gzip header")
|
||||
if len(onDisk) < 4 || string(onDisk[:4]) != "SBX1" {
|
||||
t.Fatalf("stored file does not have encrypted scratch header")
|
||||
}
|
||||
|
||||
reader, _, err := store.Open(meta.ID)
|
||||
@@ -54,10 +74,92 @@ func TestCreateStoresGzipAndOpenIsTransparent(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestCreateEncryptedStoreDoesNotExposeGzipHeaderOnDisk(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
key := make([]byte, 32)
|
||||
if _, err := rand.Read(key); err != nil {
|
||||
t.Fatalf("rand.Read() error = %v", err)
|
||||
}
|
||||
|
||||
dataDir := filepath.Join(t.TempDir(), "data")
|
||||
store, err := NewFilesystemStore(dataDir, key)
|
||||
if err != nil {
|
||||
t.Fatalf("NewEncryptedFilesystemStore() error = %v", err)
|
||||
}
|
||||
|
||||
original := bytes.Repeat([]byte("secret payload "), 1024)
|
||||
meta, err := store.Create(context.Background(), bytes.NewReader(original), "text/plain", time.Hour)
|
||||
if err != nil {
|
||||
t.Fatalf("Create() error = %v", err)
|
||||
}
|
||||
|
||||
onDisk, err := os.ReadFile(meta.FilePath)
|
||||
if err != nil {
|
||||
t.Fatalf("ReadFile() error = %v", err)
|
||||
}
|
||||
if len(onDisk) < 2 {
|
||||
t.Fatalf("encrypted file too short")
|
||||
}
|
||||
if onDisk[0] == 0x1f && onDisk[1] == 0x8b {
|
||||
t.Fatalf("encrypted file unexpectedly starts with gzip header")
|
||||
}
|
||||
|
||||
reader, _, err := store.Open(meta.ID)
|
||||
if err != nil {
|
||||
t.Fatalf("Open() error = %v", err)
|
||||
}
|
||||
defer reader.Close()
|
||||
got, err := io.ReadAll(reader)
|
||||
if err != nil {
|
||||
t.Fatalf("ReadAll() error = %v", err)
|
||||
}
|
||||
if !bytes.Equal(got, original) {
|
||||
t.Fatalf("Open() content mismatch: got %d bytes, want %d", len(got), len(original))
|
||||
}
|
||||
|
||||
filePayload, err := os.ReadFile(meta.FilePath)
|
||||
if err != nil {
|
||||
t.Fatalf("ReadFile(encrypted) error = %v", err)
|
||||
}
|
||||
sum := sha256.Sum256(filePayload)
|
||||
wantID := hex.EncodeToString(sum[:])
|
||||
if meta.ID != wantID {
|
||||
t.Fatalf("meta.ID = %q, want SHA-256 of encrypted blob %q", meta.ID, wantID)
|
||||
}
|
||||
}
|
||||
|
||||
func TestEncryptedStoreFailsToOpenWithWrongKey(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
keyA := make([]byte, 32)
|
||||
if _, err := rand.Read(keyA); err != nil {
|
||||
t.Fatalf("rand.Read(keyA) error = %v", err)
|
||||
}
|
||||
keyB := make([]byte, 32)
|
||||
if _, err := rand.Read(keyB); err != nil {
|
||||
t.Fatalf("rand.Read(keyB) error = %v", err)
|
||||
}
|
||||
|
||||
dataDir := filepath.Join(t.TempDir(), "data")
|
||||
storeA, err := NewFilesystemStore(dataDir, keyA)
|
||||
if err != nil {
|
||||
t.Fatalf("NewEncryptedFilesystemStore(keyA) error = %v", err)
|
||||
}
|
||||
_, err = storeA.Create(context.Background(), bytes.NewReader([]byte("hello")), "text/plain", time.Hour)
|
||||
if err != nil {
|
||||
t.Fatalf("Create() error = %v", err)
|
||||
}
|
||||
|
||||
if _, err := NewFilesystemStore(dataDir, keyB); err == nil {
|
||||
t.Fatalf("NewFilesystemStore() with wrong key error = nil, want non-nil")
|
||||
}
|
||||
}
|
||||
|
||||
func TestCreateWithOriginalNameNormalizesFilename(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
store, err := NewFilesystemStore(filepath.Join(t.TempDir(), "data"))
|
||||
store, err := NewFilesystemStore(filepath.Join(t.TempDir(), "data"), testEncryptionKey)
|
||||
if err != nil {
|
||||
t.Fatalf("NewFilesystemStore() error = %v", err)
|
||||
}
|
||||
@@ -70,7 +172,7 @@ func TestCreateWithOriginalNameNormalizesFilename(t *testing.T) {
|
||||
t.Fatalf("meta.OriginalName = %q, want %q", meta.OriginalName, "demo.txt")
|
||||
}
|
||||
|
||||
reloaded, err := NewFilesystemStore(store.dataDir)
|
||||
reloaded, err := NewFilesystemStore(store.dataDir, testEncryptionKey)
|
||||
if err != nil {
|
||||
t.Fatalf("NewFilesystemStore() reload error = %v", err)
|
||||
}
|
||||
@@ -83,201 +185,31 @@ func TestCreateWithOriginalNameNormalizesFilename(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestCreateWithOriginalNameRetriesOnIDCollision(t *testing.T) {
|
||||
func TestCreateWithOriginalNameSameContentGetsDifferentIDs(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
store, err := NewFilesystemStore(filepath.Join(t.TempDir(), "data"))
|
||||
store, err := NewFilesystemStore(filepath.Join(t.TempDir(), "data"), testEncryptionKey)
|
||||
if err != nil {
|
||||
t.Fatalf("NewFilesystemStore() error = %v", err)
|
||||
}
|
||||
|
||||
existingID := "aaaaaaaaaaaaaaaaaaaaaaaa"
|
||||
existingPath := filepath.Join(store.filesDir, existingID)
|
||||
if err := os.WriteFile(existingPath, []byte{0x1f, 0x8b, 0x08, 0x00}, 0o644); err != nil {
|
||||
t.Fatalf("WriteFile(existing collision path) error = %v", err)
|
||||
}
|
||||
store.mu.Lock()
|
||||
store.metadata[existingID] = Metadata{
|
||||
ID: existingID,
|
||||
FilePath: existingPath,
|
||||
CreatedAt: time.Now().UTC(),
|
||||
ExpiresAt: time.Now().UTC().Add(time.Hour),
|
||||
ContentType: "text/plain",
|
||||
Size: 4,
|
||||
}
|
||||
store.mu.Unlock()
|
||||
|
||||
nextID := "bbbbbbbbbbbbbbbbbbbbbbbb"
|
||||
calls := 0
|
||||
store.idGenerator = func() (string, error) {
|
||||
calls++
|
||||
if calls == 1 {
|
||||
return existingID, nil
|
||||
}
|
||||
return nextID, nil
|
||||
}
|
||||
|
||||
meta, err := store.CreateWithOriginalName(context.Background(), bytes.NewReader([]byte("hello")), "text/plain", "hello.txt", time.Hour)
|
||||
first, err := store.CreateWithOriginalName(context.Background(), bytes.NewReader([]byte("same body")), "text/plain", "one.txt", time.Hour)
|
||||
if err != nil {
|
||||
t.Fatalf("CreateWithOriginalName() error = %v", err)
|
||||
t.Fatalf("first CreateWithOriginalName() error = %v", err)
|
||||
}
|
||||
if meta.ID != nextID {
|
||||
t.Fatalf("meta.ID = %q, want %q", meta.ID, nextID)
|
||||
}
|
||||
if calls != 2 {
|
||||
t.Fatalf("idGenerator calls = %d, want 2", calls)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCreateWithOriginalNameFailsAfterMaxIDRetries(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
store, err := NewFilesystemStore(filepath.Join(t.TempDir(), "data"))
|
||||
if err != nil {
|
||||
t.Fatalf("NewFilesystemStore() error = %v", err)
|
||||
}
|
||||
|
||||
existingID := "cccccccccccccccccccccccc"
|
||||
existingPath := filepath.Join(store.filesDir, existingID)
|
||||
store.mu.Lock()
|
||||
store.metadata[existingID] = Metadata{
|
||||
ID: existingID,
|
||||
FilePath: existingPath,
|
||||
CreatedAt: time.Now().UTC(),
|
||||
ExpiresAt: time.Now().UTC().Add(time.Hour),
|
||||
ContentType: "text/plain",
|
||||
}
|
||||
store.mu.Unlock()
|
||||
|
||||
store.idGenerator = func() (string, error) {
|
||||
return existingID, nil
|
||||
}
|
||||
|
||||
_, err = store.CreateWithOriginalName(context.Background(), bytes.NewReader([]byte("hello")), "text/plain", "hello.txt", time.Hour)
|
||||
if err == nil {
|
||||
t.Fatalf("CreateWithOriginalName() error = nil, want retry exhaustion error")
|
||||
}
|
||||
if !strings.Contains(err.Error(), "failed to allocate unique scratch id") {
|
||||
t.Fatalf("unexpected error: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCreateWithOriginalNameRetriesWhenIDIsReservedInFlight(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
store, err := NewFilesystemStore(filepath.Join(t.TempDir(), "data"))
|
||||
if err != nil {
|
||||
t.Fatalf("NewFilesystemStore() error = %v", err)
|
||||
}
|
||||
|
||||
const collideID = "abababababababababababab"
|
||||
const uniqueID = "cdcdcdcdcdcdcdcdcdcdcdcd"
|
||||
|
||||
var idMu sync.Mutex
|
||||
idCalls := 0
|
||||
store.idGenerator = func() (string, error) {
|
||||
idMu.Lock()
|
||||
defer idMu.Unlock()
|
||||
idCalls++
|
||||
if idCalls <= 2 {
|
||||
return collideID, nil
|
||||
}
|
||||
return uniqueID, nil
|
||||
}
|
||||
|
||||
release := make(chan struct{})
|
||||
firstMetaCh := make(chan Metadata, 1)
|
||||
firstErrCh := make(chan error, 1)
|
||||
go func() {
|
||||
meta, createErr := store.CreateWithOriginalName(
|
||||
context.Background(),
|
||||
&blockingReader{release: release, payload: []byte("first upload")},
|
||||
"text/plain",
|
||||
"first.txt",
|
||||
time.Hour,
|
||||
)
|
||||
firstMetaCh <- meta
|
||||
firstErrCh <- createErr
|
||||
}()
|
||||
|
||||
deadline := time.Now().Add(2 * time.Second)
|
||||
for {
|
||||
store.mu.RLock()
|
||||
_, reserved := store.reservedIDs[collideID]
|
||||
store.mu.RUnlock()
|
||||
if reserved {
|
||||
break
|
||||
}
|
||||
if time.Now().After(deadline) {
|
||||
t.Fatalf("timed out waiting for first upload reservation")
|
||||
}
|
||||
time.Sleep(2 * time.Millisecond)
|
||||
}
|
||||
|
||||
secondMeta, err := store.CreateWithOriginalName(
|
||||
context.Background(),
|
||||
strings.NewReader("second upload"),
|
||||
"text/plain",
|
||||
"second.txt",
|
||||
time.Hour,
|
||||
)
|
||||
second, err := store.CreateWithOriginalName(context.Background(), bytes.NewReader([]byte("same body")), "text/plain", "two.txt", time.Hour)
|
||||
if err != nil {
|
||||
t.Fatalf("second CreateWithOriginalName() error = %v", err)
|
||||
}
|
||||
if secondMeta.ID != uniqueID {
|
||||
t.Fatalf("second meta.ID = %q, want %q after collision retry", secondMeta.ID, uniqueID)
|
||||
}
|
||||
if secondMeta.ID == collideID {
|
||||
t.Fatalf("second create reused in-flight reserved ID")
|
||||
}
|
||||
|
||||
close(release)
|
||||
firstMeta := <-firstMetaCh
|
||||
if err := <-firstErrCh; err != nil {
|
||||
t.Fatalf("first CreateWithOriginalName() error = %v", err)
|
||||
}
|
||||
if firstMeta.ID != collideID {
|
||||
t.Fatalf("first meta.ID = %q, want %q", firstMeta.ID, collideID)
|
||||
}
|
||||
|
||||
firstReader, _, err := store.Open(firstMeta.ID)
|
||||
if err != nil {
|
||||
t.Fatalf("Open(first) error = %v", err)
|
||||
}
|
||||
defer firstReader.Close()
|
||||
firstBody, err := io.ReadAll(firstReader)
|
||||
if err != nil {
|
||||
t.Fatalf("ReadAll(first) error = %v", err)
|
||||
}
|
||||
if string(firstBody) != "first upload" {
|
||||
t.Fatalf("first content = %q, want %q", string(firstBody), "first upload")
|
||||
}
|
||||
|
||||
secondReader, _, err := store.Open(secondMeta.ID)
|
||||
if err != nil {
|
||||
t.Fatalf("Open(second) error = %v", err)
|
||||
}
|
||||
defer secondReader.Close()
|
||||
secondBody, err := io.ReadAll(secondReader)
|
||||
if err != nil {
|
||||
t.Fatalf("ReadAll(second) error = %v", err)
|
||||
}
|
||||
if string(secondBody) != "second upload" {
|
||||
t.Fatalf("second content = %q, want %q", string(secondBody), "second upload")
|
||||
}
|
||||
|
||||
idMu.Lock()
|
||||
totalCalls := idCalls
|
||||
idMu.Unlock()
|
||||
if totalCalls < 3 {
|
||||
t.Fatalf("idGenerator call count = %d, want at least 3", totalCalls)
|
||||
if first.ID == second.ID {
|
||||
t.Fatalf("IDs should differ because encrypted blobs include random nonce")
|
||||
}
|
||||
}
|
||||
|
||||
func TestGetDeleteAndNotFound(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
store, err := NewFilesystemStore(filepath.Join(t.TempDir(), "data"))
|
||||
store, err := NewFilesystemStore(filepath.Join(t.TempDir(), "data"), testEncryptionKey)
|
||||
if err != nil {
|
||||
t.Fatalf("NewFilesystemStore() error = %v", err)
|
||||
}
|
||||
@@ -302,7 +234,7 @@ func TestGetDeleteAndNotFound(t *testing.T) {
|
||||
func TestDeleteExpired(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
store, err := NewFilesystemStore(filepath.Join(t.TempDir(), "data"))
|
||||
store, err := NewFilesystemStore(filepath.Join(t.TempDir(), "data"), testEncryptionKey)
|
||||
if err != nil {
|
||||
t.Fatalf("NewFilesystemStore() error = %v", err)
|
||||
}
|
||||
@@ -332,7 +264,7 @@ func TestDeleteExpired(t *testing.T) {
|
||||
func TestOpenInvalidGzipFails(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
store, err := NewFilesystemStore(filepath.Join(t.TempDir(), "data"))
|
||||
store, err := NewFilesystemStore(filepath.Join(t.TempDir(), "data"), testEncryptionKey)
|
||||
if err != nil {
|
||||
t.Fatalf("NewFilesystemStore() error = %v", err)
|
||||
}
|
||||
@@ -368,7 +300,7 @@ func TestNewFilesystemStoreInvalidIndex(t *testing.T) {
|
||||
if err := os.WriteFile(filepath.Join(root, indexFilename), []byte("{bad"), 0o644); err != nil {
|
||||
t.Fatalf("WriteFile() error = %v", err)
|
||||
}
|
||||
if _, err := NewFilesystemStore(root); err == nil {
|
||||
if _, err := NewFilesystemStore(root, testEncryptionKey); err == nil {
|
||||
t.Fatalf("NewFilesystemStore() error = nil, want invalid index error")
|
||||
}
|
||||
}
|
||||
@@ -419,15 +351,21 @@ func TestNewFilesystemStoreReconcileOnStartup(t *testing.T) {
|
||||
Size: 1,
|
||||
},
|
||||
}
|
||||
payload, err := json.Marshal(meta)
|
||||
payload, err := json.Marshal(indexDocument{
|
||||
Scratches: meta,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("json.Marshal() error = %v", err)
|
||||
}
|
||||
if err := os.WriteFile(filepath.Join(root, indexFilename), payload, 0o644); err != nil {
|
||||
encryptedPayload, err := encryptIndexPayload(payload, mustTestAEAD(t))
|
||||
if err != nil {
|
||||
t.Fatalf("encryptIndexPayload() error = %v", err)
|
||||
}
|
||||
if err := os.WriteFile(filepath.Join(root, indexFilename), encryptedPayload, 0o644); err != nil {
|
||||
t.Fatalf("WriteFile index error = %v", err)
|
||||
}
|
||||
|
||||
store, err := NewFilesystemStore(root)
|
||||
store, err := NewFilesystemStore(root, testEncryptionKey)
|
||||
if err != nil {
|
||||
t.Fatalf("NewFilesystemStore() error = %v", err)
|
||||
}
|
||||
@@ -442,6 +380,63 @@ func TestNewFilesystemStoreReconcileOnStartup(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestNewFilesystemStoreWithDefaultTTLShiftsExpiriesOnTTLChange(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
dataDir := filepath.Join(t.TempDir(), "data")
|
||||
store, err := NewFilesystemStoreWithDefaultTTL(dataDir, time.Hour, testEncryptionKey)
|
||||
if err != nil {
|
||||
t.Fatalf("NewFilesystemStoreWithDefaultTTL() error = %v", err)
|
||||
}
|
||||
|
||||
meta, err := store.Create(context.Background(), bytes.NewReader([]byte("hello")), "text/plain", time.Hour)
|
||||
if err != nil {
|
||||
t.Fatalf("Create() error = %v", err)
|
||||
}
|
||||
originalExpiry := meta.ExpiresAt
|
||||
|
||||
reloaded, err := NewFilesystemStoreWithDefaultTTL(dataDir, 2*time.Hour, testEncryptionKey)
|
||||
if err != nil {
|
||||
t.Fatalf("NewFilesystemStoreWithDefaultTTL() reload error = %v", err)
|
||||
}
|
||||
got, err := reloaded.Get(meta.ID)
|
||||
if err != nil {
|
||||
t.Fatalf("Get() error = %v", err)
|
||||
}
|
||||
|
||||
wantExpiry := originalExpiry.Add(time.Hour)
|
||||
if !got.ExpiresAt.Equal(wantExpiry) {
|
||||
t.Fatalf("ExpiresAt = %s, want %s", got.ExpiresAt, wantExpiry)
|
||||
}
|
||||
|
||||
again, err := NewFilesystemStoreWithDefaultTTL(dataDir, 2*time.Hour, testEncryptionKey)
|
||||
if err != nil {
|
||||
t.Fatalf("NewFilesystemStoreWithDefaultTTL() second reload error = %v", err)
|
||||
}
|
||||
gotAgain, err := again.Get(meta.ID)
|
||||
if err != nil {
|
||||
t.Fatalf("Get() second reload error = %v", err)
|
||||
}
|
||||
if !gotAgain.ExpiresAt.Equal(wantExpiry) {
|
||||
t.Fatalf("ExpiresAt after unchanged TTL = %s, want %s", gotAgain.ExpiresAt, wantExpiry)
|
||||
}
|
||||
|
||||
indexRaw, err := os.ReadFile(filepath.Join(dataDir, indexFilename))
|
||||
if err != nil {
|
||||
t.Fatalf("ReadFile(index) error = %v", err)
|
||||
}
|
||||
if bytes.Contains(indexRaw, []byte(`"default_ttl"`)) {
|
||||
t.Fatalf("index should not be plaintext JSON")
|
||||
}
|
||||
indexPlaintext, err := decryptIndexPayload(indexRaw, mustTestAEAD(t))
|
||||
if err != nil {
|
||||
t.Fatalf("decryptIndexPayload() error = %v", err)
|
||||
}
|
||||
if !bytes.Contains(indexPlaintext, []byte(`"default_ttl": "2h0m0s"`)) {
|
||||
t.Fatalf("decrypted index missing default ttl marker: %s", string(indexPlaintext))
|
||||
}
|
||||
}
|
||||
|
||||
type errCloser struct{}
|
||||
|
||||
func (errCloser) Close() error { return errors.New("close failed") }
|
||||
@@ -465,7 +460,7 @@ func (failingReader) Read(_ []byte) (int, error) { return 0, errors.New("boom")
|
||||
func TestCreateRollbackOnPersistFailure(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
store, err := NewFilesystemStore(filepath.Join(t.TempDir(), "data"))
|
||||
store, err := NewFilesystemStore(filepath.Join(t.TempDir(), "data"), testEncryptionKey)
|
||||
if err != nil {
|
||||
t.Fatalf("NewFilesystemStore() error = %v", err)
|
||||
}
|
||||
@@ -487,7 +482,7 @@ func TestCreateRollbackOnPersistFailure(t *testing.T) {
|
||||
func TestCreateReaderFailure(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
store, err := NewFilesystemStore(filepath.Join(t.TempDir(), "data"))
|
||||
store, err := NewFilesystemStore(filepath.Join(t.TempDir(), "data"), testEncryptionKey)
|
||||
if err != nil {
|
||||
t.Fatalf("NewFilesystemStore() error = %v", err)
|
||||
}
|
||||
@@ -499,7 +494,7 @@ func TestCreateReaderFailure(t *testing.T) {
|
||||
func TestDeletePersistFailureRestoresEntry(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
store, err := NewFilesystemStore(filepath.Join(t.TempDir(), "data"))
|
||||
store, err := NewFilesystemStore(filepath.Join(t.TempDir(), "data"), testEncryptionKey)
|
||||
if err != nil {
|
||||
t.Fatalf("NewFilesystemStore() error = %v", err)
|
||||
}
|
||||
@@ -525,7 +520,7 @@ func TestDeletePersistFailureRestoresEntry(t *testing.T) {
|
||||
func TestDeleteRemoveFailure(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
store, err := NewFilesystemStore(filepath.Join(t.TempDir(), "data"))
|
||||
store, err := NewFilesystemStore(filepath.Join(t.TempDir(), "data"), testEncryptionKey)
|
||||
if err != nil {
|
||||
t.Fatalf("NewFilesystemStore() error = %v", err)
|
||||
}
|
||||
@@ -557,7 +552,7 @@ func TestDeleteRemoveFailure(t *testing.T) {
|
||||
func TestDeleteExpiredPersistFailureRestoresEntries(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
store, err := NewFilesystemStore(filepath.Join(t.TempDir(), "data"))
|
||||
store, err := NewFilesystemStore(filepath.Join(t.TempDir(), "data"), testEncryptionKey)
|
||||
if err != nil {
|
||||
t.Fatalf("NewFilesystemStore() error = %v", err)
|
||||
}
|
||||
@@ -583,7 +578,7 @@ func TestDeleteExpiredPersistFailureRestoresEntries(t *testing.T) {
|
||||
func TestDeleteExpiredRemoveFailure(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
store, err := NewFilesystemStore(filepath.Join(t.TempDir(), "data"))
|
||||
store, err := NewFilesystemStore(filepath.Join(t.TempDir(), "data"), testEncryptionKey)
|
||||
if err != nil {
|
||||
t.Fatalf("NewFilesystemStore() error = %v", err)
|
||||
}
|
||||
@@ -619,106 +614,45 @@ func TestNewFilesystemStoreFailsWhenDataDirIsFile(t *testing.T) {
|
||||
if err := os.WriteFile(filePath, []byte("x"), 0o644); err != nil {
|
||||
t.Fatalf("WriteFile() error = %v", err)
|
||||
}
|
||||
if _, err := NewFilesystemStore(filePath); err == nil {
|
||||
if _, err := NewFilesystemStore(filePath, testEncryptionKey); err == nil {
|
||||
t.Fatalf("NewFilesystemStore() error = nil, want mkdir failure")
|
||||
}
|
||||
}
|
||||
|
||||
type blockingReader struct {
|
||||
release <-chan struct{}
|
||||
payload []byte
|
||||
sent bool
|
||||
}
|
||||
|
||||
func (r *blockingReader) Read(p []byte) (int, error) {
|
||||
if r.sent {
|
||||
return 0, io.EOF
|
||||
}
|
||||
<-r.release
|
||||
r.sent = true
|
||||
return copy(p, r.payload), nil
|
||||
}
|
||||
|
||||
func TestCreateReservationIsNotVisibleToCleanupOrReads(t *testing.T) {
|
||||
func TestNewFilesystemStoreRequires32ByteKey(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
store, err := NewFilesystemStore(filepath.Join(t.TempDir(), "data"))
|
||||
dataDir := filepath.Join(t.TempDir(), "data")
|
||||
if _, err := NewFilesystemStore(dataDir, nil); err == nil {
|
||||
t.Fatalf("NewFilesystemStore() with nil key error = nil, want non-nil")
|
||||
}
|
||||
if _, err := NewFilesystemStore(dataDir, []byte("short")); err == nil {
|
||||
t.Fatalf("NewFilesystemStore() with short key error = nil, want non-nil")
|
||||
}
|
||||
}
|
||||
|
||||
func TestNewFilesystemStoreFailsToLoadEncryptedIndexWithWrongKey(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
dataDir := filepath.Join(t.TempDir(), "data")
|
||||
store, err := NewFilesystemStore(dataDir, testEncryptionKey)
|
||||
if err != nil {
|
||||
t.Fatalf("NewFilesystemStore() error = %v", err)
|
||||
}
|
||||
|
||||
const reservedID = "ffffffffffffffffffffffff"
|
||||
store.idGenerator = func() (string, error) {
|
||||
return reservedID, nil
|
||||
if _, err := store.Create(context.Background(), bytes.NewReader([]byte("hello")), "text/plain", time.Hour); err != nil {
|
||||
t.Fatalf("Create() error = %v", err)
|
||||
}
|
||||
|
||||
release := make(chan struct{})
|
||||
createErrCh := make(chan error, 1)
|
||||
go func() {
|
||||
_, createErr := store.CreateWithOriginalName(
|
||||
context.Background(),
|
||||
&blockingReader{release: release, payload: []byte("in-flight")},
|
||||
"text/plain",
|
||||
"",
|
||||
time.Hour,
|
||||
)
|
||||
createErrCh <- createErr
|
||||
}()
|
||||
|
||||
deadline := time.Now().Add(2 * time.Second)
|
||||
for {
|
||||
store.mu.RLock()
|
||||
_, reserved := store.reservedIDs[reservedID]
|
||||
_, visible := store.metadata[reservedID]
|
||||
store.mu.RUnlock()
|
||||
if reserved {
|
||||
if visible {
|
||||
t.Fatalf("in-flight ID should not appear in metadata")
|
||||
}
|
||||
break
|
||||
}
|
||||
if time.Now().After(deadline) {
|
||||
t.Fatalf("timed out waiting for ID reservation")
|
||||
}
|
||||
time.Sleep(2 * time.Millisecond)
|
||||
}
|
||||
|
||||
if _, err := store.Get(reservedID); !errors.Is(err, ErrNotFound) {
|
||||
t.Fatalf("Get() during in-flight create error = %v, want ErrNotFound", err)
|
||||
}
|
||||
|
||||
deleted, err := store.DeleteExpired(time.Now().UTC())
|
||||
if err != nil {
|
||||
t.Fatalf("DeleteExpired() error = %v", err)
|
||||
}
|
||||
if deleted != 0 {
|
||||
t.Fatalf("DeleteExpired() = %d, want 0 for in-flight create", deleted)
|
||||
}
|
||||
|
||||
close(release)
|
||||
if err := <-createErrCh; err != nil {
|
||||
t.Fatalf("CreateWithOriginalName() error = %v", err)
|
||||
}
|
||||
|
||||
reader, _, err := store.Open(reservedID)
|
||||
if err != nil {
|
||||
t.Fatalf("Open() error = %v", err)
|
||||
}
|
||||
defer reader.Close()
|
||||
|
||||
body, err := io.ReadAll(reader)
|
||||
if err != nil {
|
||||
t.Fatalf("ReadAll() error = %v", err)
|
||||
}
|
||||
if string(body) != "in-flight" {
|
||||
t.Fatalf("Open() content = %q, want %q", string(body), "in-flight")
|
||||
wrongKey := []byte("abcdef0123456789abcdef0123456789")
|
||||
if _, err := NewFilesystemStore(dataDir, wrongKey); err == nil {
|
||||
t.Fatalf("NewFilesystemStore() with wrong key error = nil, want non-nil")
|
||||
}
|
||||
}
|
||||
|
||||
func TestConcurrentCreateWithOriginalNameProducesUniqueReadableEntries(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
store, err := NewFilesystemStore(filepath.Join(t.TempDir(), "data"))
|
||||
store, err := NewFilesystemStore(filepath.Join(t.TempDir(), "data"), testEncryptionKey)
|
||||
if err != nil {
|
||||
t.Fatalf("NewFilesystemStore() error = %v", err)
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user