Lots more changes

This commit is contained in:
2026-06-01 01:29:47 -05:00
parent 8bec7f0dda
commit 050ec138f3
20 changed files with 1773 additions and 689 deletions
+5 -3
View File
@@ -12,10 +12,12 @@ import (
"scratchbox/internal/storage"
)
var testStorageKey = []byte("0123456789abcdef0123456789abcdef")
func TestNewWorker(t *testing.T) {
t.Parallel()
store, err := storage.NewFilesystemStore(filepath.Join(t.TempDir(), "data"))
store, err := storage.NewFilesystemStore(filepath.Join(t.TempDir(), "data"), testStorageKey)
if err != nil {
t.Fatalf("NewFilesystemStore() error = %v", err)
}
@@ -29,7 +31,7 @@ func TestNewWorker(t *testing.T) {
func TestWorkerStartRemovesExpiredAndLogs(t *testing.T) {
t.Parallel()
store, err := storage.NewFilesystemStore(filepath.Join(t.TempDir(), "data"))
store, err := storage.NewFilesystemStore(filepath.Join(t.TempDir(), "data"), testStorageKey)
if err != nil {
t.Fatalf("NewFilesystemStore() error = %v", err)
}
@@ -53,7 +55,7 @@ func TestWorkerStartRemovesExpiredAndLogs(t *testing.T) {
func TestWorkerStartWithNoExpiredEntries(t *testing.T) {
t.Parallel()
store, err := storage.NewFilesystemStore(filepath.Join(t.TempDir(), "data"))
store, err := storage.NewFilesystemStore(filepath.Join(t.TempDir(), "data"), testStorageKey)
if err != nil {
t.Fatalf("NewFilesystemStore() error = %v", err)
}
+163 -38
View File
@@ -1,6 +1,8 @@
package config
import (
"crypto/rand"
"encoding/base64"
"errors"
"fmt"
"net/netip"
@@ -14,23 +16,34 @@ import (
)
const (
defaultListenAddr = ":8080"
defaultReadHeaderTimeout = "5s"
defaultReadTimeout = "30s"
defaultWriteTimeout = "30s"
defaultIdleTimeout = "120s"
defaultMaxHeaderBytes = 1 << 20
defaultMaxUploadSize = "100MiB"
defaultTTL = "1h"
defaultRawCacheMaxSize = "200MiB"
defaultRawCacheMaxEntries = 32
defaultDataDir = "./data"
defaultCleanupInterval = "1m"
defaultRateLimitEnabled = true
defaultRequestsPerMinute = 30
defaultRateLimitBurst = 10
maxDefaultTTLLimit = 24 * time.Hour
minCleanupInterval = 10 * time.Second
defaultListenAddr = ":8080"
defaultReadHeaderTimeout = "5s"
defaultReadTimeout = "30s"
defaultWriteTimeout = "30s"
defaultIdleTimeout = "120s"
defaultMaxHeaderBytes = 1 << 20
defaultAccessLogEnabled = true
defaultAccessLogMaxSize = "100MiB"
defaultAccessLogMaxBackups = 5
defaultMaxUploadSize = "100MiB"
defaultTTL = "15m"
defaultRawCacheMaxSize = "200MiB"
defaultRawCacheMaxEntries = 32
defaultDataDir = "./data"
defaultCleanupInterval = "1m"
defaultStorageEncryptionKey = "vVPlqMfKa8OtD3x0RKp4rVCJ4QoScf5mWdgfg4f+5GU="
defaultEncryptionKeyFile = "config.key"
defaultUIRateLimitEnabled = false
defaultAPIReadRateLimitEnabled = false
defaultAPIWriteRateLimitEnabled = true
defaultUIRequestsPerMinute = 360
defaultUIRateLimitBurst = 120
defaultAPIReadRequestsPerMinute = 300
defaultAPIReadRateLimitBurst = 100
defaultAPIWriteRequestsPerMinute = 30
defaultAPIWriteRateLimitBurst = 10
maxDefaultTTLLimit = 24 * time.Hour
minCleanupInterval = 10 * time.Second
)
type Config struct {
@@ -41,16 +54,25 @@ type Config struct {
}
type ServerConfig struct {
ListenAddr string `yaml:"listen_addr" comment:"Address to bind the HTTP server to."`
ReadHeaderTimeout string `yaml:"read_header_timeout" comment:"Maximum duration for reading request headers (for slowloris protection). Must be > 0."`
ReadTimeout string `yaml:"read_timeout" comment:"Maximum duration for reading the full request, including body. Must be > 0."`
WriteTimeout string `yaml:"write_timeout" comment:"Maximum duration before timing out writes of a response. Must be > 0."`
IdleTimeout string `yaml:"idle_timeout" comment:"Maximum amount of time to wait for the next request when keep-alives are enabled. Must be > 0."`
MaxHeaderBytes int `yaml:"max_header_bytes" comment:"Maximum size of request headers in bytes. Must be > 0."`
ReadHeaderTimeoutDur time.Duration `yaml:"-"`
ReadTimeoutDur time.Duration `yaml:"-"`
WriteTimeoutDur time.Duration `yaml:"-"`
IdleTimeoutDur time.Duration `yaml:"-"`
ListenAddr string `yaml:"listen_addr" comment:"Address to bind the HTTP server to."`
ReadHeaderTimeout string `yaml:"read_header_timeout" comment:"Maximum duration for reading request headers (for slowloris protection). Must be > 0."`
ReadTimeout string `yaml:"read_timeout" comment:"Maximum duration for reading the full request, including body. Must be > 0."`
WriteTimeout string `yaml:"write_timeout" comment:"Maximum duration before timing out writes of a response. Must be > 0."`
IdleTimeout string `yaml:"idle_timeout" comment:"Maximum amount of time to wait for the next request when keep-alives are enabled. Must be > 0."`
MaxHeaderBytes int `yaml:"max_header_bytes" comment:"Maximum size of request headers in bytes. Must be > 0."`
AccessLog AccessLogConfig `yaml:"access_log" comment:"Access log settings for all HTTP requests."`
ReadHeaderTimeoutDur time.Duration `yaml:"-"`
ReadTimeoutDur time.Duration `yaml:"-"`
WriteTimeoutDur time.Duration `yaml:"-"`
IdleTimeoutDur time.Duration `yaml:"-"`
}
type AccessLogConfig struct {
Enabled bool `yaml:"enabled" comment:"Enable HTTP access logs."`
FilePath string `yaml:"file_path" comment:"Optional file path for access logs. When set, logs are tee'd to stdout and this file."`
MaxSize string `yaml:"max_size" comment:"Maximum size for access log file before rotation. Supports B, KB, MB, GB and KiB, MiB, GiB."`
MaxBackups int `yaml:"max_backups" comment:"Number of rotated access log files to keep."`
MaxSizeBytes int64 `yaml:"-"`
}
type LimitsConfig struct {
@@ -66,14 +88,18 @@ type LimitsConfig struct {
type StorageConfig struct {
DataDir string `yaml:"data_dir" comment:"Directory where scratch files and metadata are stored."`
CleanupInterval string `yaml:"cleanup_interval" comment:"How often expired scratches are deleted (minimum 10s)."`
EncryptionKeyFile string `yaml:"encryption_key_file" comment:"Path to a file containing a base64-encoded 32-byte AES key for at-rest encryption. Relative paths are resolved from the config file directory."`
CleanupIntervalParsed time.Duration `yaml:"-"`
EncryptionKeyBytes []byte `yaml:"-"`
}
type SecurityConfig struct {
AllowedIPs []string `yaml:"allowed_ips" comment:"Allowlist for POST /api/scratch as IPs or CIDRs (examples: 127.0.0.1, 192.168.1.0/24). Keep 127.0.0.1 for local access; add trusted ranges as needed. Empty means unrestricted."`
TrustProxyHeaders bool `yaml:"trust_proxy_headers" comment:"Honor X-Forwarded-For / X-Real-IP only when request source IP matches security.trusted_proxy_ips."`
TrustedProxyIPs []string `yaml:"trusted_proxy_ips" comment:"IPs/CIDRs of reverse proxies allowed to supply forwarded client IP headers. Required when trust_proxy_headers is true."`
RateLimit RateLimitConfig `yaml:"rate_limit" comment:"Per-client token bucket on write and scratch read routes."`
RateLimitUI RateLimitConfig `yaml:"rate_limit_ui" comment:"Per-client token bucket on UI routes (/, /u, /s/{id})."`
RateLimitAPIRead RateLimitConfig `yaml:"rate_limit_api_read" comment:"Per-client token bucket on API read routes (GET /api/config, GET /api/scratch/{id}, GET /api/raw/{id})."`
RateLimitAPIWrite RateLimitConfig `yaml:"rate_limit_api_write" comment:"Per-client token bucket on API write route (POST /api/scratch)."`
AllowedPrefixes []netip.Prefix `yaml:"-"`
TrustedProxyPrefixes []netip.Prefix `yaml:"-"`
}
@@ -95,6 +121,9 @@ func Load(path string) (Config, error) {
return Config{}, fmt.Errorf("parse config yaml: %w", err)
}
}
if err := cfg.loadEncryptionKey(path); err != nil {
return Config{}, err
}
if err := cfg.Validate(); err != nil {
return Config{}, err
@@ -115,6 +144,11 @@ func defaults() Config {
WriteTimeout: defaultWriteTimeout,
IdleTimeout: defaultIdleTimeout,
MaxHeaderBytes: defaultMaxHeaderBytes,
AccessLog: AccessLogConfig{
Enabled: defaultAccessLogEnabled,
MaxSize: defaultAccessLogMaxSize,
MaxBackups: defaultAccessLogMaxBackups,
},
},
Limits: LimitsConfig{
MaxUploadSize: defaultMaxUploadSize,
@@ -123,17 +157,29 @@ func defaults() Config {
RawCacheMaxEntries: defaultRawCacheMaxEntries,
},
Storage: StorageConfig{
DataDir: defaultDataDir,
CleanupInterval: defaultCleanupInterval,
DataDir: defaultDataDir,
CleanupInterval: defaultCleanupInterval,
EncryptionKeyFile: defaultEncryptionKeyFile,
EncryptionKeyBytes: mustDecodeDefaultEncryptionKey(),
},
Security: SecurityConfig{
AllowedIPs: []string{
"127.0.0.1",
},
RateLimit: RateLimitConfig{
Enabled: defaultRateLimitEnabled,
RequestsPerMinute: defaultRequestsPerMinute,
Burst: defaultRateLimitBurst,
RateLimitUI: RateLimitConfig{
Enabled: defaultUIRateLimitEnabled,
RequestsPerMinute: defaultUIRequestsPerMinute,
Burst: defaultUIRateLimitBurst,
},
RateLimitAPIRead: RateLimitConfig{
Enabled: defaultAPIReadRateLimitEnabled,
RequestsPerMinute: defaultAPIReadRequestsPerMinute,
Burst: defaultAPIReadRateLimitBurst,
},
RateLimitAPIWrite: RateLimitConfig{
Enabled: defaultAPIWriteRateLimitEnabled,
RequestsPerMinute: defaultAPIWriteRequestsPerMinute,
Burst: defaultAPIWriteRateLimitBurst,
},
},
}
@@ -182,6 +228,14 @@ func (c *Config) Validate() error {
if c.Server.MaxHeaderBytes <= 0 {
return errors.New("server.max_header_bytes must be > 0")
}
accessLogMaxSizeBytes, err := parseHumanSize(c.Server.AccessLog.MaxSize)
if err != nil {
return fmt.Errorf("server.access_log.max_size invalid: %w", err)
}
c.Server.AccessLog.MaxSizeBytes = accessLogMaxSizeBytes
if c.Server.AccessLog.MaxBackups <= 0 {
return errors.New("server.access_log.max_backups must be > 0")
}
sizeBytes, err := parseHumanSize(c.Limits.MaxUploadSize)
if err != nil {
@@ -222,6 +276,12 @@ func (c *Config) Validate() error {
if err := ensureWritableDir(c.Storage.DataDir); err != nil {
return fmt.Errorf("storage.data_dir not writable: %w", err)
}
if strings.TrimSpace(c.Storage.EncryptionKeyFile) == "" {
return errors.New("storage.encryption_key_file is required")
}
if len(c.Storage.EncryptionKeyBytes) != 32 {
return fmt.Errorf("storage.encryption_key_file must decode to 32 bytes, got %d", len(c.Storage.EncryptionKeyBytes))
}
prefixes, err := parseAllowedPrefixes(c.Security.AllowedIPs)
if err != nil {
@@ -237,11 +297,23 @@ func (c *Config) Validate() error {
return errors.New("security.trusted_proxy_ips is required when security.trust_proxy_headers is true")
}
if c.Security.RateLimit.RequestsPerMinute <= 0 {
return errors.New("security.rate_limit.requests_per_minute must be > 0")
if c.Security.RateLimitUI.RequestsPerMinute <= 0 {
return errors.New("security.rate_limit_ui.requests_per_minute must be > 0")
}
if c.Security.RateLimit.Burst <= 0 {
return errors.New("security.rate_limit.burst must be > 0")
if c.Security.RateLimitUI.Burst <= 0 {
return errors.New("security.rate_limit_ui.burst must be > 0")
}
if c.Security.RateLimitAPIRead.RequestsPerMinute <= 0 {
return errors.New("security.rate_limit_api_read.requests_per_minute must be > 0")
}
if c.Security.RateLimitAPIRead.Burst <= 0 {
return errors.New("security.rate_limit_api_read.burst must be > 0")
}
if c.Security.RateLimitAPIWrite.RequestsPerMinute <= 0 {
return errors.New("security.rate_limit_api_write.requests_per_minute must be > 0")
}
if c.Security.RateLimitAPIWrite.Burst <= 0 {
return errors.New("security.rate_limit_api_write.burst must be > 0")
}
return nil
@@ -338,3 +410,56 @@ func parseHumanSize(raw string) (int64, error) {
}
return size, nil
}
func GenerateEncryptionKey() (string, error) {
raw := make([]byte, 32)
if _, err := rand.Read(raw); err != nil {
return "", fmt.Errorf("generate encryption key bytes: %w", err)
}
return base64.StdEncoding.EncodeToString(raw), nil
}
func (c *Config) loadEncryptionKey(configPath string) error {
if configPath == "" {
return nil
}
keyPath := strings.TrimSpace(c.Storage.EncryptionKeyFile)
if keyPath == "" {
return errors.New("storage.encryption_key_file is required")
}
if !filepath.IsAbs(keyPath) {
keyPath = filepath.Join(filepath.Dir(configPath), keyPath)
}
keyRaw, err := os.ReadFile(keyPath)
if err != nil {
return fmt.Errorf("read storage.encryption_key_file %q: %w", keyPath, err)
}
encKey, err := decodeEncryptionKey(strings.TrimSpace(string(keyRaw)))
if err != nil {
return fmt.Errorf("storage.encryption_key_file invalid: %w", err)
}
c.Storage.EncryptionKeyBytes = encKey
return nil
}
func decodeEncryptionKey(raw string) ([]byte, error) {
if raw == "" {
return nil, errors.New("key is empty")
}
encKey, err := base64.StdEncoding.DecodeString(raw)
if err != nil {
return nil, fmt.Errorf("invalid base64: %w", err)
}
if len(encKey) != 32 {
return nil, fmt.Errorf("must decode to 32 bytes, got %d", len(encKey))
}
return encKey, nil
}
func mustDecodeDefaultEncryptionKey() []byte {
encKey, err := decodeEncryptionKey(defaultStorageEncryptionKey)
if err != nil {
panic(fmt.Sprintf("invalid built-in encryption key: %v", err))
}
return encKey
}
+165 -11
View File
@@ -1,6 +1,7 @@
package config
import (
"encoding/base64"
"net/netip"
"os"
"path/filepath"
@@ -37,8 +38,17 @@ func TestValidateAppliesDefaultsAndParsedFields(t *testing.T) {
if cfg.Server.MaxHeaderBytes != 1<<20 {
t.Fatalf("MaxHeaderBytes = %d, want %d", cfg.Server.MaxHeaderBytes, 1<<20)
}
if cfg.Limits.DefaultTTLDuration != time.Hour {
t.Fatalf("DefaultTTLDuration = %s, want 1h", cfg.Limits.DefaultTTLDuration)
if !cfg.Server.AccessLog.Enabled {
t.Fatalf("AccessLog.Enabled = %v, want true", cfg.Server.AccessLog.Enabled)
}
if cfg.Server.AccessLog.MaxSizeBytes != 100*1024*1024 {
t.Fatalf("AccessLog.MaxSizeBytes = %d, want %d", cfg.Server.AccessLog.MaxSizeBytes, int64(100*1024*1024))
}
if cfg.Server.AccessLog.MaxBackups != 5 {
t.Fatalf("AccessLog.MaxBackups = %d, want 5", cfg.Server.AccessLog.MaxBackups)
}
if cfg.Limits.DefaultTTLDuration != 15*time.Minute {
t.Fatalf("DefaultTTLDuration = %s, want 15m", cfg.Limits.DefaultTTLDuration)
}
if cfg.Limits.RawCacheMaxBytes != 200*1024*1024 {
t.Fatalf("RawCacheMaxBytes = %d, want %d", cfg.Limits.RawCacheMaxBytes, int64(200*1024*1024))
@@ -49,6 +59,9 @@ func TestValidateAppliesDefaultsAndParsedFields(t *testing.T) {
if cfg.Storage.CleanupIntervalParsed != time.Minute {
t.Fatalf("CleanupIntervalParsed = %s, want 1m", cfg.Storage.CleanupIntervalParsed)
}
if len(cfg.Storage.EncryptionKeyBytes) != 32 {
t.Fatalf("EncryptionKeyBytes length = %d, want 32", len(cfg.Storage.EncryptionKeyBytes))
}
if len(cfg.Security.AllowedPrefixes) != 1 {
t.Fatalf("AllowedPrefixes len = %d, want 1", len(cfg.Security.AllowedPrefixes))
}
@@ -63,6 +76,7 @@ func TestLoadParsesConfiguredFields(t *testing.T) {
tmp := t.TempDir()
dataDir := filepath.Join(tmp, "store")
configPath := filepath.Join(tmp, "config.yaml")
keyPath := filepath.Join(tmp, "scratchbox.key")
content := `
server:
listen_addr: "127.0.0.1:9090"
@@ -71,6 +85,11 @@ server:
write_timeout: "35s"
idle_timeout: "90s"
max_header_bytes: 2097152
access_log:
enabled: true
file_path: "` + filepath.Join(tmp, "access.log") + `"
max_size: "2MiB"
max_backups: 7
limits:
max_upload_size: "2MiB"
default_ttl: "2h"
@@ -79,6 +98,7 @@ limits:
storage:
data_dir: "` + dataDir + `"
cleanup_interval: "45s"
encryption_key_file: "` + filepath.Base(keyPath) + `"
security:
allowed_ips:
- "127.0.0.1"
@@ -86,7 +106,15 @@ security:
trust_proxy_headers: true
trusted_proxy_ips:
- "10.0.0.0/8"
rate_limit:
rate_limit_ui:
enabled: true
requests_per_minute: 12
burst: 4
rate_limit_api_read:
enabled: true
requests_per_minute: 12
burst: 4
rate_limit_api_write:
enabled: true
requests_per_minute: 12
burst: 4
@@ -94,6 +122,9 @@ security:
if err := os.WriteFile(configPath, []byte(content), 0o644); err != nil {
t.Fatalf("WriteFile() error = %v", err)
}
if err := os.WriteFile(keyPath, []byte("AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=\n"), 0o600); err != nil {
t.Fatalf("WriteFile(key) error = %v", err)
}
cfg, err := Load(configPath)
if err != nil {
@@ -118,6 +149,15 @@ security:
if cfg.Server.MaxHeaderBytes != 2*1024*1024 {
t.Fatalf("MaxHeaderBytes = %d, want %d", cfg.Server.MaxHeaderBytes, 2*1024*1024)
}
if !cfg.Server.AccessLog.Enabled {
t.Fatalf("AccessLog.Enabled = false, want true")
}
if cfg.Server.AccessLog.MaxSizeBytes != 2*1024*1024 {
t.Fatalf("AccessLog.MaxSizeBytes = %d, want %d", cfg.Server.AccessLog.MaxSizeBytes, int64(2*1024*1024))
}
if cfg.Server.AccessLog.MaxBackups != 7 {
t.Fatalf("AccessLog.MaxBackups = %d, want 7", cfg.Server.AccessLog.MaxBackups)
}
if cfg.Limits.MaxUploadSizeBytes != 2*1024*1024 {
t.Fatalf("MaxUploadSizeBytes = %d, want %d", cfg.Limits.MaxUploadSizeBytes, int64(2*1024*1024))
}
@@ -133,11 +173,23 @@ security:
if cfg.Storage.CleanupIntervalParsed != 45*time.Second {
t.Fatalf("CleanupIntervalParsed = %s, want 45s", cfg.Storage.CleanupIntervalParsed)
}
if len(cfg.Storage.EncryptionKeyBytes) != 32 {
t.Fatalf("EncryptionKeyBytes length = %d, want 32", len(cfg.Storage.EncryptionKeyBytes))
}
if cfg.Storage.EncryptionKeyBytes[0] != 0 {
t.Fatalf("EncryptionKeyBytes[0] = %d, want 0", cfg.Storage.EncryptionKeyBytes[0])
}
if !cfg.Security.TrustProxyHeaders {
t.Fatalf("TrustProxyHeaders = false, want true")
}
if cfg.Security.RateLimit.RequestsPerMinute != 12 || cfg.Security.RateLimit.Burst != 4 {
t.Fatalf("unexpected rate_limit values: %+v", cfg.Security.RateLimit)
if cfg.Security.RateLimitUI.RequestsPerMinute != 12 || cfg.Security.RateLimitUI.Burst != 4 {
t.Fatalf("unexpected rate_limit_ui values: %+v", cfg.Security.RateLimitUI)
}
if cfg.Security.RateLimitAPIRead.RequestsPerMinute != 12 || cfg.Security.RateLimitAPIRead.Burst != 4 {
t.Fatalf("unexpected rate_limit_api_read values: %+v", cfg.Security.RateLimitAPIRead)
}
if cfg.Security.RateLimitAPIWrite.RequestsPerMinute != 12 || cfg.Security.RateLimitAPIWrite.Burst != 4 {
t.Fatalf("unexpected rate_limit_api_write values: %+v", cfg.Security.RateLimitAPIWrite)
}
if len(cfg.Security.AllowedPrefixes) != 2 {
t.Fatalf("AllowedPrefixes len = %d, want 2", len(cfg.Security.AllowedPrefixes))
@@ -204,6 +256,20 @@ func TestValidateRejectsInvalidValues(t *testing.T) {
},
wantErr: "server.max_header_bytes",
},
{
name: "invalid access log max size",
mutate: func(cfg *Config) {
cfg.Server.AccessLog.MaxSize = "oops"
},
wantErr: "server.access_log.max_size",
},
{
name: "invalid access log max backups",
mutate: func(cfg *Config) {
cfg.Server.AccessLog.MaxBackups = 0
},
wantErr: "server.access_log.max_backups",
},
{
name: "invalid ttl parse",
mutate: func(cfg *Config) {
@@ -255,11 +321,25 @@ func TestValidateRejectsInvalidValues(t *testing.T) {
wantErr: "security.trusted_proxy_ips is required",
},
{
name: "invalid rate limit burst",
name: "invalid ui rate limit burst",
mutate: func(cfg *Config) {
cfg.Security.RateLimit.Burst = 0
cfg.Security.RateLimitUI.Burst = 0
},
wantErr: "security.rate_limit.burst",
wantErr: "security.rate_limit_ui.burst",
},
{
name: "invalid api read rate limit burst",
mutate: func(cfg *Config) {
cfg.Security.RateLimitAPIRead.Burst = 0
},
wantErr: "security.rate_limit_api_read.burst",
},
{
name: "invalid api write rate limit burst",
mutate: func(cfg *Config) {
cfg.Security.RateLimitAPIWrite.Burst = 0
},
wantErr: "security.rate_limit_api_write.burst",
},
{
name: "data dir required",
@@ -283,11 +363,39 @@ func TestValidateRejectsInvalidValues(t *testing.T) {
wantErr: "storage.cleanup_interval",
},
{
name: "invalid rate limit rpm",
name: "encryption key file required",
mutate: func(cfg *Config) {
cfg.Security.RateLimit.RequestsPerMinute = 0
cfg.Storage.EncryptionKeyFile = " "
},
wantErr: "security.rate_limit.requests_per_minute",
wantErr: "storage.encryption_key_file is required",
},
{
name: "encryption key bytes invalid length",
mutate: func(cfg *Config) {
cfg.Storage.EncryptionKeyBytes = []byte("short")
},
wantErr: "storage.encryption_key_file must decode to 32 bytes",
},
{
name: "invalid ui rate limit rpm",
mutate: func(cfg *Config) {
cfg.Security.RateLimitUI.RequestsPerMinute = 0
},
wantErr: "security.rate_limit_ui.requests_per_minute",
},
{
name: "invalid api read rate limit rpm",
mutate: func(cfg *Config) {
cfg.Security.RateLimitAPIRead.RequestsPerMinute = 0
},
wantErr: "security.rate_limit_api_read.requests_per_minute",
},
{
name: "invalid api write rate limit rpm",
mutate: func(cfg *Config) {
cfg.Security.RateLimitAPIWrite.RequestsPerMinute = 0
},
wantErr: "security.rate_limit_api_write.requests_per_minute",
},
}
@@ -328,6 +436,29 @@ func TestLoadErrors(t *testing.T) {
if err == nil || !strings.Contains(err.Error(), "parse config yaml") {
t.Fatalf("Load() error = %v, want yaml parse error", err)
}
validConfigPath := filepath.Join(tmp, "valid.yaml")
validConfig := `
storage:
data_dir: "` + filepath.Join(tmp, "store") + `"
cleanup_interval: "10s"
`
if err := os.WriteFile(validConfigPath, []byte(validConfig), 0o644); err != nil {
t.Fatalf("WriteFile(valid config) error = %v", err)
}
_, err = Load(validConfigPath)
if err == nil || !strings.Contains(err.Error(), "read storage.encryption_key_file") {
t.Fatalf("Load() error = %v, want missing key file error", err)
}
keyPath := filepath.Join(tmp, "config.key")
if err := os.WriteFile(keyPath, []byte("not-base64"), 0o600); err != nil {
t.Fatalf("WriteFile(invalid key) error = %v", err)
}
_, err = Load(validConfigPath)
if err == nil || !strings.Contains(err.Error(), "storage.encryption_key_file invalid: invalid base64") {
t.Fatalf("Load() error = %v, want invalid key base64 error", err)
}
}
func TestParseAllowedPrefixesTrimsAndParses(t *testing.T) {
@@ -385,3 +516,26 @@ func TestParseHumanSizeVariants(t *testing.T) {
})
}
}
func TestGenerateEncryptionKey(t *testing.T) {
t.Parallel()
k1, err := GenerateEncryptionKey()
if err != nil {
t.Fatalf("GenerateEncryptionKey() error = %v", err)
}
k2, err := GenerateEncryptionKey()
if err != nil {
t.Fatalf("GenerateEncryptionKey() second error = %v", err)
}
if k1 == k2 {
t.Fatalf("GenerateEncryptionKey() produced duplicate keys")
}
b1, err := base64.StdEncoding.DecodeString(k1)
if err != nil {
t.Fatalf("DecodeString(k1) error = %v", err)
}
if len(b1) != 32 {
t.Fatalf("decoded key length = %d, want 32", len(b1))
}
}
+44 -26
View File
@@ -18,46 +18,63 @@ import (
)
type Server struct {
cfg config.Config
store *storage.FilesystemStore
logger *log.Logger
rawCache *rawContentCache
cfg config.Config
store *storage.FilesystemStore
logger *log.Logger
accessLogger *log.Logger
rawCache *rawContentCache
}
func NewServer(cfg config.Config, store *storage.FilesystemStore, logger *log.Logger) (*Server, error) {
func NewServer(cfg config.Config, store *storage.FilesystemStore, logger *log.Logger, accessLogger *log.Logger) (*Server, error) {
return &Server{
cfg: cfg,
store: store,
logger: logger,
rawCache: newRawContentCache(cfg.Limits.RawCacheMaxEntries, cfg.Limits.RawCacheMaxBytes),
cfg: cfg,
store: store,
logger: logger,
accessLogger: accessLogger,
rawCache: newRawContentCache(cfg.Limits.RawCacheMaxEntries, cfg.Limits.RawCacheMaxBytes),
}, nil
}
func (s *Server) Routes() http.Handler {
mux := http.NewServeMux()
mux.Handle("GET /{$}", http.HandlerFunc(s.serveSPA))
mux.Handle("GET /u", http.HandlerFunc(s.serveSPA))
mux.Handle("GET /api/config", http.HandlerFunc(s.getUIConfig))
uiMiddlewares := make([]func(http.Handler) http.Handler, 0, 1)
apiReadMiddlewares := make([]func(http.Handler) http.Handler, 0, 1)
apiWriteMiddlewares := make([]func(http.Handler) http.Handler, 0, 1)
if s.cfg.Security.RateLimitUI.Enabled {
uiLimiter := NewRateLimiter(s.cfg.Security.RateLimitUI.RequestsPerMinute, s.cfg.Security.RateLimitUI.Burst)
uiMiddlewares = append(uiMiddlewares, RateLimitMiddleware(uiLimiter, s.cfg.Security.TrustProxyHeaders, s.cfg.Security.TrustedProxyPrefixes))
}
if s.cfg.Security.RateLimitAPIRead.Enabled {
apiReadLimiter := NewRateLimiter(s.cfg.Security.RateLimitAPIRead.RequestsPerMinute, s.cfg.Security.RateLimitAPIRead.Burst)
apiReadMiddlewares = append(apiReadMiddlewares, RateLimitMiddleware(apiReadLimiter, s.cfg.Security.TrustProxyHeaders, s.cfg.Security.TrustedProxyPrefixes))
}
if s.cfg.Security.RateLimitAPIWrite.Enabled {
apiWriteLimiter := NewRateLimiter(s.cfg.Security.RateLimitAPIWrite.RequestsPerMinute, s.cfg.Security.RateLimitAPIWrite.Burst)
apiWriteMiddlewares = append(apiWriteMiddlewares, RateLimitMiddleware(apiWriteLimiter, s.cfg.Security.TrustProxyHeaders, s.cfg.Security.TrustedProxyPrefixes))
}
mux.Handle("GET /{$}", Chain(http.HandlerFunc(s.serveSPA), uiMiddlewares...))
mux.Handle("GET /u", Chain(http.HandlerFunc(s.serveSPA), uiMiddlewares...))
mux.Handle("GET /s/{id}", Chain(http.HandlerFunc(s.scratchPage), uiMiddlewares...))
mux.Handle("GET /api/config", Chain(http.HandlerFunc(s.getUIConfig), apiReadMiddlewares...))
mux.Handle("GET /api/scratch/{id}", Chain(http.HandlerFunc(s.getScratch), apiReadMiddlewares...))
mux.Handle("GET /api/raw/{id}", Chain(http.HandlerFunc(s.rawScratch), apiReadMiddlewares...))
writeHandler := http.HandlerFunc(s.createScratch)
middlewares := make([]func(http.Handler) http.Handler, 0, 2)
middlewares = append(middlewares, AllowlistMiddleware(s.cfg.Security.AllowedPrefixes, s.cfg.Security.TrustProxyHeaders, s.cfg.Security.TrustedProxyPrefixes, s.logger))
readMiddlewares := make([]func(http.Handler) http.Handler, 0, 1)
if s.cfg.Security.RateLimit.Enabled {
writeLimiter := NewRateLimiter(s.cfg.Security.RateLimit.RequestsPerMinute, s.cfg.Security.RateLimit.Burst)
readLimiter := NewRateLimiter(s.cfg.Security.RateLimit.RequestsPerMinute, s.cfg.Security.RateLimit.Burst)
middlewares = append(middlewares, RateLimitMiddleware(writeLimiter, s.cfg.Security.TrustProxyHeaders, s.cfg.Security.TrustedProxyPrefixes))
readMiddlewares = append(readMiddlewares, RateLimitMiddleware(readLimiter, s.cfg.Security.TrustProxyHeaders, s.cfg.Security.TrustedProxyPrefixes))
}
mux.Handle("POST /api/scratch", Chain(writeHandler, middlewares...))
mux.Handle("GET /api/scratch/{id}", Chain(http.HandlerFunc(s.getScratch), readMiddlewares...))
mux.Handle("GET /s/{id}", Chain(http.HandlerFunc(s.scratchPage), readMiddlewares...))
mux.Handle("GET /api/raw/{id}", Chain(http.HandlerFunc(s.rawScratch), readMiddlewares...))
writeMiddlewares := make([]func(http.Handler) http.Handler, 0, 2)
writeMiddlewares = append(writeMiddlewares, AllowlistMiddleware(s.cfg.Security.AllowedPrefixes, s.cfg.Security.TrustProxyHeaders, s.cfg.Security.TrustedProxyPrefixes, s.logger))
writeMiddlewares = append(writeMiddlewares, apiWriteMiddlewares...)
mux.Handle("POST /api/scratch", Chain(writeHandler, writeMiddlewares...))
mux.Handle("GET /assets/", http.FileServerFS(webassets.StaticFS()))
mux.Handle("GET /static/", http.StripPrefix("/static/", http.FileServerFS(webassets.StaticFS())))
mux.Handle("GET /{path...}", http.HandlerFunc(s.notFoundPage))
return mux
return AccessLogMiddleware(
s.accessLogger,
s.cfg.Security.TrustProxyHeaders,
s.cfg.Security.TrustedProxyPrefixes,
)(mux)
}
func (s *Server) serveSPA(w http.ResponseWriter, r *http.Request) {
@@ -68,6 +85,7 @@ func (s *Server) getUIConfig(w http.ResponseWriter, r *http.Request) {
payload := map[string]any{
"max_upload_size_bytes": s.cfg.Limits.MaxUploadSizeBytes,
"upload_allowed": s.isUploadAllowedForRequest(r),
"default_ttl": s.cfg.Limits.DefaultTTL,
}
writeJSON(w, http.StatusOK, payload)
}
+13 -3
View File
@@ -162,7 +162,7 @@ func TestReadRoutesAreRateLimitedWhenEnabled(t *testing.T) {
createReq := httptest.NewRequest(http.MethodPost, "/api/scratch", bytes.NewBufferString("hello scratch"))
createReq.Header.Set("Content-Type", "text/plain; charset=utf-8")
createReq.RemoteAddr = "127.0.0.1:9111"
createReq.RemoteAddr = "127.0.0.2:9111"
createRec := httptest.NewRecorder()
handler.ServeHTTP(createRec, createReq)
if createRec.Code != http.StatusCreated {
@@ -1041,7 +1041,7 @@ func newTestHandlerAndStore(t *testing.T, opts testServerOptions) (http.Handler,
t.Helper()
dataDir := filepath.Join(t.TempDir(), "data")
store, err := storage.NewFilesystemStore(dataDir)
store, err := storage.NewFilesystemStore(dataDir, testStorageKey)
if err != nil {
t.Fatalf("NewFilesystemStore() error = %v", err)
}
@@ -1060,7 +1060,17 @@ func newTestHandlerAndStore(t *testing.T, opts testServerOptions) (http.Handler,
DataDir: dataDir,
},
Security: config.SecurityConfig{
RateLimit: config.RateLimitConfig{
RateLimitUI: config.RateLimitConfig{
Enabled: opts.rateLimitEnable,
RequestsPerMinute: 120,
Burst: 1,
},
RateLimitAPIRead: config.RateLimitConfig{
Enabled: opts.rateLimitEnable,
RequestsPerMinute: 120,
Burst: 1,
},
RateLimitAPIWrite: config.RateLimitConfig{
Enabled: opts.rateLimitEnable,
RequestsPerMinute: 120,
Burst: 1,
+12 -6
View File
@@ -15,6 +15,8 @@ import (
"scratchbox/internal/storage"
)
var testStorageKey = []byte("0123456789abcdef0123456789abcdef")
func TestMultipartFileCount(t *testing.T) {
t.Parallel()
@@ -49,18 +51,20 @@ func TestWriteCreateErrorBranches(t *testing.T) {
func TestNewServerInitializes(t *testing.T) {
tmp := t.TempDir()
store, err := storage.NewFilesystemStore(filepath.Join(tmp, "data"))
store, err := storage.NewFilesystemStore(filepath.Join(tmp, "data"), testStorageKey)
if err != nil {
t.Fatalf("NewFilesystemStore() error = %v", err)
}
cfg := config.Config{
Limits: config.LimitsConfig{MaxUploadSizeBytes: 1024},
Security: config.SecurityConfig{
RateLimit: config.RateLimitConfig{Enabled: false},
RateLimitUI: config.RateLimitConfig{Enabled: false, RequestsPerMinute: 30, Burst: 10},
RateLimitAPIRead: config.RateLimitConfig{Enabled: false, RequestsPerMinute: 30, Burst: 10},
RateLimitAPIWrite: config.RateLimitConfig{Enabled: false, RequestsPerMinute: 30, Burst: 10},
},
}
srv, err := NewServer(cfg, store, log.New(io.Discard, "", 0))
srv, err := NewServer(cfg, store, log.New(io.Discard, "", 0), log.New(io.Discard, "", 0))
if err != nil {
t.Fatalf("NewServer() error = %v", err)
}
@@ -118,17 +122,19 @@ func TestGetUIConfigUploadDenied(t *testing.T) {
func TestWriteCreateErrorBranchesWithConfig(t *testing.T) {
tmp := t.TempDir()
store, err := storage.NewFilesystemStore(filepath.Join(tmp, "data"))
store, err := storage.NewFilesystemStore(filepath.Join(tmp, "data"), testStorageKey)
if err != nil {
t.Fatalf("NewFilesystemStore() error = %v", err)
}
cfg := config.Config{
Limits: config.LimitsConfig{MaxUploadSizeBytes: 1024},
Security: config.SecurityConfig{
RateLimit: config.RateLimitConfig{Enabled: false},
RateLimitUI: config.RateLimitConfig{Enabled: false, RequestsPerMinute: 30, Burst: 10},
RateLimitAPIRead: config.RateLimitConfig{Enabled: false, RequestsPerMinute: 30, Burst: 10},
RateLimitAPIWrite: config.RateLimitConfig{Enabled: false, RequestsPerMinute: 30, Burst: 10},
},
}
if _, err := NewServer(cfg, store, log.New(io.Discard, "", 0)); err != nil {
if _, err := NewServer(cfg, store, log.New(io.Discard, "", 0), log.New(io.Discard, "", 0)); err != nil {
t.Fatalf("NewServer() error = %v", err)
}
}
+67
View File
@@ -107,6 +107,53 @@ func RateLimitMiddleware(limiter *RateLimiter, trustProxyHeaders bool, trustedPr
}
}
func AccessLogMiddleware(logger *log.Logger, trustProxyHeaders bool, trustedProxies []netip.Prefix) func(http.Handler) http.Handler {
return func(next http.Handler) http.Handler {
if logger == nil {
return next
}
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if !shouldLogAccessRequest(r.Method, r.URL.Path) {
next.ServeHTTP(w, r)
return
}
start := time.Now()
recorder := &accessLogResponseWriter{ResponseWriter: w}
next.ServeHTTP(recorder, r)
status := recorder.status
if status == 0 {
status = http.StatusOK
}
clientIP := "-"
if ip, ok := extractClientIP(r, trustProxyHeaders, trustedProxies); ok {
clientIP = ip.String()
}
logger.Printf(
"method=%s path=%s status=%d bytes=%d duration_ms=%d ip=%s ua=%q",
r.Method,
r.URL.RequestURI(),
status,
recorder.bytes,
time.Since(start).Milliseconds(),
clientIP,
r.UserAgent(),
)
})
}
}
func shouldLogAccessRequest(method string, path string) bool {
if method == http.MethodPost && path == "/api/scratch" {
return true
}
if method == http.MethodGet && strings.HasPrefix(path, "/api/raw/") && len(path) > len("/api/raw/") {
return true
}
return false
}
func Chain(handler http.Handler, middlewares ...func(http.Handler) http.Handler) http.Handler {
wrapped := handler
for i := len(middlewares) - 1; i >= 0; i-- {
@@ -115,6 +162,26 @@ func Chain(handler http.Handler, middlewares ...func(http.Handler) http.Handler)
return wrapped
}
type accessLogResponseWriter struct {
http.ResponseWriter
status int
bytes int
}
func (w *accessLogResponseWriter) WriteHeader(statusCode int) {
w.status = statusCode
w.ResponseWriter.WriteHeader(statusCode)
}
func (w *accessLogResponseWriter) Write(p []byte) (int, error) {
if w.status == 0 {
w.status = http.StatusOK
}
n, err := w.ResponseWriter.Write(p)
w.bytes += n
return n, err
}
func extractClientIP(r *http.Request, trustProxyHeaders bool, trustedProxies []netip.Prefix) (netip.Addr, bool) {
remoteIP, ok := remoteIPFromRequest(r)
if !ok {
+93
View File
@@ -1,6 +1,7 @@
package httpapi
import (
"bytes"
"encoding/json"
"io"
"log"
@@ -228,3 +229,95 @@ func TestExtractClientIPIgnoresHeadersFromUntrustedProxy(t *testing.T) {
t.Fatalf("extractClientIP() = (%v,%v), want (203.0.113.10,true)", ip, ok)
}
}
func TestAccessLogMiddlewareWritesStructuredLog(t *testing.T) {
t.Parallel()
var buf bytes.Buffer
logger := log.New(&buf, "", 0)
handler := AccessLogMiddleware(logger, false, nil)(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
w.WriteHeader(http.StatusCreated)
_, _ = w.Write([]byte("ok"))
}))
req := httptest.NewRequest(http.MethodPost, "/api/scratch?ttl=1h", nil)
req.RemoteAddr = "198.51.100.12:4444"
req.Header.Set("User-Agent", "middleware-test")
rec := httptest.NewRecorder()
handler.ServeHTTP(rec, req)
got := buf.String()
if !strings.Contains(got, "method=POST") {
t.Fatalf("missing method field in log: %q", got)
}
if !strings.Contains(got, "path=/api/scratch?ttl=1h") {
t.Fatalf("missing path field in log: %q", got)
}
if !strings.Contains(got, "status=201") {
t.Fatalf("missing status field in log: %q", got)
}
if !strings.Contains(got, "bytes=2") {
t.Fatalf("missing bytes field in log: %q", got)
}
if !strings.Contains(got, "ip=198.51.100.12") {
t.Fatalf("missing ip field in log: %q", got)
}
}
func TestAccessLogMiddlewareSkipsConfigAndStaticPaths(t *testing.T) {
t.Parallel()
var buf bytes.Buffer
logger := log.New(&buf, "", 0)
handler := AccessLogMiddleware(logger, false, nil)(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
w.WriteHeader(http.StatusOK)
}))
reqConfig := httptest.NewRequest(http.MethodGet, "/api/config", nil)
reqConfig.RemoteAddr = "198.51.100.12:4444"
handler.ServeHTTP(httptest.NewRecorder(), reqConfig)
reqStatic := httptest.NewRequest(http.MethodGet, "/static/app.js", nil)
reqStatic.RemoteAddr = "198.51.100.12:4444"
handler.ServeHTTP(httptest.NewRecorder(), reqStatic)
reqAsset := httptest.NewRequest(http.MethodGet, "/assets/logo.svg", nil)
reqAsset.RemoteAddr = "198.51.100.12:4444"
handler.ServeHTTP(httptest.NewRecorder(), reqAsset)
reqScratchView := httptest.NewRequest(http.MethodGet, "/s/abc123", nil)
reqScratchView.RemoteAddr = "198.51.100.12:4444"
handler.ServeHTTP(httptest.NewRecorder(), reqScratchView)
reqUpload := httptest.NewRequest(http.MethodGet, "/u", nil)
reqUpload.RemoteAddr = "198.51.100.12:4444"
handler.ServeHTTP(httptest.NewRecorder(), reqUpload)
if buf.Len() != 0 {
t.Fatalf("expected no access logs for skipped paths, got %q", buf.String())
}
}
func TestAccessLogMiddlewareOnlyAllowsConfiguredRoutes(t *testing.T) {
t.Parallel()
var buf bytes.Buffer
logger := log.New(&buf, "", 0)
handler := AccessLogMiddleware(logger, false, nil)(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
w.WriteHeader(http.StatusOK)
}))
disallowed := httptest.NewRequest(http.MethodGet, "/api/scratch/some-id", nil)
disallowed.RemoteAddr = "198.51.100.12:4444"
handler.ServeHTTP(httptest.NewRecorder(), disallowed)
if buf.Len() != 0 {
t.Fatalf("expected no access log for disallowed route, got %q", buf.String())
}
allowed := httptest.NewRequest(http.MethodGet, "/api/raw/some-id", nil)
allowed.RemoteAddr = "198.51.100.12:4444"
handler.ServeHTTP(httptest.NewRecorder(), allowed)
if !strings.Contains(buf.String(), "method=GET path=/api/raw/some-id") {
t.Fatalf("expected access log for allowed route, got %q", buf.String())
}
}
+388 -134
View File
@@ -3,7 +3,11 @@ package storage
import (
"compress/gzip"
"context"
"crypto/aes"
"crypto/cipher"
"crypto/rand"
"crypto/sha256"
"encoding/binary"
"encoding/hex"
"encoding/json"
"errors"
@@ -17,14 +21,16 @@ import (
)
const (
indexFilename = "metadata.json"
filesDirName = "scratches"
idBytes = 12
maxCreateIDAttempts = 100
indexFilename = "metadata"
filesDirName = "scratches"
encryptedChunkSize = 64 * 1024
)
var ErrNotFound = errors.New("scratch not found")
var encryptedFileMagic = [4]byte{'S', 'B', 'X', '1'}
var encryptedIndexMagic = [4]byte{'S', 'M', 'D', '1'}
type Metadata struct {
ID string `json:"id"`
FilePath string `json:"file_path"`
@@ -35,35 +41,63 @@ type Metadata struct {
Size int64 `json:"size"`
}
type FilesystemStore struct {
dataDir string
filesDir string
index string
idGenerator func() (string, error)
mu sync.RWMutex
metadata map[string]Metadata
reservedIDs map[string]struct{}
type indexDocument struct {
DefaultTTL string `json:"default_ttl,omitempty"`
Scratches map[string]Metadata `json:"scratches"`
}
func NewFilesystemStore(dataDir string) (*FilesystemStore, error) {
type FilesystemStore struct {
dataDir string
filesDir string
index string
defaultTTL time.Duration
fileCipher cipher.AEAD
mu sync.RWMutex
metadata map[string]Metadata
}
func NewFilesystemStore(dataDir string, encryptionKey []byte) (*FilesystemStore, error) {
return newFilesystemStore(dataDir, 0, encryptionKey)
}
func NewFilesystemStoreWithDefaultTTL(dataDir string, defaultTTL time.Duration, encryptionKey []byte) (*FilesystemStore, error) {
return newFilesystemStore(dataDir, defaultTTL, encryptionKey)
}
func newFilesystemStore(dataDir string, defaultTTL time.Duration, encryptionKey []byte) (*FilesystemStore, error) {
filesDir := filepath.Join(dataDir, filesDirName)
if err := os.MkdirAll(filesDir, 0o755); err != nil {
return nil, fmt.Errorf("create files dir: %w", err)
}
if len(encryptionKey) != 32 {
return nil, fmt.Errorf("storage encryption key must be 32 bytes, got %d", len(encryptionKey))
}
block, err := aes.NewCipher(encryptionKey)
if err != nil {
return nil, fmt.Errorf("init storage cipher: %w", err)
}
fileCipher, err := cipher.NewGCM(block)
if err != nil {
return nil, fmt.Errorf("init storage aead: %w", err)
}
store := &FilesystemStore{
dataDir: dataDir,
filesDir: filesDir,
index: filepath.Join(dataDir, indexFilename),
idGenerator: generateID,
metadata: map[string]Metadata{},
reservedIDs: map[string]struct{}{},
dataDir: dataDir,
filesDir: filesDir,
index: filepath.Join(dataDir, indexFilename),
fileCipher: fileCipher,
metadata: map[string]Metadata{},
}
if err := store.loadIndex(); err != nil {
return nil, err
}
if err := store.reconcileOnStartup(time.Now()); err != nil {
now := time.Now().UTC()
if err := store.applyDefaultTTLOnStartup(defaultTTL); err != nil {
return nil, err
}
if err := store.reconcileOnStartup(now); err != nil {
return nil, err
}
return store, nil
@@ -74,107 +108,84 @@ func (s *FilesystemStore) Create(ctx context.Context, body io.Reader, contentTyp
}
func (s *FilesystemStore) CreateWithOriginalName(ctx context.Context, body io.Reader, contentType string, originalName string, ttl time.Duration) (Metadata, error) {
for attempt := 0; attempt < maxCreateIDAttempts; attempt++ {
id, err := s.idGenerator()
if err != nil {
return Metadata{}, fmt.Errorf("generate id: %w", err)
}
finalPath := filepath.Join(s.filesDir, id)
s.mu.RLock()
_, exists := s.metadata[id]
_, reserved := s.reservedIDs[id]
s.mu.RUnlock()
if exists || reserved {
continue
}
if _, err := os.Stat(finalPath); err == nil {
continue
} else if !errors.Is(err, os.ErrNotExist) {
return Metadata{}, fmt.Errorf("stat candidate scratch path: %w", err)
}
s.mu.Lock()
if _, exists := s.metadata[id]; exists {
s.mu.Unlock()
continue
}
if _, reserved := s.reservedIDs[id]; reserved {
s.mu.Unlock()
continue
}
// Reserve the ID before consuming the request body to avoid
// concurrent writers choosing the same ID.
s.reservedIDs[id] = struct{}{}
s.mu.Unlock()
tempFile, err := os.CreateTemp(s.filesDir, id+".tmp-*")
if err != nil {
s.mu.Lock()
delete(s.reservedIDs, id)
s.mu.Unlock()
return Metadata{}, fmt.Errorf("create temp file: %w", err)
}
gzipWriter := gzip.NewWriter(tempFile)
size, copyErr := io.Copy(gzipWriter, body)
gzipCloseErr := gzipWriter.Close()
closeErr := tempFile.Close()
if copyErr != nil {
_ = os.Remove(tempFile.Name())
s.mu.Lock()
delete(s.reservedIDs, id)
s.mu.Unlock()
return Metadata{}, fmt.Errorf("write scratch content: %w", copyErr)
}
if gzipCloseErr != nil {
_ = os.Remove(tempFile.Name())
s.mu.Lock()
delete(s.reservedIDs, id)
s.mu.Unlock()
return Metadata{}, fmt.Errorf("close gzip writer: %w", gzipCloseErr)
}
if closeErr != nil {
_ = os.Remove(tempFile.Name())
s.mu.Lock()
delete(s.reservedIDs, id)
s.mu.Unlock()
return Metadata{}, fmt.Errorf("close temp file: %w", closeErr)
}
if err := os.Rename(tempFile.Name(), finalPath); err != nil {
_ = os.Remove(tempFile.Name())
s.mu.Lock()
delete(s.reservedIDs, id)
s.mu.Unlock()
return Metadata{}, fmt.Errorf("atomically move scratch file: %w", err)
}
now := time.Now().UTC()
meta := Metadata{
ID: id,
FilePath: finalPath,
CreatedAt: now,
ExpiresAt: now.Add(ttl),
ContentType: contentType,
OriginalName: normalizeOriginalName(originalName),
Size: size,
}
s.mu.Lock()
delete(s.reservedIDs, id)
s.metadata[id] = meta
if err := s.persistLocked(); err != nil {
delete(s.metadata, id)
s.mu.Unlock()
_ = os.Remove(finalPath)
return Metadata{}, err
}
s.mu.Unlock()
return meta, nil
tempFile, err := os.CreateTemp(s.filesDir, "scratch.tmp-*")
if err != nil {
return Metadata{}, fmt.Errorf("create temp file: %w", err)
}
return Metadata{}, fmt.Errorf("failed to allocate unique scratch id after %d attempts", maxCreateIDAttempts)
blobHash := sha256.New()
blobSink := io.MultiWriter(tempFile, blobHash)
scratchWriter := io.Writer(blobSink)
encryptedWriter, err := newEncryptedFileWriter(blobSink, s.fileCipher)
if err != nil {
_ = tempFile.Close()
_ = os.Remove(tempFile.Name())
return Metadata{}, fmt.Errorf("init encrypted scratch writer: %w", err)
}
scratchWriter = encryptedWriter
gzipWriter := gzip.NewWriter(scratchWriter)
size, copyErr := io.Copy(gzipWriter, body)
gzipCloseErr := gzipWriter.Close()
encryptCloseErr := encryptedWriter.Close()
closeErr := tempFile.Close()
if copyErr != nil {
_ = os.Remove(tempFile.Name())
return Metadata{}, fmt.Errorf("write scratch content: %w", copyErr)
}
if gzipCloseErr != nil {
_ = os.Remove(tempFile.Name())
return Metadata{}, fmt.Errorf("close gzip writer: %w", gzipCloseErr)
}
if encryptCloseErr != nil {
_ = os.Remove(tempFile.Name())
return Metadata{}, fmt.Errorf("close encrypted writer: %w", encryptCloseErr)
}
if closeErr != nil {
_ = os.Remove(tempFile.Name())
return Metadata{}, fmt.Errorf("close temp file: %w", closeErr)
}
id := hex.EncodeToString(blobHash.Sum(nil))
finalPath := filepath.Join(s.filesDir, id)
if _, statErr := os.Stat(finalPath); statErr == nil {
_ = os.Remove(tempFile.Name())
return Metadata{}, errors.New("scratch id collision for content hash")
} else if !errors.Is(statErr, os.ErrNotExist) {
_ = os.Remove(tempFile.Name())
return Metadata{}, fmt.Errorf("stat candidate scratch path: %w", statErr)
}
if err := os.Rename(tempFile.Name(), finalPath); err != nil {
_ = os.Remove(tempFile.Name())
return Metadata{}, fmt.Errorf("atomically move scratch file: %w", err)
}
now := time.Now().UTC()
meta := Metadata{
ID: id,
FilePath: finalPath,
CreatedAt: now,
ExpiresAt: now.Add(ttl),
ContentType: contentType,
OriginalName: normalizeOriginalName(originalName),
Size: size,
}
s.mu.Lock()
if _, exists := s.metadata[id]; exists {
s.mu.Unlock()
_ = os.Remove(finalPath)
return Metadata{}, errors.New("scratch id collision in metadata index")
}
s.metadata[id] = meta
if err := s.persistLocked(); err != nil {
delete(s.metadata, id)
s.mu.Unlock()
_ = os.Remove(finalPath)
return Metadata{}, err
}
s.mu.Unlock()
return meta, nil
}
func normalizeOriginalName(originalName string) string {
@@ -212,7 +223,12 @@ func (s *FilesystemStore) Open(id string) (io.ReadCloser, Metadata, error) {
}
return nil, Metadata{}, fmt.Errorf("open scratch file: %w", err)
}
gzipReader, err := gzip.NewReader(file)
decryptedReader, decryptErr := newEncryptedFileReader(file, s.fileCipher)
if decryptErr != nil {
_ = file.Close()
return nil, Metadata{}, fmt.Errorf("create encrypted reader: %w", decryptErr)
}
gzipReader, err := gzip.NewReader(decryptedReader)
if err != nil {
_ = file.Close()
return nil, Metadata{}, fmt.Errorf("create gzip reader: %w", err)
@@ -286,22 +302,46 @@ func (s *FilesystemStore) loadIndex() error {
}
return fmt.Errorf("read metadata index: %w", err)
}
plaintext, err := decryptIndexPayload(b, s.fileCipher)
if err != nil {
return fmt.Errorf("decrypt metadata index: %w", err)
}
loaded := map[string]Metadata{}
if err := json.Unmarshal(b, &loaded); err != nil {
doc := indexDocument{}
if err := json.Unmarshal(plaintext, &doc); err != nil {
return fmt.Errorf("decode metadata index: %w", err)
}
s.metadata = loaded
if doc.Scratches == nil {
doc.Scratches = map[string]Metadata{}
}
s.metadata = doc.Scratches
if ttlRaw := strings.TrimSpace(doc.DefaultTTL); ttlRaw != "" {
ttl, parseErr := time.ParseDuration(ttlRaw)
if parseErr != nil {
return fmt.Errorf("decode metadata index default ttl: %w", parseErr)
}
s.defaultTTL = ttl
}
return nil
}
func (s *FilesystemStore) persistLocked() error {
tmp := s.index + ".tmp"
payload, err := json.MarshalIndent(s.metadata, "", " ")
doc := indexDocument{
Scratches: s.metadata,
}
if s.defaultTTL > 0 {
doc.DefaultTTL = s.defaultTTL.String()
}
payload, err := json.MarshalIndent(doc, "", " ")
if err != nil {
return fmt.Errorf("encode metadata index: %w", err)
}
if err := os.WriteFile(tmp, payload, 0o644); err != nil {
encryptedPayload, err := encryptIndexPayload(payload, s.fileCipher)
if err != nil {
return fmt.Errorf("encrypt metadata index: %w", err)
}
if err := os.WriteFile(tmp, encryptedPayload, 0o644); err != nil {
return fmt.Errorf("write metadata temp file: %w", err)
}
if err := os.Rename(tmp, s.index); err != nil {
@@ -310,6 +350,28 @@ func (s *FilesystemStore) persistLocked() error {
return nil
}
func (s *FilesystemStore) applyDefaultTTLOnStartup(defaultTTL time.Duration) error {
if defaultTTL <= 0 {
return nil
}
if s.defaultTTL <= 0 {
s.defaultTTL = defaultTTL
return s.persistLocked()
}
if s.defaultTTL == defaultTTL {
return nil
}
delta := defaultTTL - s.defaultTTL
for id, meta := range s.metadata {
meta.ExpiresAt = meta.ExpiresAt.Add(delta)
s.metadata[id] = meta
}
s.defaultTTL = defaultTTL
return s.persistLocked()
}
func (s *FilesystemStore) reconcileOnStartup(now time.Time) error {
s.mu.Lock()
defer s.mu.Unlock()
@@ -337,14 +399,6 @@ func (s *FilesystemStore) reconcileOnStartup(now time.Time) error {
return s.persistLocked()
}
func generateID() (string, error) {
b := make([]byte, idBytes)
if _, err := rand.Read(b); err != nil {
return "", err
}
return hex.EncodeToString(b), nil
}
type combinedReadCloser struct {
reader io.Reader
closers []io.Closer
@@ -363,3 +417,203 @@ func (c *combinedReadCloser) Close() error {
}
return firstErr
}
type encryptedFileWriter struct {
dst io.Writer
aead cipher.AEAD
noncePrefix [4]byte
counter uint64
pending []byte
}
func newEncryptedFileWriter(dst io.Writer, aead cipher.AEAD) (*encryptedFileWriter, error) {
writer := &encryptedFileWriter{
dst: dst,
aead: aead,
pending: make([]byte, 0, encryptedChunkSize),
}
if _, err := rand.Read(writer.noncePrefix[:]); err != nil {
return nil, fmt.Errorf("generate nonce prefix: %w", err)
}
header := make([]byte, 0, len(encryptedFileMagic)+len(writer.noncePrefix))
header = append(header, encryptedFileMagic[:]...)
header = append(header, writer.noncePrefix[:]...)
if _, err := writer.dst.Write(header); err != nil {
return nil, fmt.Errorf("write encrypted file header: %w", err)
}
return writer, nil
}
func (w *encryptedFileWriter) Write(p []byte) (int, error) {
written := len(p)
for len(p) > 0 {
space := encryptedChunkSize - len(w.pending)
if space > len(p) {
space = len(p)
}
w.pending = append(w.pending, p[:space]...)
p = p[space:]
if len(w.pending) == encryptedChunkSize {
if err := w.flushChunk(w.pending); err != nil {
return written - len(p), err
}
w.pending = w.pending[:0]
}
}
return written, nil
}
func (w *encryptedFileWriter) Close() error {
if len(w.pending) == 0 {
return nil
}
if err := w.flushChunk(w.pending); err != nil {
return err
}
w.pending = w.pending[:0]
return nil
}
func (w *encryptedFileWriter) flushChunk(plaintext []byte) error {
nonce := make([]byte, w.aead.NonceSize())
copy(nonce, w.noncePrefix[:])
binary.BigEndian.PutUint64(nonce[len(w.noncePrefix):], w.counter)
ciphertext := w.aead.Seal(nil, nonce, plaintext, nil)
var sizeBuf [4]byte
binary.BigEndian.PutUint32(sizeBuf[:], uint32(len(ciphertext)))
if _, err := w.dst.Write(sizeBuf[:]); err != nil {
return fmt.Errorf("write encrypted chunk size: %w", err)
}
if _, err := w.dst.Write(ciphertext); err != nil {
return fmt.Errorf("write encrypted chunk payload: %w", err)
}
w.counter++
return nil
}
type encryptedFileReader struct {
src io.Reader
aead cipher.AEAD
noncePrefix [4]byte
counter uint64
buf []byte
offset int
eof bool
}
func newEncryptedFileReader(src io.Reader, aead cipher.AEAD) (*encryptedFileReader, error) {
reader := &encryptedFileReader{
src: src,
aead: aead,
}
header := make([]byte, len(encryptedFileMagic)+len(reader.noncePrefix))
if _, err := io.ReadFull(src, header); err != nil {
return nil, fmt.Errorf("read encrypted file header: %w", err)
}
if !equalBytes(header[:len(encryptedFileMagic)], encryptedFileMagic[:]) {
return nil, errors.New("invalid encrypted file header")
}
copy(reader.noncePrefix[:], header[len(encryptedFileMagic):])
return reader, nil
}
func (r *encryptedFileReader) Read(p []byte) (int, error) {
for r.offset >= len(r.buf) {
if r.eof {
return 0, io.EOF
}
if err := r.loadNextChunk(); err != nil {
return 0, err
}
}
n := copy(p, r.buf[r.offset:])
r.offset += n
return n, nil
}
func (r *encryptedFileReader) loadNextChunk() error {
var sizeBuf [4]byte
_, err := io.ReadFull(r.src, sizeBuf[:])
if err != nil {
if errors.Is(err, io.EOF) {
r.eof = true
return io.EOF
}
if errors.Is(err, io.ErrUnexpectedEOF) {
return errors.New("truncated encrypted chunk size")
}
return fmt.Errorf("read encrypted chunk size: %w", err)
}
chunkSize := binary.BigEndian.Uint32(sizeBuf[:])
if chunkSize == 0 {
return errors.New("invalid empty encrypted chunk")
}
maxChunkSize := uint32(encryptedChunkSize + r.aead.Overhead())
if chunkSize > maxChunkSize {
return fmt.Errorf("encrypted chunk too large: %d", chunkSize)
}
ciphertext := make([]byte, chunkSize)
if _, err := io.ReadFull(r.src, ciphertext); err != nil {
return fmt.Errorf("read encrypted chunk payload: %w", err)
}
nonce := make([]byte, r.aead.NonceSize())
copy(nonce, r.noncePrefix[:])
binary.BigEndian.PutUint64(nonce[len(r.noncePrefix):], r.counter)
plaintext, err := r.aead.Open(nil, nonce, ciphertext, nil)
if err != nil {
return fmt.Errorf("decrypt chunk: %w", err)
}
r.counter++
r.buf = plaintext
r.offset = 0
return nil
}
func equalBytes(a, b []byte) bool {
if len(a) != len(b) {
return false
}
for i := range a {
if a[i] != b[i] {
return false
}
}
return true
}
func encryptIndexPayload(plaintext []byte, aead cipher.AEAD) ([]byte, error) {
nonce := make([]byte, aead.NonceSize())
if _, err := rand.Read(nonce); err != nil {
return nil, fmt.Errorf("generate index nonce: %w", err)
}
ciphertext := aead.Seal(nil, nonce, plaintext, nil)
out := make([]byte, 0, len(encryptedIndexMagic)+len(nonce)+len(ciphertext))
out = append(out, encryptedIndexMagic[:]...)
out = append(out, nonce...)
out = append(out, ciphertext...)
return out, nil
}
func decryptIndexPayload(raw []byte, aead cipher.AEAD) ([]byte, error) {
headerLen := len(encryptedIndexMagic)
nonceSize := aead.NonceSize()
if len(raw) < headerLen+nonceSize {
return nil, errors.New("encrypted index is too short")
}
if !equalBytes(raw[:headerLen], encryptedIndexMagic[:]) {
return nil, errors.New("invalid encrypted index header")
}
nonceStart := headerLen
nonceEnd := nonceStart + nonceSize
nonce := raw[nonceStart:nonceEnd]
ciphertext := raw[nonceEnd:]
if len(ciphertext) == 0 {
return nil, errors.New("encrypted index ciphertext is empty")
}
plaintext, err := aead.Open(nil, nonce, ciphertext, nil)
if err != nil {
return nil, fmt.Errorf("decrypt index payload: %w", err)
}
return plaintext, nil
}
+212 -278
View File
@@ -3,6 +3,11 @@ package storage
import (
"bytes"
"context"
"crypto/aes"
"crypto/cipher"
"crypto/rand"
"crypto/sha256"
"encoding/hex"
"encoding/json"
"errors"
"fmt"
@@ -15,10 +20,25 @@ import (
"time"
)
var testEncryptionKey = []byte("0123456789abcdef0123456789abcdef")
func mustTestAEAD(t *testing.T) cipher.AEAD {
t.Helper()
block, err := aes.NewCipher(testEncryptionKey)
if err != nil {
t.Fatalf("aes.NewCipher() error = %v", err)
}
aead, err := cipher.NewGCM(block)
if err != nil {
t.Fatalf("cipher.NewGCM() error = %v", err)
}
return aead
}
func TestCreateStoresGzipAndOpenIsTransparent(t *testing.T) {
t.Parallel()
store, err := NewFilesystemStore(filepath.Join(t.TempDir(), "data"))
store, err := NewFilesystemStore(filepath.Join(t.TempDir(), "data"), testEncryptionKey)
if err != nil {
t.Fatalf("NewFilesystemStore() error = %v", err)
}
@@ -36,8 +56,8 @@ func TestCreateStoresGzipAndOpenIsTransparent(t *testing.T) {
if err != nil {
t.Fatalf("ReadFile() error = %v", err)
}
if len(onDisk) < 2 || onDisk[0] != 0x1f || onDisk[1] != 0x8b {
t.Fatalf("stored file does not have gzip header")
if len(onDisk) < 4 || string(onDisk[:4]) != "SBX1" {
t.Fatalf("stored file does not have encrypted scratch header")
}
reader, _, err := store.Open(meta.ID)
@@ -54,10 +74,92 @@ func TestCreateStoresGzipAndOpenIsTransparent(t *testing.T) {
}
}
func TestCreateEncryptedStoreDoesNotExposeGzipHeaderOnDisk(t *testing.T) {
t.Parallel()
key := make([]byte, 32)
if _, err := rand.Read(key); err != nil {
t.Fatalf("rand.Read() error = %v", err)
}
dataDir := filepath.Join(t.TempDir(), "data")
store, err := NewFilesystemStore(dataDir, key)
if err != nil {
t.Fatalf("NewEncryptedFilesystemStore() error = %v", err)
}
original := bytes.Repeat([]byte("secret payload "), 1024)
meta, err := store.Create(context.Background(), bytes.NewReader(original), "text/plain", time.Hour)
if err != nil {
t.Fatalf("Create() error = %v", err)
}
onDisk, err := os.ReadFile(meta.FilePath)
if err != nil {
t.Fatalf("ReadFile() error = %v", err)
}
if len(onDisk) < 2 {
t.Fatalf("encrypted file too short")
}
if onDisk[0] == 0x1f && onDisk[1] == 0x8b {
t.Fatalf("encrypted file unexpectedly starts with gzip header")
}
reader, _, err := store.Open(meta.ID)
if err != nil {
t.Fatalf("Open() error = %v", err)
}
defer reader.Close()
got, err := io.ReadAll(reader)
if err != nil {
t.Fatalf("ReadAll() error = %v", err)
}
if !bytes.Equal(got, original) {
t.Fatalf("Open() content mismatch: got %d bytes, want %d", len(got), len(original))
}
filePayload, err := os.ReadFile(meta.FilePath)
if err != nil {
t.Fatalf("ReadFile(encrypted) error = %v", err)
}
sum := sha256.Sum256(filePayload)
wantID := hex.EncodeToString(sum[:])
if meta.ID != wantID {
t.Fatalf("meta.ID = %q, want SHA-256 of encrypted blob %q", meta.ID, wantID)
}
}
func TestEncryptedStoreFailsToOpenWithWrongKey(t *testing.T) {
t.Parallel()
keyA := make([]byte, 32)
if _, err := rand.Read(keyA); err != nil {
t.Fatalf("rand.Read(keyA) error = %v", err)
}
keyB := make([]byte, 32)
if _, err := rand.Read(keyB); err != nil {
t.Fatalf("rand.Read(keyB) error = %v", err)
}
dataDir := filepath.Join(t.TempDir(), "data")
storeA, err := NewFilesystemStore(dataDir, keyA)
if err != nil {
t.Fatalf("NewEncryptedFilesystemStore(keyA) error = %v", err)
}
_, err = storeA.Create(context.Background(), bytes.NewReader([]byte("hello")), "text/plain", time.Hour)
if err != nil {
t.Fatalf("Create() error = %v", err)
}
if _, err := NewFilesystemStore(dataDir, keyB); err == nil {
t.Fatalf("NewFilesystemStore() with wrong key error = nil, want non-nil")
}
}
func TestCreateWithOriginalNameNormalizesFilename(t *testing.T) {
t.Parallel()
store, err := NewFilesystemStore(filepath.Join(t.TempDir(), "data"))
store, err := NewFilesystemStore(filepath.Join(t.TempDir(), "data"), testEncryptionKey)
if err != nil {
t.Fatalf("NewFilesystemStore() error = %v", err)
}
@@ -70,7 +172,7 @@ func TestCreateWithOriginalNameNormalizesFilename(t *testing.T) {
t.Fatalf("meta.OriginalName = %q, want %q", meta.OriginalName, "demo.txt")
}
reloaded, err := NewFilesystemStore(store.dataDir)
reloaded, err := NewFilesystemStore(store.dataDir, testEncryptionKey)
if err != nil {
t.Fatalf("NewFilesystemStore() reload error = %v", err)
}
@@ -83,201 +185,31 @@ func TestCreateWithOriginalNameNormalizesFilename(t *testing.T) {
}
}
func TestCreateWithOriginalNameRetriesOnIDCollision(t *testing.T) {
func TestCreateWithOriginalNameSameContentGetsDifferentIDs(t *testing.T) {
t.Parallel()
store, err := NewFilesystemStore(filepath.Join(t.TempDir(), "data"))
store, err := NewFilesystemStore(filepath.Join(t.TempDir(), "data"), testEncryptionKey)
if err != nil {
t.Fatalf("NewFilesystemStore() error = %v", err)
}
existingID := "aaaaaaaaaaaaaaaaaaaaaaaa"
existingPath := filepath.Join(store.filesDir, existingID)
if err := os.WriteFile(existingPath, []byte{0x1f, 0x8b, 0x08, 0x00}, 0o644); err != nil {
t.Fatalf("WriteFile(existing collision path) error = %v", err)
}
store.mu.Lock()
store.metadata[existingID] = Metadata{
ID: existingID,
FilePath: existingPath,
CreatedAt: time.Now().UTC(),
ExpiresAt: time.Now().UTC().Add(time.Hour),
ContentType: "text/plain",
Size: 4,
}
store.mu.Unlock()
nextID := "bbbbbbbbbbbbbbbbbbbbbbbb"
calls := 0
store.idGenerator = func() (string, error) {
calls++
if calls == 1 {
return existingID, nil
}
return nextID, nil
}
meta, err := store.CreateWithOriginalName(context.Background(), bytes.NewReader([]byte("hello")), "text/plain", "hello.txt", time.Hour)
first, err := store.CreateWithOriginalName(context.Background(), bytes.NewReader([]byte("same body")), "text/plain", "one.txt", time.Hour)
if err != nil {
t.Fatalf("CreateWithOriginalName() error = %v", err)
t.Fatalf("first CreateWithOriginalName() error = %v", err)
}
if meta.ID != nextID {
t.Fatalf("meta.ID = %q, want %q", meta.ID, nextID)
}
if calls != 2 {
t.Fatalf("idGenerator calls = %d, want 2", calls)
}
}
func TestCreateWithOriginalNameFailsAfterMaxIDRetries(t *testing.T) {
t.Parallel()
store, err := NewFilesystemStore(filepath.Join(t.TempDir(), "data"))
if err != nil {
t.Fatalf("NewFilesystemStore() error = %v", err)
}
existingID := "cccccccccccccccccccccccc"
existingPath := filepath.Join(store.filesDir, existingID)
store.mu.Lock()
store.metadata[existingID] = Metadata{
ID: existingID,
FilePath: existingPath,
CreatedAt: time.Now().UTC(),
ExpiresAt: time.Now().UTC().Add(time.Hour),
ContentType: "text/plain",
}
store.mu.Unlock()
store.idGenerator = func() (string, error) {
return existingID, nil
}
_, err = store.CreateWithOriginalName(context.Background(), bytes.NewReader([]byte("hello")), "text/plain", "hello.txt", time.Hour)
if err == nil {
t.Fatalf("CreateWithOriginalName() error = nil, want retry exhaustion error")
}
if !strings.Contains(err.Error(), "failed to allocate unique scratch id") {
t.Fatalf("unexpected error: %v", err)
}
}
func TestCreateWithOriginalNameRetriesWhenIDIsReservedInFlight(t *testing.T) {
t.Parallel()
store, err := NewFilesystemStore(filepath.Join(t.TempDir(), "data"))
if err != nil {
t.Fatalf("NewFilesystemStore() error = %v", err)
}
const collideID = "abababababababababababab"
const uniqueID = "cdcdcdcdcdcdcdcdcdcdcdcd"
var idMu sync.Mutex
idCalls := 0
store.idGenerator = func() (string, error) {
idMu.Lock()
defer idMu.Unlock()
idCalls++
if idCalls <= 2 {
return collideID, nil
}
return uniqueID, nil
}
release := make(chan struct{})
firstMetaCh := make(chan Metadata, 1)
firstErrCh := make(chan error, 1)
go func() {
meta, createErr := store.CreateWithOriginalName(
context.Background(),
&blockingReader{release: release, payload: []byte("first upload")},
"text/plain",
"first.txt",
time.Hour,
)
firstMetaCh <- meta
firstErrCh <- createErr
}()
deadline := time.Now().Add(2 * time.Second)
for {
store.mu.RLock()
_, reserved := store.reservedIDs[collideID]
store.mu.RUnlock()
if reserved {
break
}
if time.Now().After(deadline) {
t.Fatalf("timed out waiting for first upload reservation")
}
time.Sleep(2 * time.Millisecond)
}
secondMeta, err := store.CreateWithOriginalName(
context.Background(),
strings.NewReader("second upload"),
"text/plain",
"second.txt",
time.Hour,
)
second, err := store.CreateWithOriginalName(context.Background(), bytes.NewReader([]byte("same body")), "text/plain", "two.txt", time.Hour)
if err != nil {
t.Fatalf("second CreateWithOriginalName() error = %v", err)
}
if secondMeta.ID != uniqueID {
t.Fatalf("second meta.ID = %q, want %q after collision retry", secondMeta.ID, uniqueID)
}
if secondMeta.ID == collideID {
t.Fatalf("second create reused in-flight reserved ID")
}
close(release)
firstMeta := <-firstMetaCh
if err := <-firstErrCh; err != nil {
t.Fatalf("first CreateWithOriginalName() error = %v", err)
}
if firstMeta.ID != collideID {
t.Fatalf("first meta.ID = %q, want %q", firstMeta.ID, collideID)
}
firstReader, _, err := store.Open(firstMeta.ID)
if err != nil {
t.Fatalf("Open(first) error = %v", err)
}
defer firstReader.Close()
firstBody, err := io.ReadAll(firstReader)
if err != nil {
t.Fatalf("ReadAll(first) error = %v", err)
}
if string(firstBody) != "first upload" {
t.Fatalf("first content = %q, want %q", string(firstBody), "first upload")
}
secondReader, _, err := store.Open(secondMeta.ID)
if err != nil {
t.Fatalf("Open(second) error = %v", err)
}
defer secondReader.Close()
secondBody, err := io.ReadAll(secondReader)
if err != nil {
t.Fatalf("ReadAll(second) error = %v", err)
}
if string(secondBody) != "second upload" {
t.Fatalf("second content = %q, want %q", string(secondBody), "second upload")
}
idMu.Lock()
totalCalls := idCalls
idMu.Unlock()
if totalCalls < 3 {
t.Fatalf("idGenerator call count = %d, want at least 3", totalCalls)
if first.ID == second.ID {
t.Fatalf("IDs should differ because encrypted blobs include random nonce")
}
}
func TestGetDeleteAndNotFound(t *testing.T) {
t.Parallel()
store, err := NewFilesystemStore(filepath.Join(t.TempDir(), "data"))
store, err := NewFilesystemStore(filepath.Join(t.TempDir(), "data"), testEncryptionKey)
if err != nil {
t.Fatalf("NewFilesystemStore() error = %v", err)
}
@@ -302,7 +234,7 @@ func TestGetDeleteAndNotFound(t *testing.T) {
func TestDeleteExpired(t *testing.T) {
t.Parallel()
store, err := NewFilesystemStore(filepath.Join(t.TempDir(), "data"))
store, err := NewFilesystemStore(filepath.Join(t.TempDir(), "data"), testEncryptionKey)
if err != nil {
t.Fatalf("NewFilesystemStore() error = %v", err)
}
@@ -332,7 +264,7 @@ func TestDeleteExpired(t *testing.T) {
func TestOpenInvalidGzipFails(t *testing.T) {
t.Parallel()
store, err := NewFilesystemStore(filepath.Join(t.TempDir(), "data"))
store, err := NewFilesystemStore(filepath.Join(t.TempDir(), "data"), testEncryptionKey)
if err != nil {
t.Fatalf("NewFilesystemStore() error = %v", err)
}
@@ -368,7 +300,7 @@ func TestNewFilesystemStoreInvalidIndex(t *testing.T) {
if err := os.WriteFile(filepath.Join(root, indexFilename), []byte("{bad"), 0o644); err != nil {
t.Fatalf("WriteFile() error = %v", err)
}
if _, err := NewFilesystemStore(root); err == nil {
if _, err := NewFilesystemStore(root, testEncryptionKey); err == nil {
t.Fatalf("NewFilesystemStore() error = nil, want invalid index error")
}
}
@@ -419,15 +351,21 @@ func TestNewFilesystemStoreReconcileOnStartup(t *testing.T) {
Size: 1,
},
}
payload, err := json.Marshal(meta)
payload, err := json.Marshal(indexDocument{
Scratches: meta,
})
if err != nil {
t.Fatalf("json.Marshal() error = %v", err)
}
if err := os.WriteFile(filepath.Join(root, indexFilename), payload, 0o644); err != nil {
encryptedPayload, err := encryptIndexPayload(payload, mustTestAEAD(t))
if err != nil {
t.Fatalf("encryptIndexPayload() error = %v", err)
}
if err := os.WriteFile(filepath.Join(root, indexFilename), encryptedPayload, 0o644); err != nil {
t.Fatalf("WriteFile index error = %v", err)
}
store, err := NewFilesystemStore(root)
store, err := NewFilesystemStore(root, testEncryptionKey)
if err != nil {
t.Fatalf("NewFilesystemStore() error = %v", err)
}
@@ -442,6 +380,63 @@ func TestNewFilesystemStoreReconcileOnStartup(t *testing.T) {
}
}
func TestNewFilesystemStoreWithDefaultTTLShiftsExpiriesOnTTLChange(t *testing.T) {
t.Parallel()
dataDir := filepath.Join(t.TempDir(), "data")
store, err := NewFilesystemStoreWithDefaultTTL(dataDir, time.Hour, testEncryptionKey)
if err != nil {
t.Fatalf("NewFilesystemStoreWithDefaultTTL() error = %v", err)
}
meta, err := store.Create(context.Background(), bytes.NewReader([]byte("hello")), "text/plain", time.Hour)
if err != nil {
t.Fatalf("Create() error = %v", err)
}
originalExpiry := meta.ExpiresAt
reloaded, err := NewFilesystemStoreWithDefaultTTL(dataDir, 2*time.Hour, testEncryptionKey)
if err != nil {
t.Fatalf("NewFilesystemStoreWithDefaultTTL() reload error = %v", err)
}
got, err := reloaded.Get(meta.ID)
if err != nil {
t.Fatalf("Get() error = %v", err)
}
wantExpiry := originalExpiry.Add(time.Hour)
if !got.ExpiresAt.Equal(wantExpiry) {
t.Fatalf("ExpiresAt = %s, want %s", got.ExpiresAt, wantExpiry)
}
again, err := NewFilesystemStoreWithDefaultTTL(dataDir, 2*time.Hour, testEncryptionKey)
if err != nil {
t.Fatalf("NewFilesystemStoreWithDefaultTTL() second reload error = %v", err)
}
gotAgain, err := again.Get(meta.ID)
if err != nil {
t.Fatalf("Get() second reload error = %v", err)
}
if !gotAgain.ExpiresAt.Equal(wantExpiry) {
t.Fatalf("ExpiresAt after unchanged TTL = %s, want %s", gotAgain.ExpiresAt, wantExpiry)
}
indexRaw, err := os.ReadFile(filepath.Join(dataDir, indexFilename))
if err != nil {
t.Fatalf("ReadFile(index) error = %v", err)
}
if bytes.Contains(indexRaw, []byte(`"default_ttl"`)) {
t.Fatalf("index should not be plaintext JSON")
}
indexPlaintext, err := decryptIndexPayload(indexRaw, mustTestAEAD(t))
if err != nil {
t.Fatalf("decryptIndexPayload() error = %v", err)
}
if !bytes.Contains(indexPlaintext, []byte(`"default_ttl": "2h0m0s"`)) {
t.Fatalf("decrypted index missing default ttl marker: %s", string(indexPlaintext))
}
}
type errCloser struct{}
func (errCloser) Close() error { return errors.New("close failed") }
@@ -465,7 +460,7 @@ func (failingReader) Read(_ []byte) (int, error) { return 0, errors.New("boom")
func TestCreateRollbackOnPersistFailure(t *testing.T) {
t.Parallel()
store, err := NewFilesystemStore(filepath.Join(t.TempDir(), "data"))
store, err := NewFilesystemStore(filepath.Join(t.TempDir(), "data"), testEncryptionKey)
if err != nil {
t.Fatalf("NewFilesystemStore() error = %v", err)
}
@@ -487,7 +482,7 @@ func TestCreateRollbackOnPersistFailure(t *testing.T) {
func TestCreateReaderFailure(t *testing.T) {
t.Parallel()
store, err := NewFilesystemStore(filepath.Join(t.TempDir(), "data"))
store, err := NewFilesystemStore(filepath.Join(t.TempDir(), "data"), testEncryptionKey)
if err != nil {
t.Fatalf("NewFilesystemStore() error = %v", err)
}
@@ -499,7 +494,7 @@ func TestCreateReaderFailure(t *testing.T) {
func TestDeletePersistFailureRestoresEntry(t *testing.T) {
t.Parallel()
store, err := NewFilesystemStore(filepath.Join(t.TempDir(), "data"))
store, err := NewFilesystemStore(filepath.Join(t.TempDir(), "data"), testEncryptionKey)
if err != nil {
t.Fatalf("NewFilesystemStore() error = %v", err)
}
@@ -525,7 +520,7 @@ func TestDeletePersistFailureRestoresEntry(t *testing.T) {
func TestDeleteRemoveFailure(t *testing.T) {
t.Parallel()
store, err := NewFilesystemStore(filepath.Join(t.TempDir(), "data"))
store, err := NewFilesystemStore(filepath.Join(t.TempDir(), "data"), testEncryptionKey)
if err != nil {
t.Fatalf("NewFilesystemStore() error = %v", err)
}
@@ -557,7 +552,7 @@ func TestDeleteRemoveFailure(t *testing.T) {
func TestDeleteExpiredPersistFailureRestoresEntries(t *testing.T) {
t.Parallel()
store, err := NewFilesystemStore(filepath.Join(t.TempDir(), "data"))
store, err := NewFilesystemStore(filepath.Join(t.TempDir(), "data"), testEncryptionKey)
if err != nil {
t.Fatalf("NewFilesystemStore() error = %v", err)
}
@@ -583,7 +578,7 @@ func TestDeleteExpiredPersistFailureRestoresEntries(t *testing.T) {
func TestDeleteExpiredRemoveFailure(t *testing.T) {
t.Parallel()
store, err := NewFilesystemStore(filepath.Join(t.TempDir(), "data"))
store, err := NewFilesystemStore(filepath.Join(t.TempDir(), "data"), testEncryptionKey)
if err != nil {
t.Fatalf("NewFilesystemStore() error = %v", err)
}
@@ -619,106 +614,45 @@ func TestNewFilesystemStoreFailsWhenDataDirIsFile(t *testing.T) {
if err := os.WriteFile(filePath, []byte("x"), 0o644); err != nil {
t.Fatalf("WriteFile() error = %v", err)
}
if _, err := NewFilesystemStore(filePath); err == nil {
if _, err := NewFilesystemStore(filePath, testEncryptionKey); err == nil {
t.Fatalf("NewFilesystemStore() error = nil, want mkdir failure")
}
}
type blockingReader struct {
release <-chan struct{}
payload []byte
sent bool
}
func (r *blockingReader) Read(p []byte) (int, error) {
if r.sent {
return 0, io.EOF
}
<-r.release
r.sent = true
return copy(p, r.payload), nil
}
func TestCreateReservationIsNotVisibleToCleanupOrReads(t *testing.T) {
func TestNewFilesystemStoreRequires32ByteKey(t *testing.T) {
t.Parallel()
store, err := NewFilesystemStore(filepath.Join(t.TempDir(), "data"))
dataDir := filepath.Join(t.TempDir(), "data")
if _, err := NewFilesystemStore(dataDir, nil); err == nil {
t.Fatalf("NewFilesystemStore() with nil key error = nil, want non-nil")
}
if _, err := NewFilesystemStore(dataDir, []byte("short")); err == nil {
t.Fatalf("NewFilesystemStore() with short key error = nil, want non-nil")
}
}
func TestNewFilesystemStoreFailsToLoadEncryptedIndexWithWrongKey(t *testing.T) {
t.Parallel()
dataDir := filepath.Join(t.TempDir(), "data")
store, err := NewFilesystemStore(dataDir, testEncryptionKey)
if err != nil {
t.Fatalf("NewFilesystemStore() error = %v", err)
}
const reservedID = "ffffffffffffffffffffffff"
store.idGenerator = func() (string, error) {
return reservedID, nil
if _, err := store.Create(context.Background(), bytes.NewReader([]byte("hello")), "text/plain", time.Hour); err != nil {
t.Fatalf("Create() error = %v", err)
}
release := make(chan struct{})
createErrCh := make(chan error, 1)
go func() {
_, createErr := store.CreateWithOriginalName(
context.Background(),
&blockingReader{release: release, payload: []byte("in-flight")},
"text/plain",
"",
time.Hour,
)
createErrCh <- createErr
}()
deadline := time.Now().Add(2 * time.Second)
for {
store.mu.RLock()
_, reserved := store.reservedIDs[reservedID]
_, visible := store.metadata[reservedID]
store.mu.RUnlock()
if reserved {
if visible {
t.Fatalf("in-flight ID should not appear in metadata")
}
break
}
if time.Now().After(deadline) {
t.Fatalf("timed out waiting for ID reservation")
}
time.Sleep(2 * time.Millisecond)
}
if _, err := store.Get(reservedID); !errors.Is(err, ErrNotFound) {
t.Fatalf("Get() during in-flight create error = %v, want ErrNotFound", err)
}
deleted, err := store.DeleteExpired(time.Now().UTC())
if err != nil {
t.Fatalf("DeleteExpired() error = %v", err)
}
if deleted != 0 {
t.Fatalf("DeleteExpired() = %d, want 0 for in-flight create", deleted)
}
close(release)
if err := <-createErrCh; err != nil {
t.Fatalf("CreateWithOriginalName() error = %v", err)
}
reader, _, err := store.Open(reservedID)
if err != nil {
t.Fatalf("Open() error = %v", err)
}
defer reader.Close()
body, err := io.ReadAll(reader)
if err != nil {
t.Fatalf("ReadAll() error = %v", err)
}
if string(body) != "in-flight" {
t.Fatalf("Open() content = %q, want %q", string(body), "in-flight")
wrongKey := []byte("abcdef0123456789abcdef0123456789")
if _, err := NewFilesystemStore(dataDir, wrongKey); err == nil {
t.Fatalf("NewFilesystemStore() with wrong key error = nil, want non-nil")
}
}
func TestConcurrentCreateWithOriginalNameProducesUniqueReadableEntries(t *testing.T) {
t.Parallel()
store, err := NewFilesystemStore(filepath.Join(t.TempDir(), "data"))
store, err := NewFilesystemStore(filepath.Join(t.TempDir(), "data"), testEncryptionKey)
if err != nil {
t.Fatalf("NewFilesystemStore() error = %v", err)
}