From 050ec138f33a1cb56bf8f252c48121938c7a048e Mon Sep 17 00:00:00 2001 From: Justin Harms Date: Mon, 1 Jun 2026 01:29:47 -0500 Subject: [PATCH] Lots more changes --- .gitignore | 1 + Makefile | 12 +- README.md | 53 ++- cmd/scratchbox/access_log_writer.go | 141 ++++++ cmd/scratchbox/access_log_writer_test.go | 69 +++ cmd/scratchbox/generate_key.go | 32 ++ cmd/scratchbox/main.go | 1 + cmd/scratchbox/main_test.go | 111 ++++- cmd/scratchbox/server.go | 21 +- internal/cleanup/worker_test.go | 8 +- internal/config/config.go | 201 ++++++-- internal/config/config_test.go | 176 ++++++- internal/http/handlers.go | 70 ++- internal/http/handlers_integration_test.go | 16 +- internal/http/handlers_unit_test.go | 18 +- internal/http/middleware.go | 67 +++ internal/http/middleware_test.go | 93 ++++ internal/storage/filesystem.go | 522 +++++++++++++++------ internal/storage/filesystem_test.go | 490 +++++++++---------- web/ui/src/App.svelte | 360 +++++++------- 20 files changed, 1773 insertions(+), 689 deletions(-) create mode 100644 cmd/scratchbox/access_log_writer.go create mode 100644 cmd/scratchbox/access_log_writer_test.go create mode 100644 cmd/scratchbox/generate_key.go diff --git a/.gitignore b/.gitignore index a83a3eb..aac1a8f 100644 --- a/.gitignore +++ b/.gitignore @@ -3,6 +3,7 @@ # Run Artifacts /*.yaml +/config.key /data/ # Coverage outputs diff --git a/Makefile b/Makefile index 8817d8c..ffe470b 100644 --- a/Makefile +++ b/Makefile @@ -4,13 +4,13 @@ BIN_DIR := ./bin BIN := $(BIN_DIR)/$(APP_NAME) CONFIG ?= config.yaml -.PHONY: help run dev generate-config build test test-race fmt clean +.PHONY: help run dev config build test test-race fmt clean help: @echo "Targets:" @echo " make run - Alias for make dev" @echo " make dev - Build and run server (CONFIG=$(CONFIG))" - @echo " make generate-config - Write default config to $(CONFIG)" + @echo " make config - Write default config to $(CONFIG) and generate sibling config.key if missing" @echo " make build - Build Svelte UI and binary to $(BIN)" @echo " make test - Run all Go tests with timeout and shuffle" @echo " make test-race - Run all Go tests with race detector and timeout" @@ -22,8 +22,12 @@ run: dev dev: build $(BIN) server --config $(CONFIG) -generate-config: - go run $(CMD_DIR) generate-config > $(CONFIG) +config: + mkdir -p "$(dir $(CONFIG))" + go run $(CMD_DIR) generate-config > "$(CONFIG)" + @if [ ! -f "$(dir $(CONFIG))config.key" ]; then \ + go run $(CMD_DIR) generate-key > "$(dir $(CONFIG))config.key"; \ + fi build: clean npm --prefix ./web/ui run build diff --git a/README.md b/README.md index 12bc14b..e4f09ac 100644 --- a/README.md +++ b/README.md @@ -2,17 +2,20 @@ Scratchbox is a minimal self-hosted scratch service written in Go. It is intentionally unauthenticated and optimized for temporary sharing of text or small files with an expiration time. -Each upload creates exactly one scratch. For multi-file sharing, bundle files into an archive (for example `.zip` or `.tar.gz`) and upload that single archive file. -Scratch content is stored on disk gzip-compressed transparently to reduce filesystem usage; API/UI reads return original uncompressed content. +For multi-file sharing, bundle files into an archive (for example `.zip` or `.tar.gz`) before upload. +Scratch content is gzip-compressed and AES-encrypted on disk transparently; API/UI reads return original uncompressed content. ## Quick Start 1. Generate defaults directly: - `go run ./cmd/scratchbox generate-config > config.yaml` -2. Adjust settings for your environment. -3. Start the server: +2. Generate an encryption key file next to the config: + - `go run ./cmd/scratchbox generate-key > config.key` + - or: `openssl rand -base64 32 > config.key` +3. Adjust settings for your environment. +4. Start the server: - `go run ./cmd/scratchbox server --config config.yaml` -4. Open: +5. Open: - `http://127.0.0.1:8080/` Without `--config`, built-in defaults are used. @@ -43,6 +46,16 @@ All configuration is YAML. - `max_header_bytes`: max HTTP header size in bytes. - Must be `> 0` - Default: `1048576` (1 MiB) +- `access_log`: structured request logs. + - `enabled`: enable HTTP access logs + - Default: `true` + - `file_path`: optional log file path + - When set, logs are tee'd to both stdout and the file + - `max_size`: rotation threshold for `file_path` + - Supported units: `B`, `KB`, `MB`, `GB`, `KiB`, `MiB`, `GiB` + - Default: `100MiB` + - `max_backups`: number of rotated files to retain + - Default: `5` ### `limits` @@ -53,7 +66,7 @@ All configuration is YAML. - Default: `100MiB` - `default_ttl`: expiration applied to new scratches. - Must be `> 0` and `<= 24h` - - Default: `1h` + - Default: `15m` - `raw_cache_max_size`: max total decompressed bytes cached in memory for `/api/raw` range requests. - Supported units: `B`, `KB`, `MB`, `GB`, `KiB`, `MiB`, `GiB` (case-insensitive) - Must be `> 0` @@ -70,6 +83,12 @@ All configuration is YAML. - `cleanup_interval`: background interval for deleting expired content. - Must be at least `10s` - Default: `1m` +- `encryption_key_file`: path to a file containing a base64-encoded 32-byte key used for at-rest encryption. + - Relative paths are resolved from the config file directory + - Default: `config.key` (expected alongside `config.yaml`) + - Keep this key stable across restarts + - Changing the key makes existing scratch files unreadable + - `generate-key` emits a fresh random key each run ### `security` @@ -84,14 +103,18 @@ All configuration is YAML. - `trusted_proxy_ips`: list of reverse-proxy IPs/CIDRs permitted to supply forwarded headers. - Required when `trust_proxy_headers: true` - Examples: `127.0.0.1`, `10.0.0.0/8` -- `rate_limit.enabled`: enable per-IP token-bucket rate limiting on write and scratch-read routes (`POST /api/scratch`, `GET /api/scratch/{id}`, `GET /s/{id}`, `GET /api/raw/{id}`). - - Default: `true` -- `rate_limit.requests_per_minute`: steady refill rate per client IP. - - Must be `> 0` - - Default: `30` -- `rate_limit.burst`: immediate burst capacity per client IP. - - Must be `> 0` - - Default: `10` +- `rate_limit_ui`: per-IP token-bucket for UI routes (`GET /`, `GET /u`, `GET /s/{id}`). + - `enabled` default: `false` + - `requests_per_minute` must be `> 0` (default `360`) + - `burst` must be `> 0` (default `120`) +- `rate_limit_api_read`: per-IP token-bucket for API read routes (`GET /api/config`, `GET /api/scratch/{id}`, `GET /api/raw/{id}`). + - `enabled` default: `false` + - `requests_per_minute` must be `> 0` (default `300`) + - `burst` must be `> 0` (default `100`) +- `rate_limit_api_write`: per-IP token-bucket for API write route (`POST /api/scratch`). + - `enabled` default: `true` + - `requests_per_minute` must be `> 0` (default `30`) + - `burst` must be `> 0` (default `10`) ## Security Posture @@ -129,3 +152,5 @@ Scratchbox storage is designed for a single process instance per `storage.data_d - Do not run multiple scratchbox processes against the same data directory. - Metadata/index writes are process-local and are not coordinated with cross-process locking. +- Scratch payload files and metadata index are encrypted at rest. +- Scratch IDs are derived from the SHA-256 of the stored blob (encrypted+gzip payload on disk). diff --git a/cmd/scratchbox/access_log_writer.go b/cmd/scratchbox/access_log_writer.go new file mode 100644 index 0000000..54ef090 --- /dev/null +++ b/cmd/scratchbox/access_log_writer.go @@ -0,0 +1,141 @@ +package main + +import ( + "fmt" + "io" + "os" + "path/filepath" + "sync" + + "scratchbox/internal/config" +) + +func newAccessLogWriter(cfg config.Config) (io.Writer, func() error, error) { + if !cfg.Server.AccessLog.Enabled { + return io.Discard, func() error { return nil }, nil + } + if cfg.Server.AccessLog.FilePath == "" { + return os.Stdout, func() error { return nil }, nil + } + + writer, err := newRollingFileWriter( + cfg.Server.AccessLog.FilePath, + cfg.Server.AccessLog.MaxSizeBytes, + cfg.Server.AccessLog.MaxBackups, + ) + if err != nil { + return nil, nil, err + } + return io.MultiWriter(os.Stdout, writer), writer.Close, nil +} + +type rollingFileWriter struct { + mu sync.Mutex + path string + maxSize int64 + maxBackups int + file *os.File + size int64 +} + +func newRollingFileWriter(path string, maxSize int64, maxBackups int) (*rollingFileWriter, error) { + if maxSize <= 0 { + return nil, fmt.Errorf("max size must be > 0") + } + if maxBackups <= 0 { + return nil, fmt.Errorf("max backups must be > 0") + } + + if dir := filepath.Dir(path); dir != "" && dir != "." { + if err := os.MkdirAll(dir, 0o755); err != nil { + return nil, fmt.Errorf("create log directory: %w", err) + } + } + + f, err := os.OpenFile(path, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0o644) + if err != nil { + return nil, fmt.Errorf("open log file: %w", err) + } + info, err := f.Stat() + if err != nil { + _ = f.Close() + return nil, fmt.Errorf("stat log file: %w", err) + } + + return &rollingFileWriter{ + path: path, + maxSize: maxSize, + maxBackups: maxBackups, + file: f, + size: info.Size(), + }, nil +} + +func (w *rollingFileWriter) Write(p []byte) (int, error) { + w.mu.Lock() + defer w.mu.Unlock() + + if w.size > 0 && w.size+int64(len(p)) > w.maxSize { + if err := w.rotateLocked(); err != nil { + return 0, err + } + } + + n, err := w.file.Write(p) + w.size += int64(n) + return n, err +} + +func (w *rollingFileWriter) Close() error { + w.mu.Lock() + defer w.mu.Unlock() + if w.file == nil { + return nil + } + err := w.file.Close() + w.file = nil + return err +} + +func (w *rollingFileWriter) rotateLocked() error { + if err := w.file.Close(); err != nil { + return fmt.Errorf("close before rotate: %w", err) + } + + for i := w.maxBackups - 1; i >= 1; i-- { + from := rotatedLogPath(w.path, i) + to := rotatedLogPath(w.path, i+1) + if _, err := os.Stat(from); err != nil { + if os.IsNotExist(err) { + continue + } + return fmt.Errorf("stat rotated file %s: %w", from, err) + } + _ = os.Remove(to) + if err := os.Rename(from, to); err != nil { + return fmt.Errorf("rotate backup %s -> %s: %w", from, to, err) + } + } + + firstBackup := rotatedLogPath(w.path, 1) + _ = os.Remove(firstBackup) + if _, err := os.Stat(w.path); err == nil { + if err := os.Rename(w.path, firstBackup); err != nil { + return fmt.Errorf("rotate current log to %s: %w", firstBackup, err) + } + } else if !os.IsNotExist(err) { + return fmt.Errorf("stat current log file: %w", err) + } + + f, err := os.OpenFile(w.path, os.O_TRUNC|os.O_CREATE|os.O_WRONLY, 0o644) + if err != nil { + return fmt.Errorf("open rotated log file: %w", err) + } + w.file = f + w.size = 0 + return nil +} + +func rotatedLogPath(path string, index int) string { + return fmt.Sprintf("%s.%d", path, index) +} diff --git a/cmd/scratchbox/access_log_writer_test.go b/cmd/scratchbox/access_log_writer_test.go new file mode 100644 index 0000000..f5f1a01 --- /dev/null +++ b/cmd/scratchbox/access_log_writer_test.go @@ -0,0 +1,69 @@ +package main + +import ( + "bytes" + "os" + "path/filepath" + "testing" + + "scratchbox/internal/config" +) + +func TestNewAccessLogWriterDisabled(t *testing.T) { + t.Parallel() + + cfg := config.Default() + cfg.Server.AccessLog.Enabled = false + writer, closeFn, err := newAccessLogWriter(cfg) + if err != nil { + t.Fatalf("newAccessLogWriter() error = %v", err) + } + if writer == nil { + t.Fatalf("newAccessLogWriter() writer = nil") + } + if err := closeFn(); err != nil { + t.Fatalf("closeFn() error = %v", err) + } +} + +func TestRollingFileWriterRotatesAndKeepsBackups(t *testing.T) { + t.Parallel() + + logPath := filepath.Join(t.TempDir(), "access.log") + w, err := newRollingFileWriter(logPath, 10, 2) + if err != nil { + t.Fatalf("newRollingFileWriter() error = %v", err) + } + defer w.Close() + + _, _ = w.Write([]byte("12345")) + _, _ = w.Write([]byte("67890")) + _, _ = w.Write([]byte("abc")) + _, _ = w.Write([]byte("defghijkl")) + + if err := w.Close(); err != nil { + t.Fatalf("Close() error = %v", err) + } + + current, err := os.ReadFile(logPath) + if err != nil { + t.Fatalf("ReadFile(current) error = %v", err) + } + backup1, err := os.ReadFile(logPath + ".1") + if err != nil { + t.Fatalf("ReadFile(backup1) error = %v", err) + } + backup2, err := os.ReadFile(logPath + ".2") + if err != nil { + t.Fatalf("ReadFile(backup2) error = %v", err) + } + if !bytes.Equal(current, []byte("defghijkl")) { + t.Fatalf("current log = %q, want %q", string(current), "defghijkl") + } + if !bytes.Equal(backup1, []byte("abc")) { + t.Fatalf("backup1 log = %q, want %q", string(backup1), "abc") + } + if !bytes.Equal(backup2, []byte("1234567890")) { + t.Fatalf("backup2 log = %q, want %q", string(backup2), "1234567890") + } +} diff --git a/cmd/scratchbox/generate_key.go b/cmd/scratchbox/generate_key.go new file mode 100644 index 0000000..9808ad9 --- /dev/null +++ b/cmd/scratchbox/generate_key.go @@ -0,0 +1,32 @@ +package main + +import ( + "fmt" + + "github.com/spf13/cobra" + + "scratchbox/internal/config" +) + +func newGenerateKeyCmd() *cobra.Command { + cmd := &cobra.Command{ + Use: "generate-key", + Short: "Print a random base64 encryption key to stdout", + Args: cobra.NoArgs, + RunE: func(cmd *cobra.Command, _ []string) error { + return writeEncryptionKey(cmd) + }, + } + return cmd +} + +func writeEncryptionKey(cmd *cobra.Command) error { + key, err := config.GenerateEncryptionKey() + if err != nil { + return fmt.Errorf("generate encryption key: %w", err) + } + if _, err := fmt.Fprintln(cmd.OutOrStdout(), key); err != nil { + return fmt.Errorf("write encryption key: %w", err) + } + return nil +} diff --git a/cmd/scratchbox/main.go b/cmd/scratchbox/main.go index a1c51fa..0b3fd7f 100644 --- a/cmd/scratchbox/main.go +++ b/cmd/scratchbox/main.go @@ -27,5 +27,6 @@ func newRootCmd() *cobra.Command { } cmd.AddCommand(newServerCmd()) cmd.AddCommand(newGenerateConfigCmd()) + cmd.AddCommand(newGenerateKeyCmd()) return cmd } diff --git a/cmd/scratchbox/main_test.go b/cmd/scratchbox/main_test.go index d5312c6..e33670c 100644 --- a/cmd/scratchbox/main_test.go +++ b/cmd/scratchbox/main_test.go @@ -3,6 +3,7 @@ package main import ( "bytes" "context" + "encoding/base64" "encoding/json" "io" "log" @@ -31,6 +32,18 @@ func repoRoot(t *testing.T) string { return filepath.Clean(filepath.Join(filepath.Dir(file), "..", "..")) } +func writeConfigKeyFile(t *testing.T, configPath string) { + t.Helper() + keyPath := filepath.Join(filepath.Dir(configPath), "config.key") + key, err := config.GenerateEncryptionKey() + if err != nil { + t.Fatalf("GenerateEncryptionKey() error = %v", err) + } + if err := os.WriteFile(keyPath, []byte(key+"\n"), 0o600); err != nil { + t.Fatalf("WriteFile(config.key) error = %v", err) + } +} + func TestMainProcessSuccess(t *testing.T) { t.Parallel() @@ -48,7 +61,15 @@ storage: security: allowed_ips: [] trust_proxy_headers: false - rate_limit: + rate_limit_ui: + enabled: true + requests_per_minute: 10 + burst: 1 + rate_limit_api_read: + enabled: true + requests_per_minute: 10 + burst: 1 + rate_limit_api_write: enabled: true requests_per_minute: 10 burst: 1 @@ -56,6 +77,7 @@ security: if err := os.WriteFile(cfgPath, []byte(cfg), 0o644); err != nil { t.Fatalf("WriteFile() error = %v", err) } + writeConfigKeyFile(t, cfgPath) cmd := exec.Command(os.Args[0], "-test.run=TestMainHelperProcess") cmd.Dir = repoRoot(t) @@ -104,7 +126,15 @@ storage: security: allowed_ips: [] trust_proxy_headers: false - rate_limit: + rate_limit_ui: + enabled: true + requests_per_minute: 10 + burst: 1 + rate_limit_api_read: + enabled: true + requests_per_minute: 10 + burst: 1 + rate_limit_api_write: enabled: true requests_per_minute: 10 burst: 1 @@ -112,6 +142,7 @@ security: if err := os.WriteFile(cfgPath, []byte(cfg), 0o644); err != nil { t.Fatalf("WriteFile() error = %v", err) } + writeConfigKeyFile(t, cfgPath) cmd := exec.Command(os.Args[0], "-test.run=TestMainHelperProcess") cmd.Dir = repoRoot(t) @@ -198,8 +229,26 @@ func TestGenerateConfigCommandWritesDefaultYAML(t *testing.T) { if cfg.Storage.DataDir != "./data" { t.Fatalf("storage.data_dir = %q, want %q", cfg.Storage.DataDir, "./data") } - if cfg.Security.RateLimit.RequestsPerMinute != 30 { - t.Fatalf("security.rate_limit.requests_per_minute = %d, want %d", cfg.Security.RateLimit.RequestsPerMinute, 30) + if cfg.Storage.EncryptionKeyFile != "config.key" { + t.Fatalf("storage.encryption_key_file = %q, want %q", cfg.Storage.EncryptionKeyFile, "config.key") + } + if cfg.Security.RateLimitUI.RequestsPerMinute != 360 { + t.Fatalf("security.rate_limit_ui.requests_per_minute = %d, want %d", cfg.Security.RateLimitUI.RequestsPerMinute, 360) + } + if cfg.Security.RateLimitUI.Enabled { + t.Fatalf("security.rate_limit_ui.enabled = %v, want false", cfg.Security.RateLimitUI.Enabled) + } + if cfg.Security.RateLimitAPIRead.RequestsPerMinute != 300 { + t.Fatalf("security.rate_limit_api_read.requests_per_minute = %d, want %d", cfg.Security.RateLimitAPIRead.RequestsPerMinute, 300) + } + if cfg.Security.RateLimitAPIRead.Enabled { + t.Fatalf("security.rate_limit_api_read.enabled = %v, want false", cfg.Security.RateLimitAPIRead.Enabled) + } + if cfg.Security.RateLimitAPIWrite.RequestsPerMinute != 30 { + t.Fatalf("security.rate_limit_api_write.requests_per_minute = %d, want %d", cfg.Security.RateLimitAPIWrite.RequestsPerMinute, 30) + } + if !cfg.Security.RateLimitAPIWrite.Enabled { + t.Fatalf("security.rate_limit_api_write.enabled = %v, want true", cfg.Security.RateLimitAPIWrite.Enabled) } } @@ -216,6 +265,45 @@ func TestGenerateConfigCommandRejectsUnexpectedArgs(t *testing.T) { } } +func TestGenerateKeyCommandWritesKey(t *testing.T) { + t.Parallel() + + cmd := newRootCmd() + var out bytes.Buffer + cmd.SetOut(&out) + cmd.SetErr(io.Discard) + cmd.SetArgs([]string{"generate-key"}) + + if err := cmd.Execute(); err != nil { + t.Fatalf("Execute() error = %v", err) + } + + raw := strings.TrimSpace(out.String()) + if raw == "" { + t.Fatalf("generate-key output is empty") + } + keyBytes, err := base64.StdEncoding.DecodeString(raw) + if err != nil { + t.Fatalf("generate-key output is not valid base64: %v", err) + } + if len(keyBytes) != 32 { + t.Fatalf("decoded key length = %d, want 32", len(keyBytes)) + } +} + +func TestGenerateKeyCommandRejectsUnexpectedArgs(t *testing.T) { + t.Parallel() + + cmd := newRootCmd() + cmd.SetOut(io.Discard) + cmd.SetErr(io.Discard) + cmd.SetArgs([]string{"generate-key", "extra"}) + + if err := cmd.Execute(); err == nil { + t.Fatalf("expected error for unexpected args") + } +} + func TestLiveServerEndToEndHTTP(t *testing.T) { tmp := t.TempDir() cfgPath := filepath.Join(tmp, "config.yaml") @@ -231,7 +319,15 @@ storage: security: allowed_ips: [] trust_proxy_headers: false - rate_limit: + rate_limit_ui: + enabled: true + requests_per_minute: 30 + burst: 10 + rate_limit_api_read: + enabled: true + requests_per_minute: 30 + burst: 10 + rate_limit_api_write: enabled: true requests_per_minute: 30 burst: 10 @@ -239,13 +335,14 @@ security: if err := os.WriteFile(cfgPath, []byte(cfgRaw), 0o644); err != nil { t.Fatalf("WriteFile() error = %v", err) } + writeConfigKeyFile(t, cfgPath) cfg, err := config.Load(cfgPath) if err != nil { t.Fatalf("config.Load() error = %v", err) } logger := log.New(io.Discard, "", 0) - store, err := storage.NewFilesystemStore(cfg.Storage.DataDir) + store, err := storage.NewFilesystemStore(cfg.Storage.DataDir, cfg.Storage.EncryptionKeyBytes) if err != nil { t.Fatalf("NewFilesystemStore() error = %v", err) } @@ -261,7 +358,7 @@ security: _ = os.Chdir(wd) }() - srv, err := httpapi.NewServer(cfg, store, logger) + srv, err := httpapi.NewServer(cfg, store, logger, logger) if err != nil { t.Fatalf("httpapi.NewServer() error = %v", err) } diff --git a/cmd/scratchbox/server.go b/cmd/scratchbox/server.go index b8701ef..49a404b 100644 --- a/cmd/scratchbox/server.go +++ b/cmd/scratchbox/server.go @@ -40,13 +40,30 @@ func run(configPath string) error { } logger := log.New(os.Stdout, "[scratchbox] ", log.LstdFlags|log.LUTC) + accessWriter, closeAccessWriter, err := newAccessLogWriter(cfg) + if err != nil { + return fmt.Errorf("access log init failed: %w", err) + } + defer func() { + if closeErr := closeAccessWriter(); closeErr != nil { + logger.Printf("access log writer close failed: %v", closeErr) + } + }() + var accessLogger *log.Logger + if cfg.Server.AccessLog.Enabled { + accessLogger = log.New(accessWriter, "[access] ", log.LstdFlags|log.LUTC) + } - store, err := storage.NewFilesystemStore(cfg.Storage.DataDir) + store, err := storage.NewFilesystemStoreWithDefaultTTL( + cfg.Storage.DataDir, + cfg.Limits.DefaultTTLDuration, + cfg.Storage.EncryptionKeyBytes, + ) if err != nil { return fmt.Errorf("storage init failed: %w", err) } - server, err := httpapi.NewServer(cfg, store, logger) + server, err := httpapi.NewServer(cfg, store, logger, accessLogger) if err != nil { return fmt.Errorf("http init failed: %w", err) } diff --git a/internal/cleanup/worker_test.go b/internal/cleanup/worker_test.go index 05d5cef..e4f8437 100644 --- a/internal/cleanup/worker_test.go +++ b/internal/cleanup/worker_test.go @@ -12,10 +12,12 @@ import ( "scratchbox/internal/storage" ) +var testStorageKey = []byte("0123456789abcdef0123456789abcdef") + func TestNewWorker(t *testing.T) { t.Parallel() - store, err := storage.NewFilesystemStore(filepath.Join(t.TempDir(), "data")) + store, err := storage.NewFilesystemStore(filepath.Join(t.TempDir(), "data"), testStorageKey) if err != nil { t.Fatalf("NewFilesystemStore() error = %v", err) } @@ -29,7 +31,7 @@ func TestNewWorker(t *testing.T) { func TestWorkerStartRemovesExpiredAndLogs(t *testing.T) { t.Parallel() - store, err := storage.NewFilesystemStore(filepath.Join(t.TempDir(), "data")) + store, err := storage.NewFilesystemStore(filepath.Join(t.TempDir(), "data"), testStorageKey) if err != nil { t.Fatalf("NewFilesystemStore() error = %v", err) } @@ -53,7 +55,7 @@ func TestWorkerStartRemovesExpiredAndLogs(t *testing.T) { func TestWorkerStartWithNoExpiredEntries(t *testing.T) { t.Parallel() - store, err := storage.NewFilesystemStore(filepath.Join(t.TempDir(), "data")) + store, err := storage.NewFilesystemStore(filepath.Join(t.TempDir(), "data"), testStorageKey) if err != nil { t.Fatalf("NewFilesystemStore() error = %v", err) } diff --git a/internal/config/config.go b/internal/config/config.go index 8d6242f..3d5a93c 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -1,6 +1,8 @@ package config import ( + "crypto/rand" + "encoding/base64" "errors" "fmt" "net/netip" @@ -14,23 +16,34 @@ import ( ) const ( - defaultListenAddr = ":8080" - defaultReadHeaderTimeout = "5s" - defaultReadTimeout = "30s" - defaultWriteTimeout = "30s" - defaultIdleTimeout = "120s" - defaultMaxHeaderBytes = 1 << 20 - defaultMaxUploadSize = "100MiB" - defaultTTL = "1h" - defaultRawCacheMaxSize = "200MiB" - defaultRawCacheMaxEntries = 32 - defaultDataDir = "./data" - defaultCleanupInterval = "1m" - defaultRateLimitEnabled = true - defaultRequestsPerMinute = 30 - defaultRateLimitBurst = 10 - maxDefaultTTLLimit = 24 * time.Hour - minCleanupInterval = 10 * time.Second + defaultListenAddr = ":8080" + defaultReadHeaderTimeout = "5s" + defaultReadTimeout = "30s" + defaultWriteTimeout = "30s" + defaultIdleTimeout = "120s" + defaultMaxHeaderBytes = 1 << 20 + defaultAccessLogEnabled = true + defaultAccessLogMaxSize = "100MiB" + defaultAccessLogMaxBackups = 5 + defaultMaxUploadSize = "100MiB" + defaultTTL = "15m" + defaultRawCacheMaxSize = "200MiB" + defaultRawCacheMaxEntries = 32 + defaultDataDir = "./data" + defaultCleanupInterval = "1m" + defaultStorageEncryptionKey = "vVPlqMfKa8OtD3x0RKp4rVCJ4QoScf5mWdgfg4f+5GU=" + defaultEncryptionKeyFile = "config.key" + defaultUIRateLimitEnabled = false + defaultAPIReadRateLimitEnabled = false + defaultAPIWriteRateLimitEnabled = true + defaultUIRequestsPerMinute = 360 + defaultUIRateLimitBurst = 120 + defaultAPIReadRequestsPerMinute = 300 + defaultAPIReadRateLimitBurst = 100 + defaultAPIWriteRequestsPerMinute = 30 + defaultAPIWriteRateLimitBurst = 10 + maxDefaultTTLLimit = 24 * time.Hour + minCleanupInterval = 10 * time.Second ) type Config struct { @@ -41,16 +54,25 @@ type Config struct { } type ServerConfig struct { - ListenAddr string `yaml:"listen_addr" comment:"Address to bind the HTTP server to."` - ReadHeaderTimeout string `yaml:"read_header_timeout" comment:"Maximum duration for reading request headers (for slowloris protection). Must be > 0."` - ReadTimeout string `yaml:"read_timeout" comment:"Maximum duration for reading the full request, including body. Must be > 0."` - WriteTimeout string `yaml:"write_timeout" comment:"Maximum duration before timing out writes of a response. Must be > 0."` - IdleTimeout string `yaml:"idle_timeout" comment:"Maximum amount of time to wait for the next request when keep-alives are enabled. Must be > 0."` - MaxHeaderBytes int `yaml:"max_header_bytes" comment:"Maximum size of request headers in bytes. Must be > 0."` - ReadHeaderTimeoutDur time.Duration `yaml:"-"` - ReadTimeoutDur time.Duration `yaml:"-"` - WriteTimeoutDur time.Duration `yaml:"-"` - IdleTimeoutDur time.Duration `yaml:"-"` + ListenAddr string `yaml:"listen_addr" comment:"Address to bind the HTTP server to."` + ReadHeaderTimeout string `yaml:"read_header_timeout" comment:"Maximum duration for reading request headers (for slowloris protection). Must be > 0."` + ReadTimeout string `yaml:"read_timeout" comment:"Maximum duration for reading the full request, including body. Must be > 0."` + WriteTimeout string `yaml:"write_timeout" comment:"Maximum duration before timing out writes of a response. Must be > 0."` + IdleTimeout string `yaml:"idle_timeout" comment:"Maximum amount of time to wait for the next request when keep-alives are enabled. Must be > 0."` + MaxHeaderBytes int `yaml:"max_header_bytes" comment:"Maximum size of request headers in bytes. Must be > 0."` + AccessLog AccessLogConfig `yaml:"access_log" comment:"Access log settings for all HTTP requests."` + ReadHeaderTimeoutDur time.Duration `yaml:"-"` + ReadTimeoutDur time.Duration `yaml:"-"` + WriteTimeoutDur time.Duration `yaml:"-"` + IdleTimeoutDur time.Duration `yaml:"-"` +} + +type AccessLogConfig struct { + Enabled bool `yaml:"enabled" comment:"Enable HTTP access logs."` + FilePath string `yaml:"file_path" comment:"Optional file path for access logs. When set, logs are tee'd to stdout and this file."` + MaxSize string `yaml:"max_size" comment:"Maximum size for access log file before rotation. Supports B, KB, MB, GB and KiB, MiB, GiB."` + MaxBackups int `yaml:"max_backups" comment:"Number of rotated access log files to keep."` + MaxSizeBytes int64 `yaml:"-"` } type LimitsConfig struct { @@ -66,14 +88,18 @@ type LimitsConfig struct { type StorageConfig struct { DataDir string `yaml:"data_dir" comment:"Directory where scratch files and metadata are stored."` CleanupInterval string `yaml:"cleanup_interval" comment:"How often expired scratches are deleted (minimum 10s)."` + EncryptionKeyFile string `yaml:"encryption_key_file" comment:"Path to a file containing a base64-encoded 32-byte AES key for at-rest encryption. Relative paths are resolved from the config file directory."` CleanupIntervalParsed time.Duration `yaml:"-"` + EncryptionKeyBytes []byte `yaml:"-"` } type SecurityConfig struct { AllowedIPs []string `yaml:"allowed_ips" comment:"Allowlist for POST /api/scratch as IPs or CIDRs (examples: 127.0.0.1, 192.168.1.0/24). Keep 127.0.0.1 for local access; add trusted ranges as needed. Empty means unrestricted."` TrustProxyHeaders bool `yaml:"trust_proxy_headers" comment:"Honor X-Forwarded-For / X-Real-IP only when request source IP matches security.trusted_proxy_ips."` TrustedProxyIPs []string `yaml:"trusted_proxy_ips" comment:"IPs/CIDRs of reverse proxies allowed to supply forwarded client IP headers. Required when trust_proxy_headers is true."` - RateLimit RateLimitConfig `yaml:"rate_limit" comment:"Per-client token bucket on write and scratch read routes."` + RateLimitUI RateLimitConfig `yaml:"rate_limit_ui" comment:"Per-client token bucket on UI routes (/, /u, /s/{id})."` + RateLimitAPIRead RateLimitConfig `yaml:"rate_limit_api_read" comment:"Per-client token bucket on API read routes (GET /api/config, GET /api/scratch/{id}, GET /api/raw/{id})."` + RateLimitAPIWrite RateLimitConfig `yaml:"rate_limit_api_write" comment:"Per-client token bucket on API write route (POST /api/scratch)."` AllowedPrefixes []netip.Prefix `yaml:"-"` TrustedProxyPrefixes []netip.Prefix `yaml:"-"` } @@ -95,6 +121,9 @@ func Load(path string) (Config, error) { return Config{}, fmt.Errorf("parse config yaml: %w", err) } } + if err := cfg.loadEncryptionKey(path); err != nil { + return Config{}, err + } if err := cfg.Validate(); err != nil { return Config{}, err @@ -115,6 +144,11 @@ func defaults() Config { WriteTimeout: defaultWriteTimeout, IdleTimeout: defaultIdleTimeout, MaxHeaderBytes: defaultMaxHeaderBytes, + AccessLog: AccessLogConfig{ + Enabled: defaultAccessLogEnabled, + MaxSize: defaultAccessLogMaxSize, + MaxBackups: defaultAccessLogMaxBackups, + }, }, Limits: LimitsConfig{ MaxUploadSize: defaultMaxUploadSize, @@ -123,17 +157,29 @@ func defaults() Config { RawCacheMaxEntries: defaultRawCacheMaxEntries, }, Storage: StorageConfig{ - DataDir: defaultDataDir, - CleanupInterval: defaultCleanupInterval, + DataDir: defaultDataDir, + CleanupInterval: defaultCleanupInterval, + EncryptionKeyFile: defaultEncryptionKeyFile, + EncryptionKeyBytes: mustDecodeDefaultEncryptionKey(), }, Security: SecurityConfig{ AllowedIPs: []string{ "127.0.0.1", }, - RateLimit: RateLimitConfig{ - Enabled: defaultRateLimitEnabled, - RequestsPerMinute: defaultRequestsPerMinute, - Burst: defaultRateLimitBurst, + RateLimitUI: RateLimitConfig{ + Enabled: defaultUIRateLimitEnabled, + RequestsPerMinute: defaultUIRequestsPerMinute, + Burst: defaultUIRateLimitBurst, + }, + RateLimitAPIRead: RateLimitConfig{ + Enabled: defaultAPIReadRateLimitEnabled, + RequestsPerMinute: defaultAPIReadRequestsPerMinute, + Burst: defaultAPIReadRateLimitBurst, + }, + RateLimitAPIWrite: RateLimitConfig{ + Enabled: defaultAPIWriteRateLimitEnabled, + RequestsPerMinute: defaultAPIWriteRequestsPerMinute, + Burst: defaultAPIWriteRateLimitBurst, }, }, } @@ -182,6 +228,14 @@ func (c *Config) Validate() error { if c.Server.MaxHeaderBytes <= 0 { return errors.New("server.max_header_bytes must be > 0") } + accessLogMaxSizeBytes, err := parseHumanSize(c.Server.AccessLog.MaxSize) + if err != nil { + return fmt.Errorf("server.access_log.max_size invalid: %w", err) + } + c.Server.AccessLog.MaxSizeBytes = accessLogMaxSizeBytes + if c.Server.AccessLog.MaxBackups <= 0 { + return errors.New("server.access_log.max_backups must be > 0") + } sizeBytes, err := parseHumanSize(c.Limits.MaxUploadSize) if err != nil { @@ -222,6 +276,12 @@ func (c *Config) Validate() error { if err := ensureWritableDir(c.Storage.DataDir); err != nil { return fmt.Errorf("storage.data_dir not writable: %w", err) } + if strings.TrimSpace(c.Storage.EncryptionKeyFile) == "" { + return errors.New("storage.encryption_key_file is required") + } + if len(c.Storage.EncryptionKeyBytes) != 32 { + return fmt.Errorf("storage.encryption_key_file must decode to 32 bytes, got %d", len(c.Storage.EncryptionKeyBytes)) + } prefixes, err := parseAllowedPrefixes(c.Security.AllowedIPs) if err != nil { @@ -237,11 +297,23 @@ func (c *Config) Validate() error { return errors.New("security.trusted_proxy_ips is required when security.trust_proxy_headers is true") } - if c.Security.RateLimit.RequestsPerMinute <= 0 { - return errors.New("security.rate_limit.requests_per_minute must be > 0") + if c.Security.RateLimitUI.RequestsPerMinute <= 0 { + return errors.New("security.rate_limit_ui.requests_per_minute must be > 0") } - if c.Security.RateLimit.Burst <= 0 { - return errors.New("security.rate_limit.burst must be > 0") + if c.Security.RateLimitUI.Burst <= 0 { + return errors.New("security.rate_limit_ui.burst must be > 0") + } + if c.Security.RateLimitAPIRead.RequestsPerMinute <= 0 { + return errors.New("security.rate_limit_api_read.requests_per_minute must be > 0") + } + if c.Security.RateLimitAPIRead.Burst <= 0 { + return errors.New("security.rate_limit_api_read.burst must be > 0") + } + if c.Security.RateLimitAPIWrite.RequestsPerMinute <= 0 { + return errors.New("security.rate_limit_api_write.requests_per_minute must be > 0") + } + if c.Security.RateLimitAPIWrite.Burst <= 0 { + return errors.New("security.rate_limit_api_write.burst must be > 0") } return nil @@ -338,3 +410,56 @@ func parseHumanSize(raw string) (int64, error) { } return size, nil } + +func GenerateEncryptionKey() (string, error) { + raw := make([]byte, 32) + if _, err := rand.Read(raw); err != nil { + return "", fmt.Errorf("generate encryption key bytes: %w", err) + } + return base64.StdEncoding.EncodeToString(raw), nil +} + +func (c *Config) loadEncryptionKey(configPath string) error { + if configPath == "" { + return nil + } + keyPath := strings.TrimSpace(c.Storage.EncryptionKeyFile) + if keyPath == "" { + return errors.New("storage.encryption_key_file is required") + } + if !filepath.IsAbs(keyPath) { + keyPath = filepath.Join(filepath.Dir(configPath), keyPath) + } + keyRaw, err := os.ReadFile(keyPath) + if err != nil { + return fmt.Errorf("read storage.encryption_key_file %q: %w", keyPath, err) + } + encKey, err := decodeEncryptionKey(strings.TrimSpace(string(keyRaw))) + if err != nil { + return fmt.Errorf("storage.encryption_key_file invalid: %w", err) + } + c.Storage.EncryptionKeyBytes = encKey + return nil +} + +func decodeEncryptionKey(raw string) ([]byte, error) { + if raw == "" { + return nil, errors.New("key is empty") + } + encKey, err := base64.StdEncoding.DecodeString(raw) + if err != nil { + return nil, fmt.Errorf("invalid base64: %w", err) + } + if len(encKey) != 32 { + return nil, fmt.Errorf("must decode to 32 bytes, got %d", len(encKey)) + } + return encKey, nil +} + +func mustDecodeDefaultEncryptionKey() []byte { + encKey, err := decodeEncryptionKey(defaultStorageEncryptionKey) + if err != nil { + panic(fmt.Sprintf("invalid built-in encryption key: %v", err)) + } + return encKey +} diff --git a/internal/config/config_test.go b/internal/config/config_test.go index 1259b32..8101d1b 100644 --- a/internal/config/config_test.go +++ b/internal/config/config_test.go @@ -1,6 +1,7 @@ package config import ( + "encoding/base64" "net/netip" "os" "path/filepath" @@ -37,8 +38,17 @@ func TestValidateAppliesDefaultsAndParsedFields(t *testing.T) { if cfg.Server.MaxHeaderBytes != 1<<20 { t.Fatalf("MaxHeaderBytes = %d, want %d", cfg.Server.MaxHeaderBytes, 1<<20) } - if cfg.Limits.DefaultTTLDuration != time.Hour { - t.Fatalf("DefaultTTLDuration = %s, want 1h", cfg.Limits.DefaultTTLDuration) + if !cfg.Server.AccessLog.Enabled { + t.Fatalf("AccessLog.Enabled = %v, want true", cfg.Server.AccessLog.Enabled) + } + if cfg.Server.AccessLog.MaxSizeBytes != 100*1024*1024 { + t.Fatalf("AccessLog.MaxSizeBytes = %d, want %d", cfg.Server.AccessLog.MaxSizeBytes, int64(100*1024*1024)) + } + if cfg.Server.AccessLog.MaxBackups != 5 { + t.Fatalf("AccessLog.MaxBackups = %d, want 5", cfg.Server.AccessLog.MaxBackups) + } + if cfg.Limits.DefaultTTLDuration != 15*time.Minute { + t.Fatalf("DefaultTTLDuration = %s, want 15m", cfg.Limits.DefaultTTLDuration) } if cfg.Limits.RawCacheMaxBytes != 200*1024*1024 { t.Fatalf("RawCacheMaxBytes = %d, want %d", cfg.Limits.RawCacheMaxBytes, int64(200*1024*1024)) @@ -49,6 +59,9 @@ func TestValidateAppliesDefaultsAndParsedFields(t *testing.T) { if cfg.Storage.CleanupIntervalParsed != time.Minute { t.Fatalf("CleanupIntervalParsed = %s, want 1m", cfg.Storage.CleanupIntervalParsed) } + if len(cfg.Storage.EncryptionKeyBytes) != 32 { + t.Fatalf("EncryptionKeyBytes length = %d, want 32", len(cfg.Storage.EncryptionKeyBytes)) + } if len(cfg.Security.AllowedPrefixes) != 1 { t.Fatalf("AllowedPrefixes len = %d, want 1", len(cfg.Security.AllowedPrefixes)) } @@ -63,6 +76,7 @@ func TestLoadParsesConfiguredFields(t *testing.T) { tmp := t.TempDir() dataDir := filepath.Join(tmp, "store") configPath := filepath.Join(tmp, "config.yaml") + keyPath := filepath.Join(tmp, "scratchbox.key") content := ` server: listen_addr: "127.0.0.1:9090" @@ -71,6 +85,11 @@ server: write_timeout: "35s" idle_timeout: "90s" max_header_bytes: 2097152 + access_log: + enabled: true + file_path: "` + filepath.Join(tmp, "access.log") + `" + max_size: "2MiB" + max_backups: 7 limits: max_upload_size: "2MiB" default_ttl: "2h" @@ -79,6 +98,7 @@ limits: storage: data_dir: "` + dataDir + `" cleanup_interval: "45s" + encryption_key_file: "` + filepath.Base(keyPath) + `" security: allowed_ips: - "127.0.0.1" @@ -86,7 +106,15 @@ security: trust_proxy_headers: true trusted_proxy_ips: - "10.0.0.0/8" - rate_limit: + rate_limit_ui: + enabled: true + requests_per_minute: 12 + burst: 4 + rate_limit_api_read: + enabled: true + requests_per_minute: 12 + burst: 4 + rate_limit_api_write: enabled: true requests_per_minute: 12 burst: 4 @@ -94,6 +122,9 @@ security: if err := os.WriteFile(configPath, []byte(content), 0o644); err != nil { t.Fatalf("WriteFile() error = %v", err) } + if err := os.WriteFile(keyPath, []byte("AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=\n"), 0o600); err != nil { + t.Fatalf("WriteFile(key) error = %v", err) + } cfg, err := Load(configPath) if err != nil { @@ -118,6 +149,15 @@ security: if cfg.Server.MaxHeaderBytes != 2*1024*1024 { t.Fatalf("MaxHeaderBytes = %d, want %d", cfg.Server.MaxHeaderBytes, 2*1024*1024) } + if !cfg.Server.AccessLog.Enabled { + t.Fatalf("AccessLog.Enabled = false, want true") + } + if cfg.Server.AccessLog.MaxSizeBytes != 2*1024*1024 { + t.Fatalf("AccessLog.MaxSizeBytes = %d, want %d", cfg.Server.AccessLog.MaxSizeBytes, int64(2*1024*1024)) + } + if cfg.Server.AccessLog.MaxBackups != 7 { + t.Fatalf("AccessLog.MaxBackups = %d, want 7", cfg.Server.AccessLog.MaxBackups) + } if cfg.Limits.MaxUploadSizeBytes != 2*1024*1024 { t.Fatalf("MaxUploadSizeBytes = %d, want %d", cfg.Limits.MaxUploadSizeBytes, int64(2*1024*1024)) } @@ -133,11 +173,23 @@ security: if cfg.Storage.CleanupIntervalParsed != 45*time.Second { t.Fatalf("CleanupIntervalParsed = %s, want 45s", cfg.Storage.CleanupIntervalParsed) } + if len(cfg.Storage.EncryptionKeyBytes) != 32 { + t.Fatalf("EncryptionKeyBytes length = %d, want 32", len(cfg.Storage.EncryptionKeyBytes)) + } + if cfg.Storage.EncryptionKeyBytes[0] != 0 { + t.Fatalf("EncryptionKeyBytes[0] = %d, want 0", cfg.Storage.EncryptionKeyBytes[0]) + } if !cfg.Security.TrustProxyHeaders { t.Fatalf("TrustProxyHeaders = false, want true") } - if cfg.Security.RateLimit.RequestsPerMinute != 12 || cfg.Security.RateLimit.Burst != 4 { - t.Fatalf("unexpected rate_limit values: %+v", cfg.Security.RateLimit) + if cfg.Security.RateLimitUI.RequestsPerMinute != 12 || cfg.Security.RateLimitUI.Burst != 4 { + t.Fatalf("unexpected rate_limit_ui values: %+v", cfg.Security.RateLimitUI) + } + if cfg.Security.RateLimitAPIRead.RequestsPerMinute != 12 || cfg.Security.RateLimitAPIRead.Burst != 4 { + t.Fatalf("unexpected rate_limit_api_read values: %+v", cfg.Security.RateLimitAPIRead) + } + if cfg.Security.RateLimitAPIWrite.RequestsPerMinute != 12 || cfg.Security.RateLimitAPIWrite.Burst != 4 { + t.Fatalf("unexpected rate_limit_api_write values: %+v", cfg.Security.RateLimitAPIWrite) } if len(cfg.Security.AllowedPrefixes) != 2 { t.Fatalf("AllowedPrefixes len = %d, want 2", len(cfg.Security.AllowedPrefixes)) @@ -204,6 +256,20 @@ func TestValidateRejectsInvalidValues(t *testing.T) { }, wantErr: "server.max_header_bytes", }, + { + name: "invalid access log max size", + mutate: func(cfg *Config) { + cfg.Server.AccessLog.MaxSize = "oops" + }, + wantErr: "server.access_log.max_size", + }, + { + name: "invalid access log max backups", + mutate: func(cfg *Config) { + cfg.Server.AccessLog.MaxBackups = 0 + }, + wantErr: "server.access_log.max_backups", + }, { name: "invalid ttl parse", mutate: func(cfg *Config) { @@ -255,11 +321,25 @@ func TestValidateRejectsInvalidValues(t *testing.T) { wantErr: "security.trusted_proxy_ips is required", }, { - name: "invalid rate limit burst", + name: "invalid ui rate limit burst", mutate: func(cfg *Config) { - cfg.Security.RateLimit.Burst = 0 + cfg.Security.RateLimitUI.Burst = 0 }, - wantErr: "security.rate_limit.burst", + wantErr: "security.rate_limit_ui.burst", + }, + { + name: "invalid api read rate limit burst", + mutate: func(cfg *Config) { + cfg.Security.RateLimitAPIRead.Burst = 0 + }, + wantErr: "security.rate_limit_api_read.burst", + }, + { + name: "invalid api write rate limit burst", + mutate: func(cfg *Config) { + cfg.Security.RateLimitAPIWrite.Burst = 0 + }, + wantErr: "security.rate_limit_api_write.burst", }, { name: "data dir required", @@ -283,11 +363,39 @@ func TestValidateRejectsInvalidValues(t *testing.T) { wantErr: "storage.cleanup_interval", }, { - name: "invalid rate limit rpm", + name: "encryption key file required", mutate: func(cfg *Config) { - cfg.Security.RateLimit.RequestsPerMinute = 0 + cfg.Storage.EncryptionKeyFile = " " }, - wantErr: "security.rate_limit.requests_per_minute", + wantErr: "storage.encryption_key_file is required", + }, + { + name: "encryption key bytes invalid length", + mutate: func(cfg *Config) { + cfg.Storage.EncryptionKeyBytes = []byte("short") + }, + wantErr: "storage.encryption_key_file must decode to 32 bytes", + }, + { + name: "invalid ui rate limit rpm", + mutate: func(cfg *Config) { + cfg.Security.RateLimitUI.RequestsPerMinute = 0 + }, + wantErr: "security.rate_limit_ui.requests_per_minute", + }, + { + name: "invalid api read rate limit rpm", + mutate: func(cfg *Config) { + cfg.Security.RateLimitAPIRead.RequestsPerMinute = 0 + }, + wantErr: "security.rate_limit_api_read.requests_per_minute", + }, + { + name: "invalid api write rate limit rpm", + mutate: func(cfg *Config) { + cfg.Security.RateLimitAPIWrite.RequestsPerMinute = 0 + }, + wantErr: "security.rate_limit_api_write.requests_per_minute", }, } @@ -328,6 +436,29 @@ func TestLoadErrors(t *testing.T) { if err == nil || !strings.Contains(err.Error(), "parse config yaml") { t.Fatalf("Load() error = %v, want yaml parse error", err) } + + validConfigPath := filepath.Join(tmp, "valid.yaml") + validConfig := ` +storage: + data_dir: "` + filepath.Join(tmp, "store") + `" + cleanup_interval: "10s" +` + if err := os.WriteFile(validConfigPath, []byte(validConfig), 0o644); err != nil { + t.Fatalf("WriteFile(valid config) error = %v", err) + } + _, err = Load(validConfigPath) + if err == nil || !strings.Contains(err.Error(), "read storage.encryption_key_file") { + t.Fatalf("Load() error = %v, want missing key file error", err) + } + + keyPath := filepath.Join(tmp, "config.key") + if err := os.WriteFile(keyPath, []byte("not-base64"), 0o600); err != nil { + t.Fatalf("WriteFile(invalid key) error = %v", err) + } + _, err = Load(validConfigPath) + if err == nil || !strings.Contains(err.Error(), "storage.encryption_key_file invalid: invalid base64") { + t.Fatalf("Load() error = %v, want invalid key base64 error", err) + } } func TestParseAllowedPrefixesTrimsAndParses(t *testing.T) { @@ -385,3 +516,26 @@ func TestParseHumanSizeVariants(t *testing.T) { }) } } + +func TestGenerateEncryptionKey(t *testing.T) { + t.Parallel() + + k1, err := GenerateEncryptionKey() + if err != nil { + t.Fatalf("GenerateEncryptionKey() error = %v", err) + } + k2, err := GenerateEncryptionKey() + if err != nil { + t.Fatalf("GenerateEncryptionKey() second error = %v", err) + } + if k1 == k2 { + t.Fatalf("GenerateEncryptionKey() produced duplicate keys") + } + b1, err := base64.StdEncoding.DecodeString(k1) + if err != nil { + t.Fatalf("DecodeString(k1) error = %v", err) + } + if len(b1) != 32 { + t.Fatalf("decoded key length = %d, want 32", len(b1)) + } +} diff --git a/internal/http/handlers.go b/internal/http/handlers.go index 2c679df..2e18840 100644 --- a/internal/http/handlers.go +++ b/internal/http/handlers.go @@ -18,46 +18,63 @@ import ( ) type Server struct { - cfg config.Config - store *storage.FilesystemStore - logger *log.Logger - rawCache *rawContentCache + cfg config.Config + store *storage.FilesystemStore + logger *log.Logger + accessLogger *log.Logger + rawCache *rawContentCache } -func NewServer(cfg config.Config, store *storage.FilesystemStore, logger *log.Logger) (*Server, error) { +func NewServer(cfg config.Config, store *storage.FilesystemStore, logger *log.Logger, accessLogger *log.Logger) (*Server, error) { return &Server{ - cfg: cfg, - store: store, - logger: logger, - rawCache: newRawContentCache(cfg.Limits.RawCacheMaxEntries, cfg.Limits.RawCacheMaxBytes), + cfg: cfg, + store: store, + logger: logger, + accessLogger: accessLogger, + rawCache: newRawContentCache(cfg.Limits.RawCacheMaxEntries, cfg.Limits.RawCacheMaxBytes), }, nil } func (s *Server) Routes() http.Handler { mux := http.NewServeMux() - mux.Handle("GET /{$}", http.HandlerFunc(s.serveSPA)) - mux.Handle("GET /u", http.HandlerFunc(s.serveSPA)) - mux.Handle("GET /api/config", http.HandlerFunc(s.getUIConfig)) + uiMiddlewares := make([]func(http.Handler) http.Handler, 0, 1) + apiReadMiddlewares := make([]func(http.Handler) http.Handler, 0, 1) + apiWriteMiddlewares := make([]func(http.Handler) http.Handler, 0, 1) + if s.cfg.Security.RateLimitUI.Enabled { + uiLimiter := NewRateLimiter(s.cfg.Security.RateLimitUI.RequestsPerMinute, s.cfg.Security.RateLimitUI.Burst) + uiMiddlewares = append(uiMiddlewares, RateLimitMiddleware(uiLimiter, s.cfg.Security.TrustProxyHeaders, s.cfg.Security.TrustedProxyPrefixes)) + } + if s.cfg.Security.RateLimitAPIRead.Enabled { + apiReadLimiter := NewRateLimiter(s.cfg.Security.RateLimitAPIRead.RequestsPerMinute, s.cfg.Security.RateLimitAPIRead.Burst) + apiReadMiddlewares = append(apiReadMiddlewares, RateLimitMiddleware(apiReadLimiter, s.cfg.Security.TrustProxyHeaders, s.cfg.Security.TrustedProxyPrefixes)) + } + if s.cfg.Security.RateLimitAPIWrite.Enabled { + apiWriteLimiter := NewRateLimiter(s.cfg.Security.RateLimitAPIWrite.RequestsPerMinute, s.cfg.Security.RateLimitAPIWrite.Burst) + apiWriteMiddlewares = append(apiWriteMiddlewares, RateLimitMiddleware(apiWriteLimiter, s.cfg.Security.TrustProxyHeaders, s.cfg.Security.TrustedProxyPrefixes)) + } + + mux.Handle("GET /{$}", Chain(http.HandlerFunc(s.serveSPA), uiMiddlewares...)) + mux.Handle("GET /u", Chain(http.HandlerFunc(s.serveSPA), uiMiddlewares...)) + mux.Handle("GET /s/{id}", Chain(http.HandlerFunc(s.scratchPage), uiMiddlewares...)) + + mux.Handle("GET /api/config", Chain(http.HandlerFunc(s.getUIConfig), apiReadMiddlewares...)) + mux.Handle("GET /api/scratch/{id}", Chain(http.HandlerFunc(s.getScratch), apiReadMiddlewares...)) + mux.Handle("GET /api/raw/{id}", Chain(http.HandlerFunc(s.rawScratch), apiReadMiddlewares...)) writeHandler := http.HandlerFunc(s.createScratch) - middlewares := make([]func(http.Handler) http.Handler, 0, 2) - middlewares = append(middlewares, AllowlistMiddleware(s.cfg.Security.AllowedPrefixes, s.cfg.Security.TrustProxyHeaders, s.cfg.Security.TrustedProxyPrefixes, s.logger)) - readMiddlewares := make([]func(http.Handler) http.Handler, 0, 1) - if s.cfg.Security.RateLimit.Enabled { - writeLimiter := NewRateLimiter(s.cfg.Security.RateLimit.RequestsPerMinute, s.cfg.Security.RateLimit.Burst) - readLimiter := NewRateLimiter(s.cfg.Security.RateLimit.RequestsPerMinute, s.cfg.Security.RateLimit.Burst) - middlewares = append(middlewares, RateLimitMiddleware(writeLimiter, s.cfg.Security.TrustProxyHeaders, s.cfg.Security.TrustedProxyPrefixes)) - readMiddlewares = append(readMiddlewares, RateLimitMiddleware(readLimiter, s.cfg.Security.TrustProxyHeaders, s.cfg.Security.TrustedProxyPrefixes)) - } - mux.Handle("POST /api/scratch", Chain(writeHandler, middlewares...)) - mux.Handle("GET /api/scratch/{id}", Chain(http.HandlerFunc(s.getScratch), readMiddlewares...)) - mux.Handle("GET /s/{id}", Chain(http.HandlerFunc(s.scratchPage), readMiddlewares...)) - mux.Handle("GET /api/raw/{id}", Chain(http.HandlerFunc(s.rawScratch), readMiddlewares...)) + writeMiddlewares := make([]func(http.Handler) http.Handler, 0, 2) + writeMiddlewares = append(writeMiddlewares, AllowlistMiddleware(s.cfg.Security.AllowedPrefixes, s.cfg.Security.TrustProxyHeaders, s.cfg.Security.TrustedProxyPrefixes, s.logger)) + writeMiddlewares = append(writeMiddlewares, apiWriteMiddlewares...) + mux.Handle("POST /api/scratch", Chain(writeHandler, writeMiddlewares...)) mux.Handle("GET /assets/", http.FileServerFS(webassets.StaticFS())) mux.Handle("GET /static/", http.StripPrefix("/static/", http.FileServerFS(webassets.StaticFS()))) mux.Handle("GET /{path...}", http.HandlerFunc(s.notFoundPage)) - return mux + return AccessLogMiddleware( + s.accessLogger, + s.cfg.Security.TrustProxyHeaders, + s.cfg.Security.TrustedProxyPrefixes, + )(mux) } func (s *Server) serveSPA(w http.ResponseWriter, r *http.Request) { @@ -68,6 +85,7 @@ func (s *Server) getUIConfig(w http.ResponseWriter, r *http.Request) { payload := map[string]any{ "max_upload_size_bytes": s.cfg.Limits.MaxUploadSizeBytes, "upload_allowed": s.isUploadAllowedForRequest(r), + "default_ttl": s.cfg.Limits.DefaultTTL, } writeJSON(w, http.StatusOK, payload) } diff --git a/internal/http/handlers_integration_test.go b/internal/http/handlers_integration_test.go index d0c985b..b025bfb 100644 --- a/internal/http/handlers_integration_test.go +++ b/internal/http/handlers_integration_test.go @@ -162,7 +162,7 @@ func TestReadRoutesAreRateLimitedWhenEnabled(t *testing.T) { createReq := httptest.NewRequest(http.MethodPost, "/api/scratch", bytes.NewBufferString("hello scratch")) createReq.Header.Set("Content-Type", "text/plain; charset=utf-8") - createReq.RemoteAddr = "127.0.0.1:9111" + createReq.RemoteAddr = "127.0.0.2:9111" createRec := httptest.NewRecorder() handler.ServeHTTP(createRec, createReq) if createRec.Code != http.StatusCreated { @@ -1041,7 +1041,7 @@ func newTestHandlerAndStore(t *testing.T, opts testServerOptions) (http.Handler, t.Helper() dataDir := filepath.Join(t.TempDir(), "data") - store, err := storage.NewFilesystemStore(dataDir) + store, err := storage.NewFilesystemStore(dataDir, testStorageKey) if err != nil { t.Fatalf("NewFilesystemStore() error = %v", err) } @@ -1060,7 +1060,17 @@ func newTestHandlerAndStore(t *testing.T, opts testServerOptions) (http.Handler, DataDir: dataDir, }, Security: config.SecurityConfig{ - RateLimit: config.RateLimitConfig{ + RateLimitUI: config.RateLimitConfig{ + Enabled: opts.rateLimitEnable, + RequestsPerMinute: 120, + Burst: 1, + }, + RateLimitAPIRead: config.RateLimitConfig{ + Enabled: opts.rateLimitEnable, + RequestsPerMinute: 120, + Burst: 1, + }, + RateLimitAPIWrite: config.RateLimitConfig{ Enabled: opts.rateLimitEnable, RequestsPerMinute: 120, Burst: 1, diff --git a/internal/http/handlers_unit_test.go b/internal/http/handlers_unit_test.go index f81ee20..2ed8670 100644 --- a/internal/http/handlers_unit_test.go +++ b/internal/http/handlers_unit_test.go @@ -15,6 +15,8 @@ import ( "scratchbox/internal/storage" ) +var testStorageKey = []byte("0123456789abcdef0123456789abcdef") + func TestMultipartFileCount(t *testing.T) { t.Parallel() @@ -49,18 +51,20 @@ func TestWriteCreateErrorBranches(t *testing.T) { func TestNewServerInitializes(t *testing.T) { tmp := t.TempDir() - store, err := storage.NewFilesystemStore(filepath.Join(tmp, "data")) + store, err := storage.NewFilesystemStore(filepath.Join(tmp, "data"), testStorageKey) if err != nil { t.Fatalf("NewFilesystemStore() error = %v", err) } cfg := config.Config{ Limits: config.LimitsConfig{MaxUploadSizeBytes: 1024}, Security: config.SecurityConfig{ - RateLimit: config.RateLimitConfig{Enabled: false}, + RateLimitUI: config.RateLimitConfig{Enabled: false, RequestsPerMinute: 30, Burst: 10}, + RateLimitAPIRead: config.RateLimitConfig{Enabled: false, RequestsPerMinute: 30, Burst: 10}, + RateLimitAPIWrite: config.RateLimitConfig{Enabled: false, RequestsPerMinute: 30, Burst: 10}, }, } - srv, err := NewServer(cfg, store, log.New(io.Discard, "", 0)) + srv, err := NewServer(cfg, store, log.New(io.Discard, "", 0), log.New(io.Discard, "", 0)) if err != nil { t.Fatalf("NewServer() error = %v", err) } @@ -118,17 +122,19 @@ func TestGetUIConfigUploadDenied(t *testing.T) { func TestWriteCreateErrorBranchesWithConfig(t *testing.T) { tmp := t.TempDir() - store, err := storage.NewFilesystemStore(filepath.Join(tmp, "data")) + store, err := storage.NewFilesystemStore(filepath.Join(tmp, "data"), testStorageKey) if err != nil { t.Fatalf("NewFilesystemStore() error = %v", err) } cfg := config.Config{ Limits: config.LimitsConfig{MaxUploadSizeBytes: 1024}, Security: config.SecurityConfig{ - RateLimit: config.RateLimitConfig{Enabled: false}, + RateLimitUI: config.RateLimitConfig{Enabled: false, RequestsPerMinute: 30, Burst: 10}, + RateLimitAPIRead: config.RateLimitConfig{Enabled: false, RequestsPerMinute: 30, Burst: 10}, + RateLimitAPIWrite: config.RateLimitConfig{Enabled: false, RequestsPerMinute: 30, Burst: 10}, }, } - if _, err := NewServer(cfg, store, log.New(io.Discard, "", 0)); err != nil { + if _, err := NewServer(cfg, store, log.New(io.Discard, "", 0), log.New(io.Discard, "", 0)); err != nil { t.Fatalf("NewServer() error = %v", err) } } diff --git a/internal/http/middleware.go b/internal/http/middleware.go index 6fe6781..f09c6f4 100644 --- a/internal/http/middleware.go +++ b/internal/http/middleware.go @@ -107,6 +107,53 @@ func RateLimitMiddleware(limiter *RateLimiter, trustProxyHeaders bool, trustedPr } } +func AccessLogMiddleware(logger *log.Logger, trustProxyHeaders bool, trustedProxies []netip.Prefix) func(http.Handler) http.Handler { + return func(next http.Handler) http.Handler { + if logger == nil { + return next + } + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if !shouldLogAccessRequest(r.Method, r.URL.Path) { + next.ServeHTTP(w, r) + return + } + + start := time.Now() + recorder := &accessLogResponseWriter{ResponseWriter: w} + next.ServeHTTP(recorder, r) + + status := recorder.status + if status == 0 { + status = http.StatusOK + } + clientIP := "-" + if ip, ok := extractClientIP(r, trustProxyHeaders, trustedProxies); ok { + clientIP = ip.String() + } + logger.Printf( + "method=%s path=%s status=%d bytes=%d duration_ms=%d ip=%s ua=%q", + r.Method, + r.URL.RequestURI(), + status, + recorder.bytes, + time.Since(start).Milliseconds(), + clientIP, + r.UserAgent(), + ) + }) + } +} + +func shouldLogAccessRequest(method string, path string) bool { + if method == http.MethodPost && path == "/api/scratch" { + return true + } + if method == http.MethodGet && strings.HasPrefix(path, "/api/raw/") && len(path) > len("/api/raw/") { + return true + } + return false +} + func Chain(handler http.Handler, middlewares ...func(http.Handler) http.Handler) http.Handler { wrapped := handler for i := len(middlewares) - 1; i >= 0; i-- { @@ -115,6 +162,26 @@ func Chain(handler http.Handler, middlewares ...func(http.Handler) http.Handler) return wrapped } +type accessLogResponseWriter struct { + http.ResponseWriter + status int + bytes int +} + +func (w *accessLogResponseWriter) WriteHeader(statusCode int) { + w.status = statusCode + w.ResponseWriter.WriteHeader(statusCode) +} + +func (w *accessLogResponseWriter) Write(p []byte) (int, error) { + if w.status == 0 { + w.status = http.StatusOK + } + n, err := w.ResponseWriter.Write(p) + w.bytes += n + return n, err +} + func extractClientIP(r *http.Request, trustProxyHeaders bool, trustedProxies []netip.Prefix) (netip.Addr, bool) { remoteIP, ok := remoteIPFromRequest(r) if !ok { diff --git a/internal/http/middleware_test.go b/internal/http/middleware_test.go index 8eb1490..01a51db 100644 --- a/internal/http/middleware_test.go +++ b/internal/http/middleware_test.go @@ -1,6 +1,7 @@ package httpapi import ( + "bytes" "encoding/json" "io" "log" @@ -228,3 +229,95 @@ func TestExtractClientIPIgnoresHeadersFromUntrustedProxy(t *testing.T) { t.Fatalf("extractClientIP() = (%v,%v), want (203.0.113.10,true)", ip, ok) } } + +func TestAccessLogMiddlewareWritesStructuredLog(t *testing.T) { + t.Parallel() + + var buf bytes.Buffer + logger := log.New(&buf, "", 0) + handler := AccessLogMiddleware(logger, false, nil)(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + w.WriteHeader(http.StatusCreated) + _, _ = w.Write([]byte("ok")) + })) + + req := httptest.NewRequest(http.MethodPost, "/api/scratch?ttl=1h", nil) + req.RemoteAddr = "198.51.100.12:4444" + req.Header.Set("User-Agent", "middleware-test") + rec := httptest.NewRecorder() + handler.ServeHTTP(rec, req) + + got := buf.String() + if !strings.Contains(got, "method=POST") { + t.Fatalf("missing method field in log: %q", got) + } + if !strings.Contains(got, "path=/api/scratch?ttl=1h") { + t.Fatalf("missing path field in log: %q", got) + } + if !strings.Contains(got, "status=201") { + t.Fatalf("missing status field in log: %q", got) + } + if !strings.Contains(got, "bytes=2") { + t.Fatalf("missing bytes field in log: %q", got) + } + if !strings.Contains(got, "ip=198.51.100.12") { + t.Fatalf("missing ip field in log: %q", got) + } +} + +func TestAccessLogMiddlewareSkipsConfigAndStaticPaths(t *testing.T) { + t.Parallel() + + var buf bytes.Buffer + logger := log.New(&buf, "", 0) + handler := AccessLogMiddleware(logger, false, nil)(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + w.WriteHeader(http.StatusOK) + })) + + reqConfig := httptest.NewRequest(http.MethodGet, "/api/config", nil) + reqConfig.RemoteAddr = "198.51.100.12:4444" + handler.ServeHTTP(httptest.NewRecorder(), reqConfig) + + reqStatic := httptest.NewRequest(http.MethodGet, "/static/app.js", nil) + reqStatic.RemoteAddr = "198.51.100.12:4444" + handler.ServeHTTP(httptest.NewRecorder(), reqStatic) + + reqAsset := httptest.NewRequest(http.MethodGet, "/assets/logo.svg", nil) + reqAsset.RemoteAddr = "198.51.100.12:4444" + handler.ServeHTTP(httptest.NewRecorder(), reqAsset) + + reqScratchView := httptest.NewRequest(http.MethodGet, "/s/abc123", nil) + reqScratchView.RemoteAddr = "198.51.100.12:4444" + handler.ServeHTTP(httptest.NewRecorder(), reqScratchView) + + reqUpload := httptest.NewRequest(http.MethodGet, "/u", nil) + reqUpload.RemoteAddr = "198.51.100.12:4444" + handler.ServeHTTP(httptest.NewRecorder(), reqUpload) + + if buf.Len() != 0 { + t.Fatalf("expected no access logs for skipped paths, got %q", buf.String()) + } +} + +func TestAccessLogMiddlewareOnlyAllowsConfiguredRoutes(t *testing.T) { + t.Parallel() + + var buf bytes.Buffer + logger := log.New(&buf, "", 0) + handler := AccessLogMiddleware(logger, false, nil)(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + w.WriteHeader(http.StatusOK) + })) + + disallowed := httptest.NewRequest(http.MethodGet, "/api/scratch/some-id", nil) + disallowed.RemoteAddr = "198.51.100.12:4444" + handler.ServeHTTP(httptest.NewRecorder(), disallowed) + if buf.Len() != 0 { + t.Fatalf("expected no access log for disallowed route, got %q", buf.String()) + } + + allowed := httptest.NewRequest(http.MethodGet, "/api/raw/some-id", nil) + allowed.RemoteAddr = "198.51.100.12:4444" + handler.ServeHTTP(httptest.NewRecorder(), allowed) + if !strings.Contains(buf.String(), "method=GET path=/api/raw/some-id") { + t.Fatalf("expected access log for allowed route, got %q", buf.String()) + } +} diff --git a/internal/storage/filesystem.go b/internal/storage/filesystem.go index a010a4b..1f53bb9 100644 --- a/internal/storage/filesystem.go +++ b/internal/storage/filesystem.go @@ -3,7 +3,11 @@ package storage import ( "compress/gzip" "context" + "crypto/aes" + "crypto/cipher" "crypto/rand" + "crypto/sha256" + "encoding/binary" "encoding/hex" "encoding/json" "errors" @@ -17,14 +21,16 @@ import ( ) const ( - indexFilename = "metadata.json" - filesDirName = "scratches" - idBytes = 12 - maxCreateIDAttempts = 100 + indexFilename = "metadata" + filesDirName = "scratches" + encryptedChunkSize = 64 * 1024 ) var ErrNotFound = errors.New("scratch not found") +var encryptedFileMagic = [4]byte{'S', 'B', 'X', '1'} +var encryptedIndexMagic = [4]byte{'S', 'M', 'D', '1'} + type Metadata struct { ID string `json:"id"` FilePath string `json:"file_path"` @@ -35,35 +41,63 @@ type Metadata struct { Size int64 `json:"size"` } -type FilesystemStore struct { - dataDir string - filesDir string - index string - idGenerator func() (string, error) - - mu sync.RWMutex - metadata map[string]Metadata - reservedIDs map[string]struct{} +type indexDocument struct { + DefaultTTL string `json:"default_ttl,omitempty"` + Scratches map[string]Metadata `json:"scratches"` } -func NewFilesystemStore(dataDir string) (*FilesystemStore, error) { +type FilesystemStore struct { + dataDir string + filesDir string + index string + defaultTTL time.Duration + fileCipher cipher.AEAD + + mu sync.RWMutex + metadata map[string]Metadata +} + +func NewFilesystemStore(dataDir string, encryptionKey []byte) (*FilesystemStore, error) { + return newFilesystemStore(dataDir, 0, encryptionKey) +} + +func NewFilesystemStoreWithDefaultTTL(dataDir string, defaultTTL time.Duration, encryptionKey []byte) (*FilesystemStore, error) { + return newFilesystemStore(dataDir, defaultTTL, encryptionKey) +} + +func newFilesystemStore(dataDir string, defaultTTL time.Duration, encryptionKey []byte) (*FilesystemStore, error) { filesDir := filepath.Join(dataDir, filesDirName) if err := os.MkdirAll(filesDir, 0o755); err != nil { return nil, fmt.Errorf("create files dir: %w", err) } + if len(encryptionKey) != 32 { + return nil, fmt.Errorf("storage encryption key must be 32 bytes, got %d", len(encryptionKey)) + } + block, err := aes.NewCipher(encryptionKey) + if err != nil { + return nil, fmt.Errorf("init storage cipher: %w", err) + } + fileCipher, err := cipher.NewGCM(block) + if err != nil { + return nil, fmt.Errorf("init storage aead: %w", err) + } + store := &FilesystemStore{ - dataDir: dataDir, - filesDir: filesDir, - index: filepath.Join(dataDir, indexFilename), - idGenerator: generateID, - metadata: map[string]Metadata{}, - reservedIDs: map[string]struct{}{}, + dataDir: dataDir, + filesDir: filesDir, + index: filepath.Join(dataDir, indexFilename), + fileCipher: fileCipher, + metadata: map[string]Metadata{}, } if err := store.loadIndex(); err != nil { return nil, err } - if err := store.reconcileOnStartup(time.Now()); err != nil { + now := time.Now().UTC() + if err := store.applyDefaultTTLOnStartup(defaultTTL); err != nil { + return nil, err + } + if err := store.reconcileOnStartup(now); err != nil { return nil, err } return store, nil @@ -74,107 +108,84 @@ func (s *FilesystemStore) Create(ctx context.Context, body io.Reader, contentTyp } func (s *FilesystemStore) CreateWithOriginalName(ctx context.Context, body io.Reader, contentType string, originalName string, ttl time.Duration) (Metadata, error) { - for attempt := 0; attempt < maxCreateIDAttempts; attempt++ { - id, err := s.idGenerator() - if err != nil { - return Metadata{}, fmt.Errorf("generate id: %w", err) - } - finalPath := filepath.Join(s.filesDir, id) - - s.mu.RLock() - _, exists := s.metadata[id] - _, reserved := s.reservedIDs[id] - s.mu.RUnlock() - if exists || reserved { - continue - } - if _, err := os.Stat(finalPath); err == nil { - continue - } else if !errors.Is(err, os.ErrNotExist) { - return Metadata{}, fmt.Errorf("stat candidate scratch path: %w", err) - } - - s.mu.Lock() - if _, exists := s.metadata[id]; exists { - s.mu.Unlock() - continue - } - if _, reserved := s.reservedIDs[id]; reserved { - s.mu.Unlock() - continue - } - // Reserve the ID before consuming the request body to avoid - // concurrent writers choosing the same ID. - s.reservedIDs[id] = struct{}{} - s.mu.Unlock() - - tempFile, err := os.CreateTemp(s.filesDir, id+".tmp-*") - if err != nil { - s.mu.Lock() - delete(s.reservedIDs, id) - s.mu.Unlock() - return Metadata{}, fmt.Errorf("create temp file: %w", err) - } - - gzipWriter := gzip.NewWriter(tempFile) - size, copyErr := io.Copy(gzipWriter, body) - gzipCloseErr := gzipWriter.Close() - closeErr := tempFile.Close() - if copyErr != nil { - _ = os.Remove(tempFile.Name()) - s.mu.Lock() - delete(s.reservedIDs, id) - s.mu.Unlock() - return Metadata{}, fmt.Errorf("write scratch content: %w", copyErr) - } - if gzipCloseErr != nil { - _ = os.Remove(tempFile.Name()) - s.mu.Lock() - delete(s.reservedIDs, id) - s.mu.Unlock() - return Metadata{}, fmt.Errorf("close gzip writer: %w", gzipCloseErr) - } - if closeErr != nil { - _ = os.Remove(tempFile.Name()) - s.mu.Lock() - delete(s.reservedIDs, id) - s.mu.Unlock() - return Metadata{}, fmt.Errorf("close temp file: %w", closeErr) - } - - if err := os.Rename(tempFile.Name(), finalPath); err != nil { - _ = os.Remove(tempFile.Name()) - s.mu.Lock() - delete(s.reservedIDs, id) - s.mu.Unlock() - return Metadata{}, fmt.Errorf("atomically move scratch file: %w", err) - } - - now := time.Now().UTC() - meta := Metadata{ - ID: id, - FilePath: finalPath, - CreatedAt: now, - ExpiresAt: now.Add(ttl), - ContentType: contentType, - OriginalName: normalizeOriginalName(originalName), - Size: size, - } - - s.mu.Lock() - delete(s.reservedIDs, id) - s.metadata[id] = meta - if err := s.persistLocked(); err != nil { - delete(s.metadata, id) - s.mu.Unlock() - _ = os.Remove(finalPath) - return Metadata{}, err - } - s.mu.Unlock() - return meta, nil + tempFile, err := os.CreateTemp(s.filesDir, "scratch.tmp-*") + if err != nil { + return Metadata{}, fmt.Errorf("create temp file: %w", err) } - return Metadata{}, fmt.Errorf("failed to allocate unique scratch id after %d attempts", maxCreateIDAttempts) + blobHash := sha256.New() + blobSink := io.MultiWriter(tempFile, blobHash) + scratchWriter := io.Writer(blobSink) + encryptedWriter, err := newEncryptedFileWriter(blobSink, s.fileCipher) + if err != nil { + _ = tempFile.Close() + _ = os.Remove(tempFile.Name()) + return Metadata{}, fmt.Errorf("init encrypted scratch writer: %w", err) + } + scratchWriter = encryptedWriter + + gzipWriter := gzip.NewWriter(scratchWriter) + size, copyErr := io.Copy(gzipWriter, body) + gzipCloseErr := gzipWriter.Close() + encryptCloseErr := encryptedWriter.Close() + closeErr := tempFile.Close() + if copyErr != nil { + _ = os.Remove(tempFile.Name()) + return Metadata{}, fmt.Errorf("write scratch content: %w", copyErr) + } + if gzipCloseErr != nil { + _ = os.Remove(tempFile.Name()) + return Metadata{}, fmt.Errorf("close gzip writer: %w", gzipCloseErr) + } + if encryptCloseErr != nil { + _ = os.Remove(tempFile.Name()) + return Metadata{}, fmt.Errorf("close encrypted writer: %w", encryptCloseErr) + } + if closeErr != nil { + _ = os.Remove(tempFile.Name()) + return Metadata{}, fmt.Errorf("close temp file: %w", closeErr) + } + + id := hex.EncodeToString(blobHash.Sum(nil)) + finalPath := filepath.Join(s.filesDir, id) + if _, statErr := os.Stat(finalPath); statErr == nil { + _ = os.Remove(tempFile.Name()) + return Metadata{}, errors.New("scratch id collision for content hash") + } else if !errors.Is(statErr, os.ErrNotExist) { + _ = os.Remove(tempFile.Name()) + return Metadata{}, fmt.Errorf("stat candidate scratch path: %w", statErr) + } + if err := os.Rename(tempFile.Name(), finalPath); err != nil { + _ = os.Remove(tempFile.Name()) + return Metadata{}, fmt.Errorf("atomically move scratch file: %w", err) + } + + now := time.Now().UTC() + meta := Metadata{ + ID: id, + FilePath: finalPath, + CreatedAt: now, + ExpiresAt: now.Add(ttl), + ContentType: contentType, + OriginalName: normalizeOriginalName(originalName), + Size: size, + } + + s.mu.Lock() + if _, exists := s.metadata[id]; exists { + s.mu.Unlock() + _ = os.Remove(finalPath) + return Metadata{}, errors.New("scratch id collision in metadata index") + } + s.metadata[id] = meta + if err := s.persistLocked(); err != nil { + delete(s.metadata, id) + s.mu.Unlock() + _ = os.Remove(finalPath) + return Metadata{}, err + } + s.mu.Unlock() + return meta, nil } func normalizeOriginalName(originalName string) string { @@ -212,7 +223,12 @@ func (s *FilesystemStore) Open(id string) (io.ReadCloser, Metadata, error) { } return nil, Metadata{}, fmt.Errorf("open scratch file: %w", err) } - gzipReader, err := gzip.NewReader(file) + decryptedReader, decryptErr := newEncryptedFileReader(file, s.fileCipher) + if decryptErr != nil { + _ = file.Close() + return nil, Metadata{}, fmt.Errorf("create encrypted reader: %w", decryptErr) + } + gzipReader, err := gzip.NewReader(decryptedReader) if err != nil { _ = file.Close() return nil, Metadata{}, fmt.Errorf("create gzip reader: %w", err) @@ -286,22 +302,46 @@ func (s *FilesystemStore) loadIndex() error { } return fmt.Errorf("read metadata index: %w", err) } + plaintext, err := decryptIndexPayload(b, s.fileCipher) + if err != nil { + return fmt.Errorf("decrypt metadata index: %w", err) + } - loaded := map[string]Metadata{} - if err := json.Unmarshal(b, &loaded); err != nil { + doc := indexDocument{} + if err := json.Unmarshal(plaintext, &doc); err != nil { return fmt.Errorf("decode metadata index: %w", err) } - s.metadata = loaded + if doc.Scratches == nil { + doc.Scratches = map[string]Metadata{} + } + s.metadata = doc.Scratches + if ttlRaw := strings.TrimSpace(doc.DefaultTTL); ttlRaw != "" { + ttl, parseErr := time.ParseDuration(ttlRaw) + if parseErr != nil { + return fmt.Errorf("decode metadata index default ttl: %w", parseErr) + } + s.defaultTTL = ttl + } return nil } func (s *FilesystemStore) persistLocked() error { tmp := s.index + ".tmp" - payload, err := json.MarshalIndent(s.metadata, "", " ") + doc := indexDocument{ + Scratches: s.metadata, + } + if s.defaultTTL > 0 { + doc.DefaultTTL = s.defaultTTL.String() + } + payload, err := json.MarshalIndent(doc, "", " ") if err != nil { return fmt.Errorf("encode metadata index: %w", err) } - if err := os.WriteFile(tmp, payload, 0o644); err != nil { + encryptedPayload, err := encryptIndexPayload(payload, s.fileCipher) + if err != nil { + return fmt.Errorf("encrypt metadata index: %w", err) + } + if err := os.WriteFile(tmp, encryptedPayload, 0o644); err != nil { return fmt.Errorf("write metadata temp file: %w", err) } if err := os.Rename(tmp, s.index); err != nil { @@ -310,6 +350,28 @@ func (s *FilesystemStore) persistLocked() error { return nil } +func (s *FilesystemStore) applyDefaultTTLOnStartup(defaultTTL time.Duration) error { + if defaultTTL <= 0 { + return nil + } + + if s.defaultTTL <= 0 { + s.defaultTTL = defaultTTL + return s.persistLocked() + } + if s.defaultTTL == defaultTTL { + return nil + } + + delta := defaultTTL - s.defaultTTL + for id, meta := range s.metadata { + meta.ExpiresAt = meta.ExpiresAt.Add(delta) + s.metadata[id] = meta + } + s.defaultTTL = defaultTTL + return s.persistLocked() +} + func (s *FilesystemStore) reconcileOnStartup(now time.Time) error { s.mu.Lock() defer s.mu.Unlock() @@ -337,14 +399,6 @@ func (s *FilesystemStore) reconcileOnStartup(now time.Time) error { return s.persistLocked() } -func generateID() (string, error) { - b := make([]byte, idBytes) - if _, err := rand.Read(b); err != nil { - return "", err - } - return hex.EncodeToString(b), nil -} - type combinedReadCloser struct { reader io.Reader closers []io.Closer @@ -363,3 +417,203 @@ func (c *combinedReadCloser) Close() error { } return firstErr } + +type encryptedFileWriter struct { + dst io.Writer + aead cipher.AEAD + noncePrefix [4]byte + counter uint64 + pending []byte +} + +func newEncryptedFileWriter(dst io.Writer, aead cipher.AEAD) (*encryptedFileWriter, error) { + writer := &encryptedFileWriter{ + dst: dst, + aead: aead, + pending: make([]byte, 0, encryptedChunkSize), + } + if _, err := rand.Read(writer.noncePrefix[:]); err != nil { + return nil, fmt.Errorf("generate nonce prefix: %w", err) + } + header := make([]byte, 0, len(encryptedFileMagic)+len(writer.noncePrefix)) + header = append(header, encryptedFileMagic[:]...) + header = append(header, writer.noncePrefix[:]...) + if _, err := writer.dst.Write(header); err != nil { + return nil, fmt.Errorf("write encrypted file header: %w", err) + } + return writer, nil +} + +func (w *encryptedFileWriter) Write(p []byte) (int, error) { + written := len(p) + for len(p) > 0 { + space := encryptedChunkSize - len(w.pending) + if space > len(p) { + space = len(p) + } + w.pending = append(w.pending, p[:space]...) + p = p[space:] + if len(w.pending) == encryptedChunkSize { + if err := w.flushChunk(w.pending); err != nil { + return written - len(p), err + } + w.pending = w.pending[:0] + } + } + return written, nil +} + +func (w *encryptedFileWriter) Close() error { + if len(w.pending) == 0 { + return nil + } + if err := w.flushChunk(w.pending); err != nil { + return err + } + w.pending = w.pending[:0] + return nil +} + +func (w *encryptedFileWriter) flushChunk(plaintext []byte) error { + nonce := make([]byte, w.aead.NonceSize()) + copy(nonce, w.noncePrefix[:]) + binary.BigEndian.PutUint64(nonce[len(w.noncePrefix):], w.counter) + ciphertext := w.aead.Seal(nil, nonce, plaintext, nil) + + var sizeBuf [4]byte + binary.BigEndian.PutUint32(sizeBuf[:], uint32(len(ciphertext))) + if _, err := w.dst.Write(sizeBuf[:]); err != nil { + return fmt.Errorf("write encrypted chunk size: %w", err) + } + if _, err := w.dst.Write(ciphertext); err != nil { + return fmt.Errorf("write encrypted chunk payload: %w", err) + } + w.counter++ + return nil +} + +type encryptedFileReader struct { + src io.Reader + aead cipher.AEAD + noncePrefix [4]byte + counter uint64 + buf []byte + offset int + eof bool +} + +func newEncryptedFileReader(src io.Reader, aead cipher.AEAD) (*encryptedFileReader, error) { + reader := &encryptedFileReader{ + src: src, + aead: aead, + } + header := make([]byte, len(encryptedFileMagic)+len(reader.noncePrefix)) + if _, err := io.ReadFull(src, header); err != nil { + return nil, fmt.Errorf("read encrypted file header: %w", err) + } + if !equalBytes(header[:len(encryptedFileMagic)], encryptedFileMagic[:]) { + return nil, errors.New("invalid encrypted file header") + } + copy(reader.noncePrefix[:], header[len(encryptedFileMagic):]) + return reader, nil +} + +func (r *encryptedFileReader) Read(p []byte) (int, error) { + for r.offset >= len(r.buf) { + if r.eof { + return 0, io.EOF + } + if err := r.loadNextChunk(); err != nil { + return 0, err + } + } + n := copy(p, r.buf[r.offset:]) + r.offset += n + return n, nil +} + +func (r *encryptedFileReader) loadNextChunk() error { + var sizeBuf [4]byte + _, err := io.ReadFull(r.src, sizeBuf[:]) + if err != nil { + if errors.Is(err, io.EOF) { + r.eof = true + return io.EOF + } + if errors.Is(err, io.ErrUnexpectedEOF) { + return errors.New("truncated encrypted chunk size") + } + return fmt.Errorf("read encrypted chunk size: %w", err) + } + chunkSize := binary.BigEndian.Uint32(sizeBuf[:]) + if chunkSize == 0 { + return errors.New("invalid empty encrypted chunk") + } + maxChunkSize := uint32(encryptedChunkSize + r.aead.Overhead()) + if chunkSize > maxChunkSize { + return fmt.Errorf("encrypted chunk too large: %d", chunkSize) + } + ciphertext := make([]byte, chunkSize) + if _, err := io.ReadFull(r.src, ciphertext); err != nil { + return fmt.Errorf("read encrypted chunk payload: %w", err) + } + nonce := make([]byte, r.aead.NonceSize()) + copy(nonce, r.noncePrefix[:]) + binary.BigEndian.PutUint64(nonce[len(r.noncePrefix):], r.counter) + plaintext, err := r.aead.Open(nil, nonce, ciphertext, nil) + if err != nil { + return fmt.Errorf("decrypt chunk: %w", err) + } + r.counter++ + r.buf = plaintext + r.offset = 0 + return nil +} + +func equalBytes(a, b []byte) bool { + if len(a) != len(b) { + return false + } + for i := range a { + if a[i] != b[i] { + return false + } + } + return true +} + +func encryptIndexPayload(plaintext []byte, aead cipher.AEAD) ([]byte, error) { + nonce := make([]byte, aead.NonceSize()) + if _, err := rand.Read(nonce); err != nil { + return nil, fmt.Errorf("generate index nonce: %w", err) + } + ciphertext := aead.Seal(nil, nonce, plaintext, nil) + out := make([]byte, 0, len(encryptedIndexMagic)+len(nonce)+len(ciphertext)) + out = append(out, encryptedIndexMagic[:]...) + out = append(out, nonce...) + out = append(out, ciphertext...) + return out, nil +} + +func decryptIndexPayload(raw []byte, aead cipher.AEAD) ([]byte, error) { + headerLen := len(encryptedIndexMagic) + nonceSize := aead.NonceSize() + if len(raw) < headerLen+nonceSize { + return nil, errors.New("encrypted index is too short") + } + if !equalBytes(raw[:headerLen], encryptedIndexMagic[:]) { + return nil, errors.New("invalid encrypted index header") + } + nonceStart := headerLen + nonceEnd := nonceStart + nonceSize + nonce := raw[nonceStart:nonceEnd] + ciphertext := raw[nonceEnd:] + if len(ciphertext) == 0 { + return nil, errors.New("encrypted index ciphertext is empty") + } + plaintext, err := aead.Open(nil, nonce, ciphertext, nil) + if err != nil { + return nil, fmt.Errorf("decrypt index payload: %w", err) + } + return plaintext, nil +} diff --git a/internal/storage/filesystem_test.go b/internal/storage/filesystem_test.go index 7d5e543..2ad6ffc 100644 --- a/internal/storage/filesystem_test.go +++ b/internal/storage/filesystem_test.go @@ -3,6 +3,11 @@ package storage import ( "bytes" "context" + "crypto/aes" + "crypto/cipher" + "crypto/rand" + "crypto/sha256" + "encoding/hex" "encoding/json" "errors" "fmt" @@ -15,10 +20,25 @@ import ( "time" ) +var testEncryptionKey = []byte("0123456789abcdef0123456789abcdef") + +func mustTestAEAD(t *testing.T) cipher.AEAD { + t.Helper() + block, err := aes.NewCipher(testEncryptionKey) + if err != nil { + t.Fatalf("aes.NewCipher() error = %v", err) + } + aead, err := cipher.NewGCM(block) + if err != nil { + t.Fatalf("cipher.NewGCM() error = %v", err) + } + return aead +} + func TestCreateStoresGzipAndOpenIsTransparent(t *testing.T) { t.Parallel() - store, err := NewFilesystemStore(filepath.Join(t.TempDir(), "data")) + store, err := NewFilesystemStore(filepath.Join(t.TempDir(), "data"), testEncryptionKey) if err != nil { t.Fatalf("NewFilesystemStore() error = %v", err) } @@ -36,8 +56,8 @@ func TestCreateStoresGzipAndOpenIsTransparent(t *testing.T) { if err != nil { t.Fatalf("ReadFile() error = %v", err) } - if len(onDisk) < 2 || onDisk[0] != 0x1f || onDisk[1] != 0x8b { - t.Fatalf("stored file does not have gzip header") + if len(onDisk) < 4 || string(onDisk[:4]) != "SBX1" { + t.Fatalf("stored file does not have encrypted scratch header") } reader, _, err := store.Open(meta.ID) @@ -54,10 +74,92 @@ func TestCreateStoresGzipAndOpenIsTransparent(t *testing.T) { } } +func TestCreateEncryptedStoreDoesNotExposeGzipHeaderOnDisk(t *testing.T) { + t.Parallel() + + key := make([]byte, 32) + if _, err := rand.Read(key); err != nil { + t.Fatalf("rand.Read() error = %v", err) + } + + dataDir := filepath.Join(t.TempDir(), "data") + store, err := NewFilesystemStore(dataDir, key) + if err != nil { + t.Fatalf("NewEncryptedFilesystemStore() error = %v", err) + } + + original := bytes.Repeat([]byte("secret payload "), 1024) + meta, err := store.Create(context.Background(), bytes.NewReader(original), "text/plain", time.Hour) + if err != nil { + t.Fatalf("Create() error = %v", err) + } + + onDisk, err := os.ReadFile(meta.FilePath) + if err != nil { + t.Fatalf("ReadFile() error = %v", err) + } + if len(onDisk) < 2 { + t.Fatalf("encrypted file too short") + } + if onDisk[0] == 0x1f && onDisk[1] == 0x8b { + t.Fatalf("encrypted file unexpectedly starts with gzip header") + } + + reader, _, err := store.Open(meta.ID) + if err != nil { + t.Fatalf("Open() error = %v", err) + } + defer reader.Close() + got, err := io.ReadAll(reader) + if err != nil { + t.Fatalf("ReadAll() error = %v", err) + } + if !bytes.Equal(got, original) { + t.Fatalf("Open() content mismatch: got %d bytes, want %d", len(got), len(original)) + } + + filePayload, err := os.ReadFile(meta.FilePath) + if err != nil { + t.Fatalf("ReadFile(encrypted) error = %v", err) + } + sum := sha256.Sum256(filePayload) + wantID := hex.EncodeToString(sum[:]) + if meta.ID != wantID { + t.Fatalf("meta.ID = %q, want SHA-256 of encrypted blob %q", meta.ID, wantID) + } +} + +func TestEncryptedStoreFailsToOpenWithWrongKey(t *testing.T) { + t.Parallel() + + keyA := make([]byte, 32) + if _, err := rand.Read(keyA); err != nil { + t.Fatalf("rand.Read(keyA) error = %v", err) + } + keyB := make([]byte, 32) + if _, err := rand.Read(keyB); err != nil { + t.Fatalf("rand.Read(keyB) error = %v", err) + } + + dataDir := filepath.Join(t.TempDir(), "data") + storeA, err := NewFilesystemStore(dataDir, keyA) + if err != nil { + t.Fatalf("NewEncryptedFilesystemStore(keyA) error = %v", err) + } + _, err = storeA.Create(context.Background(), bytes.NewReader([]byte("hello")), "text/plain", time.Hour) + if err != nil { + t.Fatalf("Create() error = %v", err) + } + + if _, err := NewFilesystemStore(dataDir, keyB); err == nil { + t.Fatalf("NewFilesystemStore() with wrong key error = nil, want non-nil") + } +} + func TestCreateWithOriginalNameNormalizesFilename(t *testing.T) { t.Parallel() - store, err := NewFilesystemStore(filepath.Join(t.TempDir(), "data")) + store, err := NewFilesystemStore(filepath.Join(t.TempDir(), "data"), testEncryptionKey) if err != nil { t.Fatalf("NewFilesystemStore() error = %v", err) } @@ -70,7 +172,7 @@ func TestCreateWithOriginalNameNormalizesFilename(t *testing.T) { t.Fatalf("meta.OriginalName = %q, want %q", meta.OriginalName, "demo.txt") } - reloaded, err := NewFilesystemStore(store.dataDir) + reloaded, err := NewFilesystemStore(store.dataDir, testEncryptionKey) if err != nil { t.Fatalf("NewFilesystemStore() reload error = %v", err) } @@ -83,201 +185,31 @@ func TestCreateWithOriginalNameNormalizesFilename(t *testing.T) { } } -func TestCreateWithOriginalNameRetriesOnIDCollision(t *testing.T) { +func TestCreateWithOriginalNameSameContentGetsDifferentIDs(t *testing.T) { t.Parallel() - store, err := NewFilesystemStore(filepath.Join(t.TempDir(), "data")) + store, err := NewFilesystemStore(filepath.Join(t.TempDir(), "data"), testEncryptionKey) if err != nil { t.Fatalf("NewFilesystemStore() error = %v", err) } - existingID := "aaaaaaaaaaaaaaaaaaaaaaaa" - existingPath := filepath.Join(store.filesDir, existingID) - if err := os.WriteFile(existingPath, []byte{0x1f, 0x8b, 0x08, 0x00}, 0o644); err != nil { - t.Fatalf("WriteFile(existing collision path) error = %v", err) - } - store.mu.Lock() - store.metadata[existingID] = Metadata{ - ID: existingID, - FilePath: existingPath, - CreatedAt: time.Now().UTC(), - ExpiresAt: time.Now().UTC().Add(time.Hour), - ContentType: "text/plain", - Size: 4, - } - store.mu.Unlock() - - nextID := "bbbbbbbbbbbbbbbbbbbbbbbb" - calls := 0 - store.idGenerator = func() (string, error) { - calls++ - if calls == 1 { - return existingID, nil - } - return nextID, nil - } - - meta, err := store.CreateWithOriginalName(context.Background(), bytes.NewReader([]byte("hello")), "text/plain", "hello.txt", time.Hour) + first, err := store.CreateWithOriginalName(context.Background(), bytes.NewReader([]byte("same body")), "text/plain", "one.txt", time.Hour) if err != nil { - t.Fatalf("CreateWithOriginalName() error = %v", err) + t.Fatalf("first CreateWithOriginalName() error = %v", err) } - if meta.ID != nextID { - t.Fatalf("meta.ID = %q, want %q", meta.ID, nextID) - } - if calls != 2 { - t.Fatalf("idGenerator calls = %d, want 2", calls) - } -} - -func TestCreateWithOriginalNameFailsAfterMaxIDRetries(t *testing.T) { - t.Parallel() - - store, err := NewFilesystemStore(filepath.Join(t.TempDir(), "data")) - if err != nil { - t.Fatalf("NewFilesystemStore() error = %v", err) - } - - existingID := "cccccccccccccccccccccccc" - existingPath := filepath.Join(store.filesDir, existingID) - store.mu.Lock() - store.metadata[existingID] = Metadata{ - ID: existingID, - FilePath: existingPath, - CreatedAt: time.Now().UTC(), - ExpiresAt: time.Now().UTC().Add(time.Hour), - ContentType: "text/plain", - } - store.mu.Unlock() - - store.idGenerator = func() (string, error) { - return existingID, nil - } - - _, err = store.CreateWithOriginalName(context.Background(), bytes.NewReader([]byte("hello")), "text/plain", "hello.txt", time.Hour) - if err == nil { - t.Fatalf("CreateWithOriginalName() error = nil, want retry exhaustion error") - } - if !strings.Contains(err.Error(), "failed to allocate unique scratch id") { - t.Fatalf("unexpected error: %v", err) - } -} - -func TestCreateWithOriginalNameRetriesWhenIDIsReservedInFlight(t *testing.T) { - t.Parallel() - - store, err := NewFilesystemStore(filepath.Join(t.TempDir(), "data")) - if err != nil { - t.Fatalf("NewFilesystemStore() error = %v", err) - } - - const collideID = "abababababababababababab" - const uniqueID = "cdcdcdcdcdcdcdcdcdcdcdcd" - - var idMu sync.Mutex - idCalls := 0 - store.idGenerator = func() (string, error) { - idMu.Lock() - defer idMu.Unlock() - idCalls++ - if idCalls <= 2 { - return collideID, nil - } - return uniqueID, nil - } - - release := make(chan struct{}) - firstMetaCh := make(chan Metadata, 1) - firstErrCh := make(chan error, 1) - go func() { - meta, createErr := store.CreateWithOriginalName( - context.Background(), - &blockingReader{release: release, payload: []byte("first upload")}, - "text/plain", - "first.txt", - time.Hour, - ) - firstMetaCh <- meta - firstErrCh <- createErr - }() - - deadline := time.Now().Add(2 * time.Second) - for { - store.mu.RLock() - _, reserved := store.reservedIDs[collideID] - store.mu.RUnlock() - if reserved { - break - } - if time.Now().After(deadline) { - t.Fatalf("timed out waiting for first upload reservation") - } - time.Sleep(2 * time.Millisecond) - } - - secondMeta, err := store.CreateWithOriginalName( - context.Background(), - strings.NewReader("second upload"), - "text/plain", - "second.txt", - time.Hour, - ) + second, err := store.CreateWithOriginalName(context.Background(), bytes.NewReader([]byte("same body")), "text/plain", "two.txt", time.Hour) if err != nil { t.Fatalf("second CreateWithOriginalName() error = %v", err) } - if secondMeta.ID != uniqueID { - t.Fatalf("second meta.ID = %q, want %q after collision retry", secondMeta.ID, uniqueID) - } - if secondMeta.ID == collideID { - t.Fatalf("second create reused in-flight reserved ID") - } - - close(release) - firstMeta := <-firstMetaCh - if err := <-firstErrCh; err != nil { - t.Fatalf("first CreateWithOriginalName() error = %v", err) - } - if firstMeta.ID != collideID { - t.Fatalf("first meta.ID = %q, want %q", firstMeta.ID, collideID) - } - - firstReader, _, err := store.Open(firstMeta.ID) - if err != nil { - t.Fatalf("Open(first) error = %v", err) - } - defer firstReader.Close() - firstBody, err := io.ReadAll(firstReader) - if err != nil { - t.Fatalf("ReadAll(first) error = %v", err) - } - if string(firstBody) != "first upload" { - t.Fatalf("first content = %q, want %q", string(firstBody), "first upload") - } - - secondReader, _, err := store.Open(secondMeta.ID) - if err != nil { - t.Fatalf("Open(second) error = %v", err) - } - defer secondReader.Close() - secondBody, err := io.ReadAll(secondReader) - if err != nil { - t.Fatalf("ReadAll(second) error = %v", err) - } - if string(secondBody) != "second upload" { - t.Fatalf("second content = %q, want %q", string(secondBody), "second upload") - } - - idMu.Lock() - totalCalls := idCalls - idMu.Unlock() - if totalCalls < 3 { - t.Fatalf("idGenerator call count = %d, want at least 3", totalCalls) + if first.ID == second.ID { + t.Fatalf("IDs should differ because encrypted blobs include random nonce") } } func TestGetDeleteAndNotFound(t *testing.T) { t.Parallel() - store, err := NewFilesystemStore(filepath.Join(t.TempDir(), "data")) + store, err := NewFilesystemStore(filepath.Join(t.TempDir(), "data"), testEncryptionKey) if err != nil { t.Fatalf("NewFilesystemStore() error = %v", err) } @@ -302,7 +234,7 @@ func TestGetDeleteAndNotFound(t *testing.T) { func TestDeleteExpired(t *testing.T) { t.Parallel() - store, err := NewFilesystemStore(filepath.Join(t.TempDir(), "data")) + store, err := NewFilesystemStore(filepath.Join(t.TempDir(), "data"), testEncryptionKey) if err != nil { t.Fatalf("NewFilesystemStore() error = %v", err) } @@ -332,7 +264,7 @@ func TestDeleteExpired(t *testing.T) { func TestOpenInvalidGzipFails(t *testing.T) { t.Parallel() - store, err := NewFilesystemStore(filepath.Join(t.TempDir(), "data")) + store, err := NewFilesystemStore(filepath.Join(t.TempDir(), "data"), testEncryptionKey) if err != nil { t.Fatalf("NewFilesystemStore() error = %v", err) } @@ -368,7 +300,7 @@ func TestNewFilesystemStoreInvalidIndex(t *testing.T) { if err := os.WriteFile(filepath.Join(root, indexFilename), []byte("{bad"), 0o644); err != nil { t.Fatalf("WriteFile() error = %v", err) } - if _, err := NewFilesystemStore(root); err == nil { + if _, err := NewFilesystemStore(root, testEncryptionKey); err == nil { t.Fatalf("NewFilesystemStore() error = nil, want invalid index error") } } @@ -419,15 +351,21 @@ func TestNewFilesystemStoreReconcileOnStartup(t *testing.T) { Size: 1, }, } - payload, err := json.Marshal(meta) + payload, err := json.Marshal(indexDocument{ + Scratches: meta, + }) if err != nil { t.Fatalf("json.Marshal() error = %v", err) } - if err := os.WriteFile(filepath.Join(root, indexFilename), payload, 0o644); err != nil { + encryptedPayload, err := encryptIndexPayload(payload, mustTestAEAD(t)) + if err != nil { + t.Fatalf("encryptIndexPayload() error = %v", err) + } + if err := os.WriteFile(filepath.Join(root, indexFilename), encryptedPayload, 0o644); err != nil { t.Fatalf("WriteFile index error = %v", err) } - store, err := NewFilesystemStore(root) + store, err := NewFilesystemStore(root, testEncryptionKey) if err != nil { t.Fatalf("NewFilesystemStore() error = %v", err) } @@ -442,6 +380,63 @@ func TestNewFilesystemStoreReconcileOnStartup(t *testing.T) { } } +func TestNewFilesystemStoreWithDefaultTTLShiftsExpiriesOnTTLChange(t *testing.T) { + t.Parallel() + + dataDir := filepath.Join(t.TempDir(), "data") + store, err := NewFilesystemStoreWithDefaultTTL(dataDir, time.Hour, testEncryptionKey) + if err != nil { + t.Fatalf("NewFilesystemStoreWithDefaultTTL() error = %v", err) + } + + meta, err := store.Create(context.Background(), bytes.NewReader([]byte("hello")), "text/plain", time.Hour) + if err != nil { + t.Fatalf("Create() error = %v", err) + } + originalExpiry := meta.ExpiresAt + + reloaded, err := NewFilesystemStoreWithDefaultTTL(dataDir, 2*time.Hour, testEncryptionKey) + if err != nil { + t.Fatalf("NewFilesystemStoreWithDefaultTTL() reload error = %v", err) + } + got, err := reloaded.Get(meta.ID) + if err != nil { + t.Fatalf("Get() error = %v", err) + } + + wantExpiry := originalExpiry.Add(time.Hour) + if !got.ExpiresAt.Equal(wantExpiry) { + t.Fatalf("ExpiresAt = %s, want %s", got.ExpiresAt, wantExpiry) + } + + again, err := NewFilesystemStoreWithDefaultTTL(dataDir, 2*time.Hour, testEncryptionKey) + if err != nil { + t.Fatalf("NewFilesystemStoreWithDefaultTTL() second reload error = %v", err) + } + gotAgain, err := again.Get(meta.ID) + if err != nil { + t.Fatalf("Get() second reload error = %v", err) + } + if !gotAgain.ExpiresAt.Equal(wantExpiry) { + t.Fatalf("ExpiresAt after unchanged TTL = %s, want %s", gotAgain.ExpiresAt, wantExpiry) + } + + indexRaw, err := os.ReadFile(filepath.Join(dataDir, indexFilename)) + if err != nil { + t.Fatalf("ReadFile(index) error = %v", err) + } + if bytes.Contains(indexRaw, []byte(`"default_ttl"`)) { + t.Fatalf("index should not be plaintext JSON") + } + indexPlaintext, err := decryptIndexPayload(indexRaw, mustTestAEAD(t)) + if err != nil { + t.Fatalf("decryptIndexPayload() error = %v", err) + } + if !bytes.Contains(indexPlaintext, []byte(`"default_ttl": "2h0m0s"`)) { + t.Fatalf("decrypted index missing default ttl marker: %s", string(indexPlaintext)) + } +} + type errCloser struct{} func (errCloser) Close() error { return errors.New("close failed") } @@ -465,7 +460,7 @@ func (failingReader) Read(_ []byte) (int, error) { return 0, errors.New("boom") func TestCreateRollbackOnPersistFailure(t *testing.T) { t.Parallel() - store, err := NewFilesystemStore(filepath.Join(t.TempDir(), "data")) + store, err := NewFilesystemStore(filepath.Join(t.TempDir(), "data"), testEncryptionKey) if err != nil { t.Fatalf("NewFilesystemStore() error = %v", err) } @@ -487,7 +482,7 @@ func TestCreateRollbackOnPersistFailure(t *testing.T) { func TestCreateReaderFailure(t *testing.T) { t.Parallel() - store, err := NewFilesystemStore(filepath.Join(t.TempDir(), "data")) + store, err := NewFilesystemStore(filepath.Join(t.TempDir(), "data"), testEncryptionKey) if err != nil { t.Fatalf("NewFilesystemStore() error = %v", err) } @@ -499,7 +494,7 @@ func TestCreateReaderFailure(t *testing.T) { func TestDeletePersistFailureRestoresEntry(t *testing.T) { t.Parallel() - store, err := NewFilesystemStore(filepath.Join(t.TempDir(), "data")) + store, err := NewFilesystemStore(filepath.Join(t.TempDir(), "data"), testEncryptionKey) if err != nil { t.Fatalf("NewFilesystemStore() error = %v", err) } @@ -525,7 +520,7 @@ func TestDeletePersistFailureRestoresEntry(t *testing.T) { func TestDeleteRemoveFailure(t *testing.T) { t.Parallel() - store, err := NewFilesystemStore(filepath.Join(t.TempDir(), "data")) + store, err := NewFilesystemStore(filepath.Join(t.TempDir(), "data"), testEncryptionKey) if err != nil { t.Fatalf("NewFilesystemStore() error = %v", err) } @@ -557,7 +552,7 @@ func TestDeleteRemoveFailure(t *testing.T) { func TestDeleteExpiredPersistFailureRestoresEntries(t *testing.T) { t.Parallel() - store, err := NewFilesystemStore(filepath.Join(t.TempDir(), "data")) + store, err := NewFilesystemStore(filepath.Join(t.TempDir(), "data"), testEncryptionKey) if err != nil { t.Fatalf("NewFilesystemStore() error = %v", err) } @@ -583,7 +578,7 @@ func TestDeleteExpiredPersistFailureRestoresEntries(t *testing.T) { func TestDeleteExpiredRemoveFailure(t *testing.T) { t.Parallel() - store, err := NewFilesystemStore(filepath.Join(t.TempDir(), "data")) + store, err := NewFilesystemStore(filepath.Join(t.TempDir(), "data"), testEncryptionKey) if err != nil { t.Fatalf("NewFilesystemStore() error = %v", err) } @@ -619,106 +614,45 @@ func TestNewFilesystemStoreFailsWhenDataDirIsFile(t *testing.T) { if err := os.WriteFile(filePath, []byte("x"), 0o644); err != nil { t.Fatalf("WriteFile() error = %v", err) } - if _, err := NewFilesystemStore(filePath); err == nil { + if _, err := NewFilesystemStore(filePath, testEncryptionKey); err == nil { t.Fatalf("NewFilesystemStore() error = nil, want mkdir failure") } } -type blockingReader struct { - release <-chan struct{} - payload []byte - sent bool -} - -func (r *blockingReader) Read(p []byte) (int, error) { - if r.sent { - return 0, io.EOF - } - <-r.release - r.sent = true - return copy(p, r.payload), nil -} - -func TestCreateReservationIsNotVisibleToCleanupOrReads(t *testing.T) { +func TestNewFilesystemStoreRequires32ByteKey(t *testing.T) { t.Parallel() - store, err := NewFilesystemStore(filepath.Join(t.TempDir(), "data")) + dataDir := filepath.Join(t.TempDir(), "data") + if _, err := NewFilesystemStore(dataDir, nil); err == nil { + t.Fatalf("NewFilesystemStore() with nil key error = nil, want non-nil") + } + if _, err := NewFilesystemStore(dataDir, []byte("short")); err == nil { + t.Fatalf("NewFilesystemStore() with short key error = nil, want non-nil") + } +} + +func TestNewFilesystemStoreFailsToLoadEncryptedIndexWithWrongKey(t *testing.T) { + t.Parallel() + + dataDir := filepath.Join(t.TempDir(), "data") + store, err := NewFilesystemStore(dataDir, testEncryptionKey) if err != nil { t.Fatalf("NewFilesystemStore() error = %v", err) } - - const reservedID = "ffffffffffffffffffffffff" - store.idGenerator = func() (string, error) { - return reservedID, nil + if _, err := store.Create(context.Background(), bytes.NewReader([]byte("hello")), "text/plain", time.Hour); err != nil { + t.Fatalf("Create() error = %v", err) } - release := make(chan struct{}) - createErrCh := make(chan error, 1) - go func() { - _, createErr := store.CreateWithOriginalName( - context.Background(), - &blockingReader{release: release, payload: []byte("in-flight")}, - "text/plain", - "", - time.Hour, - ) - createErrCh <- createErr - }() - - deadline := time.Now().Add(2 * time.Second) - for { - store.mu.RLock() - _, reserved := store.reservedIDs[reservedID] - _, visible := store.metadata[reservedID] - store.mu.RUnlock() - if reserved { - if visible { - t.Fatalf("in-flight ID should not appear in metadata") - } - break - } - if time.Now().After(deadline) { - t.Fatalf("timed out waiting for ID reservation") - } - time.Sleep(2 * time.Millisecond) - } - - if _, err := store.Get(reservedID); !errors.Is(err, ErrNotFound) { - t.Fatalf("Get() during in-flight create error = %v, want ErrNotFound", err) - } - - deleted, err := store.DeleteExpired(time.Now().UTC()) - if err != nil { - t.Fatalf("DeleteExpired() error = %v", err) - } - if deleted != 0 { - t.Fatalf("DeleteExpired() = %d, want 0 for in-flight create", deleted) - } - - close(release) - if err := <-createErrCh; err != nil { - t.Fatalf("CreateWithOriginalName() error = %v", err) - } - - reader, _, err := store.Open(reservedID) - if err != nil { - t.Fatalf("Open() error = %v", err) - } - defer reader.Close() - - body, err := io.ReadAll(reader) - if err != nil { - t.Fatalf("ReadAll() error = %v", err) - } - if string(body) != "in-flight" { - t.Fatalf("Open() content = %q, want %q", string(body), "in-flight") + wrongKey := []byte("abcdef0123456789abcdef0123456789") + if _, err := NewFilesystemStore(dataDir, wrongKey); err == nil { + t.Fatalf("NewFilesystemStore() with wrong key error = nil, want non-nil") } } func TestConcurrentCreateWithOriginalNameProducesUniqueReadableEntries(t *testing.T) { t.Parallel() - store, err := NewFilesystemStore(filepath.Join(t.TempDir(), "data")) + store, err := NewFilesystemStore(filepath.Join(t.TempDir(), "data"), testEncryptionKey) if err != nil { t.Fatalf("NewFilesystemStore() error = %v", err) } diff --git a/web/ui/src/App.svelte b/web/ui/src/App.svelte index 656ec44..cfe60cf 100644 --- a/web/ui/src/App.svelte +++ b/web/ui/src/App.svelte @@ -1,13 +1,12 @@