Heavy security and file splitting

2026-06-01 20:15:28 -05:00
parent bdbe1a9416
commit ad4355df17
27 changed files with 1540 additions and 1166 deletions
@@ -22,6 +22,7 @@ const (
 	defaultReadTimeout               = "30s"
 	defaultWriteTimeout              = "30s"
 	defaultIdleTimeout               = "120s"
+	defaultShutdownTimeout           = "10s"
 	defaultMaxHeaderBytes            = 1 << 20
 	defaultAccessLogEnabled          = true
 	defaultAccessLogMaxSize          = "100MiB"
@@ -42,6 +43,7 @@ const (
 	defaultAPIReadRateLimitBurst     = 100
 	defaultAPIWriteRequestsPerMinute = 30
 	defaultAPIWriteRateLimitBurst    = 10
+	defaultHSTSEnabled               = true
 	maxDefaultTTLLimit               = 24 * time.Hour
 	minCleanupInterval               = 10 * time.Second
 	envPrefix                        = "SCRATCHBOX_"
@@ -51,7 +53,7 @@ type Config struct {
 	Server   ServerConfig   `yaml:"server" comment:"HTTP listener settings."`
 	Limits   LimitsConfig   `yaml:"limits" comment:"Upload, expiration, and raw cache defaults."`
 	Storage  StorageConfig  `yaml:"storage" comment:"Filesystem storage location and cleanup cadence."`
-	Security SecurityConfig `yaml:"security" comment:"Write-route IP restrictions and rate limiting."`
+	Security SecurityConfig `yaml:"security" comment:"Write-route IP restrictions, rate limiting, and HSTS."`
 }

 type ServerConfig struct {
@@ -60,12 +62,14 @@ type ServerConfig struct {
 	ReadTimeout          string          `yaml:"read_timeout" comment:"Maximum duration for reading the full request, including body. Must be > 0."`
 	WriteTimeout         string          `yaml:"write_timeout" comment:"Maximum duration before timing out writes of a response. Must be > 0."`
 	IdleTimeout          string          `yaml:"idle_timeout" comment:"Maximum amount of time to wait for the next request when keep-alives are enabled. Must be > 0."`
+	ShutdownTimeout      string          `yaml:"shutdown_timeout" comment:"Graceful shutdown timeout for the HTTP server. Must be > 0."`
 	MaxHeaderBytes       int             `yaml:"max_header_bytes" comment:"Maximum size of request headers in bytes. Must be > 0."`
 	AccessLog            AccessLogConfig `yaml:"access_log" comment:"Access log settings for all HTTP requests."`
 	ReadHeaderTimeoutDur time.Duration   `yaml:"-"`
 	ReadTimeoutDur       time.Duration   `yaml:"-"`
 	WriteTimeoutDur      time.Duration   `yaml:"-"`
 	IdleTimeoutDur       time.Duration   `yaml:"-"`
+	ShutdownTimeoutDur   time.Duration   `yaml:"-"`
 }

 type AccessLogConfig struct {
@@ -96,10 +100,11 @@ type StorageConfig struct {
 type SecurityConfig struct {
 	AllowedIPs           []string        `yaml:"allowed_ips" comment:"Allowlist for POST /api/scratch as IPs or CIDRs (examples: 127.0.0.1, 192.168.1.0/24). Keep 127.0.0.1 for local access; add trusted ranges as needed. Empty means unrestricted."`
 	TrustProxyHeaders    bool            `yaml:"trust_proxy_headers" comment:"Honor X-Forwarded-For / X-Real-IP only when request source IP matches security.trusted_proxy_ips."`
-	TrustedProxyIPs      []string        `yaml:"trusted_proxy_ips" comment:"IPs/CIDRs of reverse proxies allowed to supply forwarded client IP headers. Required when trust_proxy_headers is true."`
+	TrustedProxyIPs      []string        `yaml:"trusted_proxy_ips" comment:"IPs/CIDRs of reverse proxies allowed to supply forwarded client IP headers. Required when trust_proxy_headers is true. Must be the *immediate* proxies only (never 0/0 or untrusted ranges)."`
 	RateLimitUI          RateLimitConfig `yaml:"rate_limit_ui" comment:"Per-client token bucket on UI routes (/, /u, /s/{id})."`
 	RateLimitAPIRead     RateLimitConfig `yaml:"rate_limit_api_read" comment:"Per-client token bucket on API read routes (GET /api/config, GET /api/scratch/{id}, GET /api/raw/{id})."`
 	RateLimitAPIWrite    RateLimitConfig `yaml:"rate_limit_api_write" comment:"Per-client token bucket on API write route (POST /api/scratch)."`
+	HSTSEnabled          bool            `yaml:"hsts_enabled" comment:"Enable Strict-Transport-Security (HSTS) header. Recommended when behind a TLS-terminating reverse proxy for public deployments (default: true). Set to false for local-only HTTP use without proxies."`
 	AllowedPrefixes      []netip.Prefix  `yaml:"-"`
 	TrustedProxyPrefixes []netip.Prefix  `yaml:"-"`
 }
@@ -150,6 +155,7 @@ func defaults() Config {
 			ReadTimeout:       defaultReadTimeout,
 			WriteTimeout:      defaultWriteTimeout,
 			IdleTimeout:       defaultIdleTimeout,
+			ShutdownTimeout:   defaultShutdownTimeout,
 			MaxHeaderBytes:    defaultMaxHeaderBytes,
 			AccessLog: AccessLogConfig{
 				Enabled:    defaultAccessLogEnabled,
@@ -186,6 +192,7 @@ func defaults() Config {
 				RequestsPerMinute: defaultAPIWriteRequestsPerMinute,
 				Burst:             defaultAPIWriteRateLimitBurst,
 			},
+			HSTSEnabled: defaultHSTSEnabled,
 		},
 	}
 }
@@ -230,6 +237,15 @@ func (c *Config) Validate() error {
 	}
 	c.Server.IdleTimeoutDur = idleTimeout

+	shutdownTimeout, err := time.ParseDuration(strings.TrimSpace(c.Server.ShutdownTimeout))
+	if err != nil {
+		return fmt.Errorf("server.shutdown_timeout invalid: %w", err)
+	}
+	if shutdownTimeout <= 0 {
+		return errors.New("server.shutdown_timeout must be > 0")
+	}
+	c.Server.ShutdownTimeoutDur = shutdownTimeout
+
 	if c.Server.MaxHeaderBytes <= 0 {
 		return errors.New("server.max_header_bytes must be > 0")
 	}
@@ -278,18 +294,10 @@ func (c *Config) Validate() error {
 	if strings.TrimSpace(c.Storage.DataDir) == "" {
 		return errors.New("storage.data_dir is required")
 	}
-	if err := ensureWritableDir(c.Storage.DataDir); err != nil {
-		return fmt.Errorf("storage.data_dir not writable: %w", err)
-	}
-	encKey, err := ensureEncryptionKeyInDataDir(c.Storage.DataDir)
-	if err != nil {
-		return fmt.Errorf("storage encryption key setup failed: %w", err)
-	}
-	if len(encKey) != 32 {
-		return fmt.Errorf("storage encryption key must decode to 32 bytes, got %d", len(encKey))
-	}
-	c.Storage.EncryptionKeyBytes = encKey
-
+	// NOTE: directory creation + encryption key provisioning (with auto-gen on empty dir)
+	// moved to PrepareDataDir() to keep Validate() free of filesystem side effects.
+	// Callers (server startup, tests) must invoke PrepareDataDir after successful Validate/Load
+	// if they need EncryptionKeyBytes populated and data dir ensured.
 	prefixes, err := parseAllowedPrefixes(c.Security.AllowedIPs)
 	if err != nil {
 		return fmt.Errorf("security.allowed_ips invalid: %w", err)
@@ -326,6 +334,31 @@ func (c *Config) Validate() error {
 	return nil
 }

+// PrepareDataDir performs the filesystem side effects that were previously inside Validate:
+// ensures the data dir is creatable/writable (via probe), and ensures the encryption key
+// file exists at data_dir/metadata.key (auto-generating a fresh one only for a truly empty dir).
+// It populates Storage.EncryptionKeyBytes. This keeps Validate pure (checks + duration/size
+// parsing only). Must be called after Load/Validate (or on a manually built Config) before
+// constructing a FilesystemStore that needs the key. Idempotent for subsequent calls if key
+// already present.
+func (c *Config) PrepareDataDir() error {
+	if strings.TrimSpace(c.Storage.DataDir) == "" {
+		return errors.New("storage.data_dir is required")
+	}
+	if err := ensureWritableDir(c.Storage.DataDir); err != nil {
+		return fmt.Errorf("storage.data_dir not writable: %w", err)
+	}
+	encKey, err := ensureEncryptionKeyInDataDir(c.Storage.DataDir)
+	if err != nil {
+		return fmt.Errorf("storage encryption key setup failed: %w", err)
+	}
+	if len(encKey) != 32 {
+		return fmt.Errorf("storage encryption key must decode to 32 bytes, got %d", len(encKey))
+	}
+	c.Storage.EncryptionKeyBytes = encKey
+	return nil
+}
+
 func ensureWritableDir(dir string) error {
 	if err := os.MkdirAll(dir, 0o755); err != nil {
 		return err
@@ -413,6 +446,11 @@ func parseHumanSize(raw string) (int64, error) {

 	size := int64(number * multiplier)
 	if size <= 0 {
+		if number > 0 {
+			// overflow or truncation from huge float (e.g. 1e20 * GiB); guard before
+			// passing surprising/negative limit downstream.
+			return 0, errors.New("size too large")
+		}
 		return 0, errors.New("calculated size must be > 0")
 	}
 	return size, nil
@@ -508,6 +546,9 @@ func (c *Config) applyEnvOverrides(prefix string) error {
 	if err := applyStringEnv(prefix+"SERVER_IDLE_TIMEOUT", &c.Server.IdleTimeout); err != nil {
 		return err
 	}
+	if err := applyStringEnv(prefix+"SERVER_SHUTDOWN_TIMEOUT", &c.Server.ShutdownTimeout); err != nil {
+		return err
+	}
 	if err := applyIntEnv(prefix+"SERVER_MAX_HEADER_BYTES", &c.Server.MaxHeaderBytes); err != nil {
 		return err
 	}
@@ -580,6 +621,10 @@ func (c *Config) applyEnvOverrides(prefix string) error {
 		return err
 	}

+	if err := applyBoolEnv(prefix+"SECURITY_HSTS_ENABLED", &c.Security.HSTSEnabled); err != nil {
+		return err
+	}
+
 	return nil
 }

@@ -19,6 +19,9 @@ func TestValidateAppliesDefaultsAndParsedFields(t *testing.T) {
 	if err := cfg.Validate(); err != nil {
 		t.Fatalf("Validate() returned error: %v", err)
 	}
+	if err := cfg.PrepareDataDir(); err != nil {
+		t.Fatalf("PrepareDataDir() returned error: %v", err)
+	}

 	if cfg.Limits.MaxUploadSizeBytes != 100*1024*1024 {
 		t.Fatalf("MaxUploadSizeBytes = %d, want %d", cfg.Limits.MaxUploadSizeBytes, int64(100*1024*1024))
@@ -35,6 +38,9 @@ func TestValidateAppliesDefaultsAndParsedFields(t *testing.T) {
 	if cfg.Server.IdleTimeoutDur != 120*time.Second {
 		t.Fatalf("IdleTimeoutDur = %s, want 120s", cfg.Server.IdleTimeoutDur)
 	}
+	if cfg.Server.ShutdownTimeoutDur != 10*time.Second {
+		t.Fatalf("ShutdownTimeoutDur = %s, want 10s", cfg.Server.ShutdownTimeoutDur)
+	}
 	if cfg.Server.MaxHeaderBytes != 1<<20 {
 		t.Fatalf("MaxHeaderBytes = %d, want %d", cfg.Server.MaxHeaderBytes, 1<<20)
 	}
@@ -68,6 +74,9 @@ func TestValidateAppliesDefaultsAndParsedFields(t *testing.T) {
 	if got := cfg.Security.AllowedPrefixes[0]; got != netip.MustParsePrefix("127.0.0.1/32") {
 		t.Fatalf("AllowedPrefixes[0] = %v, want 127.0.0.1/32", got)
 	}
+	if !cfg.Security.HSTSEnabled {
+		t.Fatalf("HSTSEnabled = false, want true (default)")
+	}
 }

 func TestLoadParsesConfiguredFields(t *testing.T) {
@@ -116,6 +125,7 @@ security:
    enabled: true
    requests_per_minute: 12
    burst: 4
+  hsts_enabled: false
 `
 	if err := os.WriteFile(configPath, []byte(content), 0o644); err != nil {
 		t.Fatalf("WriteFile() error = %v", err)
@@ -125,6 +135,9 @@ security:
 	if err != nil {
 		t.Fatalf("Load() error = %v", err)
 	}
+	if err := cfg.PrepareDataDir(); err != nil {
+		t.Fatalf("PrepareDataDir() error = %v", err)
+	}

 	if cfg.Server.ListenAddr != "127.0.0.1:9090" {
 		t.Fatalf("ListenAddr = %q, want %q", cfg.Server.ListenAddr, "127.0.0.1:9090")
@@ -141,6 +154,9 @@ security:
 	if cfg.Server.IdleTimeoutDur != 90*time.Second {
 		t.Fatalf("IdleTimeoutDur = %s, want 90s", cfg.Server.IdleTimeoutDur)
 	}
+	if cfg.Server.ShutdownTimeoutDur != 10*time.Second {
+		t.Fatalf("ShutdownTimeoutDur = %s, want 10s", cfg.Server.ShutdownTimeoutDur)
+	}
 	if cfg.Server.MaxHeaderBytes != 2*1024*1024 {
 		t.Fatalf("MaxHeaderBytes = %d, want %d", cfg.Server.MaxHeaderBytes, 2*1024*1024)
 	}
@@ -186,6 +202,9 @@ security:
 	if cfg.Security.RateLimitAPIWrite.RequestsPerMinute != 12 || cfg.Security.RateLimitAPIWrite.Burst != 4 {
 		t.Fatalf("unexpected rate_limit_api_write values: %+v", cfg.Security.RateLimitAPIWrite)
 	}
+	if cfg.Security.HSTSEnabled {
+		t.Fatalf("HSTSEnabled = true, want false (from config)")
+	}
 	if len(cfg.Security.AllowedPrefixes) != 2 {
 		t.Fatalf("AllowedPrefixes len = %d, want 2", len(cfg.Security.AllowedPrefixes))
 	}
@@ -244,6 +263,13 @@ func TestValidateRejectsInvalidValues(t *testing.T) {
 			},
 			wantErr: "server.idle_timeout",
 		},
+		{
+			name: "invalid shutdown timeout",
+			mutate: func(cfg *Config) {
+				cfg.Server.ShutdownTimeout = "bogus"
+			},
+			wantErr: "server.shutdown_timeout",
+		},
 		{
 			name: "invalid max header bytes",
 			mutate: func(cfg *Config) {
@@ -432,13 +458,23 @@ storage:
 		t.Fatalf("Load() error = %v, want nil", err)
 	}

+	// First prepare creates the data dir (mkdir probe) so we can write the bad key file for test.
+	// (Load no longer has side effects.)
+	cfgForBad, err := Load(validConfigPath)
+	if err != nil {
+		t.Fatalf("Load(valid) error = %v", err)
+	}
+	if err := cfgForBad.PrepareDataDir(); err != nil {
+		t.Fatalf("PrepareDataDir (to create dir) error = %v", err)
+	}
+
 	keyPath := filepath.Join(tmp, "store", dataDirEncryptionKeyFilename)
 	if err := os.WriteFile(keyPath, []byte("not-base64"), 0o600); err != nil {
 		t.Fatalf("WriteFile(invalid key) error = %v", err)
 	}
-	_, err = Load(validConfigPath)
-	if err == nil || !strings.Contains(err.Error(), "storage encryption key setup failed") {
-		t.Fatalf("Load() error = %v, want invalid key base64 error", err)
+	// Re-prepare (or load+prepare) now hits the bad key decode inside ensure.
+	if err := cfgForBad.PrepareDataDir(); err == nil || !strings.Contains(err.Error(), "storage encryption key setup failed") {
+		t.Fatalf("PrepareDataDir after bad key error = %v, want setup failed containing 'storage encryption key setup failed'", err)
 	}
 }

@@ -465,9 +501,13 @@ storage:
 		t.Fatalf("WriteFile(config) error = %v", err)
 	}

-	_, err := Load(configPath)
+	cfg, err := Load(configPath)
+	if err != nil {
+		t.Fatalf("Load() error = %v, want success (pure validate)", err)
+	}
+	err = cfg.PrepareDataDir()
 	if err == nil || !strings.Contains(err.Error(), "refusing to generate missing") {
-		t.Fatalf("Load() error = %v, want missing-key safety error", err)
+		t.Fatalf("PrepareDataDir() error = %v, want missing-key safety error containing 'refusing to generate missing'", err)
 	}
 }

@@ -479,6 +519,7 @@ func TestLoadFromEnvParsesConfiguredFields(t *testing.T) {
 	t.Setenv("SCRATCHBOX_SERVER_READ_TIMEOUT", "20s")
 	t.Setenv("SCRATCHBOX_SERVER_WRITE_TIMEOUT", "40s")
 	t.Setenv("SCRATCHBOX_SERVER_IDLE_TIMEOUT", "80s")
+	t.Setenv("SCRATCHBOX_SERVER_SHUTDOWN_TIMEOUT", "5s")
 	t.Setenv("SCRATCHBOX_SERVER_MAX_HEADER_BYTES", "2097152")
 	t.Setenv("SCRATCHBOX_SERVER_ACCESS_LOG_ENABLED", "false")
 	t.Setenv("SCRATCHBOX_SERVER_ACCESS_LOG_FILE_PATH", "/tmp/access.log")
@@ -505,11 +546,15 @@ func TestLoadFromEnvParsesConfiguredFields(t *testing.T) {
 	t.Setenv("SCRATCHBOX_SECURITY_RATE_LIMIT_API_WRITE_ENABLED", "true")
 	t.Setenv("SCRATCHBOX_SECURITY_RATE_LIMIT_API_WRITE_REQUESTS_PER_MINUTE", "40")
 	t.Setenv("SCRATCHBOX_SECURITY_RATE_LIMIT_API_WRITE_BURST", "12")
+	t.Setenv("SCRATCHBOX_SECURITY_HSTS_ENABLED", "false")

 	cfg, err := LoadFromEnv()
 	if err != nil {
 		t.Fatalf("LoadFromEnv() error = %v", err)
 	}
+	if err := cfg.PrepareDataDir(); err != nil {
+		t.Fatalf("PrepareDataDir() error = %v", err)
+	}

 	if cfg.Server.ListenAddr != "127.0.0.1:19090" {
 		t.Fatalf("ListenAddr = %q, want %q", cfg.Server.ListenAddr, "127.0.0.1:19090")
@@ -526,6 +571,9 @@ func TestLoadFromEnvParsesConfiguredFields(t *testing.T) {
 	if cfg.Server.IdleTimeoutDur != 80*time.Second {
 		t.Fatalf("IdleTimeoutDur = %s, want 80s", cfg.Server.IdleTimeoutDur)
 	}
+	if cfg.Server.ShutdownTimeoutDur != 5*time.Second {
+		t.Fatalf("ShutdownTimeoutDur = %s, want 5s", cfg.Server.ShutdownTimeoutDur)
+	}
 	if cfg.Server.MaxHeaderBytes != 2*1024*1024 {
 		t.Fatalf("MaxHeaderBytes = %d, want %d", cfg.Server.MaxHeaderBytes, 2*1024*1024)
 	}
@@ -571,6 +619,9 @@ func TestLoadFromEnvParsesConfiguredFields(t *testing.T) {
 	if cfg.Security.RateLimitAPIWrite.RequestsPerMinute != 40 || cfg.Security.RateLimitAPIWrite.Burst != 12 {
 		t.Fatalf("unexpected rate_limit_api_write values: %+v", cfg.Security.RateLimitAPIWrite)
 	}
+	if cfg.Security.HSTSEnabled {
+		t.Fatalf("HSTSEnabled from env = true, want false")
+	}
 }

 func TestLoadFromEnvRejectsInvalidBool(t *testing.T) {
@@ -616,6 +667,7 @@ func TestParseHumanSizeVariants(t *testing.T) {
 		{in: "abc", wantErr: "does not start with a number"},
 		{in: "5XB", wantErr: "unsupported unit"},
 		{in: "", wantErr: "size is empty"},
+		{in: "100000000000000000GiB", wantErr: "size too large"},
 	}

 	for _, tc := range tests {