s1d3sw1ped/scratchbox
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 5 Wiki Activity

Many changes

Browse Source
This commit is contained in:
Justin Harms
2026-05-31 20:17:49 -05:00
parent 9e398957a9
commit 99d46ecd2a
28 changed files with 3729 additions and 594 deletions
Download Patch File Download Diff File Expand all files Collapse all files
internal/config/config.go
+50 -29
View File
@@ -14,53 +14,58 @@ import (
)
const (
defaultListenAddr = ":8080"
defaultMaxUploadSize = "100MB"
defaultTTL = "1h"
defaultDataDir = "./data"
defaultCleanupInterval = "1m"
defaultRateLimitEnabled = true
defaultRequestsPerMinute = 30
defaultRateLimitBurst = 10
maxDefaultTTLLimit = 24 * time.Hour
minCleanupInterval = 10 * time.Second
defaultListenAddr = ":8080"
defaultMaxUploadSize = "100MiB"
defaultTTL = "1h"
defaultRawCacheMaxSize = "200MiB"
defaultRawCacheMaxEntries = 32
defaultDataDir = "./data"
defaultCleanupInterval = "1m"
defaultRateLimitEnabled = true
defaultRequestsPerMinute = 30
defaultRateLimitBurst = 10
maxDefaultTTLLimit = 24 * time.Hour
minCleanupInterval = 10 * time.Second
)
type Config struct {
Server ServerConfig `yaml:"server"`
Limits LimitsConfig `yaml:"limits"`
Storage StorageConfig `yaml:"storage"`
Security SecurityConfig `yaml:"security"`
Server ServerConfig `yaml:"server" comment:"HTTP listener settings."`
Limits LimitsConfig `yaml:"limits" comment:"Upload, expiration, and raw cache defaults."`
Storage StorageConfig `yaml:"storage" comment:"Filesystem storage location and cleanup cadence."`
Security SecurityConfig `yaml:"security" comment:"Write-route IP restrictions and rate limiting."`
}
type ServerConfig struct {
ListenAddr string `yaml:"listen_addr"`
ListenAddr string `yaml:"listen_addr" comment:"Address to bind the HTTP server to."`
}
type LimitsConfig struct {
MaxUploadSize string `yaml:"max_upload_size"`
DefaultTTL string `yaml:"default_ttl"`
MaxUploadSize string `yaml:"max_upload_size" comment:"Maximum upload request body size. Supports B, KB, MB, GB and KiB, MiB, GiB (case-insensitive). Defaults to bytes when unit is omitted (example: 1048576)."`
DefaultTTL string `yaml:"default_ttl" comment:"Default expiration for new scratches. Must be > 0 and <= 24h."`
RawCacheMaxSize string `yaml:"raw_cache_max_size" comment:"Maximum total decompressed bytes cached in memory for /api/raw range requests. Supports B, KB, MB, GB and KiB, MiB, GiB. Set this above limits.max_upload_size based on your expected concurrent large downloads."`
RawCacheMaxEntries int `yaml:"raw_cache_max_entries" comment:"Maximum number of decompressed /api/raw entries retained in memory cache. Use this as a secondary cap alongside raw_cache_max_size."`
MaxUploadSizeBytes int64 `yaml:"-"`
DefaultTTLDuration time.Duration `yaml:"-"`
RawCacheMaxBytes int64 `yaml:"-"`
}
type StorageConfig struct {
DataDir string `yaml:"data_dir"`
CleanupInterval string `yaml:"cleanup_interval"`
DataDir string `yaml:"data_dir" comment:"Directory where scratch files and metadata are stored."`
CleanupInterval string `yaml:"cleanup_interval" comment:"How often expired scratches are deleted (minimum 10s)."`
CleanupIntervalParsed time.Duration `yaml:"-"`
}
type SecurityConfig struct {
AllowedIPs []string `yaml:"allowed_ips"`
TrustProxyHeaders bool `yaml:"trust_proxy_headers"`
RateLimit RateLimitConfig `yaml:"rate_limit"`
AllowedIPs []string `yaml:"allowed_ips" comment:"Allowlist for POST /api/scratch as IPs or CIDRs (examples: 127.0.0.1, 192.168.1.0/24). Keep 127.0.0.1 for local access; add trusted ranges as needed. Empty means unrestricted."`
TrustProxyHeaders bool `yaml:"trust_proxy_headers" comment:"Honor X-Forwarded-For / X-Real-IP when true (trusted proxy only)."`
RateLimit RateLimitConfig `yaml:"rate_limit" comment:"Per-client token bucket on POST /api/scratch."`
AllowedPrefixes []netip.Prefix `yaml:"-"`
}
type RateLimitConfig struct {
Enabled bool `yaml:"enabled"`
RequestsPerMinute int `yaml:"requests_per_minute"`
Burst int `yaml:"burst"`
Enabled bool `yaml:"enabled" comment:"Enable per-client write-route rate limiting."`
RequestsPerMinute int `yaml:"requests_per_minute" comment:"Steady refill rate per client IP."`
Burst int `yaml:"burst" comment:"Maximum immediate burst capacity per client IP."`
}
func Load(path string) (Config, error) {
@@ -81,20 +86,29 @@ func Load(path string) (Config, error) {
return cfg, nil
}
func Default() Config {
return defaults()
}
func defaults() Config {
return Config{
Server: ServerConfig{
ListenAddr: defaultListenAddr,
},
Limits: LimitsConfig{
MaxUploadSize: defaultMaxUploadSize,
DefaultTTL: defaultTTL,
MaxUploadSize: defaultMaxUploadSize,
DefaultTTL: defaultTTL,
RawCacheMaxSize: defaultRawCacheMaxSize,
RawCacheMaxEntries: defaultRawCacheMaxEntries,
},
Storage: StorageConfig{
DataDir: defaultDataDir,
CleanupInterval: defaultCleanupInterval,
},
Security: SecurityConfig{
AllowedIPs: []string{
"127.0.0.1",
},
RateLimit: RateLimitConfig{
Enabled: defaultRateLimitEnabled,
RequestsPerMinute: defaultRequestsPerMinute,
@@ -124,6 +138,15 @@ func (c *Config) Validate() error {
}
c.Limits.DefaultTTLDuration = ttl
rawCacheBytes, err := parseHumanSize(c.Limits.RawCacheMaxSize)
if err != nil {
return fmt.Errorf("limits.raw_cache_max_size invalid: %w", err)
}
c.Limits.RawCacheMaxBytes = rawCacheBytes
if c.Limits.RawCacheMaxEntries <= 0 {
return errors.New("limits.raw_cache_max_entries must be > 0")
}
cleanupInterval, err := time.ParseDuration(strings.TrimSpace(c.Storage.CleanupInterval))
if err != nil {
return fmt.Errorf("storage.cleanup_interval invalid: %w", err)
@@ -231,11 +254,9 @@ func parseHumanSize(raw string) (int64, error) {
"KB": 1_000,
"MB": 1_000_000,
"GB": 1_000_000_000,
"TB": 1_000_000_000_000,
"KIB": 1024,
"MIB": 1024 * 1024,
"GIB": 1024 * 1024 * 1024,
"TIB": 1024 * 1024 * 1024 * 1024,
}
multiplier, ok := multipliers[unitPart]
internal/config/config_test.go
+37 -4
View File
@@ -19,17 +19,26 @@ func TestValidateAppliesDefaultsAndParsedFields(t *testing.T) {
t.Fatalf("Validate() returned error: %v", err)
}
if cfg.Limits.MaxUploadSizeBytes != 100_000_000 {
t.Fatalf("MaxUploadSizeBytes = %d, want %d", cfg.Limits.MaxUploadSizeBytes, int64(100_000_000))
if cfg.Limits.MaxUploadSizeBytes != 100*1024*1024 {
t.Fatalf("MaxUploadSizeBytes = %d, want %d", cfg.Limits.MaxUploadSizeBytes, int64(100*1024*1024))
}
if cfg.Limits.DefaultTTLDuration != time.Hour {
t.Fatalf("DefaultTTLDuration = %s, want 1h", cfg.Limits.DefaultTTLDuration)
}
if cfg.Limits.RawCacheMaxBytes != 200*1024*1024 {
t.Fatalf("RawCacheMaxBytes = %d, want %d", cfg.Limits.RawCacheMaxBytes, int64(200*1024*1024))
}
if cfg.Limits.RawCacheMaxEntries != 32 {
t.Fatalf("RawCacheMaxEntries = %d, want %d", cfg.Limits.RawCacheMaxEntries, 32)
}
if cfg.Storage.CleanupIntervalParsed != time.Minute {
t.Fatalf("CleanupIntervalParsed = %s, want 1m", cfg.Storage.CleanupIntervalParsed)
}
if len(cfg.Security.AllowedPrefixes) != 0 {
t.Fatalf("AllowedPrefixes len = %d, want 0", len(cfg.Security.AllowedPrefixes))
if len(cfg.Security.AllowedPrefixes) != 1 {
t.Fatalf("AllowedPrefixes len = %d, want 1", len(cfg.Security.AllowedPrefixes))
}
if got := cfg.Security.AllowedPrefixes[0]; got != netip.MustParsePrefix("127.0.0.1/32") {
t.Fatalf("AllowedPrefixes[0] = %v, want 127.0.0.1/32", got)
}
}
@@ -45,6 +54,8 @@ server:
limits:
max_upload_size: "2MiB"
default_ttl: "2h"
raw_cache_max_size: "16MiB"
raw_cache_max_entries: 5
storage:
data_dir: "` + dataDir + `"
cleanup_interval: "45s"
@@ -76,6 +87,12 @@ security:
if cfg.Limits.DefaultTTLDuration != 2*time.Hour {
t.Fatalf("DefaultTTLDuration = %s, want 2h", cfg.Limits.DefaultTTLDuration)
}
if cfg.Limits.RawCacheMaxBytes != 16*1024*1024 {
t.Fatalf("RawCacheMaxBytes = %d, want %d", cfg.Limits.RawCacheMaxBytes, int64(16*1024*1024))
}
if cfg.Limits.RawCacheMaxEntries != 5 {
t.Fatalf("RawCacheMaxEntries = %d, want %d", cfg.Limits.RawCacheMaxEntries, 5)
}
if cfg.Storage.CleanupIntervalParsed != 45*time.Second {
t.Fatalf("CleanupIntervalParsed = %s, want 45s", cfg.Storage.CleanupIntervalParsed)
}
@@ -119,6 +136,20 @@ func TestValidateRejectsInvalidValues(t *testing.T) {
},
wantErr: "limits.default_ttl",
},
{
name: "invalid raw cache size",
mutate: func(cfg *Config) {
cfg.Limits.RawCacheMaxSize = "oops"
},
wantErr: "limits.raw_cache_max_size",
},
{
name: "invalid raw cache max entries",
mutate: func(cfg *Config) {
cfg.Limits.RawCacheMaxEntries = 0
},
wantErr: "limits.raw_cache_max_entries",
},
{
name: "ttl above max",
mutate: func(cfg *Config) {
@@ -236,6 +267,8 @@ func TestParseHumanSizeVariants(t *testing.T) {
{in: "10", want: 10},
{in: "1.5KB", want: 1500},
{in: "2MiB", want: 2 * 1024 * 1024},
{in: "1TB", wantErr: "unsupported unit"},
{in: "1TiB", wantErr: "unsupported unit"},
{in: "0", wantErr: "size must be > 0"},
{in: "abc", wantErr: "does not start with a number"},
{in: "5XB", wantErr: "unsupported unit"},