Add example environment configuration and Docker Compose setup
CI / Go Tests (push) Failing after 9s
CI / Build (push) Successful in 9s
Format / gofmt (push) Successful in 7s
Release Artifacts / Validate release tag (push) Successful in 1s
Release Artifacts / Build and release executables (push) Failing after 8s
Release Artifacts / Build and release Docker image (push) Has been skipped

- Introduced a new `.env.example` file to provide a template for environment variables used by the Scratchbox server.
- Added a `docker-compose.example.yml` file to facilitate easy deployment of the Scratchbox service with Docker.
- Updated `docker-entrypoint.sh` to enforce environment variable settings for server configuration.
- Removed the generation of a config key file from the workflow, simplifying the configuration process.
- Adjusted the README.md to reflect changes in configuration management and usage instructions.
This commit is contained in:
2026-06-01 04:36:53 -05:00
parent f74e028227
commit bc9077660b
10 changed files with 672 additions and 112 deletions
+39
View File
@@ -0,0 +1,39 @@
# Scratchbox env-mode example for `scratchbox server --env`.
# Set values here and pass this file with `--env-file .env`.
#
# IMPORTANT:
# - Scratchbox stores its key at $SCRATCHBOX_STORAGE_DATA_DIR/metadata.key.
# - Keep that data directory persistent across restarts.
# server
SCRATCHBOX_SERVER_READ_HEADER_TIMEOUT=5s
SCRATCHBOX_SERVER_READ_TIMEOUT=30s
SCRATCHBOX_SERVER_WRITE_TIMEOUT=30s
SCRATCHBOX_SERVER_IDLE_TIMEOUT=120s
SCRATCHBOX_SERVER_MAX_HEADER_BYTES=1048576
SCRATCHBOX_SERVER_ACCESS_LOG_ENABLED=true
# limits
SCRATCHBOX_LIMITS_MAX_UPLOAD_SIZE=100MiB
SCRATCHBOX_LIMITS_DEFAULT_TTL=15m
SCRATCHBOX_LIMITS_RAW_CACHE_MAX_SIZE=200MiB
SCRATCHBOX_LIMITS_RAW_CACHE_MAX_ENTRIES=32
# storage
SCRATCHBOX_STORAGE_CLEANUP_INTERVAL=1m
# security
# Comma-separated IP/CIDR list. Empty = unrestricted.
SCRATCHBOX_SECURITY_ALLOWED_IPS=127.0.0.1 # This should be set to your networks CIDR like 192.168.2.0/24 or a particulat IP like 192.168.2.55.
SCRATCHBOX_SECURITY_TRUST_PROXY_HEADERS=false
# Comma-separated IP/CIDR list.
SCRATCHBOX_SECURITY_TRUSTED_PROXY_IPS= # Same as SCRATCHBOX_SECURITY_ALLOWED_IPS just Trusted Proxies you expect clients to come from instead.
SCRATCHBOX_SECURITY_RATE_LIMIT_UI_ENABLED=false
SCRATCHBOX_SECURITY_RATE_LIMIT_UI_REQUESTS_PER_MINUTE=360
SCRATCHBOX_SECURITY_RATE_LIMIT_UI_BURST=120
SCRATCHBOX_SECURITY_RATE_LIMIT_API_READ_ENABLED=false
SCRATCHBOX_SECURITY_RATE_LIMIT_API_READ_REQUESTS_PER_MINUTE=300
SCRATCHBOX_SECURITY_RATE_LIMIT_API_READ_BURST=100
SCRATCHBOX_SECURITY_RATE_LIMIT_API_WRITE_ENABLED=true
SCRATCHBOX_SECURITY_RATE_LIMIT_API_WRITE_REQUESTS_PER_MINUTE=30
SCRATCHBOX_SECURITY_RATE_LIMIT_API_WRITE_BURST=10
-1
View File
@@ -3,7 +3,6 @@
# Run Artifacts
/*.yaml
/config.key
/data/
# Coverage outputs
+3 -9
View File
@@ -5,13 +5,12 @@ BIN := $(BIN_DIR)/$(APP_NAME)
CONFIG ?= config.yaml
DOCKER_IMAGE ?= scratchbox:dev
.PHONY: help run dev config build docker test test-race fmt clean
.PHONY: help run dev build docker test test-race fmt clean
help:
@echo "Targets:"
@echo " make run - Alias for make dev"
@echo " make dev - Build and run server (CONFIG=$(CONFIG))"
@echo " make config - Write default config to $(CONFIG) and generate sibling config.key if missing"
@echo " make build - Build Svelte UI and binary to $(BIN)"
@echo " make docker - Build Docker image (DOCKER_IMAGE=$(DOCKER_IMAGE))"
@echo " make test - Run all Go tests with timeout and shuffle"
@@ -24,13 +23,6 @@ run: dev
dev: build
$(BIN) server --config $(CONFIG)
config:
mkdir -p "$(dir $(CONFIG))"
go run $(CMD_DIR) generate-config > "$(CONFIG)"
@if [ ! -f "$(dir $(CONFIG))config.key" ]; then \
go run $(CMD_DIR) generate-key > "$(dir $(CONFIG))config.key"; \
fi
build: clean
npm --prefix ./web/ui run build
mkdir -p $(BIN_DIR)
@@ -49,4 +41,6 @@ fmt:
go fmt ./...
clean:
rm -rf web/static
rm -rf $(CONFIG)
rm -rf $(BIN_DIR)
+105 -20
View File
@@ -9,13 +9,10 @@ Scratch content is gzip-compressed and AES-encrypted on disk transparently; API/
1. Generate defaults directly:
- `go run ./cmd/scratchbox generate-config > config.yaml`
2. Generate an encryption key file next to the config:
- `go run ./cmd/scratchbox generate-key > config.key`
- or: `openssl rand -base64 32 > config.key`
3. Adjust settings for your environment.
4. Start the server:
2. Adjust settings for your environment.
3. Start the server:
- `go run ./cmd/scratchbox server --config config.yaml`
5. Open:
4. Open:
- `http://127.0.0.1:8080/`
Without `--config`, built-in defaults are used.
@@ -34,19 +31,109 @@ Optionally set a custom image name/tag:
- `make docker DOCKER_IMAGE=scratchbox:latest`
Run with persistent data and config directories:
Run with persistent data and env-based config:
- `mkdir -p ./docker-data ./docker-config`
- `docker run --rm -p 8080:8080 -v "$(pwd)/docker-data:/data" -v "$(pwd)/docker-config:/config" scratchbox:dev`
- `mkdir -p ./docker-data`
- `cp .env.example .env`
- `docker run --rm -p 8080:8080 --env-file .env -v "$(pwd)/docker-data:/data" scratchbox:dev`
- Without `.env`, the image still starts with default config and uses `SCRATCHBOX_STORAGE_DATA_DIR=/data`.
- The image always forces `SCRATCHBOX_STORAGE_DATA_DIR=/data`.
- The image always forces `SCRATCHBOX_SERVER_LISTEN_ADDR=:8080`.
- The image always forces `SCRATCHBOX_SERVER_ACCESS_LOG_FILE_PATH` to empty (stdout-only access logs).
Use the included Compose example:
- `cp docker-compose.example.yml docker-compose.yml`
- `cp .env.example .env`
- `mkdir -p ./docker-data`
- `docker compose up -d`
Inside the container, the default command is:
- `scratchbox server --config /config/config.yaml`
- `scratchbox server --env`
- The image forces `SCRATCHBOX_STORAGE_DATA_DIR=/data`.
- The image forces `SCRATCHBOX_SERVER_LISTEN_ADDR=:8080`.
- The image forces `SCRATCHBOX_SERVER_ACCESS_LOG_FILE_PATH` to empty.
Create config files with the image (writes to mounted `./docker-config`):
To run with a YAML config instead, pass `--config` explicitly (mutually exclusive with `--env`):
- `docker run --rm -v "$(pwd)/docker-config:/config" scratchbox:dev generate-config > ./docker-config/config.yaml`
- `docker run --rm -v "$(pwd)/docker-config:/config" scratchbox:dev generate-key > ./docker-config/config.key`
- `docker run --rm -p 8080:8080 -v "$(pwd)/docker-data:/data" -v "$(pwd)/docker-config:/config" scratchbox:dev server --config /config/config.yaml`
Storage encryption key behavior:
- The key file is always stored at `storage.data_dir/metadata.key` (next to `metadata`).
- Scratchbox only auto-generates this key when `storage.data_dir` has no existing files.
- If the key is missing and files already exist in `storage.data_dir`, startup fails to prevent corruption.
- On subsequent startups, Scratchbox reuses the same key file from that data directory.
- Keep `storage.data_dir` persistent; losing or replacing `metadata.key` makes existing scratches unreadable.
Environment variables supported by `scratchbox server --env`:
- `SCRATCHBOX_SERVER_LISTEN_ADDR`
- `SCRATCHBOX_SERVER_READ_HEADER_TIMEOUT`
- `SCRATCHBOX_SERVER_READ_TIMEOUT`
- `SCRATCHBOX_SERVER_WRITE_TIMEOUT`
- `SCRATCHBOX_SERVER_IDLE_TIMEOUT`
- `SCRATCHBOX_SERVER_MAX_HEADER_BYTES`
- `SCRATCHBOX_SERVER_ACCESS_LOG_ENABLED`
- `SCRATCHBOX_SERVER_ACCESS_LOG_FILE_PATH` (forced empty by Docker image)
- `SCRATCHBOX_SERVER_ACCESS_LOG_MAX_SIZE`
- `SCRATCHBOX_SERVER_ACCESS_LOG_MAX_BACKUPS`
- `SCRATCHBOX_LIMITS_MAX_UPLOAD_SIZE`
- `SCRATCHBOX_LIMITS_DEFAULT_TTL`
- `SCRATCHBOX_LIMITS_RAW_CACHE_MAX_SIZE`
- `SCRATCHBOX_LIMITS_RAW_CACHE_MAX_ENTRIES`
- `SCRATCHBOX_STORAGE_DATA_DIR`
- `SCRATCHBOX_STORAGE_CLEANUP_INTERVAL`
- `SCRATCHBOX_SECURITY_ALLOWED_IPS` (comma-separated)
- `SCRATCHBOX_SECURITY_TRUST_PROXY_HEADERS`
- `SCRATCHBOX_SECURITY_TRUSTED_PROXY_IPS` (comma-separated)
- `SCRATCHBOX_SECURITY_RATE_LIMIT_UI_ENABLED`
- `SCRATCHBOX_SECURITY_RATE_LIMIT_UI_REQUESTS_PER_MINUTE`
- `SCRATCHBOX_SECURITY_RATE_LIMIT_UI_BURST`
- `SCRATCHBOX_SECURITY_RATE_LIMIT_API_READ_ENABLED`
- `SCRATCHBOX_SECURITY_RATE_LIMIT_API_READ_REQUESTS_PER_MINUTE`
- `SCRATCHBOX_SECURITY_RATE_LIMIT_API_READ_BURST`
- `SCRATCHBOX_SECURITY_RATE_LIMIT_API_WRITE_ENABLED`
- `SCRATCHBOX_SECURITY_RATE_LIMIT_API_WRITE_REQUESTS_PER_MINUTE`
- `SCRATCHBOX_SECURITY_RATE_LIMIT_API_WRITE_BURST`
Example `.env` for Docker:
```dotenv
# server
SCRATCHBOX_SERVER_LISTEN_ADDR=:8080
SCRATCHBOX_SERVER_READ_HEADER_TIMEOUT=5s
SCRATCHBOX_SERVER_READ_TIMEOUT=30s
SCRATCHBOX_SERVER_WRITE_TIMEOUT=30s
SCRATCHBOX_SERVER_IDLE_TIMEOUT=120s
SCRATCHBOX_SERVER_MAX_HEADER_BYTES=1048576
SCRATCHBOX_SERVER_ACCESS_LOG_ENABLED=true
# limits
SCRATCHBOX_LIMITS_MAX_UPLOAD_SIZE=100MiB
SCRATCHBOX_LIMITS_DEFAULT_TTL=15m
SCRATCHBOX_LIMITS_RAW_CACHE_MAX_SIZE=200MiB
SCRATCHBOX_LIMITS_RAW_CACHE_MAX_ENTRIES=32
# storage
SCRATCHBOX_STORAGE_DATA_DIR=/data
SCRATCHBOX_STORAGE_CLEANUP_INTERVAL=1m
# security
SCRATCHBOX_SECURITY_ALLOWED_IPS=127.0.0.1
SCRATCHBOX_SECURITY_TRUST_PROXY_HEADERS=false
SCRATCHBOX_SECURITY_TRUSTED_PROXY_IPS=
SCRATCHBOX_SECURITY_RATE_LIMIT_UI_ENABLED=false
SCRATCHBOX_SECURITY_RATE_LIMIT_UI_REQUESTS_PER_MINUTE=360
SCRATCHBOX_SECURITY_RATE_LIMIT_UI_BURST=120
SCRATCHBOX_SECURITY_RATE_LIMIT_API_READ_ENABLED=false
SCRATCHBOX_SECURITY_RATE_LIMIT_API_READ_REQUESTS_PER_MINUTE=300
SCRATCHBOX_SECURITY_RATE_LIMIT_API_READ_BURST=100
SCRATCHBOX_SECURITY_RATE_LIMIT_API_WRITE_ENABLED=true
SCRATCHBOX_SECURITY_RATE_LIMIT_API_WRITE_REQUESTS_PER_MINUTE=30
SCRATCHBOX_SECURITY_RATE_LIMIT_API_WRITE_BURST=10
```
## Release Artifacts
@@ -80,7 +167,6 @@ To run from an unpacked artifact:
If you do not have a config file yet, generate one first:
- `go run ./cmd/scratchbox generate-config > config.yaml`
- `go run ./cmd/scratchbox generate-key > config.key`
## Configuration
@@ -142,12 +228,11 @@ All configuration is YAML.
- `cleanup_interval`: background interval for deleting expired content.
- Must be at least `10s`
- Default: `1m`
- `encryption_key_file`: path to a file containing a base64-encoded 32-byte key used for at-rest encryption.
- Relative paths are resolved from the config file directory
- Default: `config.key` (expected alongside `config.yaml`)
- Keep this key stable across restarts
- Changing the key makes existing scratch files unreadable
- `generate-key` emits a fresh random key each run
- Encryption key file is managed automatically at `storage.data_dir/metadata.key`.
- Auto-generated only when `storage.data_dir` has no existing files
- If missing while files exist in `storage.data_dir`, startup fails
- Reused on subsequent startups for the same data directory
- Changing/removing this file makes existing scratch files unreadable
### `security`
+88 -18
View File
@@ -32,18 +32,6 @@ func repoRoot(t *testing.T) string {
return filepath.Clean(filepath.Join(filepath.Dir(file), "..", ".."))
}
func writeConfigKeyFile(t *testing.T, configPath string) {
t.Helper()
keyPath := filepath.Join(filepath.Dir(configPath), "config.key")
key, err := config.GenerateEncryptionKey()
if err != nil {
t.Fatalf("GenerateEncryptionKey() error = %v", err)
}
if err := os.WriteFile(keyPath, []byte(key+"\n"), 0o600); err != nil {
t.Fatalf("WriteFile(config.key) error = %v", err)
}
}
func TestMainProcessSuccess(t *testing.T) {
t.Parallel()
@@ -77,7 +65,6 @@ security:
if err := os.WriteFile(cfgPath, []byte(cfg), 0o644); err != nil {
t.Fatalf("WriteFile() error = %v", err)
}
writeConfigKeyFile(t, cfgPath)
cmd := exec.Command(os.Args[0], "-test.run=TestMainHelperProcess")
cmd.Dir = repoRoot(t)
@@ -105,6 +92,75 @@ func TestMainProcessBadConfigFails(t *testing.T) {
}
}
func TestMainProcessMissingConfigAutoGenerates(t *testing.T) {
t.Parallel()
tmp := t.TempDir()
cfgPath := filepath.Join(tmp, "nested", "config.yaml")
ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second)
defer cancel()
cmd := exec.CommandContext(ctx, os.Args[0], "-test.run=TestMainHelperProcess")
cmd.Dir = repoRoot(t)
cmd.Env = append(os.Environ(),
"GO_WANT_MAIN_HELPER=1",
"MAIN_HELPER_MODE=missing-config-autogen",
"MAIN_HELPER_CONFIG="+cfgPath,
)
out, err := cmd.CombinedOutput()
if err != nil {
t.Fatalf("helper process failed: %v, output=%s", err, string(out))
}
if !strings.Contains(string(out), "Review and adjust it, then re-run the same command.") {
t.Fatalf("helper output missing expected guidance, output=%q", string(out))
}
raw, err := os.ReadFile(cfgPath)
if err != nil {
t.Fatalf("ReadFile(generated config) error = %v", err)
}
if !strings.Contains(string(raw), "server:") {
t.Fatalf("generated config missing expected section")
}
}
func TestMainProcessEnvSuccess(t *testing.T) {
t.Parallel()
tmp := t.TempDir()
cmd := exec.Command(os.Args[0], "-test.run=TestMainHelperProcess")
cmd.Dir = repoRoot(t)
cmd.Env = append(os.Environ(),
"GO_WANT_MAIN_HELPER=1",
"MAIN_HELPER_MODE=env-success",
"SCRATCHBOX_SERVER_LISTEN_ADDR=127.0.0.1:0",
"SCRATCHBOX_STORAGE_DATA_DIR="+filepath.Join(tmp, "data"),
"SCRATCHBOX_SECURITY_ALLOWED_IPS=",
)
if out, err := cmd.CombinedOutput(); err != nil {
t.Fatalf("helper process failed: %v, output=%s", err, string(out))
}
}
func TestServerConfigAndEnvMutuallyExclusive(t *testing.T) {
t.Parallel()
cmd := newRootCmd()
cmd.SetOut(io.Discard)
cmd.SetErr(io.Discard)
cmd.SetArgs([]string{"server", "--config", "config.yaml", "--env"})
err := cmd.Execute()
if err == nil {
t.Fatalf("expected error when both --config and --env are set")
}
if !strings.Contains(err.Error(), "none of the others can be") {
t.Fatalf("error = %q, want mutually exclusive flags error", err.Error())
}
}
func TestMainProcessStorageInitFails(t *testing.T) {
t.Parallel()
@@ -142,7 +198,6 @@ security:
if err := os.WriteFile(cfgPath, []byte(cfg), 0o644); err != nil {
t.Fatalf("WriteFile() error = %v", err)
}
writeConfigKeyFile(t, cfgPath)
cmd := exec.Command(os.Args[0], "-test.run=TestMainHelperProcess")
cmd.Dir = repoRoot(t)
@@ -181,6 +236,25 @@ func TestMainHelperProcess(t *testing.T) {
os.Args = []string{"scratchbox", "server", "--config", "/no/such/config.yaml"}
main()
os.Exit(0)
case "missing-config-autogen":
cfgPath := os.Getenv("MAIN_HELPER_CONFIG")
if cfgPath == "" {
os.Exit(2)
}
os.Args = []string{"scratchbox", "server", "--config", cfgPath}
main()
os.Exit(0)
case "env-success":
go func() {
time.Sleep(120 * time.Millisecond)
proc, err := os.FindProcess(os.Getpid())
if err == nil {
_ = proc.Signal(os.Interrupt)
}
}()
os.Args = []string{"scratchbox", "server", "--env"}
main()
os.Exit(0)
case "storage-fail":
cfgPath := os.Getenv("MAIN_HELPER_CONFIG")
os.Args = []string{"scratchbox", "server", "--config", cfgPath}
@@ -229,9 +303,6 @@ func TestGenerateConfigCommandWritesDefaultYAML(t *testing.T) {
if cfg.Storage.DataDir != "./data" {
t.Fatalf("storage.data_dir = %q, want %q", cfg.Storage.DataDir, "./data")
}
if cfg.Storage.EncryptionKeyFile != "config.key" {
t.Fatalf("storage.encryption_key_file = %q, want %q", cfg.Storage.EncryptionKeyFile, "config.key")
}
if cfg.Security.RateLimitUI.RequestsPerMinute != 360 {
t.Fatalf("security.rate_limit_ui.requests_per_minute = %d, want %d", cfg.Security.RateLimitUI.RequestsPerMinute, 360)
}
@@ -335,7 +406,6 @@ security:
if err := os.WriteFile(cfgPath, []byte(cfgRaw), 0o644); err != nil {
t.Fatalf("WriteFile() error = %v", err)
}
writeConfigKeyFile(t, cfgPath)
cfg, err := config.Load(cfgPath)
if err != nil {
+56 -3
View File
@@ -2,11 +2,14 @@ package main
import (
"context"
"errors"
"fmt"
"log"
"net/http"
"os"
"os/signal"
"path/filepath"
"strings"
"syscall"
"time"
@@ -20,21 +23,40 @@ import (
func newServerCmd() *cobra.Command {
var configPath string
var useEnv bool
cmd := &cobra.Command{
Use: "server",
Short: "Run the scratchbox service",
Args: cobra.NoArgs,
RunE: func(_ *cobra.Command, _ []string) error {
return run(configPath)
return run(configPath, useEnv)
},
}
cmd.Flags().StringVarP(&configPath, "config", "c", "", "path to config file")
cmd.Flags().BoolVar(&useEnv, "env", false, "load config from SCRATCHBOX_* environment variables")
cmd.MarkFlagsMutuallyExclusive("config", "env")
return cmd
}
func run(configPath string) error {
cfg, err := config.Load(configPath)
func run(configPath string, useEnv bool) error {
var (
cfg config.Config
err error
)
if useEnv {
cfg, err = config.LoadFromEnv()
} else {
generated, err := ensureConfigFile(configPath)
if err != nil {
return fmt.Errorf("config init failed: %w", err)
}
if generated {
fmt.Fprintf(os.Stdout, "Generated default config at %s\nReview and adjust it, then re-run the same command.\n", strings.TrimSpace(configPath))
return nil
}
cfg, err = config.Load(configPath)
}
if err != nil {
return fmt.Errorf("config load failed: %w", err)
}
@@ -100,3 +122,34 @@ func run(configPath string) error {
}
return nil
}
func ensureConfigFile(configPath string) (bool, error) {
path := strings.TrimSpace(configPath)
if path == "" {
return false, nil
}
info, err := os.Stat(path)
if err == nil {
if info.IsDir() {
return false, fmt.Errorf("config path %q is a directory", path)
}
return false, nil
}
if !errors.Is(err, os.ErrNotExist) {
return false, fmt.Errorf("stat config path %q: %w", path, err)
}
cfg := config.Default()
content, err := marshalYAMLWithComments(cfg)
if err != nil {
return false, fmt.Errorf("marshal default config: %w", err)
}
if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil {
return false, fmt.Errorf("create config dir: %w", err)
}
if err := os.WriteFile(path, content, 0o644); err != nil {
return false, fmt.Errorf("write default config: %w", err)
}
return true, nil
}
+10
View File
@@ -0,0 +1,10 @@
services:
scratchbox:
image: git.s1d3sw1ped.com/s1d3sw1ped/scratchbox:latest
env_file:
- .env
ports:
- "8080:8080"
volumes:
- ./docker-data:/data
restart: unless-stopped
+16 -4
View File
@@ -1,12 +1,24 @@
#!/bin/sh
set -eu
# Default to running the server with mounted config.
# Container-enforced data directory.
SCRATCHBOX_STORAGE_DATA_DIR=/data
export SCRATCHBOX_STORAGE_DATA_DIR
# Container-enforced listen address/port.
SCRATCHBOX_SERVER_LISTEN_ADDR=:8080
export SCRATCHBOX_SERVER_LISTEN_ADDR
# Container-enforced access log behavior: stdout only (no file sink).
SCRATCHBOX_SERVER_ACCESS_LOG_FILE_PATH=
export SCRATCHBOX_SERVER_ACCESS_LOG_FILE_PATH
# Default to running the server from SCRATCHBOX_* env vars.
if [ "$#" -eq 0 ]; then
set -- server --config /config/config.yaml
set -- server --env
elif [ "${1#-}" != "$1" ]; then
# Allow passing server flags directly, e.g. `docker run ... --config ...`.
set -- server "$@"
# Allow passing server flags directly, e.g. `docker run ... --env`.
set -- server --env "$@"
fi
exec scratchbox "$@"
+208 -32
View File
@@ -5,6 +5,7 @@ import (
"encoding/base64"
"errors"
"fmt"
"io/fs"
"net/netip"
"os"
"path/filepath"
@@ -31,8 +32,7 @@ const (
defaultRawCacheMaxEntries = 32
defaultDataDir = "./data"
defaultCleanupInterval = "1m"
defaultStorageEncryptionKey = "vVPlqMfKa8OtD3x0RKp4rVCJ4QoScf5mWdgfg4f+5GU="
defaultEncryptionKeyFile = "config.key"
dataDirEncryptionKeyFilename = "metadata.key"
defaultUIRateLimitEnabled = false
defaultAPIReadRateLimitEnabled = false
defaultAPIWriteRateLimitEnabled = true
@@ -44,6 +44,7 @@ const (
defaultAPIWriteRateLimitBurst = 10
maxDefaultTTLLimit = 24 * time.Hour
minCleanupInterval = 10 * time.Second
envPrefix = "SCRATCHBOX_"
)
type Config struct {
@@ -88,7 +89,6 @@ type LimitsConfig struct {
type StorageConfig struct {
DataDir string `yaml:"data_dir" comment:"Directory where scratch files and metadata are stored."`
CleanupInterval string `yaml:"cleanup_interval" comment:"How often expired scratches are deleted (minimum 10s)."`
EncryptionKeyFile string `yaml:"encryption_key_file" comment:"Path to a file containing a base64-encoded 32-byte AES key for at-rest encryption. Relative paths are resolved from the config file directory."`
CleanupIntervalParsed time.Duration `yaml:"-"`
EncryptionKeyBytes []byte `yaml:"-"`
}
@@ -121,10 +121,17 @@ func Load(path string) (Config, error) {
return Config{}, fmt.Errorf("parse config yaml: %w", err)
}
}
if err := cfg.loadEncryptionKey(path); err != nil {
if err := cfg.Validate(); err != nil {
return Config{}, err
}
return cfg, nil
}
func LoadFromEnv() (Config, error) {
cfg := defaults()
if err := cfg.applyEnvOverrides(envPrefix); err != nil {
return Config{}, err
}
if err := cfg.Validate(); err != nil {
return Config{}, err
}
@@ -157,10 +164,8 @@ func defaults() Config {
RawCacheMaxEntries: defaultRawCacheMaxEntries,
},
Storage: StorageConfig{
DataDir: defaultDataDir,
CleanupInterval: defaultCleanupInterval,
EncryptionKeyFile: defaultEncryptionKeyFile,
EncryptionKeyBytes: mustDecodeDefaultEncryptionKey(),
DataDir: defaultDataDir,
CleanupInterval: defaultCleanupInterval,
},
Security: SecurityConfig{
AllowedIPs: []string{
@@ -276,12 +281,14 @@ func (c *Config) Validate() error {
if err := ensureWritableDir(c.Storage.DataDir); err != nil {
return fmt.Errorf("storage.data_dir not writable: %w", err)
}
if strings.TrimSpace(c.Storage.EncryptionKeyFile) == "" {
return errors.New("storage.encryption_key_file is required")
encKey, err := ensureEncryptionKeyInDataDir(c.Storage.DataDir)
if err != nil {
return fmt.Errorf("storage encryption key setup failed: %w", err)
}
if len(c.Storage.EncryptionKeyBytes) != 32 {
return fmt.Errorf("storage.encryption_key_file must decode to 32 bytes, got %d", len(c.Storage.EncryptionKeyBytes))
if len(encKey) != 32 {
return fmt.Errorf("storage encryption key must decode to 32 bytes, got %d", len(encKey))
}
c.Storage.EncryptionKeyBytes = encKey
prefixes, err := parseAllowedPrefixes(c.Security.AllowedIPs)
if err != nil {
@@ -419,27 +426,56 @@ func GenerateEncryptionKey() (string, error) {
return base64.StdEncoding.EncodeToString(raw), nil
}
func (c *Config) loadEncryptionKey(configPath string) error {
if configPath == "" {
return nil
}
keyPath := strings.TrimSpace(c.Storage.EncryptionKeyFile)
if keyPath == "" {
return errors.New("storage.encryption_key_file is required")
}
if !filepath.IsAbs(keyPath) {
keyPath = filepath.Join(filepath.Dir(configPath), keyPath)
}
func ensureEncryptionKeyInDataDir(dataDir string) ([]byte, error) {
keyPath := filepath.Join(dataDir, dataDirEncryptionKeyFilename)
keyRaw, err := os.ReadFile(keyPath)
if err != nil {
return fmt.Errorf("read storage.encryption_key_file %q: %w", keyPath, err)
if !errors.Is(err, os.ErrNotExist) {
return nil, fmt.Errorf("read %q: %w", keyPath, err)
}
hasFiles, checkErr := dataDirContainsFiles(dataDir, keyPath)
if checkErr != nil {
return nil, fmt.Errorf("scan %q for existing files: %w", dataDir, checkErr)
}
if hasFiles {
return nil, fmt.Errorf("refusing to generate missing %q because %q contains files; restore the original key file to avoid data corruption", dataDirEncryptionKeyFilename, dataDir)
}
generated, genErr := GenerateEncryptionKey()
if genErr != nil {
return nil, fmt.Errorf("generate key: %w", genErr)
}
if writeErr := os.WriteFile(keyPath, []byte(generated+"\n"), 0o600); writeErr != nil {
return nil, fmt.Errorf("write %q: %w", keyPath, writeErr)
}
return decodeEncryptionKey(generated)
}
encKey, err := decodeEncryptionKey(strings.TrimSpace(string(keyRaw)))
if err != nil {
return fmt.Errorf("storage.encryption_key_file invalid: %w", err)
return nil, fmt.Errorf("%q invalid: %w", keyPath, err)
}
c.Storage.EncryptionKeyBytes = encKey
return nil
return encKey, nil
}
func dataDirContainsFiles(dataDir string, keyPath string) (bool, error) {
keyPath = filepath.Clean(keyPath)
foundFile := false
err := filepath.WalkDir(dataDir, func(path string, d fs.DirEntry, walkErr error) error {
if walkErr != nil {
return walkErr
}
if d.IsDir() {
return nil
}
if filepath.Clean(path) == keyPath {
return nil
}
foundFile = true
return fs.SkipAll
})
if err != nil && !errors.Is(err, fs.SkipAll) {
return false, err
}
return foundFile, nil
}
func decodeEncryptionKey(raw string) ([]byte, error) {
@@ -456,10 +492,150 @@ func decodeEncryptionKey(raw string) ([]byte, error) {
return encKey, nil
}
func mustDecodeDefaultEncryptionKey() []byte {
encKey, err := decodeEncryptionKey(defaultStorageEncryptionKey)
if err != nil {
panic(fmt.Sprintf("invalid built-in encryption key: %v", err))
func (c *Config) applyEnvOverrides(prefix string) error {
if err := applyStringEnv(prefix+"SERVER_LISTEN_ADDR", &c.Server.ListenAddr); err != nil {
return err
}
return encKey
if err := applyStringEnv(prefix+"SERVER_READ_HEADER_TIMEOUT", &c.Server.ReadHeaderTimeout); err != nil {
return err
}
if err := applyStringEnv(prefix+"SERVER_READ_TIMEOUT", &c.Server.ReadTimeout); err != nil {
return err
}
if err := applyStringEnv(prefix+"SERVER_WRITE_TIMEOUT", &c.Server.WriteTimeout); err != nil {
return err
}
if err := applyStringEnv(prefix+"SERVER_IDLE_TIMEOUT", &c.Server.IdleTimeout); err != nil {
return err
}
if err := applyIntEnv(prefix+"SERVER_MAX_HEADER_BYTES", &c.Server.MaxHeaderBytes); err != nil {
return err
}
if err := applyBoolEnv(prefix+"SERVER_ACCESS_LOG_ENABLED", &c.Server.AccessLog.Enabled); err != nil {
return err
}
if err := applyStringEnv(prefix+"SERVER_ACCESS_LOG_FILE_PATH", &c.Server.AccessLog.FilePath); err != nil {
return err
}
if err := applyStringEnv(prefix+"SERVER_ACCESS_LOG_MAX_SIZE", &c.Server.AccessLog.MaxSize); err != nil {
return err
}
if err := applyIntEnv(prefix+"SERVER_ACCESS_LOG_MAX_BACKUPS", &c.Server.AccessLog.MaxBackups); err != nil {
return err
}
if err := applyStringEnv(prefix+"LIMITS_MAX_UPLOAD_SIZE", &c.Limits.MaxUploadSize); err != nil {
return err
}
if err := applyStringEnv(prefix+"LIMITS_DEFAULT_TTL", &c.Limits.DefaultTTL); err != nil {
return err
}
if err := applyStringEnv(prefix+"LIMITS_RAW_CACHE_MAX_SIZE", &c.Limits.RawCacheMaxSize); err != nil {
return err
}
if err := applyIntEnv(prefix+"LIMITS_RAW_CACHE_MAX_ENTRIES", &c.Limits.RawCacheMaxEntries); err != nil {
return err
}
if err := applyStringEnv(prefix+"STORAGE_DATA_DIR", &c.Storage.DataDir); err != nil {
return err
}
if err := applyStringEnv(prefix+"STORAGE_CLEANUP_INTERVAL", &c.Storage.CleanupInterval); err != nil {
return err
}
applyListEnv(prefix+"SECURITY_ALLOWED_IPS", &c.Security.AllowedIPs)
if err := applyBoolEnv(prefix+"SECURITY_TRUST_PROXY_HEADERS", &c.Security.TrustProxyHeaders); err != nil {
return err
}
applyListEnv(prefix+"SECURITY_TRUSTED_PROXY_IPS", &c.Security.TrustedProxyIPs)
if err := applyBoolEnv(prefix+"SECURITY_RATE_LIMIT_UI_ENABLED", &c.Security.RateLimitUI.Enabled); err != nil {
return err
}
if err := applyIntEnv(prefix+"SECURITY_RATE_LIMIT_UI_REQUESTS_PER_MINUTE", &c.Security.RateLimitUI.RequestsPerMinute); err != nil {
return err
}
if err := applyIntEnv(prefix+"SECURITY_RATE_LIMIT_UI_BURST", &c.Security.RateLimitUI.Burst); err != nil {
return err
}
if err := applyBoolEnv(prefix+"SECURITY_RATE_LIMIT_API_READ_ENABLED", &c.Security.RateLimitAPIRead.Enabled); err != nil {
return err
}
if err := applyIntEnv(prefix+"SECURITY_RATE_LIMIT_API_READ_REQUESTS_PER_MINUTE", &c.Security.RateLimitAPIRead.RequestsPerMinute); err != nil {
return err
}
if err := applyIntEnv(prefix+"SECURITY_RATE_LIMIT_API_READ_BURST", &c.Security.RateLimitAPIRead.Burst); err != nil {
return err
}
if err := applyBoolEnv(prefix+"SECURITY_RATE_LIMIT_API_WRITE_ENABLED", &c.Security.RateLimitAPIWrite.Enabled); err != nil {
return err
}
if err := applyIntEnv(prefix+"SECURITY_RATE_LIMIT_API_WRITE_REQUESTS_PER_MINUTE", &c.Security.RateLimitAPIWrite.RequestsPerMinute); err != nil {
return err
}
if err := applyIntEnv(prefix+"SECURITY_RATE_LIMIT_API_WRITE_BURST", &c.Security.RateLimitAPIWrite.Burst); err != nil {
return err
}
return nil
}
func applyStringEnv(name string, target *string) error {
value, ok := os.LookupEnv(name)
if !ok {
return nil
}
*target = strings.TrimSpace(value)
return nil
}
func applyBoolEnv(name string, target *bool) error {
value, ok := os.LookupEnv(name)
if !ok {
return nil
}
parsed, err := strconv.ParseBool(strings.TrimSpace(value))
if err != nil {
return fmt.Errorf("%s invalid bool: %w", name, err)
}
*target = parsed
return nil
}
func applyIntEnv(name string, target *int) error {
value, ok := os.LookupEnv(name)
if !ok {
return nil
}
parsed, err := strconv.Atoi(strings.TrimSpace(value))
if err != nil {
return fmt.Errorf("%s invalid int: %w", name, err)
}
*target = parsed
return nil
}
func applyListEnv(name string, target *[]string) {
value, ok := os.LookupEnv(name)
if !ok {
return
}
if strings.TrimSpace(value) == "" {
*target = nil
return
}
parts := strings.Split(value, ",")
out := make([]string, 0, len(parts))
for _, part := range parts {
item := strings.TrimSpace(part)
if item == "" {
continue
}
out = append(out, item)
}
*target = out
}
+147 -25
View File
@@ -76,7 +76,6 @@ func TestLoadParsesConfiguredFields(t *testing.T) {
tmp := t.TempDir()
dataDir := filepath.Join(tmp, "store")
configPath := filepath.Join(tmp, "config.yaml")
keyPath := filepath.Join(tmp, "scratchbox.key")
content := `
server:
listen_addr: "127.0.0.1:9090"
@@ -98,7 +97,6 @@ limits:
storage:
data_dir: "` + dataDir + `"
cleanup_interval: "45s"
encryption_key_file: "` + filepath.Base(keyPath) + `"
security:
allowed_ips:
- "127.0.0.1"
@@ -122,9 +120,6 @@ security:
if err := os.WriteFile(configPath, []byte(content), 0o644); err != nil {
t.Fatalf("WriteFile() error = %v", err)
}
if err := os.WriteFile(keyPath, []byte("AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=\n"), 0o600); err != nil {
t.Fatalf("WriteFile(key) error = %v", err)
}
cfg, err := Load(configPath)
if err != nil {
@@ -176,8 +171,8 @@ security:
if len(cfg.Storage.EncryptionKeyBytes) != 32 {
t.Fatalf("EncryptionKeyBytes length = %d, want 32", len(cfg.Storage.EncryptionKeyBytes))
}
if cfg.Storage.EncryptionKeyBytes[0] != 0 {
t.Fatalf("EncryptionKeyBytes[0] = %d, want 0", cfg.Storage.EncryptionKeyBytes[0])
if _, err := os.Stat(filepath.Join(dataDir, dataDirEncryptionKeyFilename)); err != nil {
t.Fatalf("expected data-dir key file to exist: %v", err)
}
if !cfg.Security.TrustProxyHeaders {
t.Fatalf("TrustProxyHeaders = false, want true")
@@ -362,20 +357,6 @@ func TestValidateRejectsInvalidValues(t *testing.T) {
},
wantErr: "storage.cleanup_interval",
},
{
name: "encryption key file required",
mutate: func(cfg *Config) {
cfg.Storage.EncryptionKeyFile = " "
},
wantErr: "storage.encryption_key_file is required",
},
{
name: "encryption key bytes invalid length",
mutate: func(cfg *Config) {
cfg.Storage.EncryptionKeyBytes = []byte("short")
},
wantErr: "storage.encryption_key_file must decode to 32 bytes",
},
{
name: "invalid ui rate limit rpm",
mutate: func(cfg *Config) {
@@ -447,20 +428,161 @@ storage:
t.Fatalf("WriteFile(valid config) error = %v", err)
}
_, err = Load(validConfigPath)
if err == nil || !strings.Contains(err.Error(), "read storage.encryption_key_file") {
t.Fatalf("Load() error = %v, want missing key file error", err)
if err != nil {
t.Fatalf("Load() error = %v, want nil", err)
}
keyPath := filepath.Join(tmp, "config.key")
keyPath := filepath.Join(tmp, "store", dataDirEncryptionKeyFilename)
if err := os.WriteFile(keyPath, []byte("not-base64"), 0o600); err != nil {
t.Fatalf("WriteFile(invalid key) error = %v", err)
}
_, err = Load(validConfigPath)
if err == nil || !strings.Contains(err.Error(), "storage.encryption_key_file invalid: invalid base64") {
if err == nil || !strings.Contains(err.Error(), "storage encryption key setup failed") {
t.Fatalf("Load() error = %v, want invalid key base64 error", err)
}
}
func TestLoadFailsWhenKeyMissingAndDataFilesExist(t *testing.T) {
t.Parallel()
tmp := t.TempDir()
dataDir := filepath.Join(tmp, "store")
scratchesDir := filepath.Join(dataDir, "scratches")
if err := os.MkdirAll(scratchesDir, 0o755); err != nil {
t.Fatalf("MkdirAll() error = %v", err)
}
if err := os.WriteFile(filepath.Join(scratchesDir, "existing-scratch"), []byte("payload"), 0o600); err != nil {
t.Fatalf("WriteFile() error = %v", err)
}
configPath := filepath.Join(tmp, "config.yaml")
configRaw := `
storage:
data_dir: "` + dataDir + `"
cleanup_interval: "10s"
`
if err := os.WriteFile(configPath, []byte(configRaw), 0o644); err != nil {
t.Fatalf("WriteFile(config) error = %v", err)
}
_, err := Load(configPath)
if err == nil || !strings.Contains(err.Error(), "refusing to generate missing") {
t.Fatalf("Load() error = %v, want missing-key safety error", err)
}
}
func TestLoadFromEnvParsesConfiguredFields(t *testing.T) {
dataDir := filepath.Join(t.TempDir(), "store")
t.Setenv("SCRATCHBOX_SERVER_LISTEN_ADDR", "127.0.0.1:19090")
t.Setenv("SCRATCHBOX_SERVER_READ_HEADER_TIMEOUT", "6s")
t.Setenv("SCRATCHBOX_SERVER_READ_TIMEOUT", "20s")
t.Setenv("SCRATCHBOX_SERVER_WRITE_TIMEOUT", "40s")
t.Setenv("SCRATCHBOX_SERVER_IDLE_TIMEOUT", "80s")
t.Setenv("SCRATCHBOX_SERVER_MAX_HEADER_BYTES", "2097152")
t.Setenv("SCRATCHBOX_SERVER_ACCESS_LOG_ENABLED", "false")
t.Setenv("SCRATCHBOX_SERVER_ACCESS_LOG_FILE_PATH", "/tmp/access.log")
t.Setenv("SCRATCHBOX_SERVER_ACCESS_LOG_MAX_SIZE", "4MiB")
t.Setenv("SCRATCHBOX_SERVER_ACCESS_LOG_MAX_BACKUPS", "8")
t.Setenv("SCRATCHBOX_LIMITS_MAX_UPLOAD_SIZE", "8MiB")
t.Setenv("SCRATCHBOX_LIMITS_DEFAULT_TTL", "30m")
t.Setenv("SCRATCHBOX_LIMITS_RAW_CACHE_MAX_SIZE", "64MiB")
t.Setenv("SCRATCHBOX_LIMITS_RAW_CACHE_MAX_ENTRIES", "16")
t.Setenv("SCRATCHBOX_STORAGE_DATA_DIR", dataDir)
t.Setenv("SCRATCHBOX_STORAGE_CLEANUP_INTERVAL", "15s")
t.Setenv("SCRATCHBOX_SECURITY_ALLOWED_IPS", "127.0.0.1,10.0.0.0/8")
t.Setenv("SCRATCHBOX_SECURITY_TRUST_PROXY_HEADERS", "true")
t.Setenv("SCRATCHBOX_SECURITY_TRUSTED_PROXY_IPS", "10.0.0.0/8")
t.Setenv("SCRATCHBOX_SECURITY_RATE_LIMIT_UI_ENABLED", "true")
t.Setenv("SCRATCHBOX_SECURITY_RATE_LIMIT_UI_REQUESTS_PER_MINUTE", "50")
t.Setenv("SCRATCHBOX_SECURITY_RATE_LIMIT_UI_BURST", "20")
t.Setenv("SCRATCHBOX_SECURITY_RATE_LIMIT_API_READ_ENABLED", "true")
t.Setenv("SCRATCHBOX_SECURITY_RATE_LIMIT_API_READ_REQUESTS_PER_MINUTE", "60")
t.Setenv("SCRATCHBOX_SECURITY_RATE_LIMIT_API_READ_BURST", "30")
t.Setenv("SCRATCHBOX_SECURITY_RATE_LIMIT_API_WRITE_ENABLED", "true")
t.Setenv("SCRATCHBOX_SECURITY_RATE_LIMIT_API_WRITE_REQUESTS_PER_MINUTE", "40")
t.Setenv("SCRATCHBOX_SECURITY_RATE_LIMIT_API_WRITE_BURST", "12")
cfg, err := LoadFromEnv()
if err != nil {
t.Fatalf("LoadFromEnv() error = %v", err)
}
if cfg.Server.ListenAddr != "127.0.0.1:19090" {
t.Fatalf("ListenAddr = %q, want %q", cfg.Server.ListenAddr, "127.0.0.1:19090")
}
if cfg.Server.ReadHeaderTimeoutDur != 6*time.Second {
t.Fatalf("ReadHeaderTimeoutDur = %s, want 6s", cfg.Server.ReadHeaderTimeoutDur)
}
if cfg.Server.ReadTimeoutDur != 20*time.Second {
t.Fatalf("ReadTimeoutDur = %s, want 20s", cfg.Server.ReadTimeoutDur)
}
if cfg.Server.WriteTimeoutDur != 40*time.Second {
t.Fatalf("WriteTimeoutDur = %s, want 40s", cfg.Server.WriteTimeoutDur)
}
if cfg.Server.IdleTimeoutDur != 80*time.Second {
t.Fatalf("IdleTimeoutDur = %s, want 80s", cfg.Server.IdleTimeoutDur)
}
if cfg.Server.MaxHeaderBytes != 2*1024*1024 {
t.Fatalf("MaxHeaderBytes = %d, want %d", cfg.Server.MaxHeaderBytes, 2*1024*1024)
}
if cfg.Server.AccessLog.Enabled {
t.Fatalf("AccessLog.Enabled = %v, want false", cfg.Server.AccessLog.Enabled)
}
if cfg.Limits.MaxUploadSizeBytes != 8*1024*1024 {
t.Fatalf("MaxUploadSizeBytes = %d, want %d", cfg.Limits.MaxUploadSizeBytes, int64(8*1024*1024))
}
if cfg.Limits.DefaultTTLDuration != 30*time.Minute {
t.Fatalf("DefaultTTLDuration = %s, want 30m", cfg.Limits.DefaultTTLDuration)
}
if cfg.Limits.RawCacheMaxBytes != 64*1024*1024 {
t.Fatalf("RawCacheMaxBytes = %d, want %d", cfg.Limits.RawCacheMaxBytes, int64(64*1024*1024))
}
if cfg.Limits.RawCacheMaxEntries != 16 {
t.Fatalf("RawCacheMaxEntries = %d, want 16", cfg.Limits.RawCacheMaxEntries)
}
if cfg.Storage.DataDir != dataDir {
t.Fatalf("DataDir = %q, want %q", cfg.Storage.DataDir, dataDir)
}
if cfg.Storage.CleanupIntervalParsed != 15*time.Second {
t.Fatalf("CleanupIntervalParsed = %s, want 15s", cfg.Storage.CleanupIntervalParsed)
}
if len(cfg.Storage.EncryptionKeyBytes) != 32 {
t.Fatalf("EncryptionKeyBytes length = %d, want 32", len(cfg.Storage.EncryptionKeyBytes))
}
if len(cfg.Security.AllowedPrefixes) != 2 {
t.Fatalf("AllowedPrefixes len = %d, want 2", len(cfg.Security.AllowedPrefixes))
}
if len(cfg.Security.TrustedProxyPrefixes) != 1 {
t.Fatalf("TrustedProxyPrefixes len = %d, want 1", len(cfg.Security.TrustedProxyPrefixes))
}
if !cfg.Security.TrustProxyHeaders {
t.Fatalf("TrustProxyHeaders = false, want true")
}
if cfg.Security.RateLimitUI.RequestsPerMinute != 50 || cfg.Security.RateLimitUI.Burst != 20 {
t.Fatalf("unexpected rate_limit_ui values: %+v", cfg.Security.RateLimitUI)
}
if cfg.Security.RateLimitAPIRead.RequestsPerMinute != 60 || cfg.Security.RateLimitAPIRead.Burst != 30 {
t.Fatalf("unexpected rate_limit_api_read values: %+v", cfg.Security.RateLimitAPIRead)
}
if cfg.Security.RateLimitAPIWrite.RequestsPerMinute != 40 || cfg.Security.RateLimitAPIWrite.Burst != 12 {
t.Fatalf("unexpected rate_limit_api_write values: %+v", cfg.Security.RateLimitAPIWrite)
}
}
func TestLoadFromEnvRejectsInvalidBool(t *testing.T) {
t.Setenv("SCRATCHBOX_STORAGE_DATA_DIR", filepath.Join(t.TempDir(), "store"))
t.Setenv("SCRATCHBOX_SERVER_ACCESS_LOG_ENABLED", "not-bool")
_, err := LoadFromEnv()
if err == nil || !strings.Contains(err.Error(), "SCRATCHBOX_SERVER_ACCESS_LOG_ENABLED invalid bool") {
t.Fatalf("LoadFromEnv() error = %v, want invalid bool env error", err)
}
}
func TestParseAllowedPrefixesTrimsAndParses(t *testing.T) {
t.Parallel()