Compare commits
1 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| 90f1cf8bdf |
@@ -1,26 +1,38 @@
|
||||
# Scratchbox env-mode example for `scratchbox server --env`.
|
||||
# Set values here and pass this file with `--env-file .env`.
|
||||
#
|
||||
# The complete list of supported SCRATCHBOX_* env vars is the single source of
|
||||
# truth in internal/config/config.go (var envSuffixes). See the comment on
|
||||
# envSuffixes for maintenance rules when adding new vars.
|
||||
#
|
||||
# IMPORTANT:
|
||||
# - Scratchbox stores its key at $SCRATCHBOX_STORAGE_DATA_DIR/metadata.key.
|
||||
# - Keep that data directory persistent across restarts.
|
||||
|
||||
# server
|
||||
SCRATCHBOX_SERVER_LISTEN_ADDR=:8080
|
||||
SCRATCHBOX_SERVER_READ_HEADER_TIMEOUT=5s
|
||||
# Body/response deadlines. 0 = disabled. Raise or set 0 when LIMITS_MAX_UPLOAD_SIZE is multi-GiB.
|
||||
SCRATCHBOX_SERVER_READ_TIMEOUT=30s
|
||||
SCRATCHBOX_SERVER_WRITE_TIMEOUT=30s
|
||||
SCRATCHBOX_SERVER_IDLE_TIMEOUT=120s
|
||||
SCRATCHBOX_SERVER_SHUTDOWN_TIMEOUT=10s
|
||||
SCRATCHBOX_SERVER_MAX_HEADER_BYTES=1048576
|
||||
SCRATCHBOX_SERVER_ACCESS_LOG_ENABLED=true
|
||||
SCRATCHBOX_SERVER_ACCESS_LOG_FILE_PATH=
|
||||
SCRATCHBOX_SERVER_ACCESS_LOG_MAX_SIZE=100MiB
|
||||
SCRATCHBOX_SERVER_ACCESS_LOG_MAX_BACKUPS=5
|
||||
|
||||
# limits
|
||||
# When raising MAX_UPLOAD_SIZE to multi-GiB, also set READ/WRITE_TIMEOUT high or 0 (see above).
|
||||
SCRATCHBOX_LIMITS_MAX_UPLOAD_SIZE=100MiB
|
||||
SCRATCHBOX_LIMITS_DEFAULT_TTL=15m
|
||||
# In-memory cache for small /api/raw range hits. Larger scratches stream from disk.
|
||||
SCRATCHBOX_LIMITS_RAW_CACHE_MAX_SIZE=200MiB
|
||||
SCRATCHBOX_LIMITS_RAW_CACHE_MAX_ENTRIES=32
|
||||
|
||||
# storage
|
||||
SCRATCHBOX_STORAGE_DATA_DIR=./data
|
||||
SCRATCHBOX_STORAGE_CLEANUP_INTERVAL=1m
|
||||
|
||||
# security
|
||||
@@ -29,12 +41,21 @@ SCRATCHBOX_SECURITY_ALLOWED_IPS=127.0.0.1 # This should be set to your networks
|
||||
SCRATCHBOX_SECURITY_TRUST_PROXY_HEADERS=false
|
||||
# Comma-separated IP/CIDR list.
|
||||
SCRATCHBOX_SECURITY_TRUSTED_PROXY_IPS= # Same as SCRATCHBOX_SECURITY_ALLOWED_IPS just Trusted Proxies you expect clients to come from instead. Must be *immediate* reverse proxies only; avoid 0/0.
|
||||
|
||||
# Per-client token bucket on UI routes (/, /u, /s/{id}).
|
||||
SCRATCHBOX_SECURITY_RATE_LIMIT_UI_ENABLED=false
|
||||
SCRATCHBOX_SECURITY_RATE_LIMIT_UI_REQUESTS_PER_MINUTE=360
|
||||
SCRATCHBOX_SECURITY_RATE_LIMIT_UI_BURST=120
|
||||
|
||||
# Per-client token bucket on API read routes (GET /api/config, GET /api/scratch/{id}, GET /api/raw/{id}).
|
||||
SCRATCHBOX_SECURITY_RATE_LIMIT_API_READ_ENABLED=false
|
||||
SCRATCHBOX_SECURITY_RATE_LIMIT_API_READ_REQUESTS_PER_MINUTE=300
|
||||
SCRATCHBOX_SECURITY_RATE_LIMIT_API_READ_BURST=100
|
||||
|
||||
# Per-client token bucket on API write route (POST /api/scratch).
|
||||
SCRATCHBOX_SECURITY_RATE_LIMIT_API_WRITE_ENABLED=true
|
||||
SCRATCHBOX_SECURITY_RATE_LIMIT_API_WRITE_REQUESTS_PER_MINUTE=30
|
||||
SCRATCHBOX_SECURITY_RATE_LIMIT_API_WRITE_BURST=10
|
||||
|
||||
# Enable Strict-Transport-Security (HSTS) header (recommended for public TLS deploys).
|
||||
SCRATCHBOX_SECURITY_HSTS_ENABLED=true
|
||||
|
||||
+3
-1
@@ -12,4 +12,6 @@
|
||||
/web/ui/node_modules/
|
||||
|
||||
# Frontend build artifacts
|
||||
/web/static/
|
||||
/web/static/
|
||||
# Local/large test fixtures (do not commit)
|
||||
/testdata/*.zip
|
||||
|
||||
@@ -64,7 +64,7 @@ Storage encryption key behavior:
|
||||
- On subsequent startups, Scratchbox reuses the same key file from that data directory.
|
||||
- Keep `storage.data_dir` persistent; losing or replacing `metadata.key` makes existing scratches unreadable.
|
||||
|
||||
Environment variables supported by `scratchbox server --env` (note: when `--env` is used, the variables and their values are printed to stdout on startup for diagnostics, with values of IP allow/trusted lists redacted):
|
||||
Environment variables supported by `scratchbox server --env` (note: when `--env` is used, the variables and their values are printed to stdout on startup for diagnostics, with values of IP allow/trusted lists redacted). The canonical list of supported vars is envSuffixes in internal/config/config.go — see the comment there for how to keep this documentation in sync when adding new configuration.
|
||||
|
||||
- `SCRATCHBOX_SERVER_LISTEN_ADDR`
|
||||
- `SCRATCHBOX_SERVER_READ_HEADER_TIMEOUT`
|
||||
@@ -97,7 +97,7 @@ Environment variables supported by `scratchbox server --env` (note: when `--env`
|
||||
- `SCRATCHBOX_SECURITY_RATE_LIMIT_API_WRITE_BURST`
|
||||
- `SCRATCHBOX_SECURITY_HSTS_ENABLED`
|
||||
|
||||
Example `.env` for Docker:
|
||||
Example `.env` for Docker (must be kept in sync with envSuffixes in internal/config/config.go):
|
||||
|
||||
```dotenv
|
||||
# server
|
||||
@@ -109,6 +109,9 @@ SCRATCHBOX_SERVER_IDLE_TIMEOUT=120s
|
||||
SCRATCHBOX_SERVER_SHUTDOWN_TIMEOUT=10s
|
||||
SCRATCHBOX_SERVER_MAX_HEADER_BYTES=1048576
|
||||
SCRATCHBOX_SERVER_ACCESS_LOG_ENABLED=true
|
||||
SCRATCHBOX_SERVER_ACCESS_LOG_FILE_PATH=
|
||||
SCRATCHBOX_SERVER_ACCESS_LOG_MAX_SIZE=100MiB
|
||||
SCRATCHBOX_SERVER_ACCESS_LOG_MAX_BACKUPS=5
|
||||
|
||||
# limits
|
||||
SCRATCHBOX_LIMITS_MAX_UPLOAD_SIZE=100MiB
|
||||
@@ -180,11 +183,13 @@ All configuration is YAML.
|
||||
- `read_header_timeout`: maximum time for reading request headers.
|
||||
- Must be `> 0`
|
||||
- Default: `5s`
|
||||
- `read_timeout`: maximum time for reading the full request.
|
||||
- Must be `> 0`
|
||||
- `read_timeout`: maximum time for reading the full request (including body).
|
||||
- Must be `>= 0`. `0` disables the deadline (same as Go `http.Server`).
|
||||
- Raise this (or set `0`) when `limits.max_upload_size` is large or clients are slow: a 30s body timeout cannot complete multi-GiB uploads.
|
||||
- Default: `30s`
|
||||
- `write_timeout`: maximum time allowed for writing the response.
|
||||
- Must be `> 0`
|
||||
- `write_timeout`: maximum time allowed for writing the response (also bounds slow request handling after headers are read).
|
||||
- Must be `>= 0`. `0` disables the deadline.
|
||||
- Raise this (or set `0`) for large uploads/downloads; after request headers are read the write deadline covers the rest of the handler.
|
||||
- Default: `30s`
|
||||
- `idle_timeout`: maximum keep-alive idle time between requests.
|
||||
- Must be `> 0`
|
||||
|
||||
@@ -135,6 +135,7 @@ func TestMainProcessEnvSuccess(t *testing.T) {
|
||||
cmd.Env = append(os.Environ(),
|
||||
"GO_WANT_MAIN_HELPER=1",
|
||||
"MAIN_HELPER_MODE=env-success",
|
||||
// These are from the canonical envSuffixes list (internal/config/config.go)
|
||||
"SCRATCHBOX_SERVER_LISTEN_ADDR=127.0.0.1:0",
|
||||
"SCRATCHBOX_STORAGE_DATA_DIR="+filepath.Join(tmp, "data"),
|
||||
"SCRATCHBOX_SECURITY_ALLOWED_IPS=",
|
||||
|
||||
@@ -34,7 +34,7 @@ func newServerCmd() *cobra.Command {
|
||||
},
|
||||
}
|
||||
cmd.Flags().StringVarP(&configPath, "config", "c", "", "path to config file")
|
||||
cmd.Flags().BoolVar(&useEnv, "env", false, "load config from SCRATCHBOX_* environment variables")
|
||||
cmd.Flags().BoolVar(&useEnv, "env", false, "load config from SCRATCHBOX_* environment variables (see envSuffixes in internal/config/config.go)")
|
||||
cmd.MarkFlagsMutuallyExclusive("config", "env")
|
||||
return cmd
|
||||
}
|
||||
@@ -134,6 +134,8 @@ func run(configPath string, useEnv bool) error {
|
||||
}
|
||||
|
||||
func printEnvStartup() int {
|
||||
// All SCRATCHBOX_* variables are defined by envSuffixes (single source of truth)
|
||||
// in internal/config/config.go. See the comment there for update rules.
|
||||
entries := make([]string, 0)
|
||||
for _, item := range os.Environ() {
|
||||
if strings.HasPrefix(item, "SCRATCHBOX_") {
|
||||
|
||||
@@ -1,6 +1,10 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
|
||||
# This script forces certain SCRATCHBOX_* vars for Docker.
|
||||
# The full list of supported env vars (single source of truth) is in
|
||||
# internal/config/config.go:envSuffixes . Update that list + docs when changing env support.
|
||||
|
||||
# Container-enforced data directory.
|
||||
SCRATCHBOX_STORAGE_DATA_DIR=/data
|
||||
export SCRATCHBOX_STORAGE_DATA_DIR
|
||||
|
||||
@@ -49,6 +49,62 @@ const (
|
||||
envPrefix = "SCRATCHBOX_"
|
||||
)
|
||||
|
||||
// envSuffixes is the single source of truth for every SCRATCHBOX_* environment
|
||||
// variable suffix supported by applyEnvOverrides (without the "SCRATCHBOX_" prefix).
|
||||
//
|
||||
// Usage sites that hardcode or document these vars MUST have a comment pointing here:
|
||||
// - applyEnvOverrides (this file)
|
||||
// - printEnvStartup + flag docs in cmd/scratchbox/server.go
|
||||
// - docker-entrypoint.sh
|
||||
// - .env.example
|
||||
// - README.md (supported list + example block)
|
||||
// - tests that setenv them (e.g. config_test.go, main_test.go)
|
||||
//
|
||||
// When you add, remove, or rename support in applyEnvOverrides (or the corresponding
|
||||
// struct fields), you MUST:
|
||||
// 1. Update this list.
|
||||
// 2. Add/update the pointer comment at every usage site.
|
||||
// 3. Update .env.example with the new var (plus a sensible default + comment).
|
||||
// 4. Update the "Environment variables supported by..." list in README.md.
|
||||
// 5. Update the example .env block in README.md.
|
||||
//
|
||||
// A test in config_test.go will fail if .env.example or README.md are missing any
|
||||
// entry from this list. This prevents the documentation drift that has happened
|
||||
// in the past when new config fields (e.g. shutdown_timeout, access_log details)
|
||||
// were added.
|
||||
var envSuffixes = []string{
|
||||
"SERVER_LISTEN_ADDR",
|
||||
"SERVER_READ_HEADER_TIMEOUT",
|
||||
"SERVER_READ_TIMEOUT",
|
||||
"SERVER_WRITE_TIMEOUT",
|
||||
"SERVER_IDLE_TIMEOUT",
|
||||
"SERVER_SHUTDOWN_TIMEOUT",
|
||||
"SERVER_MAX_HEADER_BYTES",
|
||||
"SERVER_ACCESS_LOG_ENABLED",
|
||||
"SERVER_ACCESS_LOG_FILE_PATH",
|
||||
"SERVER_ACCESS_LOG_MAX_SIZE",
|
||||
"SERVER_ACCESS_LOG_MAX_BACKUPS",
|
||||
"LIMITS_MAX_UPLOAD_SIZE",
|
||||
"LIMITS_DEFAULT_TTL",
|
||||
"LIMITS_RAW_CACHE_MAX_SIZE",
|
||||
"LIMITS_RAW_CACHE_MAX_ENTRIES",
|
||||
"STORAGE_DATA_DIR",
|
||||
"STORAGE_CLEANUP_INTERVAL",
|
||||
"SECURITY_ALLOWED_IPS",
|
||||
"SECURITY_TRUST_PROXY_HEADERS",
|
||||
"SECURITY_TRUSTED_PROXY_IPS",
|
||||
"SECURITY_RATE_LIMIT_UI_ENABLED",
|
||||
"SECURITY_RATE_LIMIT_UI_REQUESTS_PER_MINUTE",
|
||||
"SECURITY_RATE_LIMIT_UI_BURST",
|
||||
"SECURITY_RATE_LIMIT_API_READ_ENABLED",
|
||||
"SECURITY_RATE_LIMIT_API_READ_REQUESTS_PER_MINUTE",
|
||||
"SECURITY_RATE_LIMIT_API_READ_BURST",
|
||||
"SECURITY_RATE_LIMIT_API_WRITE_ENABLED",
|
||||
"SECURITY_RATE_LIMIT_API_WRITE_REQUESTS_PER_MINUTE",
|
||||
"SECURITY_RATE_LIMIT_API_WRITE_BURST",
|
||||
"SECURITY_HSTS_ENABLED",
|
||||
}
|
||||
|
||||
type Config struct {
|
||||
Server ServerConfig `yaml:"server" comment:"HTTP listener settings."`
|
||||
Limits LimitsConfig `yaml:"limits" comment:"Upload, expiration, and raw cache defaults."`
|
||||
@@ -59,8 +115,8 @@ type Config struct {
|
||||
type ServerConfig struct {
|
||||
ListenAddr string `yaml:"listen_addr" comment:"Address to bind the HTTP server to."`
|
||||
ReadHeaderTimeout string `yaml:"read_header_timeout" comment:"Maximum duration for reading request headers (for slowloris protection). Must be > 0."`
|
||||
ReadTimeout string `yaml:"read_timeout" comment:"Maximum duration for reading the full request, including body. Must be > 0."`
|
||||
WriteTimeout string `yaml:"write_timeout" comment:"Maximum duration before timing out writes of a response. Must be > 0."`
|
||||
ReadTimeout string `yaml:"read_timeout" comment:"Maximum duration for reading the full request, including body. Use 0 to disable (recommended when limits.max_upload_size is multi-GiB or clients are slow). Must be >= 0."`
|
||||
WriteTimeout string `yaml:"write_timeout" comment:"Maximum duration before timing out writes of a response (also bounds slow handlers after headers are read). Use 0 to disable (recommended for large uploads/downloads). Must be >= 0."`
|
||||
IdleTimeout string `yaml:"idle_timeout" comment:"Maximum amount of time to wait for the next request when keep-alives are enabled. Must be > 0."`
|
||||
ShutdownTimeout string `yaml:"shutdown_timeout" comment:"Graceful shutdown timeout for the HTTP server. Must be > 0."`
|
||||
MaxHeaderBytes int `yaml:"max_header_bytes" comment:"Maximum size of request headers in bytes. Must be > 0."`
|
||||
@@ -214,8 +270,9 @@ func (c *Config) Validate() error {
|
||||
if err != nil {
|
||||
return fmt.Errorf("server.read_timeout invalid: %w", err)
|
||||
}
|
||||
if readTimeout <= 0 {
|
||||
return errors.New("server.read_timeout must be > 0")
|
||||
// 0 means no timeout (matches net/http.Server.ReadTimeout). Negative is invalid.
|
||||
if readTimeout < 0 {
|
||||
return errors.New("server.read_timeout must be >= 0")
|
||||
}
|
||||
c.Server.ReadTimeoutDur = readTimeout
|
||||
|
||||
@@ -223,8 +280,9 @@ func (c *Config) Validate() error {
|
||||
if err != nil {
|
||||
return fmt.Errorf("server.write_timeout invalid: %w", err)
|
||||
}
|
||||
if writeTimeout <= 0 {
|
||||
return errors.New("server.write_timeout must be > 0")
|
||||
// 0 means no timeout (matches net/http.Server.WriteTimeout). Negative is invalid.
|
||||
if writeTimeout < 0 {
|
||||
return errors.New("server.write_timeout must be >= 0")
|
||||
}
|
||||
c.Server.WriteTimeoutDur = writeTimeout
|
||||
|
||||
@@ -530,6 +588,9 @@ func decodeEncryptionKey(raw string) ([]byte, error) {
|
||||
return encKey, nil
|
||||
}
|
||||
|
||||
// applyEnvOverrides applies all supported SCRATCHBOX_* vars.
|
||||
// The canonical list of supported suffixes is in envSuffixes (defined earlier in this file).
|
||||
// See the comment on envSuffixes for the full maintenance rules (update list + .env.example + README).
|
||||
func (c *Config) applyEnvOverrides(prefix string) error {
|
||||
if err := applyStringEnv(prefix+"SERVER_LISTEN_ADDR", &c.Server.ListenAddr); err != nil {
|
||||
return err
|
||||
|
||||
@@ -79,6 +79,27 @@ func TestValidateAppliesDefaultsAndParsedFields(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestValidateAllowsZeroReadAndWriteTimeout(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
cfg := defaults()
|
||||
cfg.Storage.DataDir = t.TempDir()
|
||||
// 0 disables the deadline (net/http semantics). Needed for multi-GiB uploads
|
||||
// where a fixed 30s body/response timeout cannot work.
|
||||
cfg.Server.ReadTimeout = "0s"
|
||||
cfg.Server.WriteTimeout = "0"
|
||||
|
||||
if err := cfg.Validate(); err != nil {
|
||||
t.Fatalf("Validate() returned error: %v", err)
|
||||
}
|
||||
if cfg.Server.ReadTimeoutDur != 0 {
|
||||
t.Fatalf("ReadTimeoutDur = %s, want 0", cfg.Server.ReadTimeoutDur)
|
||||
}
|
||||
if cfg.Server.WriteTimeoutDur != 0 {
|
||||
t.Fatalf("WriteTimeoutDur = %s, want 0", cfg.Server.WriteTimeoutDur)
|
||||
}
|
||||
}
|
||||
|
||||
func TestLoadParsesConfiguredFields(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
@@ -249,6 +270,13 @@ func TestValidateRejectsInvalidValues(t *testing.T) {
|
||||
},
|
||||
wantErr: "server.read_timeout",
|
||||
},
|
||||
{
|
||||
name: "negative read timeout",
|
||||
mutate: func(cfg *Config) {
|
||||
cfg.Server.ReadTimeout = "-1s"
|
||||
},
|
||||
wantErr: "server.read_timeout",
|
||||
},
|
||||
{
|
||||
name: "invalid write timeout",
|
||||
mutate: func(cfg *Config) {
|
||||
@@ -256,6 +284,13 @@ func TestValidateRejectsInvalidValues(t *testing.T) {
|
||||
},
|
||||
wantErr: "server.write_timeout",
|
||||
},
|
||||
{
|
||||
name: "negative write timeout",
|
||||
mutate: func(cfg *Config) {
|
||||
cfg.Server.WriteTimeout = "-1s"
|
||||
},
|
||||
wantErr: "server.write_timeout",
|
||||
},
|
||||
{
|
||||
name: "invalid idle timeout",
|
||||
mutate: func(cfg *Config) {
|
||||
@@ -713,3 +748,46 @@ func TestGenerateEncryptionKey(t *testing.T) {
|
||||
t.Fatalf("decoded key length = %d, want 32", len(b1))
|
||||
}
|
||||
}
|
||||
|
||||
// TestEnvDocumentationIsUpToDate ensures .env.example and the env var
|
||||
// documentation + example in README.md stay in sync with the env vars
|
||||
// actually supported by the code (envSuffixes in config.go).
|
||||
// See the comment on envSuffixes for the list of places that must point back to it.
|
||||
//
|
||||
// This is the guard against the recurring "example env and readme have
|
||||
// become out of sync" problem. When you add a new field that is loaded via
|
||||
// applyEnvOverrides, you must:
|
||||
// - add its suffix to envSuffixes
|
||||
// - add a line (with example value + comment) to .env.example
|
||||
// - add it to the list and the ```dotenv example in README.md
|
||||
//
|
||||
// The test will fail loudly until the docs are updated.
|
||||
func TestEnvDocumentationIsUpToDate(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
// Check .env.example (at repo root). Note: go test for this package
|
||||
// runs with cwd set to the package directory (internal/config), so we
|
||||
// need to go up two levels.
|
||||
envExample, err := os.ReadFile("../../.env.example")
|
||||
if err != nil {
|
||||
t.Fatalf("failed to read .env.example: %v", err)
|
||||
}
|
||||
for _, suffix := range envSuffixes {
|
||||
name := "SCRATCHBOX_" + suffix
|
||||
if !strings.Contains(string(envExample), name) {
|
||||
t.Errorf(".env.example is missing %s (add it with a default/example value and a comment matching the style of the others)", name)
|
||||
}
|
||||
}
|
||||
|
||||
// Check README.md env documentation (the supported list and the example block)
|
||||
readme, err := os.ReadFile("../../README.md")
|
||||
if err != nil {
|
||||
t.Fatalf("failed to read README.md: %v", err)
|
||||
}
|
||||
for _, suffix := range envSuffixes {
|
||||
name := "SCRATCHBOX_" + suffix
|
||||
if !strings.Contains(string(readme), name) {
|
||||
t.Errorf("README.md env docs (supported list or example .env block) are missing %s; update both the bullet list and the ```dotenv block", name)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -172,6 +172,12 @@ func (s *Server) scratchPage(w http.ResponseWriter, r *http.Request) {
|
||||
})
|
||||
}
|
||||
|
||||
// multipartMaxMemory is how much of a multipart upload may live in RAM before
|
||||
// the remainder spills to temporary files. It must NOT be tied to
|
||||
// limits.max_upload_size: when that limit is multi-GiB, using it as maxMemory
|
||||
// forces entire large files into process memory before storage compression.
|
||||
const multipartMaxMemory = 32 << 20 // 32 MiB
|
||||
|
||||
func (s *Server) createScratch(w http.ResponseWriter, r *http.Request) {
|
||||
r.Body = http.MaxBytesReader(w, r.Body, s.cfg.Limits.MaxUploadSizeBytes)
|
||||
|
||||
@@ -180,7 +186,7 @@ func (s *Server) createScratch(w http.ResponseWriter, r *http.Request) {
|
||||
contentType := strings.TrimSpace(r.Header.Get("Content-Type"))
|
||||
|
||||
if strings.HasPrefix(contentType, "multipart/form-data") {
|
||||
if err := r.ParseMultipartForm(s.cfg.Limits.MaxUploadSizeBytes); err != nil {
|
||||
if err := r.ParseMultipartForm(multipartMaxMemory); err != nil {
|
||||
s.writeCreateError(w, err)
|
||||
return
|
||||
}
|
||||
@@ -315,6 +321,29 @@ func (s *Server) rawScratch(w http.ResponseWriter, r *http.Request) {
|
||||
w.Header().Set("Content-Disposition", disposition)
|
||||
}
|
||||
w.Header().Set("Content-Type", meta.ContentType)
|
||||
|
||||
// Scratches larger than the raw cache cannot be retained in memory. Stream
|
||||
// them from storage instead of io.ReadAll so multi-GiB downloads do not OOM.
|
||||
// Range requests are only supported via the cache path (ServeContent needs a Seeker).
|
||||
if meta.Size > s.cfg.Limits.RawCacheMaxBytes {
|
||||
file, _, openErr := s.store.Open(meta.ID)
|
||||
if openErr != nil {
|
||||
if errors.Is(openErr, storage.ErrNotFound) {
|
||||
s.rawCache.Delete(id)
|
||||
w.WriteHeader(http.StatusGone)
|
||||
return
|
||||
}
|
||||
http.Error(w, "failed to read scratch", http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
defer file.Close()
|
||||
w.Header().Set("Accept-Ranges", "none")
|
||||
w.Header().Set("Content-Length", fmt.Sprintf("%d", meta.Size))
|
||||
w.WriteHeader(http.StatusOK)
|
||||
_, _ = io.Copy(w, file)
|
||||
return
|
||||
}
|
||||
|
||||
content, err, _ := s.rawCache.GetOrLoad(meta.ID, func() ([]byte, error) {
|
||||
file, _, openErr := s.store.Open(meta.ID)
|
||||
if openErr != nil {
|
||||
|
||||
@@ -373,3 +373,49 @@ func min(a, b int) int {
|
||||
}
|
||||
return b
|
||||
}
|
||||
|
||||
// Scratches larger than limits.raw_cache_max_size must stream from storage
|
||||
// (no full in-memory buffer) so multi-GiB downloads stay viable.
|
||||
func TestRawStreamsWhenLargerThanCache(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
payload := bytes.Repeat([]byte("Z"), 8*1024) // 8 KiB body
|
||||
handler := newTestHandler(t, testServerOptions{
|
||||
maxUploadBytes: int64(len(payload) * 2),
|
||||
defaultTTL: time.Hour,
|
||||
rateLimitEnable: false,
|
||||
rawCacheBytes: 1024, // smaller than payload => stream path
|
||||
})
|
||||
|
||||
createReq := httptest.NewRequest(http.MethodPost, "/api/scratch", bytes.NewReader(payload))
|
||||
createReq.Header.Set("Content-Type", "application/octet-stream")
|
||||
createReq.RemoteAddr = "127.0.0.1:7100"
|
||||
createRec := httptest.NewRecorder()
|
||||
handler.ServeHTTP(createRec, createReq)
|
||||
if createRec.Code != http.StatusCreated {
|
||||
t.Fatalf("create status = %d, want 201 body=%q", createRec.Code, createRec.Body.String())
|
||||
}
|
||||
|
||||
var created map[string]any
|
||||
if err := json.Unmarshal(createRec.Body.Bytes(), &created); err != nil {
|
||||
t.Fatalf("decode create payload: %v", err)
|
||||
}
|
||||
id, _ := created["id"].(string)
|
||||
if id == "" {
|
||||
t.Fatal("create payload missing id")
|
||||
}
|
||||
|
||||
rawReq := httptest.NewRequest(http.MethodGet, "/api/raw/"+id, nil)
|
||||
rawReq.RemoteAddr = "127.0.0.1:7101"
|
||||
rawRec := httptest.NewRecorder()
|
||||
handler.ServeHTTP(rawRec, rawReq)
|
||||
if rawRec.Code != http.StatusOK {
|
||||
t.Fatalf("raw status = %d, want 200", rawRec.Code)
|
||||
}
|
||||
if got := rawRec.Header().Get("Accept-Ranges"); got != "none" {
|
||||
t.Fatalf("Accept-Ranges = %q, want none (stream path)", got)
|
||||
}
|
||||
if got := rawRec.Body.Bytes(); !bytes.Equal(got, payload) {
|
||||
t.Fatalf("raw body len=%d, want %d (content mismatch or truncated)", len(got), len(payload))
|
||||
}
|
||||
}
|
||||
|
||||
@@ -18,6 +18,7 @@ type testServerOptions struct {
|
||||
maxUploadBytes int64
|
||||
defaultTTL time.Duration
|
||||
rateLimitEnable bool
|
||||
rawCacheBytes int64 // 0 => default 64MiB
|
||||
}
|
||||
|
||||
func newTestHandler(t *testing.T, opts testServerOptions) http.Handler {
|
||||
@@ -36,6 +37,11 @@ func newTestHandlerAndStore(t *testing.T, opts testServerOptions) (http.Handler,
|
||||
t.Fatalf("NewFilesystemStore() error = %v", err)
|
||||
}
|
||||
|
||||
rawCacheBytes := opts.rawCacheBytes
|
||||
if rawCacheBytes <= 0 {
|
||||
rawCacheBytes = 64 * 1024 * 1024
|
||||
}
|
||||
|
||||
cfg := config.Config{
|
||||
Server: config.ServerConfig{
|
||||
ListenAddr: ":0",
|
||||
@@ -43,7 +49,7 @@ func newTestHandlerAndStore(t *testing.T, opts testServerOptions) (http.Handler,
|
||||
Limits: config.LimitsConfig{
|
||||
MaxUploadSizeBytes: opts.maxUploadBytes,
|
||||
DefaultTTLDuration: opts.defaultTTL,
|
||||
RawCacheMaxBytes: 64 * 1024 * 1024,
|
||||
RawCacheMaxBytes: rawCacheBytes,
|
||||
RawCacheMaxEntries: 32,
|
||||
},
|
||||
Storage: config.StorageConfig{
|
||||
|
||||
Reference in New Issue
Block a user