Enhance environment variable documentation and configuration
- Updated `.env.example` to include new environment variables for server access logging and security settings, ensuring comprehensive configuration options. - Revised `docker-entrypoint.sh` to clarify the source of supported environment variables. - Improved `README.md` to reflect the latest environment variable changes and maintain synchronization with the canonical list in `internal/config/config.go`. - Added tests to ensure documentation consistency across `.env.example` and `README.md`, preventing future discrepancies.
This commit is contained in:
@@ -172,6 +172,12 @@ func (s *Server) scratchPage(w http.ResponseWriter, r *http.Request) {
|
||||
})
|
||||
}
|
||||
|
||||
// multipartMaxMemory is how much of a multipart upload may live in RAM before
|
||||
// the remainder spills to temporary files. It must NOT be tied to
|
||||
// limits.max_upload_size: when that limit is multi-GiB, using it as maxMemory
|
||||
// forces entire large files into process memory before storage compression.
|
||||
const multipartMaxMemory = 32 << 20 // 32 MiB
|
||||
|
||||
func (s *Server) createScratch(w http.ResponseWriter, r *http.Request) {
|
||||
r.Body = http.MaxBytesReader(w, r.Body, s.cfg.Limits.MaxUploadSizeBytes)
|
||||
|
||||
@@ -180,7 +186,7 @@ func (s *Server) createScratch(w http.ResponseWriter, r *http.Request) {
|
||||
contentType := strings.TrimSpace(r.Header.Get("Content-Type"))
|
||||
|
||||
if strings.HasPrefix(contentType, "multipart/form-data") {
|
||||
if err := r.ParseMultipartForm(s.cfg.Limits.MaxUploadSizeBytes); err != nil {
|
||||
if err := r.ParseMultipartForm(multipartMaxMemory); err != nil {
|
||||
s.writeCreateError(w, err)
|
||||
return
|
||||
}
|
||||
@@ -315,6 +321,29 @@ func (s *Server) rawScratch(w http.ResponseWriter, r *http.Request) {
|
||||
w.Header().Set("Content-Disposition", disposition)
|
||||
}
|
||||
w.Header().Set("Content-Type", meta.ContentType)
|
||||
|
||||
// Scratches larger than the raw cache cannot be retained in memory. Stream
|
||||
// them from storage instead of io.ReadAll so multi-GiB downloads do not OOM.
|
||||
// Range requests are only supported via the cache path (ServeContent needs a Seeker).
|
||||
if meta.Size > s.cfg.Limits.RawCacheMaxBytes {
|
||||
file, _, openErr := s.store.Open(meta.ID)
|
||||
if openErr != nil {
|
||||
if errors.Is(openErr, storage.ErrNotFound) {
|
||||
s.rawCache.Delete(id)
|
||||
w.WriteHeader(http.StatusGone)
|
||||
return
|
||||
}
|
||||
http.Error(w, "failed to read scratch", http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
defer file.Close()
|
||||
w.Header().Set("Accept-Ranges", "none")
|
||||
w.Header().Set("Content-Length", fmt.Sprintf("%d", meta.Size))
|
||||
w.WriteHeader(http.StatusOK)
|
||||
_, _ = io.Copy(w, file)
|
||||
return
|
||||
}
|
||||
|
||||
content, err, _ := s.rawCache.GetOrLoad(meta.ID, func() ([]byte, error) {
|
||||
file, _, openErr := s.store.Open(meta.ID)
|
||||
if openErr != nil {
|
||||
|
||||
@@ -373,3 +373,49 @@ func min(a, b int) int {
|
||||
}
|
||||
return b
|
||||
}
|
||||
|
||||
// Scratches larger than limits.raw_cache_max_size must stream from storage
|
||||
// (no full in-memory buffer) so multi-GiB downloads stay viable.
|
||||
func TestRawStreamsWhenLargerThanCache(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
payload := bytes.Repeat([]byte("Z"), 8*1024) // 8 KiB body
|
||||
handler := newTestHandler(t, testServerOptions{
|
||||
maxUploadBytes: int64(len(payload) * 2),
|
||||
defaultTTL: time.Hour,
|
||||
rateLimitEnable: false,
|
||||
rawCacheBytes: 1024, // smaller than payload => stream path
|
||||
})
|
||||
|
||||
createReq := httptest.NewRequest(http.MethodPost, "/api/scratch", bytes.NewReader(payload))
|
||||
createReq.Header.Set("Content-Type", "application/octet-stream")
|
||||
createReq.RemoteAddr = "127.0.0.1:7100"
|
||||
createRec := httptest.NewRecorder()
|
||||
handler.ServeHTTP(createRec, createReq)
|
||||
if createRec.Code != http.StatusCreated {
|
||||
t.Fatalf("create status = %d, want 201 body=%q", createRec.Code, createRec.Body.String())
|
||||
}
|
||||
|
||||
var created map[string]any
|
||||
if err := json.Unmarshal(createRec.Body.Bytes(), &created); err != nil {
|
||||
t.Fatalf("decode create payload: %v", err)
|
||||
}
|
||||
id, _ := created["id"].(string)
|
||||
if id == "" {
|
||||
t.Fatal("create payload missing id")
|
||||
}
|
||||
|
||||
rawReq := httptest.NewRequest(http.MethodGet, "/api/raw/"+id, nil)
|
||||
rawReq.RemoteAddr = "127.0.0.1:7101"
|
||||
rawRec := httptest.NewRecorder()
|
||||
handler.ServeHTTP(rawRec, rawReq)
|
||||
if rawRec.Code != http.StatusOK {
|
||||
t.Fatalf("raw status = %d, want 200", rawRec.Code)
|
||||
}
|
||||
if got := rawRec.Header().Get("Accept-Ranges"); got != "none" {
|
||||
t.Fatalf("Accept-Ranges = %q, want none (stream path)", got)
|
||||
}
|
||||
if got := rawRec.Body.Bytes(); !bytes.Equal(got, payload) {
|
||||
t.Fatalf("raw body len=%d, want %d (content mismatch or truncated)", len(got), len(payload))
|
||||
}
|
||||
}
|
||||
|
||||
@@ -18,6 +18,7 @@ type testServerOptions struct {
|
||||
maxUploadBytes int64
|
||||
defaultTTL time.Duration
|
||||
rateLimitEnable bool
|
||||
rawCacheBytes int64 // 0 => default 64MiB
|
||||
}
|
||||
|
||||
func newTestHandler(t *testing.T, opts testServerOptions) http.Handler {
|
||||
@@ -36,6 +37,11 @@ func newTestHandlerAndStore(t *testing.T, opts testServerOptions) (http.Handler,
|
||||
t.Fatalf("NewFilesystemStore() error = %v", err)
|
||||
}
|
||||
|
||||
rawCacheBytes := opts.rawCacheBytes
|
||||
if rawCacheBytes <= 0 {
|
||||
rawCacheBytes = 64 * 1024 * 1024
|
||||
}
|
||||
|
||||
cfg := config.Config{
|
||||
Server: config.ServerConfig{
|
||||
ListenAddr: ":0",
|
||||
@@ -43,7 +49,7 @@ func newTestHandlerAndStore(t *testing.T, opts testServerOptions) (http.Handler,
|
||||
Limits: config.LimitsConfig{
|
||||
MaxUploadSizeBytes: opts.maxUploadBytes,
|
||||
DefaultTTLDuration: opts.defaultTTL,
|
||||
RawCacheMaxBytes: 64 * 1024 * 1024,
|
||||
RawCacheMaxBytes: rawCacheBytes,
|
||||
RawCacheMaxEntries: 32,
|
||||
},
|
||||
Storage: config.StorageConfig{
|
||||
|
||||
Reference in New Issue
Block a user