initial commit

This commit is contained in:
2026-05-29 22:39:50 -05:00
commit 47c767f9e8
23 changed files with 3361 additions and 0 deletions
+44
View File
@@ -0,0 +1,44 @@
package cleanup
import (
"context"
"log"
"time"
"scratchbox/internal/storage"
)
type Worker struct {
store *storage.FilesystemStore
interval time.Duration
logger *log.Logger
}
func NewWorker(store *storage.FilesystemStore, interval time.Duration, logger *log.Logger) *Worker {
return &Worker{
store: store,
interval: interval,
logger: logger,
}
}
func (w *Worker) Start(ctx context.Context) {
ticker := time.NewTicker(w.interval)
defer ticker.Stop()
for {
select {
case <-ctx.Done():
return
case now := <-ticker.C:
deleted, err := w.store.DeleteExpired(now)
if err != nil {
w.logger.Printf("cleanup worker failed: %v", err)
continue
}
if deleted > 0 {
w.logger.Printf("cleanup worker removed %d expired scratches", deleted)
}
}
}
}
+74
View File
@@ -0,0 +1,74 @@
package cleanup
import (
"bytes"
"context"
"log"
"path/filepath"
"strings"
"testing"
"time"
"scratchbox/internal/storage"
)
func TestNewWorker(t *testing.T) {
t.Parallel()
store, err := storage.NewFilesystemStore(filepath.Join(t.TempDir(), "data"))
if err != nil {
t.Fatalf("NewFilesystemStore() error = %v", err)
}
logger := log.New(&bytes.Buffer{}, "", 0)
w := NewWorker(store, 20*time.Millisecond, logger)
if w == nil {
t.Fatalf("NewWorker() = nil")
}
}
func TestWorkerStartRemovesExpiredAndLogs(t *testing.T) {
t.Parallel()
store, err := storage.NewFilesystemStore(filepath.Join(t.TempDir(), "data"))
if err != nil {
t.Fatalf("NewFilesystemStore() error = %v", err)
}
if _, err := store.Create(context.Background(), strings.NewReader("expired"), "text/plain", -time.Second); err != nil {
t.Fatalf("Create() error = %v", err)
}
var buf bytes.Buffer
logger := log.New(&buf, "", 0)
w := NewWorker(store, 10*time.Millisecond, logger)
ctx, cancel := context.WithTimeout(context.Background(), 80*time.Millisecond)
defer cancel()
w.Start(ctx)
if !strings.Contains(buf.String(), "cleanup worker removed") {
t.Fatalf("expected cleanup removal log, got %q", buf.String())
}
}
func TestWorkerStartWithNoExpiredEntries(t *testing.T) {
t.Parallel()
store, err := storage.NewFilesystemStore(filepath.Join(t.TempDir(), "data"))
if err != nil {
t.Fatalf("NewFilesystemStore() error = %v", err)
}
if _, err := store.Create(context.Background(), strings.NewReader("active"), "text/plain", time.Hour); err != nil {
t.Fatalf("Create() error = %v", err)
}
var buf bytes.Buffer
logger := log.New(&buf, "", 0)
w := NewWorker(store, 10*time.Millisecond, logger)
ctx, cancel := context.WithTimeout(context.Background(), 35*time.Millisecond)
defer cancel()
w.Start(ctx)
if strings.Contains(buf.String(), "removed") {
t.Fatalf("expected no removal log, got %q", buf.String())
}
}
+251
View File
@@ -0,0 +1,251 @@
package config
import (
"errors"
"fmt"
"net/netip"
"os"
"path/filepath"
"strconv"
"strings"
"time"
"gopkg.in/yaml.v3"
)
const (
defaultListenAddr = ":8080"
defaultMaxUploadSize = "100MB"
defaultTTL = "1h"
defaultDataDir = "./data"
defaultCleanupInterval = "1m"
defaultRateLimitEnabled = true
defaultRequestsPerMinute = 30
defaultRateLimitBurst = 10
maxDefaultTTLLimit = 24 * time.Hour
minCleanupInterval = 10 * time.Second
)
type Config struct {
Server ServerConfig `yaml:"server"`
Limits LimitsConfig `yaml:"limits"`
Storage StorageConfig `yaml:"storage"`
Security SecurityConfig `yaml:"security"`
}
type ServerConfig struct {
ListenAddr string `yaml:"listen_addr"`
}
type LimitsConfig struct {
MaxUploadSize string `yaml:"max_upload_size"`
DefaultTTL string `yaml:"default_ttl"`
MaxUploadSizeBytes int64 `yaml:"-"`
DefaultTTLDuration time.Duration `yaml:"-"`
}
type StorageConfig struct {
DataDir string `yaml:"data_dir"`
CleanupInterval string `yaml:"cleanup_interval"`
CleanupIntervalParsed time.Duration `yaml:"-"`
}
type SecurityConfig struct {
AllowedIPs []string `yaml:"allowed_ips"`
TrustProxyHeaders bool `yaml:"trust_proxy_headers"`
RateLimit RateLimitConfig `yaml:"rate_limit"`
AllowedPrefixes []netip.Prefix `yaml:"-"`
}
type RateLimitConfig struct {
Enabled bool `yaml:"enabled"`
RequestsPerMinute int `yaml:"requests_per_minute"`
Burst int `yaml:"burst"`
}
func Load(path string) (Config, error) {
cfg := defaults()
if path != "" {
b, err := os.ReadFile(path)
if err != nil {
return Config{}, fmt.Errorf("read config: %w", err)
}
if err := yaml.Unmarshal(b, &cfg); err != nil {
return Config{}, fmt.Errorf("parse config yaml: %w", err)
}
}
if err := cfg.Validate(); err != nil {
return Config{}, err
}
return cfg, nil
}
func defaults() Config {
return Config{
Server: ServerConfig{
ListenAddr: defaultListenAddr,
},
Limits: LimitsConfig{
MaxUploadSize: defaultMaxUploadSize,
DefaultTTL: defaultTTL,
},
Storage: StorageConfig{
DataDir: defaultDataDir,
CleanupInterval: defaultCleanupInterval,
},
Security: SecurityConfig{
RateLimit: RateLimitConfig{
Enabled: defaultRateLimitEnabled,
RequestsPerMinute: defaultRequestsPerMinute,
Burst: defaultRateLimitBurst,
},
},
}
}
func (c *Config) Validate() error {
if strings.TrimSpace(c.Server.ListenAddr) == "" {
return errors.New("server.listen_addr is required")
}
sizeBytes, err := parseHumanSize(c.Limits.MaxUploadSize)
if err != nil {
return fmt.Errorf("limits.max_upload_size invalid: %w", err)
}
c.Limits.MaxUploadSizeBytes = sizeBytes
ttl, err := time.ParseDuration(strings.TrimSpace(c.Limits.DefaultTTL))
if err != nil {
return fmt.Errorf("limits.default_ttl invalid: %w", err)
}
if ttl <= 0 || ttl > maxDefaultTTLLimit {
return fmt.Errorf("limits.default_ttl must be in (0, %s]", maxDefaultTTLLimit)
}
c.Limits.DefaultTTLDuration = ttl
cleanupInterval, err := time.ParseDuration(strings.TrimSpace(c.Storage.CleanupInterval))
if err != nil {
return fmt.Errorf("storage.cleanup_interval invalid: %w", err)
}
if cleanupInterval < minCleanupInterval {
return fmt.Errorf("storage.cleanup_interval must be at least %s", minCleanupInterval)
}
c.Storage.CleanupIntervalParsed = cleanupInterval
if strings.TrimSpace(c.Storage.DataDir) == "" {
return errors.New("storage.data_dir is required")
}
if err := ensureWritableDir(c.Storage.DataDir); err != nil {
return fmt.Errorf("storage.data_dir not writable: %w", err)
}
prefixes, err := parseAllowedPrefixes(c.Security.AllowedIPs)
if err != nil {
return fmt.Errorf("security.allowed_ips invalid: %w", err)
}
c.Security.AllowedPrefixes = prefixes
if c.Security.RateLimit.RequestsPerMinute <= 0 {
return errors.New("security.rate_limit.requests_per_minute must be > 0")
}
if c.Security.RateLimit.Burst <= 0 {
return errors.New("security.rate_limit.burst must be > 0")
}
return nil
}
func ensureWritableDir(dir string) error {
if err := os.MkdirAll(dir, 0o755); err != nil {
return err
}
probe := filepath.Join(dir, ".write_probe")
if err := os.WriteFile(probe, []byte("ok"), 0o644); err != nil {
return err
}
return os.Remove(probe)
}
func parseAllowedPrefixes(values []string) ([]netip.Prefix, error) {
if len(values) == 0 {
return nil, nil
}
out := make([]netip.Prefix, 0, len(values))
for _, value := range values {
item := strings.TrimSpace(value)
if item == "" {
continue
}
if strings.Contains(item, "/") {
prefix, err := netip.ParsePrefix(item)
if err != nil {
return nil, fmt.Errorf("parse cidr %q: %w", item, err)
}
out = append(out, prefix)
continue
}
ip, err := netip.ParseAddr(item)
if err != nil {
return nil, fmt.Errorf("parse ip %q: %w", item, err)
}
out = append(out, netip.PrefixFrom(ip, ip.BitLen()))
}
return out, nil
}
func parseHumanSize(raw string) (int64, error) {
value := strings.TrimSpace(strings.ToUpper(raw))
if value == "" {
return 0, errors.New("size is empty")
}
idx := 0
for idx < len(value) && (value[idx] == '.' || (value[idx] >= '0' && value[idx] <= '9')) {
idx++
}
if idx == 0 {
return 0, fmt.Errorf("size %q does not start with a number", raw)
}
numPart := value[:idx]
unitPart := strings.TrimSpace(value[idx:])
if unitPart == "" {
unitPart = "B"
}
number, err := strconv.ParseFloat(numPart, 64)
if err != nil {
return 0, fmt.Errorf("parse number %q: %w", numPart, err)
}
if number <= 0 {
return 0, errors.New("size must be > 0")
}
multipliers := map[string]float64{
"B": 1,
"KB": 1_000,
"MB": 1_000_000,
"GB": 1_000_000_000,
"TB": 1_000_000_000_000,
"KIB": 1024,
"MIB": 1024 * 1024,
"GIB": 1024 * 1024 * 1024,
"TIB": 1024 * 1024 * 1024 * 1024,
}
multiplier, ok := multipliers[unitPart]
if !ok {
return 0, fmt.Errorf("unsupported unit %q", unitPart)
}
size := int64(number * multiplier)
if size <= 0 {
return 0, errors.New("calculated size must be > 0")
}
return size, nil
}
+264
View File
@@ -0,0 +1,264 @@
package config
import (
"net/netip"
"os"
"path/filepath"
"strings"
"testing"
"time"
)
func TestValidateAppliesDefaultsAndParsedFields(t *testing.T) {
t.Parallel()
cfg := defaults()
cfg.Storage.DataDir = t.TempDir()
if err := cfg.Validate(); err != nil {
t.Fatalf("Validate() returned error: %v", err)
}
if cfg.Limits.MaxUploadSizeBytes != 100_000_000 {
t.Fatalf("MaxUploadSizeBytes = %d, want %d", cfg.Limits.MaxUploadSizeBytes, int64(100_000_000))
}
if cfg.Limits.DefaultTTLDuration != time.Hour {
t.Fatalf("DefaultTTLDuration = %s, want 1h", cfg.Limits.DefaultTTLDuration)
}
if cfg.Storage.CleanupIntervalParsed != time.Minute {
t.Fatalf("CleanupIntervalParsed = %s, want 1m", cfg.Storage.CleanupIntervalParsed)
}
if len(cfg.Security.AllowedPrefixes) != 0 {
t.Fatalf("AllowedPrefixes len = %d, want 0", len(cfg.Security.AllowedPrefixes))
}
}
func TestLoadParsesConfiguredFields(t *testing.T) {
t.Parallel()
tmp := t.TempDir()
dataDir := filepath.Join(tmp, "store")
configPath := filepath.Join(tmp, "config.yaml")
content := `
server:
listen_addr: "127.0.0.1:9090"
limits:
max_upload_size: "2MiB"
default_ttl: "2h"
storage:
data_dir: "` + dataDir + `"
cleanup_interval: "45s"
security:
allowed_ips:
- "127.0.0.1"
- "10.0.0.0/8"
trust_proxy_headers: true
rate_limit:
enabled: true
requests_per_minute: 12
burst: 4
`
if err := os.WriteFile(configPath, []byte(content), 0o644); err != nil {
t.Fatalf("WriteFile() error = %v", err)
}
cfg, err := Load(configPath)
if err != nil {
t.Fatalf("Load() error = %v", err)
}
if cfg.Server.ListenAddr != "127.0.0.1:9090" {
t.Fatalf("ListenAddr = %q, want %q", cfg.Server.ListenAddr, "127.0.0.1:9090")
}
if cfg.Limits.MaxUploadSizeBytes != 2*1024*1024 {
t.Fatalf("MaxUploadSizeBytes = %d, want %d", cfg.Limits.MaxUploadSizeBytes, int64(2*1024*1024))
}
if cfg.Limits.DefaultTTLDuration != 2*time.Hour {
t.Fatalf("DefaultTTLDuration = %s, want 2h", cfg.Limits.DefaultTTLDuration)
}
if cfg.Storage.CleanupIntervalParsed != 45*time.Second {
t.Fatalf("CleanupIntervalParsed = %s, want 45s", cfg.Storage.CleanupIntervalParsed)
}
if !cfg.Security.TrustProxyHeaders {
t.Fatalf("TrustProxyHeaders = false, want true")
}
if cfg.Security.RateLimit.RequestsPerMinute != 12 || cfg.Security.RateLimit.Burst != 4 {
t.Fatalf("unexpected rate_limit values: %+v", cfg.Security.RateLimit)
}
if len(cfg.Security.AllowedPrefixes) != 2 {
t.Fatalf("AllowedPrefixes len = %d, want 2", len(cfg.Security.AllowedPrefixes))
}
}
func TestValidateRejectsInvalidValues(t *testing.T) {
t.Parallel()
tests := []struct {
name string
mutate func(*Config)
wantErr string
}{
{
name: "listen addr required",
mutate: func(cfg *Config) {
cfg.Server.ListenAddr = " "
},
wantErr: "server.listen_addr",
},
{
name: "invalid max upload size",
mutate: func(cfg *Config) {
cfg.Limits.MaxUploadSize = "oops"
},
wantErr: "limits.max_upload_size",
},
{
name: "invalid ttl parse",
mutate: func(cfg *Config) {
cfg.Limits.DefaultTTL = "not-a-duration"
},
wantErr: "limits.default_ttl",
},
{
name: "ttl above max",
mutate: func(cfg *Config) {
cfg.Limits.DefaultTTL = "25h"
},
wantErr: "limits.default_ttl",
},
{
name: "invalid allowlist entry",
mutate: func(cfg *Config) {
cfg.Security.AllowedIPs = []string{"not-an-ip"}
},
wantErr: "security.allowed_ips",
},
{
name: "invalid rate limit burst",
mutate: func(cfg *Config) {
cfg.Security.RateLimit.Burst = 0
},
wantErr: "security.rate_limit.burst",
},
{
name: "data dir required",
mutate: func(cfg *Config) {
cfg.Storage.DataDir = " "
},
wantErr: "storage.data_dir",
},
{
name: "cleanup parse failure",
mutate: func(cfg *Config) {
cfg.Storage.CleanupInterval = "not-a-duration"
},
wantErr: "storage.cleanup_interval",
},
{
name: "cleanup too small",
mutate: func(cfg *Config) {
cfg.Storage.CleanupInterval = "5s"
},
wantErr: "storage.cleanup_interval",
},
{
name: "invalid rate limit rpm",
mutate: func(cfg *Config) {
cfg.Security.RateLimit.RequestsPerMinute = 0
},
wantErr: "security.rate_limit.requests_per_minute",
},
}
for _, tc := range tests {
tc := tc
t.Run(tc.name, func(t *testing.T) {
t.Parallel()
cfg := defaults()
cfg.Storage.DataDir = t.TempDir()
tc.mutate(&cfg)
err := cfg.Validate()
if err == nil {
t.Fatalf("Validate() error = nil, want non-nil")
}
if !strings.Contains(err.Error(), tc.wantErr) {
t.Fatalf("Validate() error = %q, want substring %q", err.Error(), tc.wantErr)
}
})
}
}
func TestLoadErrors(t *testing.T) {
t.Parallel()
_, err := Load(filepath.Join(t.TempDir(), "missing.yaml"))
if err == nil || !strings.Contains(err.Error(), "read config") {
t.Fatalf("Load() error = %v, want read config error", err)
}
tmp := t.TempDir()
configPath := filepath.Join(tmp, "bad.yaml")
if err := os.WriteFile(configPath, []byte("limits: [bad"), 0o644); err != nil {
t.Fatalf("WriteFile() error = %v", err)
}
_, err = Load(configPath)
if err == nil || !strings.Contains(err.Error(), "parse config yaml") {
t.Fatalf("Load() error = %v, want yaml parse error", err)
}
}
func TestParseAllowedPrefixesTrimsAndParses(t *testing.T) {
t.Parallel()
prefixes, err := parseAllowedPrefixes([]string{" 192.0.2.4 ", " ", "2001:db8::/32"})
if err != nil {
t.Fatalf("parseAllowedPrefixes() error = %v", err)
}
if len(prefixes) != 2 {
t.Fatalf("prefixes len = %d, want 2", len(prefixes))
}
wantIP := netip.MustParsePrefix("192.0.2.4/32")
if prefixes[0] != wantIP {
t.Fatalf("prefixes[0] = %v, want %v", prefixes[0], wantIP)
}
}
func TestParseHumanSizeVariants(t *testing.T) {
t.Parallel()
tests := []struct {
in string
want int64
wantErr string
}{
{in: "10", want: 10},
{in: "1.5KB", want: 1500},
{in: "2MiB", want: 2 * 1024 * 1024},
{in: "0", wantErr: "size must be > 0"},
{in: "abc", wantErr: "does not start with a number"},
{in: "5XB", wantErr: "unsupported unit"},
{in: "", wantErr: "size is empty"},
}
for _, tc := range tests {
tc := tc
t.Run(tc.in, func(t *testing.T) {
t.Parallel()
got, err := parseHumanSize(tc.in)
if tc.wantErr != "" {
if err == nil || !strings.Contains(err.Error(), tc.wantErr) {
t.Fatalf("parseHumanSize(%q) err=%v, want contains %q", tc.in, err, tc.wantErr)
}
return
}
if err != nil {
t.Fatalf("parseHumanSize(%q) error = %v", tc.in, err)
}
if got != tc.want {
t.Fatalf("parseHumanSize(%q) = %d, want %d", tc.in, got, tc.want)
}
})
}
}
+294
View File
@@ -0,0 +1,294 @@
package httpapi
import (
"errors"
"fmt"
"html/template"
"io"
"log"
"mime"
"net/http"
"path/filepath"
"strings"
"time"
"unicode/utf8"
"scratchbox/internal/config"
"scratchbox/internal/storage"
)
const maxViewPreviewBytes int64 = 2 * 1024 * 1024
type Server struct {
cfg config.Config
store *storage.FilesystemStore
logger *log.Logger
templates *template.Template
}
type templateViewData struct {
ID string
Content string
ContentType string
RawURL string
ExpiresAt string
Truncated bool
Binary bool
}
func NewServer(cfg config.Config, store *storage.FilesystemStore, logger *log.Logger) (*Server, error) {
tmpl, err := template.ParseFiles(
filepath.Join("web", "templates", "index.html"),
filepath.Join("web", "templates", "view.html"),
)
if err != nil {
return nil, fmt.Errorf("parse templates: %w", err)
}
return &Server{
cfg: cfg,
store: store,
logger: logger,
templates: tmpl,
}, nil
}
func (s *Server) Routes() http.Handler {
mux := http.NewServeMux()
mux.Handle("GET /", http.HandlerFunc(s.index))
mux.Handle("GET /api/scratch/{id}", http.HandlerFunc(s.getScratch))
mux.Handle("GET /s/{id}", http.HandlerFunc(s.viewScratch))
mux.Handle("GET /raw/{id}", http.HandlerFunc(s.rawScratch))
writeHandler := http.HandlerFunc(s.createScratch)
middlewares := make([]func(http.Handler) http.Handler, 0, 2)
middlewares = append(middlewares, AllowlistMiddleware(s.cfg.Security.AllowedPrefixes, s.cfg.Security.TrustProxyHeaders, s.logger))
if s.cfg.Security.RateLimit.Enabled {
limiter := NewRateLimiter(s.cfg.Security.RateLimit.RequestsPerMinute, s.cfg.Security.RateLimit.Burst)
middlewares = append(middlewares, RateLimitMiddleware(limiter, s.cfg.Security.TrustProxyHeaders))
}
mux.Handle("POST /api/scratch", Chain(writeHandler, middlewares...))
mux.Handle("GET /static/", http.StripPrefix("/static/", http.FileServer(http.Dir(filepath.Join("web", "static")))))
return mux
}
func (s *Server) index(w http.ResponseWriter, r *http.Request) {
if err := s.templates.ExecuteTemplate(w, "index.html", nil); err != nil {
http.Error(w, "failed to render page", http.StatusInternalServerError)
}
}
func (s *Server) createScratch(w http.ResponseWriter, r *http.Request) {
r.Body = http.MaxBytesReader(w, r.Body, s.cfg.Limits.MaxUploadSizeBytes)
var reader io.Reader
contentType := strings.TrimSpace(r.Header.Get("Content-Type"))
if strings.HasPrefix(contentType, "multipart/form-data") {
if err := r.ParseMultipartForm(s.cfg.Limits.MaxUploadSizeBytes); err != nil {
s.writeCreateError(w, err)
return
}
if fileCount := multipartFileCount(r); fileCount > 1 {
http.Error(w, "only one file is allowed per scratch upload", http.StatusBadRequest)
return
}
file, header, err := r.FormFile("file")
if err == nil {
defer file.Close()
if strings.TrimSpace(r.FormValue("content")) != "" {
http.Error(w, "multipart request must include either file or content, not both", http.StatusBadRequest)
return
}
reader = file
contentType = header.Header.Get("Content-Type")
if contentType == "" {
contentType = "application/octet-stream"
}
} else {
body := strings.TrimSpace(r.FormValue("content"))
if body == "" {
http.Error(w, "multipart request requires file or content field", http.StatusBadRequest)
return
}
reader = strings.NewReader(body)
contentType = "text/plain; charset=utf-8"
}
} else {
reader = r.Body
if contentType == "" {
contentType = "text/plain; charset=utf-8"
}
}
meta, err := s.store.Create(r.Context(), reader, contentType, s.cfg.Limits.DefaultTTLDuration)
if err != nil {
if strings.Contains(err.Error(), "request body too large") {
http.Error(w, "upload exceeds limits.max_upload_size", http.StatusRequestEntityTooLarge)
return
}
http.Error(w, "failed to save scratch", http.StatusInternalServerError)
return
}
payload := map[string]any{
"id": meta.ID,
"created_at": meta.CreatedAt,
"expires_at": meta.ExpiresAt,
"size": meta.Size,
"content_type": meta.ContentType,
"view_url": fmt.Sprintf("/s/%s", meta.ID),
"raw_url": fmt.Sprintf("/raw/%s", meta.ID),
"api_url": fmt.Sprintf("/api/scratch/%s", meta.ID),
}
writeJSON(w, http.StatusCreated, payload)
}
func multipartFileCount(r *http.Request) int {
if r.MultipartForm == nil || len(r.MultipartForm.File) == 0 {
return 0
}
count := 0
for _, headers := range r.MultipartForm.File {
count += len(headers)
}
return count
}
func (s *Server) getScratch(w http.ResponseWriter, r *http.Request) {
id := strings.TrimSpace(r.PathValue("id"))
if id == "" {
http.NotFound(w, r)
return
}
meta, err := s.store.Get(id)
if err != nil {
http.NotFound(w, r)
return
}
if s.isExpired(meta) {
_ = s.store.Delete(id)
http.Error(w, "scratch expired", http.StatusGone)
return
}
payload := map[string]any{
"id": meta.ID,
"created_at": meta.CreatedAt,
"expires_at": meta.ExpiresAt,
"size": meta.Size,
"content_type": meta.ContentType,
"view_url": fmt.Sprintf("/s/%s", meta.ID),
"raw_url": fmt.Sprintf("/raw/%s", meta.ID),
}
writeJSON(w, http.StatusOK, payload)
}
func (s *Server) viewScratch(w http.ResponseWriter, r *http.Request) {
id := strings.TrimSpace(r.PathValue("id"))
file, meta, err := s.store.Open(id)
if err != nil {
http.NotFound(w, r)
return
}
defer file.Close()
if s.isExpired(meta) {
_ = s.store.Delete(id)
http.Error(w, "scratch expired", http.StatusGone)
return
}
limited := io.LimitReader(file, maxViewPreviewBytes+1)
b, err := io.ReadAll(limited)
if err != nil {
http.Error(w, "failed to read scratch", http.StatusInternalServerError)
return
}
truncated := int64(len(b)) > maxViewPreviewBytes
if truncated {
b = b[:maxViewPreviewBytes]
}
isBinary := !isLikelyText(meta.ContentType, b)
data := templateViewData{
ID: meta.ID,
ContentType: meta.ContentType,
RawURL: fmt.Sprintf("/raw/%s", meta.ID),
ExpiresAt: meta.ExpiresAt.Format("2006-01-02 15:04:05 MST"),
Truncated: truncated,
Binary: isBinary,
}
if !isBinary {
data.Content = string(b)
}
if err := s.templates.ExecuteTemplate(w, "view.html", data); err != nil {
http.Error(w, "failed to render scratch view", http.StatusInternalServerError)
}
}
func (s *Server) rawScratch(w http.ResponseWriter, r *http.Request) {
id := strings.TrimSpace(r.PathValue("id"))
file, meta, err := s.store.Open(id)
if err != nil {
http.NotFound(w, r)
return
}
defer file.Close()
if s.isExpired(meta) {
_ = s.store.Delete(id)
http.Error(w, "scratch expired", http.StatusGone)
return
}
if meta.ContentType == "" {
meta.ContentType = "application/octet-stream"
}
w.Header().Set("Content-Type", meta.ContentType)
w.Header().Set("Content-Length", fmt.Sprintf("%d", meta.Size))
if _, err := io.Copy(w, file); err != nil {
s.logger.Printf("failed to stream raw scratch id=%s err=%v", id, err)
}
}
func (s *Server) writeCreateError(w http.ResponseWriter, err error) {
var maxErr *http.MaxBytesError
if errors.As(err, &maxErr) || strings.Contains(err.Error(), "request body too large") {
http.Error(w, "upload exceeds limits.max_upload_size", http.StatusRequestEntityTooLarge)
return
}
http.Error(w, "invalid upload payload", http.StatusBadRequest)
}
func (s *Server) isExpired(meta storage.Metadata) bool {
return !meta.ExpiresAt.After(nowUTC())
}
func nowUTC() time.Time {
return time.Now().UTC()
}
func isLikelyText(contentType string, content []byte) bool {
if ct := strings.TrimSpace(contentType); ct != "" {
mediaType, _, err := mime.ParseMediaType(ct)
if err == nil {
if strings.HasPrefix(mediaType, "text/") {
return true
}
switch mediaType {
case "application/json", "application/xml", "application/javascript", "application/x-www-form-urlencoded":
return true
}
}
}
if len(content) == 0 {
return true
}
return utf8.Valid(content)
}
+454
View File
@@ -0,0 +1,454 @@
package httpapi
import (
"bytes"
"encoding/json"
"html/template"
"io"
"log"
"mime/multipart"
"net/http"
"net/http/httptest"
"path/filepath"
"runtime"
"strings"
"testing"
"time"
"scratchbox/internal/config"
"scratchbox/internal/storage"
)
func TestCreateReadThenExpireScratch(t *testing.T) {
t.Parallel()
handler := newTestHandler(t, testServerOptions{
maxUploadBytes: 1024,
defaultTTL: 120 * time.Millisecond,
rateLimitEnable: false,
})
createReq := httptest.NewRequest(http.MethodPost, "/api/scratch", bytes.NewBufferString("hello scratch"))
createReq.Header.Set("Content-Type", "text/plain; charset=utf-8")
createReq.RemoteAddr = "127.0.0.1:9999"
createRec := httptest.NewRecorder()
handler.ServeHTTP(createRec, createReq)
if createRec.Code != http.StatusCreated {
t.Fatalf("create status = %d, want %d", createRec.Code, http.StatusCreated)
}
var payload map[string]any
if err := json.Unmarshal(createRec.Body.Bytes(), &payload); err != nil {
t.Fatalf("decode create payload error = %v", err)
}
id, _ := payload["id"].(string)
if id == "" {
t.Fatalf("create payload missing id")
}
metaReq := httptest.NewRequest(http.MethodGet, "/api/scratch/"+id, nil)
metaReq.RemoteAddr = "127.0.0.1:9999"
metaRec := httptest.NewRecorder()
handler.ServeHTTP(metaRec, metaReq)
if metaRec.Code != http.StatusOK {
t.Fatalf("read metadata status = %d, want %d", metaRec.Code, http.StatusOK)
}
rawReq := httptest.NewRequest(http.MethodGet, "/raw/"+id, nil)
rawReq.RemoteAddr = "127.0.0.1:9999"
rawRec := httptest.NewRecorder()
handler.ServeHTTP(rawRec, rawReq)
if rawRec.Code != http.StatusOK {
t.Fatalf("read raw status = %d, want %d", rawRec.Code, http.StatusOK)
}
if got := rawRec.Body.String(); got != "hello scratch" {
t.Fatalf("raw body = %q, want %q", got, "hello scratch")
}
time.Sleep(180 * time.Millisecond)
expiredReq := httptest.NewRequest(http.MethodGet, "/api/scratch/"+id, nil)
expiredReq.RemoteAddr = "127.0.0.1:9999"
expiredRec := httptest.NewRecorder()
handler.ServeHTTP(expiredRec, expiredReq)
if expiredRec.Code != http.StatusGone {
t.Fatalf("expired status = %d, want %d", expiredRec.Code, http.StatusGone)
}
}
func TestCreateScratchRejectsBodyOverLimit(t *testing.T) {
t.Parallel()
handler := newTestHandler(t, testServerOptions{
maxUploadBytes: 8,
defaultTTL: time.Hour,
rateLimitEnable: false,
})
req := httptest.NewRequest(http.MethodPost, "/api/scratch", bytes.NewBufferString("this is too large"))
req.Header.Set("Content-Type", "text/plain; charset=utf-8")
req.RemoteAddr = "127.0.0.1:5050"
rec := httptest.NewRecorder()
handler.ServeHTTP(rec, req)
if rec.Code != http.StatusRequestEntityTooLarge {
t.Fatalf("status = %d, want %d", rec.Code, http.StatusRequestEntityTooLarge)
}
if !bytes.Contains(rec.Body.Bytes(), []byte("upload exceeds limits.max_upload_size")) {
t.Fatalf("unexpected body: %q", rec.Body.String())
}
}
func TestCreateScratchRejectsMultipleMultipartFiles(t *testing.T) {
t.Parallel()
handler := newTestHandler(t, testServerOptions{
maxUploadBytes: 1024,
defaultTTL: time.Hour,
rateLimitEnable: false,
})
var body bytes.Buffer
writer := multipart.NewWriter(&body)
file1, err := writer.CreateFormFile("file", "first.txt")
if err != nil {
t.Fatalf("CreateFormFile first error = %v", err)
}
if _, err := file1.Write([]byte("first")); err != nil {
t.Fatalf("write first file error = %v", err)
}
file2, err := writer.CreateFormFile("file", "second.txt")
if err != nil {
t.Fatalf("CreateFormFile second error = %v", err)
}
if _, err := file2.Write([]byte("second")); err != nil {
t.Fatalf("write second file error = %v", err)
}
if err := writer.Close(); err != nil {
t.Fatalf("multipart close error = %v", err)
}
req := httptest.NewRequest(http.MethodPost, "/api/scratch", &body)
req.Header.Set("Content-Type", writer.FormDataContentType())
req.RemoteAddr = "127.0.0.1:5051"
rec := httptest.NewRecorder()
handler.ServeHTTP(rec, req)
if rec.Code != http.StatusBadRequest {
t.Fatalf("status = %d, want %d", rec.Code, http.StatusBadRequest)
}
if !bytes.Contains(rec.Body.Bytes(), []byte("only one file is allowed per scratch upload")) {
t.Fatalf("unexpected body: %q", rec.Body.String())
}
}
func TestCreateScratchMultipartContentOnly(t *testing.T) {
t.Parallel()
handler := newTestHandler(t, testServerOptions{
maxUploadBytes: 1024,
defaultTTL: time.Hour,
rateLimitEnable: false,
})
var body bytes.Buffer
writer := multipart.NewWriter(&body)
if err := writer.WriteField("content", "hello from content field"); err != nil {
t.Fatalf("WriteField() error = %v", err)
}
if err := writer.Close(); err != nil {
t.Fatalf("multipart close error = %v", err)
}
req := httptest.NewRequest(http.MethodPost, "/api/scratch", &body)
req.Header.Set("Content-Type", writer.FormDataContentType())
req.RemoteAddr = "127.0.0.1:5052"
rec := httptest.NewRecorder()
handler.ServeHTTP(rec, req)
if rec.Code != http.StatusCreated {
t.Fatalf("status = %d, want %d", rec.Code, http.StatusCreated)
}
}
func TestCreateScratchMultipartRejectsFileAndContent(t *testing.T) {
t.Parallel()
handler := newTestHandler(t, testServerOptions{
maxUploadBytes: 1024,
defaultTTL: time.Hour,
rateLimitEnable: false,
})
var body bytes.Buffer
writer := multipart.NewWriter(&body)
file, err := writer.CreateFormFile("file", "one.txt")
if err != nil {
t.Fatalf("CreateFormFile() error = %v", err)
}
if _, err := file.Write([]byte("file content")); err != nil {
t.Fatalf("write file error = %v", err)
}
if err := writer.WriteField("content", "extra content"); err != nil {
t.Fatalf("WriteField() error = %v", err)
}
if err := writer.Close(); err != nil {
t.Fatalf("multipart close error = %v", err)
}
req := httptest.NewRequest(http.MethodPost, "/api/scratch", &body)
req.Header.Set("Content-Type", writer.FormDataContentType())
req.RemoteAddr = "127.0.0.1:5053"
rec := httptest.NewRecorder()
handler.ServeHTTP(rec, req)
if rec.Code != http.StatusBadRequest {
t.Fatalf("status = %d, want %d", rec.Code, http.StatusBadRequest)
}
}
func TestCreateScratchMultipartRequiresFileOrContent(t *testing.T) {
t.Parallel()
handler := newTestHandler(t, testServerOptions{
maxUploadBytes: 1024,
defaultTTL: time.Hour,
rateLimitEnable: false,
})
var body bytes.Buffer
writer := multipart.NewWriter(&body)
if err := writer.Close(); err != nil {
t.Fatalf("multipart close error = %v", err)
}
req := httptest.NewRequest(http.MethodPost, "/api/scratch", &body)
req.Header.Set("Content-Type", writer.FormDataContentType())
req.RemoteAddr = "127.0.0.1:5054"
rec := httptest.NewRecorder()
handler.ServeHTTP(rec, req)
if rec.Code != http.StatusBadRequest {
t.Fatalf("status = %d, want %d", rec.Code, http.StatusBadRequest)
}
}
func TestGetScratchNotFound(t *testing.T) {
t.Parallel()
handler := newTestHandler(t, testServerOptions{
maxUploadBytes: 1024,
defaultTTL: time.Hour,
rateLimitEnable: false,
})
req := httptest.NewRequest(http.MethodGet, "/api/scratch/does-not-exist", nil)
req.RemoteAddr = "127.0.0.1:5055"
rec := httptest.NewRecorder()
handler.ServeHTTP(rec, req)
if rec.Code != http.StatusNotFound {
t.Fatalf("status = %d, want %d", rec.Code, http.StatusNotFound)
}
}
func TestRawScratchDefaultsContentType(t *testing.T) {
t.Parallel()
handler, store := newTestHandlerAndStore(t, testServerOptions{
maxUploadBytes: 1024,
defaultTTL: time.Hour,
rateLimitEnable: false,
})
meta, err := store.Create(t.Context(), strings.NewReader("no content type"), "", time.Hour)
if err != nil {
t.Fatalf("store.Create() error = %v", err)
}
req := httptest.NewRequest(http.MethodGet, "/raw/"+meta.ID, nil)
req.RemoteAddr = "127.0.0.1:5056"
rec := httptest.NewRecorder()
handler.ServeHTTP(rec, req)
if rec.Code != http.StatusOK {
t.Fatalf("status = %d, want %d", rec.Code, http.StatusOK)
}
if got := rec.Header().Get("Content-Type"); got != "application/octet-stream" {
t.Fatalf("Content-Type = %q, want %q", got, "application/octet-stream")
}
}
func TestViewScratchAndIndexRender(t *testing.T) {
t.Parallel()
handler, store := newTemplateHandlerAndStore(t, testServerOptions{
maxUploadBytes: 10 * 1024 * 1024,
defaultTTL: time.Hour,
rateLimitEnable: false,
})
meta, err := store.Create(t.Context(), strings.NewReader("hello view"), "text/plain; charset=utf-8", time.Hour)
if err != nil {
t.Fatalf("store.Create() error = %v", err)
}
indexReq := httptest.NewRequest(http.MethodGet, "/", nil)
indexRec := httptest.NewRecorder()
handler.ServeHTTP(indexRec, indexReq)
if indexRec.Code != http.StatusOK {
t.Fatalf("index status = %d, want %d", indexRec.Code, http.StatusOK)
}
viewReq := httptest.NewRequest(http.MethodGet, "/s/"+meta.ID, nil)
viewRec := httptest.NewRecorder()
handler.ServeHTTP(viewRec, viewReq)
if viewRec.Code != http.StatusOK {
t.Fatalf("view status = %d, want %d", viewRec.Code, http.StatusOK)
}
if !strings.Contains(viewRec.Body.String(), "hello view") {
t.Fatalf("expected view body to include scratch content")
}
}
func TestViewAndRawScratchExpired(t *testing.T) {
t.Parallel()
handler, store := newTemplateHandlerAndStore(t, testServerOptions{
maxUploadBytes: 1024,
defaultTTL: time.Hour,
rateLimitEnable: false,
})
meta, err := store.Create(t.Context(), strings.NewReader("expired body"), "text/plain", -time.Second)
if err != nil {
t.Fatalf("store.Create() error = %v", err)
}
viewReq := httptest.NewRequest(http.MethodGet, "/s/"+meta.ID, nil)
viewRec := httptest.NewRecorder()
handler.ServeHTTP(viewRec, viewReq)
if viewRec.Code != http.StatusGone {
t.Fatalf("view status = %d, want %d", viewRec.Code, http.StatusGone)
}
rawReq := httptest.NewRequest(http.MethodGet, "/raw/"+meta.ID, nil)
rawRec := httptest.NewRecorder()
handler.ServeHTTP(rawRec, rawReq)
if rawRec.Code != http.StatusNotFound {
t.Fatalf("raw status after lazy delete = %d, want %d", rawRec.Code, http.StatusNotFound)
}
}
func TestCreateScratchRejectsMalformedMultipart(t *testing.T) {
t.Parallel()
handler := newTestHandler(t, testServerOptions{
maxUploadBytes: 1024,
defaultTTL: time.Hour,
rateLimitEnable: false,
})
req := httptest.NewRequest(http.MethodPost, "/api/scratch", strings.NewReader("not-multipart"))
req.Header.Set("Content-Type", "multipart/form-data; boundary=missing")
req.RemoteAddr = "127.0.0.1:5057"
rec := httptest.NewRecorder()
handler.ServeHTTP(rec, req)
if rec.Code != http.StatusBadRequest {
t.Fatalf("status = %d, want %d", rec.Code, http.StatusBadRequest)
}
}
type testServerOptions struct {
maxUploadBytes int64
defaultTTL time.Duration
rateLimitEnable bool
}
func newTestHandler(t *testing.T, opts testServerOptions) http.Handler {
t.Helper()
handler, _ := newTestHandlerAndStore(t, opts)
return handler
}
func newTestHandlerAndStore(t *testing.T, opts testServerOptions) (http.Handler, *storage.FilesystemStore) {
t.Helper()
dataDir := filepath.Join(t.TempDir(), "data")
store, err := storage.NewFilesystemStore(dataDir)
if err != nil {
t.Fatalf("NewFilesystemStore() error = %v", err)
}
cfg := config.Config{
Server: config.ServerConfig{
ListenAddr: ":0",
},
Limits: config.LimitsConfig{
MaxUploadSizeBytes: opts.maxUploadBytes,
DefaultTTLDuration: opts.defaultTTL,
},
Storage: config.StorageConfig{
DataDir: dataDir,
},
Security: config.SecurityConfig{
RateLimit: config.RateLimitConfig{
Enabled: opts.rateLimitEnable,
RequestsPerMinute: 120,
Burst: 1,
},
},
}
srv := &Server{
cfg: cfg,
store: store,
logger: log.New(io.Discard, "", 0),
templates: nil,
}
return srv.Routes(), store
}
func newTemplateHandlerAndStore(t *testing.T, opts testServerOptions) (http.Handler, *storage.FilesystemStore) {
t.Helper()
handler, store := newTestHandlerAndStore(t, opts)
_ = handler // keep shared setup path
_, filePath, _, ok := runtime.Caller(0)
if !ok {
t.Fatal("runtime.Caller failed")
}
repoRoot := filepath.Clean(filepath.Join(filepath.Dir(filePath), "..", ".."))
tmpl, err := template.ParseFiles(
filepath.Join(repoRoot, "web", "templates", "index.html"),
filepath.Join(repoRoot, "web", "templates", "view.html"),
)
if err != nil {
t.Fatalf("ParseFiles() error = %v", err)
}
cfg := config.Config{
Server: config.ServerConfig{
ListenAddr: ":0",
},
Limits: config.LimitsConfig{
MaxUploadSizeBytes: opts.maxUploadBytes,
DefaultTTLDuration: opts.defaultTTL,
},
Storage: config.StorageConfig{
DataDir: filepath.Join(t.TempDir(), "unused"),
},
Security: config.SecurityConfig{
RateLimit: config.RateLimitConfig{
Enabled: opts.rateLimitEnable,
RequestsPerMinute: 120,
Burst: 1,
},
},
}
srv := &Server{
cfg: cfg,
store: store,
logger: log.New(io.Discard, "", 0),
templates: tmpl,
}
return srv.Routes(), store
}
+161
View File
@@ -0,0 +1,161 @@
package httpapi
import (
"errors"
"html/template"
"io"
"log"
"net/http"
"net/http/httptest"
"os"
"path/filepath"
"runtime"
"testing"
"scratchbox/internal/config"
"scratchbox/internal/storage"
)
func TestMultipartFileCount(t *testing.T) {
t.Parallel()
req := httptest.NewRequest(http.MethodPost, "/api/scratch", nil)
if got := multipartFileCount(req); got != 0 {
t.Fatalf("multipartFileCount() = %d, want 0", got)
}
}
func TestWriteCreateErrorBranches(t *testing.T) {
t.Parallel()
s := &Server{}
rec1 := httptest.NewRecorder()
s.writeCreateError(rec1, &http.MaxBytesError{Limit: 5})
if rec1.Code != http.StatusRequestEntityTooLarge {
t.Fatalf("max bytes error status = %d, want %d", rec1.Code, http.StatusRequestEntityTooLarge)
}
rec2 := httptest.NewRecorder()
s.writeCreateError(rec2, errors.New("request body too large"))
if rec2.Code != http.StatusRequestEntityTooLarge {
t.Fatalf("body too large string status = %d, want %d", rec2.Code, http.StatusRequestEntityTooLarge)
}
rec3 := httptest.NewRecorder()
s.writeCreateError(rec3, errors.New("other parse failure"))
if rec3.Code != http.StatusBadRequest {
t.Fatalf("generic error status = %d, want %d", rec3.Code, http.StatusBadRequest)
}
}
func TestIsLikelyText(t *testing.T) {
t.Parallel()
if !isLikelyText("text/plain; charset=utf-8", []byte{0x00}) {
t.Fatalf("text/plain should be treated as text")
}
if !isLikelyText("application/json", []byte{0x00}) {
t.Fatalf("application/json should be treated as text")
}
if isLikelyText("application/octet-stream", []byte{0xff, 0xfe, 0xfd}) {
t.Fatalf("binary octet-stream should not be treated as text")
}
if !isLikelyText("", []byte("hello")) {
t.Fatalf("utf8 payload should be treated as text")
}
if !isLikelyText("", nil) {
t.Fatalf("empty payload should be treated as text")
}
}
func TestNewServerLoadsTemplates(t *testing.T) {
tmp := t.TempDir()
store, err := storage.NewFilesystemStore(filepath.Join(tmp, "data"))
if err != nil {
t.Fatalf("NewFilesystemStore() error = %v", err)
}
cfg := config.Config{
Limits: config.LimitsConfig{MaxUploadSizeBytes: 1024},
Security: config.SecurityConfig{
RateLimit: config.RateLimitConfig{Enabled: false},
},
}
_, file, _, ok := runtime.Caller(0)
if !ok {
t.Fatal("runtime.Caller failed")
}
repoRoot := filepath.Clean(filepath.Join(filepath.Dir(file), "..", ".."))
wd, err := os.Getwd()
if err != nil {
t.Fatalf("Getwd() error = %v", err)
}
if err := os.Chdir(repoRoot); err != nil {
t.Fatalf("Chdir() error = %v", err)
}
defer func() {
_ = os.Chdir(wd)
}()
srv, err := NewServer(cfg, store, log.New(io.Discard, "", 0))
if err != nil {
t.Fatalf("NewServer() error = %v", err)
}
if srv == nil || srv.templates == nil {
t.Fatalf("expected initialized server with templates")
}
}
func TestNewServerTemplateErrorAndIndexError(t *testing.T) {
tmp := t.TempDir()
store, err := storage.NewFilesystemStore(filepath.Join(tmp, "data"))
if err != nil {
t.Fatalf("NewFilesystemStore() error = %v", err)
}
cfg := config.Config{
Limits: config.LimitsConfig{MaxUploadSizeBytes: 1024},
Security: config.SecurityConfig{
RateLimit: config.RateLimitConfig{Enabled: false},
},
}
wd, err := os.Getwd()
if err != nil {
t.Fatalf("Getwd() error = %v", err)
}
if err := os.Chdir(tmp); err != nil {
t.Fatalf("Chdir() error = %v", err)
}
defer func() {
_ = os.Chdir(wd)
}()
if _, err := NewServer(cfg, store, log.New(io.Discard, "", 0)); err == nil {
t.Fatalf("expected NewServer() template parse error")
}
s := &Server{}
func() {
defer func() {
if recover() == nil {
t.Fatalf("expected panic when templates are nil")
}
}()
rec := httptest.NewRecorder()
req := httptest.NewRequest(http.MethodGet, "/", nil)
s.index(rec, req)
}()
}
func TestIndexExecuteTemplateError(t *testing.T) {
t.Parallel()
s := &Server{
templates: template.New("empty"),
}
rec := httptest.NewRecorder()
req := httptest.NewRequest(http.MethodGet, "/", nil)
s.index(rec, req)
if rec.Code != http.StatusInternalServerError {
t.Fatalf("index status = %d, want %d", rec.Code, http.StatusInternalServerError)
}
}
+153
View File
@@ -0,0 +1,153 @@
package httpapi
import (
"encoding/json"
"log"
"net"
"net/http"
"net/netip"
"strings"
"sync"
"time"
"golang.org/x/time/rate"
)
type visitor struct {
limiter *rate.Limiter
lastSeen time.Time
}
type RateLimiter struct {
mu sync.Mutex
visitors map[string]*visitor
rate rate.Limit
burst int
ttl time.Duration
}
func NewRateLimiter(requestsPerMinute int, burst int) *RateLimiter {
return &RateLimiter{
visitors: map[string]*visitor{},
rate: rate.Limit(float64(requestsPerMinute) / 60.0),
burst: burst,
ttl: 10 * time.Minute,
}
}
func (r *RateLimiter) Allow(ip string) bool {
now := time.Now()
r.mu.Lock()
defer r.mu.Unlock()
for key, v := range r.visitors {
if now.Sub(v.lastSeen) > r.ttl {
delete(r.visitors, key)
}
}
v, ok := r.visitors[ip]
if !ok {
v = &visitor{
limiter: rate.NewLimiter(r.rate, r.burst),
}
r.visitors[ip] = v
}
v.lastSeen = now
return v.limiter.Allow()
}
func AllowlistMiddleware(allowed []netip.Prefix, trustProxyHeaders bool, logger *log.Logger) func(http.Handler) http.Handler {
return func(next http.Handler) http.Handler {
if len(allowed) == 0 {
return next
}
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
clientIP, ok := extractClientIP(r, trustProxyHeaders)
if !ok {
http.Error(w, "unable to determine client ip", http.StatusForbidden)
return
}
for _, prefix := range allowed {
if prefix.Contains(clientIP) {
next.ServeHTTP(w, r)
return
}
}
logger.Printf("allowlist denied request method=%s path=%s ip=%s", r.Method, r.URL.Path, clientIP.String())
http.Error(w, "ip not allowed", http.StatusForbidden)
})
}
}
func RateLimitMiddleware(limiter *RateLimiter, trustProxyHeaders bool) func(http.Handler) http.Handler {
return func(next http.Handler) http.Handler {
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
clientIP, ok := extractClientIP(r, trustProxyHeaders)
if !ok {
http.Error(w, "unable to determine client ip", http.StatusTooManyRequests)
return
}
if limiter.Allow(clientIP.String()) {
next.ServeHTTP(w, r)
return
}
w.Header().Set("Retry-After", "60")
writeJSON(w, http.StatusTooManyRequests, map[string]string{
"error": "too many requests from this IP, retry later",
})
})
}
}
func Chain(handler http.Handler, middlewares ...func(http.Handler) http.Handler) http.Handler {
wrapped := handler
for i := len(middlewares) - 1; i >= 0; i-- {
wrapped = middlewares[i](wrapped)
}
return wrapped
}
func extractClientIP(r *http.Request, trustProxyHeaders bool) (netip.Addr, bool) {
if trustProxyHeaders {
if xff := strings.TrimSpace(r.Header.Get("X-Forwarded-For")); xff != "" {
parts := strings.Split(xff, ",")
if len(parts) > 0 {
if ip, err := netip.ParseAddr(strings.TrimSpace(parts[0])); err == nil {
return ip, true
}
}
}
if xrip := strings.TrimSpace(r.Header.Get("X-Real-IP")); xrip != "" {
if ip, err := netip.ParseAddr(xrip); err == nil {
return ip, true
}
}
}
host, _, err := net.SplitHostPort(strings.TrimSpace(r.RemoteAddr))
if err != nil {
if ip, parseErr := netip.ParseAddr(strings.TrimSpace(r.RemoteAddr)); parseErr == nil {
return ip, true
}
return netip.Addr{}, false
}
ip, err := netip.ParseAddr(host)
if err != nil {
return netip.Addr{}, false
}
return ip, true
}
func writeJSON(w http.ResponseWriter, status int, payload any) {
w.Header().Set("Content-Type", "application/json")
w.WriteHeader(status)
_ = json.NewEncoder(w).Encode(payload)
}
+217
View File
@@ -0,0 +1,217 @@
package httpapi
import (
"encoding/json"
"io"
"log"
"net/http"
"net/http/httptest"
"net/netip"
"strings"
"testing"
"time"
)
func TestAllowlistMiddlewareDeniesAndAllowsByIP(t *testing.T) {
t.Parallel()
logger := log.New(io.Discard, "", 0)
allowed := []netip.Prefix{netip.MustParsePrefix("10.0.0.0/8")}
mw := AllowlistMiddleware(allowed, false, logger)
next := http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
w.WriteHeader(http.StatusNoContent)
})
handler := mw(next)
reqAllowed := httptest.NewRequest(http.MethodPost, "/api/scratch", nil)
reqAllowed.RemoteAddr = "10.1.2.3:1234"
recAllowed := httptest.NewRecorder()
handler.ServeHTTP(recAllowed, reqAllowed)
if recAllowed.Code != http.StatusNoContent {
t.Fatalf("allowed request status = %d, want %d", recAllowed.Code, http.StatusNoContent)
}
reqDenied := httptest.NewRequest(http.MethodPost, "/api/scratch", nil)
reqDenied.RemoteAddr = "192.168.10.5:1234"
recDenied := httptest.NewRecorder()
handler.ServeHTTP(recDenied, reqDenied)
if recDenied.Code != http.StatusForbidden {
t.Fatalf("denied request status = %d, want %d", recDenied.Code, http.StatusForbidden)
}
}
func TestAllowlistMiddlewareHonorsProxyHeaderWhenTrusted(t *testing.T) {
t.Parallel()
logger := log.New(io.Discard, "", 0)
allowed := []netip.Prefix{netip.MustParsePrefix("203.0.113.5/32")}
mw := AllowlistMiddleware(allowed, true, logger)
handler := mw(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
w.WriteHeader(http.StatusNoContent)
}))
req := httptest.NewRequest(http.MethodPost, "/api/scratch", nil)
req.RemoteAddr = "10.0.0.10:9999"
req.Header.Set("X-Forwarded-For", "203.0.113.5, 198.51.100.2")
rec := httptest.NewRecorder()
handler.ServeHTTP(rec, req)
if rec.Code != http.StatusNoContent {
t.Fatalf("status = %d, want %d", rec.Code, http.StatusNoContent)
}
}
func TestRateLimitMiddlewareKeysByClientIP(t *testing.T) {
t.Parallel()
limiter := NewRateLimiter(60, 1)
handler := RateLimitMiddleware(limiter, false)(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
w.WriteHeader(http.StatusNoContent)
}))
req1 := httptest.NewRequest(http.MethodPost, "/api/scratch", nil)
req1.RemoteAddr = "198.51.100.7:1111"
rec1 := httptest.NewRecorder()
handler.ServeHTTP(rec1, req1)
if rec1.Code != http.StatusNoContent {
t.Fatalf("first request status = %d, want %d", rec1.Code, http.StatusNoContent)
}
req2 := httptest.NewRequest(http.MethodPost, "/api/scratch", nil)
req2.RemoteAddr = "198.51.100.7:2222"
rec2 := httptest.NewRecorder()
handler.ServeHTTP(rec2, req2)
if rec2.Code != http.StatusTooManyRequests {
t.Fatalf("second same-ip request status = %d, want %d", rec2.Code, http.StatusTooManyRequests)
}
req3 := httptest.NewRequest(http.MethodPost, "/api/scratch", nil)
req3.RemoteAddr = "198.51.100.8:3333"
rec3 := httptest.NewRecorder()
handler.ServeHTTP(rec3, req3)
if rec3.Code != http.StatusNoContent {
t.Fatalf("different-ip request status = %d, want %d", rec3.Code, http.StatusNoContent)
}
}
func TestAllowlistMiddlewareNoRestrictionsReturnsNext(t *testing.T) {
t.Parallel()
handler := AllowlistMiddleware(nil, false, log.New(io.Discard, "", 0))(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
w.WriteHeader(http.StatusAccepted)
}))
req := httptest.NewRequest(http.MethodPost, "/api/scratch", nil)
req.RemoteAddr = "not-a-valid-remote-addr"
rec := httptest.NewRecorder()
handler.ServeHTTP(rec, req)
if rec.Code != http.StatusAccepted {
t.Fatalf("status = %d, want %d", rec.Code, http.StatusAccepted)
}
}
func TestAllowlistMiddlewareRejectsUnknownClientIP(t *testing.T) {
t.Parallel()
logger := log.New(io.Discard, "", 0)
allowed := []netip.Prefix{netip.MustParsePrefix("10.0.0.0/8")}
handler := AllowlistMiddleware(allowed, false, logger)(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
w.WriteHeader(http.StatusNoContent)
}))
req := httptest.NewRequest(http.MethodPost, "/api/scratch", nil)
req.RemoteAddr = "garbage"
rec := httptest.NewRecorder()
handler.ServeHTTP(rec, req)
if rec.Code != http.StatusForbidden {
t.Fatalf("status = %d, want %d", rec.Code, http.StatusForbidden)
}
}
func TestRateLimitMiddlewareUnknownIP(t *testing.T) {
t.Parallel()
limiter := NewRateLimiter(60, 1)
handler := RateLimitMiddleware(limiter, false)(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
w.WriteHeader(http.StatusNoContent)
}))
req := httptest.NewRequest(http.MethodPost, "/api/scratch", nil)
req.RemoteAddr = "bad-addr"
rec := httptest.NewRecorder()
handler.ServeHTTP(rec, req)
if rec.Code != http.StatusTooManyRequests {
t.Fatalf("status = %d, want %d", rec.Code, http.StatusTooManyRequests)
}
}
func TestRateLimitMiddleware429PayloadAndRetryHeader(t *testing.T) {
t.Parallel()
limiter := NewRateLimiter(1, 1)
handler := RateLimitMiddleware(limiter, false)(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
w.WriteHeader(http.StatusNoContent)
}))
req1 := httptest.NewRequest(http.MethodPost, "/api/scratch", nil)
req1.RemoteAddr = "198.18.0.1:1111"
handler.ServeHTTP(httptest.NewRecorder(), req1)
req2 := httptest.NewRequest(http.MethodPost, "/api/scratch", nil)
req2.RemoteAddr = "198.18.0.1:1112"
rec2 := httptest.NewRecorder()
handler.ServeHTTP(rec2, req2)
if rec2.Code != http.StatusTooManyRequests {
t.Fatalf("status = %d, want %d", rec2.Code, http.StatusTooManyRequests)
}
if got := rec2.Header().Get("Retry-After"); got != "60" {
t.Fatalf("Retry-After = %q, want %q", got, "60")
}
if ct := rec2.Header().Get("Content-Type"); !strings.Contains(ct, "application/json") {
t.Fatalf("Content-Type = %q, want application/json", ct)
}
var payload map[string]string
if err := json.Unmarshal(rec2.Body.Bytes(), &payload); err != nil {
t.Fatalf("json unmarshal error = %v", err)
}
if payload["error"] == "" {
t.Fatalf("expected error payload")
}
}
func TestRateLimiterPrunesStaleVisitors(t *testing.T) {
t.Parallel()
limiter := NewRateLimiter(60, 1)
limiter.ttl = time.Nanosecond
limiter.visitors["old"] = &visitor{lastSeen: time.Now().Add(-time.Hour)}
if !limiter.Allow("new") {
t.Fatalf("Allow(new) = false, want true")
}
if _, ok := limiter.visitors["old"]; ok {
t.Fatalf("stale visitor should be removed")
}
}
func TestExtractClientIPFallbacks(t *testing.T) {
t.Parallel()
req := httptest.NewRequest(http.MethodGet, "/", nil)
req.RemoteAddr = "203.0.113.9"
ip, ok := extractClientIP(req, false)
if !ok || ip.String() != "203.0.113.9" {
t.Fatalf("extract direct ip = (%v,%v), want (203.0.113.9,true)", ip, ok)
}
req2 := httptest.NewRequest(http.MethodGet, "/", nil)
req2.RemoteAddr = "192.0.2.50:8080"
req2.Header.Set("X-Forwarded-For", "bad,198.51.100.1")
req2.Header.Set("X-Real-IP", "198.51.100.5")
ip2, ok2 := extractClientIP(req2, true)
if !ok2 || ip2.String() != "198.51.100.5" {
t.Fatalf("extract x-real-ip = (%v,%v), want (198.51.100.5,true)", ip2, ok2)
}
req3 := httptest.NewRequest(http.MethodGet, "/", nil)
req3.RemoteAddr = "still-not-an-ip"
if _, ok3 := extractClientIP(req3, false); ok3 {
t.Fatalf("extractClientIP() ok = true, want false")
}
}
+293
View File
@@ -0,0 +1,293 @@
package storage
import (
"compress/gzip"
"context"
"crypto/rand"
"encoding/hex"
"encoding/json"
"errors"
"fmt"
"io"
"os"
"path/filepath"
"sync"
"time"
)
const (
indexFilename = "metadata.json"
filesDirName = "scratches"
idBytes = 12
)
var ErrNotFound = errors.New("scratch not found")
type Metadata struct {
ID string `json:"id"`
FilePath string `json:"file_path"`
CreatedAt time.Time `json:"created_at"`
ExpiresAt time.Time `json:"expires_at"`
ContentType string `json:"content_type"`
Size int64 `json:"size"`
}
type FilesystemStore struct {
dataDir string
filesDir string
index string
mu sync.RWMutex
metadata map[string]Metadata
}
func NewFilesystemStore(dataDir string) (*FilesystemStore, error) {
filesDir := filepath.Join(dataDir, filesDirName)
if err := os.MkdirAll(filesDir, 0o755); err != nil {
return nil, fmt.Errorf("create files dir: %w", err)
}
store := &FilesystemStore{
dataDir: dataDir,
filesDir: filesDir,
index: filepath.Join(dataDir, indexFilename),
metadata: map[string]Metadata{},
}
if err := store.loadIndex(); err != nil {
return nil, err
}
if err := store.reconcileOnStartup(time.Now()); err != nil {
return nil, err
}
return store, nil
}
func (s *FilesystemStore) Create(ctx context.Context, body io.Reader, contentType string, ttl time.Duration) (Metadata, error) {
id, err := generateID()
if err != nil {
return Metadata{}, fmt.Errorf("generate id: %w", err)
}
tempFile, err := os.CreateTemp(s.filesDir, id+".tmp-*")
if err != nil {
return Metadata{}, fmt.Errorf("create temp file: %w", err)
}
gzipWriter := gzip.NewWriter(tempFile)
size, copyErr := io.Copy(gzipWriter, body)
gzipCloseErr := gzipWriter.Close()
closeErr := tempFile.Close()
if copyErr != nil {
_ = os.Remove(tempFile.Name())
return Metadata{}, fmt.Errorf("write scratch content: %w", copyErr)
}
if gzipCloseErr != nil {
_ = os.Remove(tempFile.Name())
return Metadata{}, fmt.Errorf("close gzip writer: %w", gzipCloseErr)
}
if closeErr != nil {
_ = os.Remove(tempFile.Name())
return Metadata{}, fmt.Errorf("close temp file: %w", closeErr)
}
finalPath := filepath.Join(s.filesDir, id)
if err := os.Rename(tempFile.Name(), finalPath); err != nil {
_ = os.Remove(tempFile.Name())
return Metadata{}, fmt.Errorf("atomically move scratch file: %w", err)
}
now := time.Now().UTC()
meta := Metadata{
ID: id,
FilePath: finalPath,
CreatedAt: now,
ExpiresAt: now.Add(ttl),
ContentType: contentType,
Size: size,
}
s.mu.Lock()
s.metadata[id] = meta
if err := s.persistLocked(); err != nil {
delete(s.metadata, id)
s.mu.Unlock()
_ = os.Remove(finalPath)
return Metadata{}, err
}
s.mu.Unlock()
return meta, nil
}
func (s *FilesystemStore) Get(id string) (Metadata, error) {
s.mu.RLock()
meta, ok := s.metadata[id]
s.mu.RUnlock()
if !ok {
return Metadata{}, ErrNotFound
}
return meta, nil
}
func (s *FilesystemStore) Open(id string) (io.ReadCloser, Metadata, error) {
meta, err := s.Get(id)
if err != nil {
return nil, Metadata{}, err
}
file, err := os.Open(meta.FilePath)
if err != nil {
if errors.Is(err, os.ErrNotExist) {
return nil, Metadata{}, ErrNotFound
}
return nil, Metadata{}, fmt.Errorf("open scratch file: %w", err)
}
gzipReader, err := gzip.NewReader(file)
if err != nil {
_ = file.Close()
return nil, Metadata{}, fmt.Errorf("create gzip reader: %w", err)
}
return &combinedReadCloser{
reader: gzipReader,
closers: []io.Closer{
gzipReader,
file,
},
}, meta, nil
}
func (s *FilesystemStore) Delete(id string) error {
s.mu.Lock()
meta, ok := s.metadata[id]
if !ok {
s.mu.Unlock()
return ErrNotFound
}
delete(s.metadata, id)
if err := s.persistLocked(); err != nil {
s.metadata[id] = meta
s.mu.Unlock()
return err
}
s.mu.Unlock()
if err := os.Remove(meta.FilePath); err != nil && !errors.Is(err, os.ErrNotExist) {
return fmt.Errorf("remove scratch file: %w", err)
}
return nil
}
func (s *FilesystemStore) DeleteExpired(now time.Time) (int, error) {
s.mu.Lock()
expired := make([]Metadata, 0)
for id, meta := range s.metadata {
if !meta.ExpiresAt.After(now) {
expired = append(expired, meta)
delete(s.metadata, id)
}
}
if len(expired) == 0 {
s.mu.Unlock()
return 0, nil
}
if err := s.persistLocked(); err != nil {
for _, meta := range expired {
s.metadata[meta.ID] = meta
}
s.mu.Unlock()
return 0, err
}
s.mu.Unlock()
for _, meta := range expired {
if err := os.Remove(meta.FilePath); err != nil && !errors.Is(err, os.ErrNotExist) {
return 0, fmt.Errorf("remove expired file %s: %w", meta.ID, err)
}
}
return len(expired), nil
}
func (s *FilesystemStore) loadIndex() error {
b, err := os.ReadFile(s.index)
if err != nil {
if errors.Is(err, os.ErrNotExist) {
return nil
}
return fmt.Errorf("read metadata index: %w", err)
}
loaded := map[string]Metadata{}
if err := json.Unmarshal(b, &loaded); err != nil {
return fmt.Errorf("decode metadata index: %w", err)
}
s.metadata = loaded
return nil
}
func (s *FilesystemStore) persistLocked() error {
tmp := s.index + ".tmp"
payload, err := json.MarshalIndent(s.metadata, "", " ")
if err != nil {
return fmt.Errorf("encode metadata index: %w", err)
}
if err := os.WriteFile(tmp, payload, 0o644); err != nil {
return fmt.Errorf("write metadata temp file: %w", err)
}
if err := os.Rename(tmp, s.index); err != nil {
return fmt.Errorf("replace metadata index: %w", err)
}
return nil
}
func (s *FilesystemStore) reconcileOnStartup(now time.Time) error {
s.mu.Lock()
defer s.mu.Unlock()
changed := false
for id, meta := range s.metadata {
if !meta.ExpiresAt.After(now) {
delete(s.metadata, id)
_ = os.Remove(meta.FilePath)
changed = true
continue
}
if _, err := os.Stat(meta.FilePath); err != nil {
if errors.Is(err, os.ErrNotExist) {
delete(s.metadata, id)
changed = true
continue
}
return fmt.Errorf("stat scratch %s: %w", id, err)
}
}
if !changed {
return nil
}
return s.persistLocked()
}
func generateID() (string, error) {
b := make([]byte, idBytes)
if _, err := rand.Read(b); err != nil {
return "", err
}
return hex.EncodeToString(b), nil
}
type combinedReadCloser struct {
reader io.Reader
closers []io.Closer
}
func (c *combinedReadCloser) Read(p []byte) (int, error) {
return c.reader.Read(p)
}
func (c *combinedReadCloser) Close() error {
var firstErr error
for _, closer := range c.closers {
if err := closer.Close(); err != nil && firstErr == nil {
firstErr = err
}
}
return firstErr
}
+402
View File
@@ -0,0 +1,402 @@
package storage
import (
"bytes"
"context"
"encoding/json"
"errors"
"io"
"os"
"path/filepath"
"testing"
"time"
)
func TestCreateStoresGzipAndOpenIsTransparent(t *testing.T) {
t.Parallel()
store, err := NewFilesystemStore(filepath.Join(t.TempDir(), "data"))
if err != nil {
t.Fatalf("NewFilesystemStore() error = %v", err)
}
original := bytes.Repeat([]byte("hello scratch "), 1024)
meta, err := store.Create(context.Background(), bytes.NewReader(original), "text/plain; charset=utf-8", time.Hour)
if err != nil {
t.Fatalf("Create() error = %v", err)
}
if meta.Size != int64(len(original)) {
t.Fatalf("meta.Size = %d, want %d", meta.Size, len(original))
}
onDisk, err := os.ReadFile(meta.FilePath)
if err != nil {
t.Fatalf("ReadFile() error = %v", err)
}
if len(onDisk) < 2 || onDisk[0] != 0x1f || onDisk[1] != 0x8b {
t.Fatalf("stored file does not have gzip header")
}
reader, _, err := store.Open(meta.ID)
if err != nil {
t.Fatalf("Open() error = %v", err)
}
defer reader.Close()
got, err := io.ReadAll(reader)
if err != nil {
t.Fatalf("ReadAll() error = %v", err)
}
if !bytes.Equal(got, original) {
t.Fatalf("open content mismatch: got %d bytes, want %d", len(got), len(original))
}
}
func TestGetDeleteAndNotFound(t *testing.T) {
t.Parallel()
store, err := NewFilesystemStore(filepath.Join(t.TempDir(), "data"))
if err != nil {
t.Fatalf("NewFilesystemStore() error = %v", err)
}
meta, err := store.Create(context.Background(), bytes.NewReader([]byte("hello")), "text/plain", time.Hour)
if err != nil {
t.Fatalf("Create() error = %v", err)
}
if _, err := store.Get(meta.ID); err != nil {
t.Fatalf("Get() error = %v", err)
}
if err := store.Delete(meta.ID); err != nil {
t.Fatalf("Delete() error = %v", err)
}
if _, err := store.Get(meta.ID); !errors.Is(err, ErrNotFound) {
t.Fatalf("Get() error = %v, want ErrNotFound", err)
}
if err := store.Delete(meta.ID); !errors.Is(err, ErrNotFound) {
t.Fatalf("Delete() error = %v, want ErrNotFound", err)
}
}
func TestDeleteExpired(t *testing.T) {
t.Parallel()
store, err := NewFilesystemStore(filepath.Join(t.TempDir(), "data"))
if err != nil {
t.Fatalf("NewFilesystemStore() error = %v", err)
}
expired, err := store.Create(context.Background(), bytes.NewReader([]byte("expired")), "text/plain", -time.Second)
if err != nil {
t.Fatalf("Create expired() error = %v", err)
}
active, err := store.Create(context.Background(), bytes.NewReader([]byte("active")), "text/plain", time.Hour)
if err != nil {
t.Fatalf("Create active() error = %v", err)
}
deleted, err := store.DeleteExpired(time.Now().UTC())
if err != nil {
t.Fatalf("DeleteExpired() error = %v", err)
}
if deleted != 1 {
t.Fatalf("DeleteExpired() = %d, want 1", deleted)
}
if _, err := store.Get(expired.ID); !errors.Is(err, ErrNotFound) {
t.Fatalf("expired Get() error = %v, want ErrNotFound", err)
}
if _, err := store.Get(active.ID); err != nil {
t.Fatalf("active Get() error = %v", err)
}
}
func TestOpenInvalidGzipFails(t *testing.T) {
t.Parallel()
store, err := NewFilesystemStore(filepath.Join(t.TempDir(), "data"))
if err != nil {
t.Fatalf("NewFilesystemStore() error = %v", err)
}
id := "deadbeefdeadbeefdeadbeef"
badPath := filepath.Join(store.filesDir, id)
if err := os.WriteFile(badPath, []byte("not-gzip"), 0o644); err != nil {
t.Fatalf("WriteFile() error = %v", err)
}
store.metadata[id] = Metadata{
ID: id,
FilePath: badPath,
CreatedAt: time.Now().UTC(),
ExpiresAt: time.Now().UTC().Add(time.Hour),
ContentType: "text/plain",
Size: int64(len("not-gzip")),
}
if err := store.persistLocked(); err != nil {
t.Fatalf("persistLocked() error = %v", err)
}
if _, _, err := store.Open(id); err == nil {
t.Fatalf("Open() error = nil, want gzip reader error")
}
}
func TestNewFilesystemStoreInvalidIndex(t *testing.T) {
t.Parallel()
root := t.TempDir()
scratches := filepath.Join(root, filesDirName)
if err := os.MkdirAll(scratches, 0o755); err != nil {
t.Fatalf("MkdirAll() error = %v", err)
}
if err := os.WriteFile(filepath.Join(root, indexFilename), []byte("{bad"), 0o644); err != nil {
t.Fatalf("WriteFile() error = %v", err)
}
if _, err := NewFilesystemStore(root); err == nil {
t.Fatalf("NewFilesystemStore() error = nil, want invalid index error")
}
}
func TestNewFilesystemStoreReconcileOnStartup(t *testing.T) {
t.Parallel()
root := t.TempDir()
scratches := filepath.Join(root, filesDirName)
if err := os.MkdirAll(scratches, 0o755); err != nil {
t.Fatalf("MkdirAll() error = %v", err)
}
keepID := "aaaaaaaaaaaaaaaaaaaaaaaa"
missingID := "bbbbbbbbbbbbbbbbbbbbbbbb"
expiredID := "cccccccccccccccccccccccc"
keepPath := filepath.Join(scratches, keepID)
expiredPath := filepath.Join(scratches, expiredID)
if err := os.WriteFile(keepPath, []byte{0x1f, 0x8b, 0x08, 0x00}, 0o644); err != nil {
t.Fatalf("WriteFile keep error = %v", err)
}
if err := os.WriteFile(expiredPath, []byte{0x1f, 0x8b, 0x08, 0x00}, 0o644); err != nil {
t.Fatalf("WriteFile expired error = %v", err)
}
meta := map[string]Metadata{
keepID: {
ID: keepID,
FilePath: keepPath,
CreatedAt: time.Now().UTC(),
ExpiresAt: time.Now().UTC().Add(time.Hour),
ContentType: "text/plain",
Size: 1,
},
missingID: {
ID: missingID,
FilePath: filepath.Join(scratches, missingID),
CreatedAt: time.Now().UTC(),
ExpiresAt: time.Now().UTC().Add(time.Hour),
ContentType: "text/plain",
Size: 1,
},
expiredID: {
ID: expiredID,
FilePath: expiredPath,
CreatedAt: time.Now().UTC(),
ExpiresAt: time.Now().UTC().Add(-time.Minute),
ContentType: "text/plain",
Size: 1,
},
}
payload, err := json.Marshal(meta)
if err != nil {
t.Fatalf("json.Marshal() error = %v", err)
}
if err := os.WriteFile(filepath.Join(root, indexFilename), payload, 0o644); err != nil {
t.Fatalf("WriteFile index error = %v", err)
}
store, err := NewFilesystemStore(root)
if err != nil {
t.Fatalf("NewFilesystemStore() error = %v", err)
}
if _, err := store.Get(keepID); err != nil {
t.Fatalf("keep entry missing: %v", err)
}
if _, err := store.Get(missingID); !errors.Is(err, ErrNotFound) {
t.Fatalf("missing entry error = %v, want ErrNotFound", err)
}
if _, err := store.Get(expiredID); !errors.Is(err, ErrNotFound) {
t.Fatalf("expired entry error = %v, want ErrNotFound", err)
}
}
type errCloser struct{}
func (errCloser) Close() error { return errors.New("close failed") }
func TestCombinedReadCloserClosePropagatesFirstError(t *testing.T) {
t.Parallel()
c := &combinedReadCloser{
reader: bytes.NewReader([]byte("ok")),
closers: []io.Closer{errCloser{}, errCloser{}},
}
if err := c.Close(); err == nil {
t.Fatalf("Close() error = nil, want non-nil")
}
}
type failingReader struct{}
func (failingReader) Read(_ []byte) (int, error) { return 0, errors.New("boom") }
func TestCreateRollbackOnPersistFailure(t *testing.T) {
t.Parallel()
store, err := NewFilesystemStore(filepath.Join(t.TempDir(), "data"))
if err != nil {
t.Fatalf("NewFilesystemStore() error = %v", err)
}
badIndexDir := filepath.Join(t.TempDir(), "index-dir")
if err := os.MkdirAll(badIndexDir, 0o755); err != nil {
t.Fatalf("MkdirAll() error = %v", err)
}
store.index = badIndexDir
if _, err := store.Create(context.Background(), bytes.NewReader([]byte("x")), "text/plain", time.Hour); err == nil {
t.Fatalf("Create() error = nil, want persist failure")
}
if len(store.metadata) != 0 {
t.Fatalf("metadata should be rolled back on persist failure")
}
}
func TestCreateReaderFailure(t *testing.T) {
t.Parallel()
store, err := NewFilesystemStore(filepath.Join(t.TempDir(), "data"))
if err != nil {
t.Fatalf("NewFilesystemStore() error = %v", err)
}
if _, err := store.Create(context.Background(), failingReader{}, "text/plain", time.Hour); err == nil {
t.Fatalf("Create() error = nil, want reader failure")
}
}
func TestDeletePersistFailureRestoresEntry(t *testing.T) {
t.Parallel()
store, err := NewFilesystemStore(filepath.Join(t.TempDir(), "data"))
if err != nil {
t.Fatalf("NewFilesystemStore() error = %v", err)
}
meta, err := store.Create(context.Background(), bytes.NewReader([]byte("hello")), "text/plain", time.Hour)
if err != nil {
t.Fatalf("Create() error = %v", err)
}
badIndexDir := filepath.Join(t.TempDir(), "index-dir")
if err := os.MkdirAll(badIndexDir, 0o755); err != nil {
t.Fatalf("MkdirAll() error = %v", err)
}
store.index = badIndexDir
if err := store.Delete(meta.ID); err == nil {
t.Fatalf("Delete() error = nil, want persist failure")
}
if _, err := store.Get(meta.ID); err != nil {
t.Fatalf("entry should be restored after delete persist failure: %v", err)
}
}
func TestDeleteRemoveFailure(t *testing.T) {
t.Parallel()
store, err := NewFilesystemStore(filepath.Join(t.TempDir(), "data"))
if err != nil {
t.Fatalf("NewFilesystemStore() error = %v", err)
}
nonEmptyDir := filepath.Join(t.TempDir(), "non-empty")
if err := os.MkdirAll(nonEmptyDir, 0o755); err != nil {
t.Fatalf("MkdirAll() error = %v", err)
}
if err := os.WriteFile(filepath.Join(nonEmptyDir, "child"), []byte("x"), 0o644); err != nil {
t.Fatalf("WriteFile() error = %v", err)
}
id := "dddddddddddddddddddddddd"
store.metadata[id] = Metadata{
ID: id,
FilePath: nonEmptyDir,
CreatedAt: time.Now().UTC(),
ExpiresAt: time.Now().UTC().Add(time.Hour),
ContentType: "text/plain",
Size: 1,
}
if err := store.persistLocked(); err != nil {
t.Fatalf("persistLocked() error = %v", err)
}
if err := store.Delete(id); err == nil {
t.Fatalf("Delete() error = nil, want remove failure")
}
}
func TestDeleteExpiredPersistFailureRestoresEntries(t *testing.T) {
t.Parallel()
store, err := NewFilesystemStore(filepath.Join(t.TempDir(), "data"))
if err != nil {
t.Fatalf("NewFilesystemStore() error = %v", err)
}
meta, err := store.Create(context.Background(), bytes.NewReader([]byte("expired")), "text/plain", -time.Second)
if err != nil {
t.Fatalf("Create() error = %v", err)
}
badIndexDir := filepath.Join(t.TempDir(), "index-dir")
if err := os.MkdirAll(badIndexDir, 0o755); err != nil {
t.Fatalf("MkdirAll() error = %v", err)
}
store.index = badIndexDir
if _, err := store.DeleteExpired(time.Now().UTC()); err == nil {
t.Fatalf("DeleteExpired() error = nil, want persist failure")
}
if _, err := store.Get(meta.ID); err != nil {
t.Fatalf("entry should be restored after DeleteExpired persist failure: %v", err)
}
}
func TestDeleteExpiredRemoveFailure(t *testing.T) {
t.Parallel()
store, err := NewFilesystemStore(filepath.Join(t.TempDir(), "data"))
if err != nil {
t.Fatalf("NewFilesystemStore() error = %v", err)
}
nonEmptyDir := filepath.Join(t.TempDir(), "expired-non-empty")
if err := os.MkdirAll(nonEmptyDir, 0o755); err != nil {
t.Fatalf("MkdirAll() error = %v", err)
}
if err := os.WriteFile(filepath.Join(nonEmptyDir, "child"), []byte("x"), 0o644); err != nil {
t.Fatalf("WriteFile() error = %v", err)
}
id := "eeeeeeeeeeeeeeeeeeeeeeee"
store.metadata[id] = Metadata{
ID: id,
FilePath: nonEmptyDir,
CreatedAt: time.Now().UTC(),
ExpiresAt: time.Now().UTC().Add(-time.Second),
ContentType: "text/plain",
Size: 1,
}
if err := store.persistLocked(); err != nil {
t.Fatalf("persistLocked() error = %v", err)
}
if _, err := store.DeleteExpired(time.Now().UTC()); err == nil {
t.Fatalf("DeleteExpired() error = nil, want remove failure")
}
}
func TestNewFilesystemStoreFailsWhenDataDirIsFile(t *testing.T) {
t.Parallel()
tmp := t.TempDir()
filePath := filepath.Join(tmp, "not-dir")
if err := os.WriteFile(filePath, []byte("x"), 0o644); err != nil {
t.Fatalf("WriteFile() error = %v", err)
}
if _, err := NewFilesystemStore(filePath); err == nil {
t.Fatalf("NewFilesystemStore() error = nil, want mkdir failure")
}
}