s1d3sw1ped/scratchbox
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 5 Wiki Activity

Many changes

Browse Source
This commit is contained in:
Justin Harms
2026-05-31 20:17:49 -05:00
parent 9e398957a9
commit 99d46ecd2a
28 changed files with 3729 additions and 594 deletions
Download Patch File Download Diff File Expand all files Collapse all files
internal/storage/filesystem.go
+138 -66
View File
@@ -11,34 +11,39 @@ import (
"io"
"os"
"path/filepath"
"strings"
"sync"
"time"
)
const (
indexFilename = "metadata.json"
filesDirName = "scratches"
idBytes = 12
indexFilename = "metadata.json"
filesDirName = "scratches"
idBytes = 12
maxCreateIDAttempts = 100
)
var ErrNotFound = errors.New("scratch not found")
type Metadata struct {
ID string `json:"id"`
FilePath string `json:"file_path"`
CreatedAt time.Time `json:"created_at"`
ExpiresAt time.Time `json:"expires_at"`
ContentType string `json:"content_type"`
Size int64 `json:"size"`
ID string `json:"id"`
FilePath string `json:"file_path"`
CreatedAt time.Time `json:"created_at"`
ExpiresAt time.Time `json:"expires_at"`
ContentType string `json:"content_type"`
OriginalName string `json:"original_name,omitempty"`
Size int64 `json:"size"`
}
type FilesystemStore struct {
dataDir string
filesDir string
index string
dataDir string
filesDir string
index string
idGenerator func() (string, error)
mu sync.RWMutex
metadata map[string]Metadata
mu sync.RWMutex
metadata map[string]Metadata
reservedIDs map[string]struct{}
}
func NewFilesystemStore(dataDir string) (*FilesystemStore, error) {
@@ -48,10 +53,12 @@ func NewFilesystemStore(dataDir string) (*FilesystemStore, error) {
}
store := &FilesystemStore{
dataDir: dataDir,
filesDir: filesDir,
index: filepath.Join(dataDir, indexFilename),
metadata: map[string]Metadata{},
dataDir: dataDir,
filesDir: filesDir,
index: filepath.Join(dataDir, indexFilename),
idGenerator: generateID,
metadata: map[string]Metadata{},
reservedIDs: map[string]struct{}{},
}
if err := store.loadIndex(); err != nil {
return nil, err
@@ -63,59 +70,124 @@ func NewFilesystemStore(dataDir string) (*FilesystemStore, error) {
}
func (s *FilesystemStore) Create(ctx context.Context, body io.Reader, contentType string, ttl time.Duration) (Metadata, error) {
id, err := generateID()
if err != nil {
return Metadata{}, fmt.Errorf("generate id: %w", err)
}
return s.CreateWithOriginalName(ctx, body, contentType, "", ttl)
}
tempFile, err := os.CreateTemp(s.filesDir, id+".tmp-*")
if err != nil {
return Metadata{}, fmt.Errorf("create temp file: %w", err)
}
func (s *FilesystemStore) CreateWithOriginalName(ctx context.Context, body io.Reader, contentType string, originalName string, ttl time.Duration) (Metadata, error) {
for attempt := 0; attempt < maxCreateIDAttempts; attempt++ {
id, err := s.idGenerator()
if err != nil {
return Metadata{}, fmt.Errorf("generate id: %w", err)
}
finalPath := filepath.Join(s.filesDir, id)
gzipWriter := gzip.NewWriter(tempFile)
size, copyErr := io.Copy(gzipWriter, body)
gzipCloseErr := gzipWriter.Close()
closeErr := tempFile.Close()
if copyErr != nil {
_ = os.Remove(tempFile.Name())
return Metadata{}, fmt.Errorf("write scratch content: %w", copyErr)
}
if gzipCloseErr != nil {
_ = os.Remove(tempFile.Name())
return Metadata{}, fmt.Errorf("close gzip writer: %w", gzipCloseErr)
}
if closeErr != nil {
_ = os.Remove(tempFile.Name())
return Metadata{}, fmt.Errorf("close temp file: %w", closeErr)
}
s.mu.RLock()
_, exists := s.metadata[id]
_, reserved := s.reservedIDs[id]
s.mu.RUnlock()
if exists || reserved {
continue
}
if _, err := os.Stat(finalPath); err == nil {
continue
} else if !errors.Is(err, os.ErrNotExist) {
return Metadata{}, fmt.Errorf("stat candidate scratch path: %w", err)
}
finalPath := filepath.Join(s.filesDir, id)
if err := os.Rename(tempFile.Name(), finalPath); err != nil {
_ = os.Remove(tempFile.Name())
return Metadata{}, fmt.Errorf("atomically move scratch file: %w", err)
}
now := time.Now().UTC()
meta := Metadata{
ID: id,
FilePath: finalPath,
CreatedAt: now,
ExpiresAt: now.Add(ttl),
ContentType: contentType,
Size: size,
}
s.mu.Lock()
s.metadata[id] = meta
if err := s.persistLocked(); err != nil {
delete(s.metadata, id)
s.mu.Lock()
if _, exists := s.metadata[id]; exists {
s.mu.Unlock()
continue
}
if _, reserved := s.reservedIDs[id]; reserved {
s.mu.Unlock()
continue
}
// Reserve the ID before consuming the request body to avoid
// concurrent writers choosing the same ID.
s.reservedIDs[id] = struct{}{}
s.mu.Unlock()
_ = os.Remove(finalPath)
return Metadata{}, err
tempFile, err := os.CreateTemp(s.filesDir, id+".tmp-*")
if err != nil {
s.mu.Lock()
delete(s.reservedIDs, id)
s.mu.Unlock()
return Metadata{}, fmt.Errorf("create temp file: %w", err)
}
gzipWriter := gzip.NewWriter(tempFile)
size, copyErr := io.Copy(gzipWriter, body)
gzipCloseErr := gzipWriter.Close()
closeErr := tempFile.Close()
if copyErr != nil {
_ = os.Remove(tempFile.Name())
s.mu.Lock()
delete(s.reservedIDs, id)
s.mu.Unlock()
return Metadata{}, fmt.Errorf("write scratch content: %w", copyErr)
}
if gzipCloseErr != nil {
_ = os.Remove(tempFile.Name())
s.mu.Lock()
delete(s.reservedIDs, id)
s.mu.Unlock()
return Metadata{}, fmt.Errorf("close gzip writer: %w", gzipCloseErr)
}
if closeErr != nil {
_ = os.Remove(tempFile.Name())
s.mu.Lock()
delete(s.reservedIDs, id)
s.mu.Unlock()
return Metadata{}, fmt.Errorf("close temp file: %w", closeErr)
}
if err := os.Rename(tempFile.Name(), finalPath); err != nil {
_ = os.Remove(tempFile.Name())
s.mu.Lock()
delete(s.reservedIDs, id)
s.mu.Unlock()
return Metadata{}, fmt.Errorf("atomically move scratch file: %w", err)
}
now := time.Now().UTC()
meta := Metadata{
ID: id,
FilePath: finalPath,
CreatedAt: now,
ExpiresAt: now.Add(ttl),
ContentType: contentType,
OriginalName: normalizeOriginalName(originalName),
Size: size,
}
s.mu.Lock()
delete(s.reservedIDs, id)
s.metadata[id] = meta
if err := s.persistLocked(); err != nil {
delete(s.metadata, id)
s.mu.Unlock()
_ = os.Remove(finalPath)
return Metadata{}, err
}
s.mu.Unlock()
return meta, nil
}
s.mu.Unlock()
return meta, nil
return Metadata{}, fmt.Errorf("failed to allocate unique scratch id after %d attempts", maxCreateIDAttempts)
}
func normalizeOriginalName(originalName string) string {
name := strings.TrimSpace(originalName)
if name == "" {
return ""
}
name = filepath.Base(name)
name = strings.TrimSpace(name)
if name == "." || name == string(filepath.Separator) {
return ""
}
return name
}
func (s *FilesystemStore) Get(id string) (Metadata, error) {
internal/storage/filesystem_test.go
+386
View File
@@ -5,9 +5,12 @@ import (
"context"
"encoding/json"
"errors"
"fmt"
"io"
"os"
"path/filepath"
"strings"
"sync"
"testing"
"time"
)
@@ -51,6 +54,226 @@ func TestCreateStoresGzipAndOpenIsTransparent(t *testing.T) {
}
}
func TestCreateWithOriginalNameNormalizesFilename(t *testing.T) {
t.Parallel()
store, err := NewFilesystemStore(filepath.Join(t.TempDir(), "data"))
if err != nil {
t.Fatalf("NewFilesystemStore() error = %v", err)
}
meta, err := store.CreateWithOriginalName(context.Background(), bytes.NewReader([]byte("hello")), "text/plain", " ../../demo.txt ", time.Hour)
if err != nil {
t.Fatalf("CreateWithOriginalName() error = %v", err)
}
if meta.OriginalName != "demo.txt" {
t.Fatalf("meta.OriginalName = %q, want %q", meta.OriginalName, "demo.txt")
}
reloaded, err := NewFilesystemStore(store.dataDir)
if err != nil {
t.Fatalf("NewFilesystemStore() reload error = %v", err)
}
got, err := reloaded.Get(meta.ID)
if err != nil {
t.Fatalf("Get() error = %v", err)
}
if got.OriginalName != "demo.txt" {
t.Fatalf("persisted OriginalName = %q, want %q", got.OriginalName, "demo.txt")
}
}
func TestCreateWithOriginalNameRetriesOnIDCollision(t *testing.T) {
t.Parallel()
store, err := NewFilesystemStore(filepath.Join(t.TempDir(), "data"))
if err != nil {
t.Fatalf("NewFilesystemStore() error = %v", err)
}
existingID := "aaaaaaaaaaaaaaaaaaaaaaaa"
existingPath := filepath.Join(store.filesDir, existingID)
if err := os.WriteFile(existingPath, []byte{0x1f, 0x8b, 0x08, 0x00}, 0o644); err != nil {
t.Fatalf("WriteFile(existing collision path) error = %v", err)
}
store.mu.Lock()
store.metadata[existingID] = Metadata{
ID: existingID,
FilePath: existingPath,
CreatedAt: time.Now().UTC(),
ExpiresAt: time.Now().UTC().Add(time.Hour),
ContentType: "text/plain",
Size: 4,
}
store.mu.Unlock()
nextID := "bbbbbbbbbbbbbbbbbbbbbbbb"
calls := 0
store.idGenerator = func() (string, error) {
calls++
if calls == 1 {
return existingID, nil
}
return nextID, nil
}
meta, err := store.CreateWithOriginalName(context.Background(), bytes.NewReader([]byte("hello")), "text/plain", "hello.txt", time.Hour)
if err != nil {
t.Fatalf("CreateWithOriginalName() error = %v", err)
}
if meta.ID != nextID {
t.Fatalf("meta.ID = %q, want %q", meta.ID, nextID)
}
if calls != 2 {
t.Fatalf("idGenerator calls = %d, want 2", calls)
}
}
func TestCreateWithOriginalNameFailsAfterMaxIDRetries(t *testing.T) {
t.Parallel()
store, err := NewFilesystemStore(filepath.Join(t.TempDir(), "data"))
if err != nil {
t.Fatalf("NewFilesystemStore() error = %v", err)
}
existingID := "cccccccccccccccccccccccc"
existingPath := filepath.Join(store.filesDir, existingID)
store.mu.Lock()
store.metadata[existingID] = Metadata{
ID: existingID,
FilePath: existingPath,
CreatedAt: time.Now().UTC(),
ExpiresAt: time.Now().UTC().Add(time.Hour),
ContentType: "text/plain",
}
store.mu.Unlock()
store.idGenerator = func() (string, error) {
return existingID, nil
}
_, err = store.CreateWithOriginalName(context.Background(), bytes.NewReader([]byte("hello")), "text/plain", "hello.txt", time.Hour)
if err == nil {
t.Fatalf("CreateWithOriginalName() error = nil, want retry exhaustion error")
}
if !strings.Contains(err.Error(), "failed to allocate unique scratch id") {
t.Fatalf("unexpected error: %v", err)
}
}
func TestCreateWithOriginalNameRetriesWhenIDIsReservedInFlight(t *testing.T) {
t.Parallel()
store, err := NewFilesystemStore(filepath.Join(t.TempDir(), "data"))
if err != nil {
t.Fatalf("NewFilesystemStore() error = %v", err)
}
const collideID = "abababababababababababab"
const uniqueID = "cdcdcdcdcdcdcdcdcdcdcdcd"
var idMu sync.Mutex
idCalls := 0
store.idGenerator = func() (string, error) {
idMu.Lock()
defer idMu.Unlock()
idCalls++
if idCalls <= 2 {
return collideID, nil
}
return uniqueID, nil
}
release := make(chan struct{})
firstMetaCh := make(chan Metadata, 1)
firstErrCh := make(chan error, 1)
go func() {
meta, createErr := store.CreateWithOriginalName(
context.Background(),
&blockingReader{release: release, payload: []byte("first upload")},
"text/plain",
"first.txt",
time.Hour,
)
firstMetaCh <- meta
firstErrCh <- createErr
}()
deadline := time.Now().Add(2 * time.Second)
for {
store.mu.RLock()
_, reserved := store.reservedIDs[collideID]
store.mu.RUnlock()
if reserved {
break
}
if time.Now().After(deadline) {
t.Fatalf("timed out waiting for first upload reservation")
}
time.Sleep(2 * time.Millisecond)
}
secondMeta, err := store.CreateWithOriginalName(
context.Background(),
strings.NewReader("second upload"),
"text/plain",
"second.txt",
time.Hour,
)
if err != nil {
t.Fatalf("second CreateWithOriginalName() error = %v", err)
}
if secondMeta.ID != uniqueID {
t.Fatalf("second meta.ID = %q, want %q after collision retry", secondMeta.ID, uniqueID)
}
if secondMeta.ID == collideID {
t.Fatalf("second create reused in-flight reserved ID")
}
close(release)
firstMeta := <-firstMetaCh
if err := <-firstErrCh; err != nil {
t.Fatalf("first CreateWithOriginalName() error = %v", err)
}
if firstMeta.ID != collideID {
t.Fatalf("first meta.ID = %q, want %q", firstMeta.ID, collideID)
}
firstReader, _, err := store.Open(firstMeta.ID)
if err != nil {
t.Fatalf("Open(first) error = %v", err)
}
defer firstReader.Close()
firstBody, err := io.ReadAll(firstReader)
if err != nil {
t.Fatalf("ReadAll(first) error = %v", err)
}
if string(firstBody) != "first upload" {
t.Fatalf("first content = %q, want %q", string(firstBody), "first upload")
}
secondReader, _, err := store.Open(secondMeta.ID)
if err != nil {
t.Fatalf("Open(second) error = %v", err)
}
defer secondReader.Close()
secondBody, err := io.ReadAll(secondReader)
if err != nil {
t.Fatalf("ReadAll(second) error = %v", err)
}
if string(secondBody) != "second upload" {
t.Fatalf("second content = %q, want %q", string(secondBody), "second upload")
}
idMu.Lock()
totalCalls := idCalls
idMu.Unlock()
if totalCalls < 3 {
t.Fatalf("idGenerator call count = %d, want at least 3", totalCalls)
}
}
func TestGetDeleteAndNotFound(t *testing.T) {
t.Parallel()
@@ -400,3 +623,166 @@ func TestNewFilesystemStoreFailsWhenDataDirIsFile(t *testing.T) {
t.Fatalf("NewFilesystemStore() error = nil, want mkdir failure")
}
}
type blockingReader struct {
release <-chan struct{}
payload []byte
sent bool
}
func (r *blockingReader) Read(p []byte) (int, error) {
if r.sent {
return 0, io.EOF
}
<-r.release
r.sent = true
return copy(p, r.payload), nil
}
func TestCreateReservationIsNotVisibleToCleanupOrReads(t *testing.T) {
t.Parallel()
store, err := NewFilesystemStore(filepath.Join(t.TempDir(), "data"))
if err != nil {
t.Fatalf("NewFilesystemStore() error = %v", err)
}
const reservedID = "ffffffffffffffffffffffff"
store.idGenerator = func() (string, error) {
return reservedID, nil
}
release := make(chan struct{})
createErrCh := make(chan error, 1)
go func() {
_, createErr := store.CreateWithOriginalName(
context.Background(),
&blockingReader{release: release, payload: []byte("in-flight")},
"text/plain",
"",
time.Hour,
)
createErrCh <- createErr
}()
deadline := time.Now().Add(2 * time.Second)
for {
store.mu.RLock()
_, reserved := store.reservedIDs[reservedID]
_, visible := store.metadata[reservedID]
store.mu.RUnlock()
if reserved {
if visible {
t.Fatalf("in-flight ID should not appear in metadata")
}
break
}
if time.Now().After(deadline) {
t.Fatalf("timed out waiting for ID reservation")
}
time.Sleep(2 * time.Millisecond)
}
if _, err := store.Get(reservedID); !errors.Is(err, ErrNotFound) {
t.Fatalf("Get() during in-flight create error = %v, want ErrNotFound", err)
}
deleted, err := store.DeleteExpired(time.Now().UTC())
if err != nil {
t.Fatalf("DeleteExpired() error = %v", err)
}
if deleted != 0 {
t.Fatalf("DeleteExpired() = %d, want 0 for in-flight create", deleted)
}
close(release)
if err := <-createErrCh; err != nil {
t.Fatalf("CreateWithOriginalName() error = %v", err)
}
reader, _, err := store.Open(reservedID)
if err != nil {
t.Fatalf("Open() error = %v", err)
}
defer reader.Close()
body, err := io.ReadAll(reader)
if err != nil {
t.Fatalf("ReadAll() error = %v", err)
}
if string(body) != "in-flight" {
t.Fatalf("Open() content = %q, want %q", string(body), "in-flight")
}
}
func TestConcurrentCreateWithOriginalNameProducesUniqueReadableEntries(t *testing.T) {
t.Parallel()
store, err := NewFilesystemStore(filepath.Join(t.TempDir(), "data"))
if err != nil {
t.Fatalf("NewFilesystemStore() error = %v", err)
}
const writers = 64
errCh := make(chan error, writers)
idsCh := make(chan string, writers)
var wg sync.WaitGroup
for i := 0; i < writers; i++ {
wg.Add(1)
go func(i int) {
defer wg.Done()
payload := fmt.Sprintf("storage-writer-%03d-%s", i, strings.Repeat("z", i%13))
meta, createErr := store.CreateWithOriginalName(
context.Background(),
strings.NewReader(payload),
"text/plain; charset=utf-8",
fmt.Sprintf("w-%03d.txt", i),
time.Hour,
)
if createErr != nil {
errCh <- fmt.Errorf("CreateWithOriginalName() error: %w", createErr)
return
}
reader, _, openErr := store.Open(meta.ID)
if openErr != nil {
errCh <- fmt.Errorf("Open() error: %w", openErr)
return
}
defer reader.Close()
got, readErr := io.ReadAll(reader)
if readErr != nil {
errCh <- fmt.Errorf("ReadAll() error: %w", readErr)
return
}
if string(got) != payload {
errCh <- fmt.Errorf("content mismatch = %q, want %q", string(got), payload)
return
}
idsCh <- meta.ID
}(i)
}
wg.Wait()
close(errCh)
close(idsCh)
for err := range errCh {
t.Error(err)
}
seen := make(map[string]struct{}, writers)
for id := range idsCh {
if _, exists := seen[id]; exists {
t.Errorf("duplicate id detected: %s", id)
continue
}
seen[id] = struct{}{}
}
if len(seen) != writers {
t.Fatalf("unique IDs = %d, want %d", len(seen), writers)
}
}