Many changes

2026-05-31 20:17:49 -05:00
parent 9e398957a9
commit 99d46ecd2a
28 changed files with 3729 additions and 594 deletions
@@ -5,9 +5,12 @@ import (
 	"context"
 	"encoding/json"
 	"errors"
+	"fmt"
 	"io"
 	"os"
 	"path/filepath"
+	"strings"
+	"sync"
 	"testing"
 	"time"
 )
@@ -51,6 +54,226 @@ func TestCreateStoresGzipAndOpenIsTransparent(t *testing.T) {
 	}
 }

+func TestCreateWithOriginalNameNormalizesFilename(t *testing.T) {
+	t.Parallel()
+
+	store, err := NewFilesystemStore(filepath.Join(t.TempDir(), "data"))
+	if err != nil {
+		t.Fatalf("NewFilesystemStore() error = %v", err)
+	}
+
+	meta, err := store.CreateWithOriginalName(context.Background(), bytes.NewReader([]byte("hello")), "text/plain", "  ../../demo.txt  ", time.Hour)
+	if err != nil {
+		t.Fatalf("CreateWithOriginalName() error = %v", err)
+	}
+	if meta.OriginalName != "demo.txt" {
+		t.Fatalf("meta.OriginalName = %q, want %q", meta.OriginalName, "demo.txt")
+	}
+
+	reloaded, err := NewFilesystemStore(store.dataDir)
+	if err != nil {
+		t.Fatalf("NewFilesystemStore() reload error = %v", err)
+	}
+	got, err := reloaded.Get(meta.ID)
+	if err != nil {
+		t.Fatalf("Get() error = %v", err)
+	}
+	if got.OriginalName != "demo.txt" {
+		t.Fatalf("persisted OriginalName = %q, want %q", got.OriginalName, "demo.txt")
+	}
+}
+
+func TestCreateWithOriginalNameRetriesOnIDCollision(t *testing.T) {
+	t.Parallel()
+
+	store, err := NewFilesystemStore(filepath.Join(t.TempDir(), "data"))
+	if err != nil {
+		t.Fatalf("NewFilesystemStore() error = %v", err)
+	}
+
+	existingID := "aaaaaaaaaaaaaaaaaaaaaaaa"
+	existingPath := filepath.Join(store.filesDir, existingID)
+	if err := os.WriteFile(existingPath, []byte{0x1f, 0x8b, 0x08, 0x00}, 0o644); err != nil {
+		t.Fatalf("WriteFile(existing collision path) error = %v", err)
+	}
+	store.mu.Lock()
+	store.metadata[existingID] = Metadata{
+		ID:          existingID,
+		FilePath:    existingPath,
+		CreatedAt:   time.Now().UTC(),
+		ExpiresAt:   time.Now().UTC().Add(time.Hour),
+		ContentType: "text/plain",
+		Size:        4,
+	}
+	store.mu.Unlock()
+
+	nextID := "bbbbbbbbbbbbbbbbbbbbbbbb"
+	calls := 0
+	store.idGenerator = func() (string, error) {
+		calls++
+		if calls == 1 {
+			return existingID, nil
+		}
+		return nextID, nil
+	}
+
+	meta, err := store.CreateWithOriginalName(context.Background(), bytes.NewReader([]byte("hello")), "text/plain", "hello.txt", time.Hour)
+	if err != nil {
+		t.Fatalf("CreateWithOriginalName() error = %v", err)
+	}
+	if meta.ID != nextID {
+		t.Fatalf("meta.ID = %q, want %q", meta.ID, nextID)
+	}
+	if calls != 2 {
+		t.Fatalf("idGenerator calls = %d, want 2", calls)
+	}
+}
+
+func TestCreateWithOriginalNameFailsAfterMaxIDRetries(t *testing.T) {
+	t.Parallel()
+
+	store, err := NewFilesystemStore(filepath.Join(t.TempDir(), "data"))
+	if err != nil {
+		t.Fatalf("NewFilesystemStore() error = %v", err)
+	}
+
+	existingID := "cccccccccccccccccccccccc"
+	existingPath := filepath.Join(store.filesDir, existingID)
+	store.mu.Lock()
+	store.metadata[existingID] = Metadata{
+		ID:          existingID,
+		FilePath:    existingPath,
+		CreatedAt:   time.Now().UTC(),
+		ExpiresAt:   time.Now().UTC().Add(time.Hour),
+		ContentType: "text/plain",
+	}
+	store.mu.Unlock()
+
+	store.idGenerator = func() (string, error) {
+		return existingID, nil
+	}
+
+	_, err = store.CreateWithOriginalName(context.Background(), bytes.NewReader([]byte("hello")), "text/plain", "hello.txt", time.Hour)
+	if err == nil {
+		t.Fatalf("CreateWithOriginalName() error = nil, want retry exhaustion error")
+	}
+	if !strings.Contains(err.Error(), "failed to allocate unique scratch id") {
+		t.Fatalf("unexpected error: %v", err)
+	}
+}
+
+func TestCreateWithOriginalNameRetriesWhenIDIsReservedInFlight(t *testing.T) {
+	t.Parallel()
+
+	store, err := NewFilesystemStore(filepath.Join(t.TempDir(), "data"))
+	if err != nil {
+		t.Fatalf("NewFilesystemStore() error = %v", err)
+	}
+
+	const collideID = "abababababababababababab"
+	const uniqueID = "cdcdcdcdcdcdcdcdcdcdcdcd"
+
+	var idMu sync.Mutex
+	idCalls := 0
+	store.idGenerator = func() (string, error) {
+		idMu.Lock()
+		defer idMu.Unlock()
+		idCalls++
+		if idCalls <= 2 {
+			return collideID, nil
+		}
+		return uniqueID, nil
+	}
+
+	release := make(chan struct{})
+	firstMetaCh := make(chan Metadata, 1)
+	firstErrCh := make(chan error, 1)
+	go func() {
+		meta, createErr := store.CreateWithOriginalName(
+			context.Background(),
+			&blockingReader{release: release, payload: []byte("first upload")},
+			"text/plain",
+			"first.txt",
+			time.Hour,
+		)
+		firstMetaCh <- meta
+		firstErrCh <- createErr
+	}()
+
+	deadline := time.Now().Add(2 * time.Second)
+	for {
+		store.mu.RLock()
+		_, reserved := store.reservedIDs[collideID]
+		store.mu.RUnlock()
+		if reserved {
+			break
+		}
+		if time.Now().After(deadline) {
+			t.Fatalf("timed out waiting for first upload reservation")
+		}
+		time.Sleep(2 * time.Millisecond)
+	}
+
+	secondMeta, err := store.CreateWithOriginalName(
+		context.Background(),
+		strings.NewReader("second upload"),
+		"text/plain",
+		"second.txt",
+		time.Hour,
+	)
+	if err != nil {
+		t.Fatalf("second CreateWithOriginalName() error = %v", err)
+	}
+	if secondMeta.ID != uniqueID {
+		t.Fatalf("second meta.ID = %q, want %q after collision retry", secondMeta.ID, uniqueID)
+	}
+	if secondMeta.ID == collideID {
+		t.Fatalf("second create reused in-flight reserved ID")
+	}
+
+	close(release)
+	firstMeta := <-firstMetaCh
+	if err := <-firstErrCh; err != nil {
+		t.Fatalf("first CreateWithOriginalName() error = %v", err)
+	}
+	if firstMeta.ID != collideID {
+		t.Fatalf("first meta.ID = %q, want %q", firstMeta.ID, collideID)
+	}
+
+	firstReader, _, err := store.Open(firstMeta.ID)
+	if err != nil {
+		t.Fatalf("Open(first) error = %v", err)
+	}
+	defer firstReader.Close()
+	firstBody, err := io.ReadAll(firstReader)
+	if err != nil {
+		t.Fatalf("ReadAll(first) error = %v", err)
+	}
+	if string(firstBody) != "first upload" {
+		t.Fatalf("first content = %q, want %q", string(firstBody), "first upload")
+	}
+
+	secondReader, _, err := store.Open(secondMeta.ID)
+	if err != nil {
+		t.Fatalf("Open(second) error = %v", err)
+	}
+	defer secondReader.Close()
+	secondBody, err := io.ReadAll(secondReader)
+	if err != nil {
+		t.Fatalf("ReadAll(second) error = %v", err)
+	}
+	if string(secondBody) != "second upload" {
+		t.Fatalf("second content = %q, want %q", string(secondBody), "second upload")
+	}
+
+	idMu.Lock()
+	totalCalls := idCalls
+	idMu.Unlock()
+	if totalCalls < 3 {
+		t.Fatalf("idGenerator call count = %d, want at least 3", totalCalls)
+	}
+}
+
 func TestGetDeleteAndNotFound(t *testing.T) {
 	t.Parallel()

@@ -400,3 +623,166 @@ func TestNewFilesystemStoreFailsWhenDataDirIsFile(t *testing.T) {
 		t.Fatalf("NewFilesystemStore() error = nil, want mkdir failure")
 	}
 }
+
+type blockingReader struct {
+	release <-chan struct{}
+	payload []byte
+	sent    bool
+}
+
+func (r *blockingReader) Read(p []byte) (int, error) {
+	if r.sent {
+		return 0, io.EOF
+	}
+	<-r.release
+	r.sent = true
+	return copy(p, r.payload), nil
+}
+
+func TestCreateReservationIsNotVisibleToCleanupOrReads(t *testing.T) {
+	t.Parallel()
+
+	store, err := NewFilesystemStore(filepath.Join(t.TempDir(), "data"))
+	if err != nil {
+		t.Fatalf("NewFilesystemStore() error = %v", err)
+	}
+
+	const reservedID = "ffffffffffffffffffffffff"
+	store.idGenerator = func() (string, error) {
+		return reservedID, nil
+	}
+
+	release := make(chan struct{})
+	createErrCh := make(chan error, 1)
+	go func() {
+		_, createErr := store.CreateWithOriginalName(
+			context.Background(),
+			&blockingReader{release: release, payload: []byte("in-flight")},
+			"text/plain",
+			"",
+			time.Hour,
+		)
+		createErrCh <- createErr
+	}()
+
+	deadline := time.Now().Add(2 * time.Second)
+	for {
+		store.mu.RLock()
+		_, reserved := store.reservedIDs[reservedID]
+		_, visible := store.metadata[reservedID]
+		store.mu.RUnlock()
+		if reserved {
+			if visible {
+				t.Fatalf("in-flight ID should not appear in metadata")
+			}
+			break
+		}
+		if time.Now().After(deadline) {
+			t.Fatalf("timed out waiting for ID reservation")
+		}
+		time.Sleep(2 * time.Millisecond)
+	}
+
+	if _, err := store.Get(reservedID); !errors.Is(err, ErrNotFound) {
+		t.Fatalf("Get() during in-flight create error = %v, want ErrNotFound", err)
+	}
+
+	deleted, err := store.DeleteExpired(time.Now().UTC())
+	if err != nil {
+		t.Fatalf("DeleteExpired() error = %v", err)
+	}
+	if deleted != 0 {
+		t.Fatalf("DeleteExpired() = %d, want 0 for in-flight create", deleted)
+	}
+
+	close(release)
+	if err := <-createErrCh; err != nil {
+		t.Fatalf("CreateWithOriginalName() error = %v", err)
+	}
+
+	reader, _, err := store.Open(reservedID)
+	if err != nil {
+		t.Fatalf("Open() error = %v", err)
+	}
+	defer reader.Close()
+
+	body, err := io.ReadAll(reader)
+	if err != nil {
+		t.Fatalf("ReadAll() error = %v", err)
+	}
+	if string(body) != "in-flight" {
+		t.Fatalf("Open() content = %q, want %q", string(body), "in-flight")
+	}
+}
+
+func TestConcurrentCreateWithOriginalNameProducesUniqueReadableEntries(t *testing.T) {
+	t.Parallel()
+
+	store, err := NewFilesystemStore(filepath.Join(t.TempDir(), "data"))
+	if err != nil {
+		t.Fatalf("NewFilesystemStore() error = %v", err)
+	}
+
+	const writers = 64
+	errCh := make(chan error, writers)
+	idsCh := make(chan string, writers)
+	var wg sync.WaitGroup
+
+	for i := 0; i < writers; i++ {
+		wg.Add(1)
+		go func(i int) {
+			defer wg.Done()
+
+			payload := fmt.Sprintf("storage-writer-%03d-%s", i, strings.Repeat("z", i%13))
+			meta, createErr := store.CreateWithOriginalName(
+				context.Background(),
+				strings.NewReader(payload),
+				"text/plain; charset=utf-8",
+				fmt.Sprintf("w-%03d.txt", i),
+				time.Hour,
+			)
+			if createErr != nil {
+				errCh <- fmt.Errorf("CreateWithOriginalName() error: %w", createErr)
+				return
+			}
+
+			reader, _, openErr := store.Open(meta.ID)
+			if openErr != nil {
+				errCh <- fmt.Errorf("Open() error: %w", openErr)
+				return
+			}
+			defer reader.Close()
+
+			got, readErr := io.ReadAll(reader)
+			if readErr != nil {
+				errCh <- fmt.Errorf("ReadAll() error: %w", readErr)
+				return
+			}
+			if string(got) != payload {
+				errCh <- fmt.Errorf("content mismatch = %q, want %q", string(got), payload)
+				return
+			}
+
+			idsCh <- meta.ID
+		}(i)
+	}
+
+	wg.Wait()
+	close(errCh)
+	close(idsCh)
+	for err := range errCh {
+		t.Error(err)
+	}
+
+	seen := make(map[string]struct{}, writers)
+	for id := range idsCh {
+		if _, exists := seen[id]; exists {
+			t.Errorf("duplicate id detected: %s", id)
+			continue
+		}
+		seen[id] = struct{}{}
+	}
+	if len(seen) != writers {
+		t.Fatalf("unique IDs = %d, want %d", len(seen), writers)
+	}
+}