package storage import ( "bytes" "context" "encoding/json" "errors" "fmt" "io" "os" "path/filepath" "strings" "sync" "testing" "time" ) func TestCreateStoresGzipAndOpenIsTransparent(t *testing.T) { t.Parallel() store, err := NewFilesystemStore(filepath.Join(t.TempDir(), "data")) if err != nil { t.Fatalf("NewFilesystemStore() error = %v", err) } original := bytes.Repeat([]byte("hello scratch "), 1024) meta, err := store.Create(context.Background(), bytes.NewReader(original), "text/plain; charset=utf-8", time.Hour) if err != nil { t.Fatalf("Create() error = %v", err) } if meta.Size != int64(len(original)) { t.Fatalf("meta.Size = %d, want %d", meta.Size, len(original)) } onDisk, err := os.ReadFile(meta.FilePath) if err != nil { t.Fatalf("ReadFile() error = %v", err) } if len(onDisk) < 2 || onDisk[0] != 0x1f || onDisk[1] != 0x8b { t.Fatalf("stored file does not have gzip header") } reader, _, err := store.Open(meta.ID) if err != nil { t.Fatalf("Open() error = %v", err) } defer reader.Close() got, err := io.ReadAll(reader) if err != nil { t.Fatalf("ReadAll() error = %v", err) } if !bytes.Equal(got, original) { t.Fatalf("open content mismatch: got %d bytes, want %d", len(got), len(original)) } } func TestCreateWithOriginalNameNormalizesFilename(t *testing.T) { t.Parallel() store, err := NewFilesystemStore(filepath.Join(t.TempDir(), "data")) if err != nil { t.Fatalf("NewFilesystemStore() error = %v", err) } meta, err := store.CreateWithOriginalName(context.Background(), bytes.NewReader([]byte("hello")), "text/plain", " ../../demo.txt ", time.Hour) if err != nil { t.Fatalf("CreateWithOriginalName() error = %v", err) } if meta.OriginalName != "demo.txt" { t.Fatalf("meta.OriginalName = %q, want %q", meta.OriginalName, "demo.txt") } reloaded, err := NewFilesystemStore(store.dataDir) if err != nil { t.Fatalf("NewFilesystemStore() reload error = %v", err) } got, err := reloaded.Get(meta.ID) if err != nil { t.Fatalf("Get() error = %v", err) } if got.OriginalName != "demo.txt" { t.Fatalf("persisted OriginalName = %q, want %q", got.OriginalName, "demo.txt") } } func TestCreateWithOriginalNameRetriesOnIDCollision(t *testing.T) { t.Parallel() store, err := NewFilesystemStore(filepath.Join(t.TempDir(), "data")) if err != nil { t.Fatalf("NewFilesystemStore() error = %v", err) } existingID := "aaaaaaaaaaaaaaaaaaaaaaaa" existingPath := filepath.Join(store.filesDir, existingID) if err := os.WriteFile(existingPath, []byte{0x1f, 0x8b, 0x08, 0x00}, 0o644); err != nil { t.Fatalf("WriteFile(existing collision path) error = %v", err) } store.mu.Lock() store.metadata[existingID] = Metadata{ ID: existingID, FilePath: existingPath, CreatedAt: time.Now().UTC(), ExpiresAt: time.Now().UTC().Add(time.Hour), ContentType: "text/plain", Size: 4, } store.mu.Unlock() nextID := "bbbbbbbbbbbbbbbbbbbbbbbb" calls := 0 store.idGenerator = func() (string, error) { calls++ if calls == 1 { return existingID, nil } return nextID, nil } meta, err := store.CreateWithOriginalName(context.Background(), bytes.NewReader([]byte("hello")), "text/plain", "hello.txt", time.Hour) if err != nil { t.Fatalf("CreateWithOriginalName() error = %v", err) } if meta.ID != nextID { t.Fatalf("meta.ID = %q, want %q", meta.ID, nextID) } if calls != 2 { t.Fatalf("idGenerator calls = %d, want 2", calls) } } func TestCreateWithOriginalNameFailsAfterMaxIDRetries(t *testing.T) { t.Parallel() store, err := NewFilesystemStore(filepath.Join(t.TempDir(), "data")) if err != nil { t.Fatalf("NewFilesystemStore() error = %v", err) } existingID := "cccccccccccccccccccccccc" existingPath := filepath.Join(store.filesDir, existingID) store.mu.Lock() store.metadata[existingID] = Metadata{ ID: existingID, FilePath: existingPath, CreatedAt: time.Now().UTC(), ExpiresAt: time.Now().UTC().Add(time.Hour), ContentType: "text/plain", } store.mu.Unlock() store.idGenerator = func() (string, error) { return existingID, nil } _, err = store.CreateWithOriginalName(context.Background(), bytes.NewReader([]byte("hello")), "text/plain", "hello.txt", time.Hour) if err == nil { t.Fatalf("CreateWithOriginalName() error = nil, want retry exhaustion error") } if !strings.Contains(err.Error(), "failed to allocate unique scratch id") { t.Fatalf("unexpected error: %v", err) } } func TestCreateWithOriginalNameRetriesWhenIDIsReservedInFlight(t *testing.T) { t.Parallel() store, err := NewFilesystemStore(filepath.Join(t.TempDir(), "data")) if err != nil { t.Fatalf("NewFilesystemStore() error = %v", err) } const collideID = "abababababababababababab" const uniqueID = "cdcdcdcdcdcdcdcdcdcdcdcd" var idMu sync.Mutex idCalls := 0 store.idGenerator = func() (string, error) { idMu.Lock() defer idMu.Unlock() idCalls++ if idCalls <= 2 { return collideID, nil } return uniqueID, nil } release := make(chan struct{}) firstMetaCh := make(chan Metadata, 1) firstErrCh := make(chan error, 1) go func() { meta, createErr := store.CreateWithOriginalName( context.Background(), &blockingReader{release: release, payload: []byte("first upload")}, "text/plain", "first.txt", time.Hour, ) firstMetaCh <- meta firstErrCh <- createErr }() deadline := time.Now().Add(2 * time.Second) for { store.mu.RLock() _, reserved := store.reservedIDs[collideID] store.mu.RUnlock() if reserved { break } if time.Now().After(deadline) { t.Fatalf("timed out waiting for first upload reservation") } time.Sleep(2 * time.Millisecond) } secondMeta, err := store.CreateWithOriginalName( context.Background(), strings.NewReader("second upload"), "text/plain", "second.txt", time.Hour, ) if err != nil { t.Fatalf("second CreateWithOriginalName() error = %v", err) } if secondMeta.ID != uniqueID { t.Fatalf("second meta.ID = %q, want %q after collision retry", secondMeta.ID, uniqueID) } if secondMeta.ID == collideID { t.Fatalf("second create reused in-flight reserved ID") } close(release) firstMeta := <-firstMetaCh if err := <-firstErrCh; err != nil { t.Fatalf("first CreateWithOriginalName() error = %v", err) } if firstMeta.ID != collideID { t.Fatalf("first meta.ID = %q, want %q", firstMeta.ID, collideID) } firstReader, _, err := store.Open(firstMeta.ID) if err != nil { t.Fatalf("Open(first) error = %v", err) } defer firstReader.Close() firstBody, err := io.ReadAll(firstReader) if err != nil { t.Fatalf("ReadAll(first) error = %v", err) } if string(firstBody) != "first upload" { t.Fatalf("first content = %q, want %q", string(firstBody), "first upload") } secondReader, _, err := store.Open(secondMeta.ID) if err != nil { t.Fatalf("Open(second) error = %v", err) } defer secondReader.Close() secondBody, err := io.ReadAll(secondReader) if err != nil { t.Fatalf("ReadAll(second) error = %v", err) } if string(secondBody) != "second upload" { t.Fatalf("second content = %q, want %q", string(secondBody), "second upload") } idMu.Lock() totalCalls := idCalls idMu.Unlock() if totalCalls < 3 { t.Fatalf("idGenerator call count = %d, want at least 3", totalCalls) } } func TestGetDeleteAndNotFound(t *testing.T) { t.Parallel() store, err := NewFilesystemStore(filepath.Join(t.TempDir(), "data")) if err != nil { t.Fatalf("NewFilesystemStore() error = %v", err) } meta, err := store.Create(context.Background(), bytes.NewReader([]byte("hello")), "text/plain", time.Hour) if err != nil { t.Fatalf("Create() error = %v", err) } if _, err := store.Get(meta.ID); err != nil { t.Fatalf("Get() error = %v", err) } if err := store.Delete(meta.ID); err != nil { t.Fatalf("Delete() error = %v", err) } if _, err := store.Get(meta.ID); !errors.Is(err, ErrNotFound) { t.Fatalf("Get() error = %v, want ErrNotFound", err) } if err := store.Delete(meta.ID); !errors.Is(err, ErrNotFound) { t.Fatalf("Delete() error = %v, want ErrNotFound", err) } } func TestDeleteExpired(t *testing.T) { t.Parallel() store, err := NewFilesystemStore(filepath.Join(t.TempDir(), "data")) if err != nil { t.Fatalf("NewFilesystemStore() error = %v", err) } expired, err := store.Create(context.Background(), bytes.NewReader([]byte("expired")), "text/plain", -time.Second) if err != nil { t.Fatalf("Create expired() error = %v", err) } active, err := store.Create(context.Background(), bytes.NewReader([]byte("active")), "text/plain", time.Hour) if err != nil { t.Fatalf("Create active() error = %v", err) } deleted, err := store.DeleteExpired(time.Now().UTC()) if err != nil { t.Fatalf("DeleteExpired() error = %v", err) } if deleted != 1 { t.Fatalf("DeleteExpired() = %d, want 1", deleted) } if _, err := store.Get(expired.ID); !errors.Is(err, ErrNotFound) { t.Fatalf("expired Get() error = %v, want ErrNotFound", err) } if _, err := store.Get(active.ID); err != nil { t.Fatalf("active Get() error = %v", err) } } func TestOpenInvalidGzipFails(t *testing.T) { t.Parallel() store, err := NewFilesystemStore(filepath.Join(t.TempDir(), "data")) if err != nil { t.Fatalf("NewFilesystemStore() error = %v", err) } id := "deadbeefdeadbeefdeadbeef" badPath := filepath.Join(store.filesDir, id) if err := os.WriteFile(badPath, []byte("not-gzip"), 0o644); err != nil { t.Fatalf("WriteFile() error = %v", err) } store.metadata[id] = Metadata{ ID: id, FilePath: badPath, CreatedAt: time.Now().UTC(), ExpiresAt: time.Now().UTC().Add(time.Hour), ContentType: "text/plain", Size: int64(len("not-gzip")), } if err := store.persistLocked(); err != nil { t.Fatalf("persistLocked() error = %v", err) } if _, _, err := store.Open(id); err == nil { t.Fatalf("Open() error = nil, want gzip reader error") } } func TestNewFilesystemStoreInvalidIndex(t *testing.T) { t.Parallel() root := t.TempDir() scratches := filepath.Join(root, filesDirName) if err := os.MkdirAll(scratches, 0o755); err != nil { t.Fatalf("MkdirAll() error = %v", err) } if err := os.WriteFile(filepath.Join(root, indexFilename), []byte("{bad"), 0o644); err != nil { t.Fatalf("WriteFile() error = %v", err) } if _, err := NewFilesystemStore(root); err == nil { t.Fatalf("NewFilesystemStore() error = nil, want invalid index error") } } func TestNewFilesystemStoreReconcileOnStartup(t *testing.T) { t.Parallel() root := t.TempDir() scratches := filepath.Join(root, filesDirName) if err := os.MkdirAll(scratches, 0o755); err != nil { t.Fatalf("MkdirAll() error = %v", err) } keepID := "aaaaaaaaaaaaaaaaaaaaaaaa" missingID := "bbbbbbbbbbbbbbbbbbbbbbbb" expiredID := "cccccccccccccccccccccccc" keepPath := filepath.Join(scratches, keepID) expiredPath := filepath.Join(scratches, expiredID) if err := os.WriteFile(keepPath, []byte{0x1f, 0x8b, 0x08, 0x00}, 0o644); err != nil { t.Fatalf("WriteFile keep error = %v", err) } if err := os.WriteFile(expiredPath, []byte{0x1f, 0x8b, 0x08, 0x00}, 0o644); err != nil { t.Fatalf("WriteFile expired error = %v", err) } meta := map[string]Metadata{ keepID: { ID: keepID, FilePath: keepPath, CreatedAt: time.Now().UTC(), ExpiresAt: time.Now().UTC().Add(time.Hour), ContentType: "text/plain", Size: 1, }, missingID: { ID: missingID, FilePath: filepath.Join(scratches, missingID), CreatedAt: time.Now().UTC(), ExpiresAt: time.Now().UTC().Add(time.Hour), ContentType: "text/plain", Size: 1, }, expiredID: { ID: expiredID, FilePath: expiredPath, CreatedAt: time.Now().UTC(), ExpiresAt: time.Now().UTC().Add(-time.Minute), ContentType: "text/plain", Size: 1, }, } payload, err := json.Marshal(meta) if err != nil { t.Fatalf("json.Marshal() error = %v", err) } if err := os.WriteFile(filepath.Join(root, indexFilename), payload, 0o644); err != nil { t.Fatalf("WriteFile index error = %v", err) } store, err := NewFilesystemStore(root) if err != nil { t.Fatalf("NewFilesystemStore() error = %v", err) } if _, err := store.Get(keepID); err != nil { t.Fatalf("keep entry missing: %v", err) } if _, err := store.Get(missingID); !errors.Is(err, ErrNotFound) { t.Fatalf("missing entry error = %v, want ErrNotFound", err) } if _, err := store.Get(expiredID); !errors.Is(err, ErrNotFound) { t.Fatalf("expired entry error = %v, want ErrNotFound", err) } } type errCloser struct{} func (errCloser) Close() error { return errors.New("close failed") } func TestCombinedReadCloserClosePropagatesFirstError(t *testing.T) { t.Parallel() c := &combinedReadCloser{ reader: bytes.NewReader([]byte("ok")), closers: []io.Closer{errCloser{}, errCloser{}}, } if err := c.Close(); err == nil { t.Fatalf("Close() error = nil, want non-nil") } } type failingReader struct{} func (failingReader) Read(_ []byte) (int, error) { return 0, errors.New("boom") } func TestCreateRollbackOnPersistFailure(t *testing.T) { t.Parallel() store, err := NewFilesystemStore(filepath.Join(t.TempDir(), "data")) if err != nil { t.Fatalf("NewFilesystemStore() error = %v", err) } badIndexDir := filepath.Join(t.TempDir(), "index-dir") if err := os.MkdirAll(badIndexDir, 0o755); err != nil { t.Fatalf("MkdirAll() error = %v", err) } store.index = badIndexDir if _, err := store.Create(context.Background(), bytes.NewReader([]byte("x")), "text/plain", time.Hour); err == nil { t.Fatalf("Create() error = nil, want persist failure") } if len(store.metadata) != 0 { t.Fatalf("metadata should be rolled back on persist failure") } } func TestCreateReaderFailure(t *testing.T) { t.Parallel() store, err := NewFilesystemStore(filepath.Join(t.TempDir(), "data")) if err != nil { t.Fatalf("NewFilesystemStore() error = %v", err) } if _, err := store.Create(context.Background(), failingReader{}, "text/plain", time.Hour); err == nil { t.Fatalf("Create() error = nil, want reader failure") } } func TestDeletePersistFailureRestoresEntry(t *testing.T) { t.Parallel() store, err := NewFilesystemStore(filepath.Join(t.TempDir(), "data")) if err != nil { t.Fatalf("NewFilesystemStore() error = %v", err) } meta, err := store.Create(context.Background(), bytes.NewReader([]byte("hello")), "text/plain", time.Hour) if err != nil { t.Fatalf("Create() error = %v", err) } badIndexDir := filepath.Join(t.TempDir(), "index-dir") if err := os.MkdirAll(badIndexDir, 0o755); err != nil { t.Fatalf("MkdirAll() error = %v", err) } store.index = badIndexDir if err := store.Delete(meta.ID); err == nil { t.Fatalf("Delete() error = nil, want persist failure") } if _, err := store.Get(meta.ID); err != nil { t.Fatalf("entry should be restored after delete persist failure: %v", err) } } func TestDeleteRemoveFailure(t *testing.T) { t.Parallel() store, err := NewFilesystemStore(filepath.Join(t.TempDir(), "data")) if err != nil { t.Fatalf("NewFilesystemStore() error = %v", err) } nonEmptyDir := filepath.Join(t.TempDir(), "non-empty") if err := os.MkdirAll(nonEmptyDir, 0o755); err != nil { t.Fatalf("MkdirAll() error = %v", err) } if err := os.WriteFile(filepath.Join(nonEmptyDir, "child"), []byte("x"), 0o644); err != nil { t.Fatalf("WriteFile() error = %v", err) } id := "dddddddddddddddddddddddd" store.metadata[id] = Metadata{ ID: id, FilePath: nonEmptyDir, CreatedAt: time.Now().UTC(), ExpiresAt: time.Now().UTC().Add(time.Hour), ContentType: "text/plain", Size: 1, } if err := store.persistLocked(); err != nil { t.Fatalf("persistLocked() error = %v", err) } if err := store.Delete(id); err == nil { t.Fatalf("Delete() error = nil, want remove failure") } } func TestDeleteExpiredPersistFailureRestoresEntries(t *testing.T) { t.Parallel() store, err := NewFilesystemStore(filepath.Join(t.TempDir(), "data")) if err != nil { t.Fatalf("NewFilesystemStore() error = %v", err) } meta, err := store.Create(context.Background(), bytes.NewReader([]byte("expired")), "text/plain", -time.Second) if err != nil { t.Fatalf("Create() error = %v", err) } badIndexDir := filepath.Join(t.TempDir(), "index-dir") if err := os.MkdirAll(badIndexDir, 0o755); err != nil { t.Fatalf("MkdirAll() error = %v", err) } store.index = badIndexDir if _, err := store.DeleteExpired(time.Now().UTC()); err == nil { t.Fatalf("DeleteExpired() error = nil, want persist failure") } if _, err := store.Get(meta.ID); err != nil { t.Fatalf("entry should be restored after DeleteExpired persist failure: %v", err) } } func TestDeleteExpiredRemoveFailure(t *testing.T) { t.Parallel() store, err := NewFilesystemStore(filepath.Join(t.TempDir(), "data")) if err != nil { t.Fatalf("NewFilesystemStore() error = %v", err) } nonEmptyDir := filepath.Join(t.TempDir(), "expired-non-empty") if err := os.MkdirAll(nonEmptyDir, 0o755); err != nil { t.Fatalf("MkdirAll() error = %v", err) } if err := os.WriteFile(filepath.Join(nonEmptyDir, "child"), []byte("x"), 0o644); err != nil { t.Fatalf("WriteFile() error = %v", err) } id := "eeeeeeeeeeeeeeeeeeeeeeee" store.metadata[id] = Metadata{ ID: id, FilePath: nonEmptyDir, CreatedAt: time.Now().UTC(), ExpiresAt: time.Now().UTC().Add(-time.Second), ContentType: "text/plain", Size: 1, } if err := store.persistLocked(); err != nil { t.Fatalf("persistLocked() error = %v", err) } if _, err := store.DeleteExpired(time.Now().UTC()); err == nil { t.Fatalf("DeleteExpired() error = nil, want remove failure") } } func TestNewFilesystemStoreFailsWhenDataDirIsFile(t *testing.T) { t.Parallel() tmp := t.TempDir() filePath := filepath.Join(tmp, "not-dir") if err := os.WriteFile(filePath, []byte("x"), 0o644); err != nil { t.Fatalf("WriteFile() error = %v", err) } if _, err := NewFilesystemStore(filePath); err == nil { t.Fatalf("NewFilesystemStore() error = nil, want mkdir failure") } } type blockingReader struct { release <-chan struct{} payload []byte sent bool } func (r *blockingReader) Read(p []byte) (int, error) { if r.sent { return 0, io.EOF } <-r.release r.sent = true return copy(p, r.payload), nil } func TestCreateReservationIsNotVisibleToCleanupOrReads(t *testing.T) { t.Parallel() store, err := NewFilesystemStore(filepath.Join(t.TempDir(), "data")) if err != nil { t.Fatalf("NewFilesystemStore() error = %v", err) } const reservedID = "ffffffffffffffffffffffff" store.idGenerator = func() (string, error) { return reservedID, nil } release := make(chan struct{}) createErrCh := make(chan error, 1) go func() { _, createErr := store.CreateWithOriginalName( context.Background(), &blockingReader{release: release, payload: []byte("in-flight")}, "text/plain", "", time.Hour, ) createErrCh <- createErr }() deadline := time.Now().Add(2 * time.Second) for { store.mu.RLock() _, reserved := store.reservedIDs[reservedID] _, visible := store.metadata[reservedID] store.mu.RUnlock() if reserved { if visible { t.Fatalf("in-flight ID should not appear in metadata") } break } if time.Now().After(deadline) { t.Fatalf("timed out waiting for ID reservation") } time.Sleep(2 * time.Millisecond) } if _, err := store.Get(reservedID); !errors.Is(err, ErrNotFound) { t.Fatalf("Get() during in-flight create error = %v, want ErrNotFound", err) } deleted, err := store.DeleteExpired(time.Now().UTC()) if err != nil { t.Fatalf("DeleteExpired() error = %v", err) } if deleted != 0 { t.Fatalf("DeleteExpired() = %d, want 0 for in-flight create", deleted) } close(release) if err := <-createErrCh; err != nil { t.Fatalf("CreateWithOriginalName() error = %v", err) } reader, _, err := store.Open(reservedID) if err != nil { t.Fatalf("Open() error = %v", err) } defer reader.Close() body, err := io.ReadAll(reader) if err != nil { t.Fatalf("ReadAll() error = %v", err) } if string(body) != "in-flight" { t.Fatalf("Open() content = %q, want %q", string(body), "in-flight") } } func TestConcurrentCreateWithOriginalNameProducesUniqueReadableEntries(t *testing.T) { t.Parallel() store, err := NewFilesystemStore(filepath.Join(t.TempDir(), "data")) if err != nil { t.Fatalf("NewFilesystemStore() error = %v", err) } const writers = 64 errCh := make(chan error, writers) idsCh := make(chan string, writers) var wg sync.WaitGroup for i := 0; i < writers; i++ { wg.Add(1) go func(i int) { defer wg.Done() payload := fmt.Sprintf("storage-writer-%03d-%s", i, strings.Repeat("z", i%13)) meta, createErr := store.CreateWithOriginalName( context.Background(), strings.NewReader(payload), "text/plain; charset=utf-8", fmt.Sprintf("w-%03d.txt", i), time.Hour, ) if createErr != nil { errCh <- fmt.Errorf("CreateWithOriginalName() error: %w", createErr) return } reader, _, openErr := store.Open(meta.ID) if openErr != nil { errCh <- fmt.Errorf("Open() error: %w", openErr) return } defer reader.Close() got, readErr := io.ReadAll(reader) if readErr != nil { errCh <- fmt.Errorf("ReadAll() error: %w", readErr) return } if string(got) != payload { errCh <- fmt.Errorf("content mismatch = %q, want %q", string(got), payload) return } idsCh <- meta.ID }(i) } wg.Wait() close(errCh) close(idsCh) for err := range errCh { t.Error(err) } seen := make(map[string]struct{}, writers) for id := range idsCh { if _, exists := seen[id]; exists { t.Errorf("duplicate id detected: %s", id) continue } seen[id] = struct{}{} } if len(seen) != writers { t.Fatalf("unique IDs = %d, want %d", len(seen), writers) } }