package storage import ( "bytes" "context" "crypto/aes" "crypto/cipher" "crypto/rand" "crypto/sha256" "encoding/hex" "encoding/json" "errors" "fmt" "io" "os" "path/filepath" "strings" "sync" "testing" "time" ) var testEncryptionKey = []byte("0123456789abcdef0123456789abcdef") func mustTestAEAD(t *testing.T) cipher.AEAD { t.Helper() block, err := aes.NewCipher(testEncryptionKey) if err != nil { t.Fatalf("aes.NewCipher() error = %v", err) } aead, err := cipher.NewGCM(block) if err != nil { t.Fatalf("cipher.NewGCM() error = %v", err) } return aead } func TestCreateStoresGzipAndOpenIsTransparent(t *testing.T) { t.Parallel() store, err := NewFilesystemStore(filepath.Join(t.TempDir(), "data"), testEncryptionKey) if err != nil { t.Fatalf("NewFilesystemStore() error = %v", err) } original := bytes.Repeat([]byte("hello scratch "), 1024) meta, err := store.Create(context.Background(), bytes.NewReader(original), "text/plain; charset=utf-8", time.Hour) if err != nil { t.Fatalf("Create() error = %v", err) } if meta.Size != int64(len(original)) { t.Fatalf("meta.Size = %d, want %d", meta.Size, len(original)) } onDisk, err := os.ReadFile(meta.FilePath) if err != nil { t.Fatalf("ReadFile() error = %v", err) } if len(onDisk) < 4 || string(onDisk[:4]) != "SBX1" { t.Fatalf("stored file does not have encrypted scratch header") } reader, _, err := store.Open(meta.ID) if err != nil { t.Fatalf("Open() error = %v", err) } defer reader.Close() got, err := io.ReadAll(reader) if err != nil { t.Fatalf("ReadAll() error = %v", err) } if !bytes.Equal(got, original) { t.Fatalf("open content mismatch: got %d bytes, want %d", len(got), len(original)) } } func TestCreateEncryptedStoreDoesNotExposeGzipHeaderOnDisk(t *testing.T) { t.Parallel() key := make([]byte, 32) if _, err := rand.Read(key); err != nil { t.Fatalf("rand.Read() error = %v", err) } dataDir := filepath.Join(t.TempDir(), "data") store, err := NewFilesystemStore(dataDir, key) if err != nil { t.Fatalf("NewEncryptedFilesystemStore() error = %v", err) } original := bytes.Repeat([]byte("secret payload "), 1024) meta, err := store.Create(context.Background(), bytes.NewReader(original), "text/plain", time.Hour) if err != nil { t.Fatalf("Create() error = %v", err) } onDisk, err := os.ReadFile(meta.FilePath) if err != nil { t.Fatalf("ReadFile() error = %v", err) } if len(onDisk) < 2 { t.Fatalf("encrypted file too short") } if onDisk[0] == 0x1f && onDisk[1] == 0x8b { t.Fatalf("encrypted file unexpectedly starts with gzip header") } reader, _, err := store.Open(meta.ID) if err != nil { t.Fatalf("Open() error = %v", err) } defer reader.Close() got, err := io.ReadAll(reader) if err != nil { t.Fatalf("ReadAll() error = %v", err) } if !bytes.Equal(got, original) { t.Fatalf("Open() content mismatch: got %d bytes, want %d", len(got), len(original)) } filePayload, err := os.ReadFile(meta.FilePath) if err != nil { t.Fatalf("ReadFile(encrypted) error = %v", err) } sum := sha256.Sum256(filePayload) wantBlobID := hex.EncodeToString(sum[:]) if meta.BlobID != wantBlobID { t.Fatalf("meta.BlobID = %q, want SHA-256 of encrypted blob %q", meta.BlobID, wantBlobID) } if meta.ID == wantBlobID { t.Fatalf("meta.ID should be a short public id, got blob hash id %q", meta.ID) } } func TestEncryptedStoreFailsToOpenWithWrongKey(t *testing.T) { t.Parallel() keyA := make([]byte, 32) if _, err := rand.Read(keyA); err != nil { t.Fatalf("rand.Read(keyA) error = %v", err) } keyB := make([]byte, 32) if _, err := rand.Read(keyB); err != nil { t.Fatalf("rand.Read(keyB) error = %v", err) } dataDir := filepath.Join(t.TempDir(), "data") storeA, err := NewFilesystemStore(dataDir, keyA) if err != nil { t.Fatalf("NewEncryptedFilesystemStore(keyA) error = %v", err) } _, err = storeA.Create(context.Background(), bytes.NewReader([]byte("hello")), "text/plain", time.Hour) if err != nil { t.Fatalf("Create() error = %v", err) } if _, err := NewFilesystemStore(dataDir, keyB); err == nil { t.Fatalf("NewFilesystemStore() with wrong key error = nil, want non-nil") } } func TestCreateWithOriginalNameNormalizesFilename(t *testing.T) { t.Parallel() store, err := NewFilesystemStore(filepath.Join(t.TempDir(), "data"), testEncryptionKey) if err != nil { t.Fatalf("NewFilesystemStore() error = %v", err) } meta, err := store.CreateWithOriginalName(context.Background(), bytes.NewReader([]byte("hello")), "text/plain", " ../../demo.txt ", time.Hour) if err != nil { t.Fatalf("CreateWithOriginalName() error = %v", err) } if meta.OriginalName != "demo.txt" { t.Fatalf("meta.OriginalName = %q, want %q", meta.OriginalName, "demo.txt") } reloaded, err := NewFilesystemStore(store.dataDir, testEncryptionKey) if err != nil { t.Fatalf("NewFilesystemStore() reload error = %v", err) } got, err := reloaded.Get(meta.ID) if err != nil { t.Fatalf("Get() error = %v", err) } if got.OriginalName != "demo.txt" { t.Fatalf("persisted OriginalName = %q, want %q", got.OriginalName, "demo.txt") } } func TestCreateWithOriginalNameSameContentGetsDifferentIDs(t *testing.T) { t.Parallel() store, err := NewFilesystemStore(filepath.Join(t.TempDir(), "data"), testEncryptionKey) if err != nil { t.Fatalf("NewFilesystemStore() error = %v", err) } first, err := store.CreateWithOriginalName(context.Background(), bytes.NewReader([]byte("same body")), "text/plain", "one.txt", time.Hour) if err != nil { t.Fatalf("first CreateWithOriginalName() error = %v", err) } second, err := store.CreateWithOriginalName(context.Background(), bytes.NewReader([]byte("same body")), "text/plain", "two.txt", time.Hour) if err != nil { t.Fatalf("second CreateWithOriginalName() error = %v", err) } if first.ID == second.ID { t.Fatalf("IDs should differ because encrypted blobs include random nonce") } } func TestGetDeleteAndNotFound(t *testing.T) { t.Parallel() store, err := NewFilesystemStore(filepath.Join(t.TempDir(), "data"), testEncryptionKey) if err != nil { t.Fatalf("NewFilesystemStore() error = %v", err) } meta, err := store.Create(context.Background(), bytes.NewReader([]byte("hello")), "text/plain", time.Hour) if err != nil { t.Fatalf("Create() error = %v", err) } if _, err := store.Get(meta.ID); err != nil { t.Fatalf("Get() error = %v", err) } if err := store.Delete(meta.ID); err != nil { t.Fatalf("Delete() error = %v", err) } if _, err := store.Get(meta.ID); !errors.Is(err, ErrNotFound) { t.Fatalf("Get() error = %v, want ErrNotFound", err) } if err := store.Delete(meta.ID); !errors.Is(err, ErrNotFound) { t.Fatalf("Delete() error = %v, want ErrNotFound", err) } } func TestDeleteExpired(t *testing.T) { t.Parallel() store, err := NewFilesystemStore(filepath.Join(t.TempDir(), "data"), testEncryptionKey) if err != nil { t.Fatalf("NewFilesystemStore() error = %v", err) } expired, err := store.Create(context.Background(), bytes.NewReader([]byte("expired")), "text/plain", -time.Second) if err != nil { t.Fatalf("Create expired() error = %v", err) } active, err := store.Create(context.Background(), bytes.NewReader([]byte("active")), "text/plain", time.Hour) if err != nil { t.Fatalf("Create active() error = %v", err) } deleted, err := store.DeleteExpired(time.Now().UTC()) if err != nil { t.Fatalf("DeleteExpired() error = %v", err) } if deleted != 1 { t.Fatalf("DeleteExpired() = %d, want 1", deleted) } if _, err := store.Get(expired.ID); !errors.Is(err, ErrNotFound) { t.Fatalf("expired Get() error = %v, want ErrNotFound", err) } if _, err := store.Get(active.ID); err != nil { t.Fatalf("active Get() error = %v", err) } } func TestOpenInvalidGzipFails(t *testing.T) { t.Parallel() store, err := NewFilesystemStore(filepath.Join(t.TempDir(), "data"), testEncryptionKey) if err != nil { t.Fatalf("NewFilesystemStore() error = %v", err) } id := "deadbeefdeadbeefdeadbeef" badPath := filepath.Join(store.filesDir, id) if err := os.WriteFile(badPath, []byte("not-gzip"), 0o644); err != nil { t.Fatalf("WriteFile() error = %v", err) } store.metadata[id] = Metadata{ ID: id, FilePath: badPath, CreatedAt: time.Now().UTC(), ExpiresAt: time.Now().UTC().Add(time.Hour), ContentType: "text/plain", Size: int64(len("not-gzip")), } if err := store.persistLocked(); err != nil { t.Fatalf("persistLocked() error = %v", err) } if _, _, err := store.Open(id); err == nil { t.Fatalf("Open() error = nil, want gzip reader error") } } func TestNewFilesystemStoreInvalidIndex(t *testing.T) { t.Parallel() root := t.TempDir() scratches := filepath.Join(root, filesDirName) if err := os.MkdirAll(scratches, 0o755); err != nil { t.Fatalf("MkdirAll() error = %v", err) } if err := os.WriteFile(filepath.Join(root, indexFilename), []byte("{bad"), 0o644); err != nil { t.Fatalf("WriteFile() error = %v", err) } if _, err := NewFilesystemStore(root, testEncryptionKey); err == nil { t.Fatalf("NewFilesystemStore() error = nil, want invalid index error") } } func TestNewFilesystemStoreReconcileOnStartup(t *testing.T) { t.Parallel() root := t.TempDir() scratches := filepath.Join(root, filesDirName) if err := os.MkdirAll(scratches, 0o755); err != nil { t.Fatalf("MkdirAll() error = %v", err) } keepID := "aaaaaaaaaaaaaaaaaaaaaaaa" missingID := "bbbbbbbbbbbbbbbbbbbbbbbb" expiredID := "cccccccccccccccccccccccc" keepPath := filepath.Join(scratches, keepID) expiredPath := filepath.Join(scratches, expiredID) if err := os.WriteFile(keepPath, []byte{0x1f, 0x8b, 0x08, 0x00}, 0o644); err != nil { t.Fatalf("WriteFile keep error = %v", err) } if err := os.WriteFile(expiredPath, []byte{0x1f, 0x8b, 0x08, 0x00}, 0o644); err != nil { t.Fatalf("WriteFile expired error = %v", err) } meta := map[string]Metadata{ keepID: { ID: keepID, FilePath: keepPath, CreatedAt: time.Now().UTC(), ExpiresAt: time.Now().UTC().Add(time.Hour), ContentType: "text/plain", Size: 1, }, missingID: { ID: missingID, FilePath: filepath.Join(scratches, missingID), CreatedAt: time.Now().UTC(), ExpiresAt: time.Now().UTC().Add(time.Hour), ContentType: "text/plain", Size: 1, }, expiredID: { ID: expiredID, FilePath: expiredPath, CreatedAt: time.Now().UTC(), ExpiresAt: time.Now().UTC().Add(-time.Minute), ContentType: "text/plain", Size: 1, }, } payload, err := json.Marshal(indexDocument{ Scratches: meta, }) if err != nil { t.Fatalf("json.Marshal() error = %v", err) } encryptedPayload, err := encryptIndexPayload(payload, mustTestAEAD(t)) if err != nil { t.Fatalf("encryptIndexPayload() error = %v", err) } if err := os.WriteFile(filepath.Join(root, indexFilename), encryptedPayload, 0o644); err != nil { t.Fatalf("WriteFile index error = %v", err) } store, err := NewFilesystemStore(root, testEncryptionKey) if err != nil { t.Fatalf("NewFilesystemStore() error = %v", err) } if _, err := store.Get(keepID); err != nil { t.Fatalf("keep entry missing: %v", err) } if _, err := store.Get(missingID); !errors.Is(err, ErrNotFound) { t.Fatalf("missing entry error = %v, want ErrNotFound", err) } if _, err := store.Get(expiredID); !errors.Is(err, ErrNotFound) { t.Fatalf("expired entry error = %v, want ErrNotFound", err) } } func TestNewFilesystemStoreWithDefaultTTLShiftsExpiriesOnTTLChange(t *testing.T) { t.Parallel() dataDir := filepath.Join(t.TempDir(), "data") store, err := NewFilesystemStoreWithDefaultTTL(dataDir, time.Hour, testEncryptionKey) if err != nil { t.Fatalf("NewFilesystemStoreWithDefaultTTL() error = %v", err) } meta, err := store.Create(context.Background(), bytes.NewReader([]byte("hello")), "text/plain", time.Hour) if err != nil { t.Fatalf("Create() error = %v", err) } originalExpiry := meta.ExpiresAt reloaded, err := NewFilesystemStoreWithDefaultTTL(dataDir, 2*time.Hour, testEncryptionKey) if err != nil { t.Fatalf("NewFilesystemStoreWithDefaultTTL() reload error = %v", err) } got, err := reloaded.Get(meta.ID) if err != nil { t.Fatalf("Get() error = %v", err) } wantExpiry := originalExpiry.Add(time.Hour) if !got.ExpiresAt.Equal(wantExpiry) { t.Fatalf("ExpiresAt = %s, want %s", got.ExpiresAt, wantExpiry) } again, err := NewFilesystemStoreWithDefaultTTL(dataDir, 2*time.Hour, testEncryptionKey) if err != nil { t.Fatalf("NewFilesystemStoreWithDefaultTTL() second reload error = %v", err) } gotAgain, err := again.Get(meta.ID) if err != nil { t.Fatalf("Get() second reload error = %v", err) } if !gotAgain.ExpiresAt.Equal(wantExpiry) { t.Fatalf("ExpiresAt after unchanged TTL = %s, want %s", gotAgain.ExpiresAt, wantExpiry) } indexRaw, err := os.ReadFile(filepath.Join(dataDir, indexFilename)) if err != nil { t.Fatalf("ReadFile(index) error = %v", err) } if bytes.Contains(indexRaw, []byte(`"default_ttl"`)) { t.Fatalf("index should not be plaintext JSON") } indexPlaintext, err := decryptIndexPayload(indexRaw, mustTestAEAD(t)) if err != nil { t.Fatalf("decryptIndexPayload() error = %v", err) } if !bytes.Contains(indexPlaintext, []byte(`"default_ttl": "2h0m0s"`)) { t.Fatalf("decrypted index missing default ttl marker: %s", string(indexPlaintext)) } } type errCloser struct{} func (errCloser) Close() error { return errors.New("close failed") } func TestCombinedReadCloserClosePropagatesFirstError(t *testing.T) { t.Parallel() c := &combinedReadCloser{ reader: bytes.NewReader([]byte("ok")), closers: []io.Closer{errCloser{}, errCloser{}}, } if err := c.Close(); err == nil { t.Fatalf("Close() error = nil, want non-nil") } } type failingReader struct{} func (failingReader) Read(_ []byte) (int, error) { return 0, errors.New("boom") } func TestCreateRollbackOnPersistFailure(t *testing.T) { t.Parallel() store, err := NewFilesystemStore(filepath.Join(t.TempDir(), "data"), testEncryptionKey) if err != nil { t.Fatalf("NewFilesystemStore() error = %v", err) } badIndexDir := filepath.Join(t.TempDir(), "index-dir") if err := os.MkdirAll(badIndexDir, 0o755); err != nil { t.Fatalf("MkdirAll() error = %v", err) } store.index = badIndexDir if _, err := store.Create(context.Background(), bytes.NewReader([]byte("x")), "text/plain", time.Hour); err == nil { t.Fatalf("Create() error = nil, want persist failure") } if len(store.metadata) != 0 { t.Fatalf("metadata should be rolled back on persist failure") } } func TestCreateReaderFailure(t *testing.T) { t.Parallel() store, err := NewFilesystemStore(filepath.Join(t.TempDir(), "data"), testEncryptionKey) if err != nil { t.Fatalf("NewFilesystemStore() error = %v", err) } if _, err := store.Create(context.Background(), failingReader{}, "text/plain", time.Hour); err == nil { t.Fatalf("Create() error = nil, want reader failure") } } func TestDeletePersistFailureRestoresEntry(t *testing.T) { t.Parallel() store, err := NewFilesystemStore(filepath.Join(t.TempDir(), "data"), testEncryptionKey) if err != nil { t.Fatalf("NewFilesystemStore() error = %v", err) } meta, err := store.Create(context.Background(), bytes.NewReader([]byte("hello")), "text/plain", time.Hour) if err != nil { t.Fatalf("Create() error = %v", err) } badIndexDir := filepath.Join(t.TempDir(), "index-dir") if err := os.MkdirAll(badIndexDir, 0o755); err != nil { t.Fatalf("MkdirAll() error = %v", err) } store.index = badIndexDir if err := store.Delete(meta.ID); err == nil { t.Fatalf("Delete() error = nil, want persist failure") } if _, err := store.Get(meta.ID); err != nil { t.Fatalf("entry should be restored after delete persist failure: %v", err) } } func TestDeleteRemoveFailure(t *testing.T) { t.Parallel() store, err := NewFilesystemStore(filepath.Join(t.TempDir(), "data"), testEncryptionKey) if err != nil { t.Fatalf("NewFilesystemStore() error = %v", err) } nonEmptyDir := filepath.Join(t.TempDir(), "non-empty") if err := os.MkdirAll(nonEmptyDir, 0o755); err != nil { t.Fatalf("MkdirAll() error = %v", err) } if err := os.WriteFile(filepath.Join(nonEmptyDir, "child"), []byte("x"), 0o644); err != nil { t.Fatalf("WriteFile() error = %v", err) } id := "dddddddddddddddddddddddd" store.metadata[id] = Metadata{ ID: id, FilePath: nonEmptyDir, CreatedAt: time.Now().UTC(), ExpiresAt: time.Now().UTC().Add(time.Hour), ContentType: "text/plain", Size: 1, } if err := store.persistLocked(); err != nil { t.Fatalf("persistLocked() error = %v", err) } if err := store.Delete(id); err == nil { t.Fatalf("Delete() error = nil, want remove failure") } } func TestDeleteExpiredPersistFailureRestoresEntries(t *testing.T) { t.Parallel() store, err := NewFilesystemStore(filepath.Join(t.TempDir(), "data"), testEncryptionKey) if err != nil { t.Fatalf("NewFilesystemStore() error = %v", err) } meta, err := store.Create(context.Background(), bytes.NewReader([]byte("expired")), "text/plain", -time.Second) if err != nil { t.Fatalf("Create() error = %v", err) } badIndexDir := filepath.Join(t.TempDir(), "index-dir") if err := os.MkdirAll(badIndexDir, 0o755); err != nil { t.Fatalf("MkdirAll() error = %v", err) } store.index = badIndexDir if _, err := store.DeleteExpired(time.Now().UTC()); err == nil { t.Fatalf("DeleteExpired() error = nil, want persist failure") } if _, err := store.Get(meta.ID); err != nil { t.Fatalf("entry should be restored after DeleteExpired persist failure: %v", err) } } func TestDeleteExpiredRemoveFailure(t *testing.T) { t.Parallel() store, err := NewFilesystemStore(filepath.Join(t.TempDir(), "data"), testEncryptionKey) if err != nil { t.Fatalf("NewFilesystemStore() error = %v", err) } nonEmptyDir := filepath.Join(t.TempDir(), "expired-non-empty") if err := os.MkdirAll(nonEmptyDir, 0o755); err != nil { t.Fatalf("MkdirAll() error = %v", err) } if err := os.WriteFile(filepath.Join(nonEmptyDir, "child"), []byte("x"), 0o644); err != nil { t.Fatalf("WriteFile() error = %v", err) } id := "eeeeeeeeeeeeeeeeeeeeeeee" store.metadata[id] = Metadata{ ID: id, FilePath: nonEmptyDir, CreatedAt: time.Now().UTC(), ExpiresAt: time.Now().UTC().Add(-time.Second), ContentType: "text/plain", Size: 1, } if err := store.persistLocked(); err != nil { t.Fatalf("persistLocked() error = %v", err) } if _, err := store.DeleteExpired(time.Now().UTC()); err == nil { t.Fatalf("DeleteExpired() error = nil, want remove failure") } } func TestNewFilesystemStoreFailsWhenDataDirIsFile(t *testing.T) { t.Parallel() tmp := t.TempDir() filePath := filepath.Join(tmp, "not-dir") if err := os.WriteFile(filePath, []byte("x"), 0o644); err != nil { t.Fatalf("WriteFile() error = %v", err) } if _, err := NewFilesystemStore(filePath, testEncryptionKey); err == nil { t.Fatalf("NewFilesystemStore() error = nil, want mkdir failure") } } func TestNewFilesystemStoreRequires32ByteKey(t *testing.T) { t.Parallel() dataDir := filepath.Join(t.TempDir(), "data") if _, err := NewFilesystemStore(dataDir, nil); err == nil { t.Fatalf("NewFilesystemStore() with nil key error = nil, want non-nil") } if _, err := NewFilesystemStore(dataDir, []byte("short")); err == nil { t.Fatalf("NewFilesystemStore() with short key error = nil, want non-nil") } } func TestNewFilesystemStoreFailsToLoadEncryptedIndexWithWrongKey(t *testing.T) { t.Parallel() dataDir := filepath.Join(t.TempDir(), "data") store, err := NewFilesystemStore(dataDir, testEncryptionKey) if err != nil { t.Fatalf("NewFilesystemStore() error = %v", err) } if _, err := store.Create(context.Background(), bytes.NewReader([]byte("hello")), "text/plain", time.Hour); err != nil { t.Fatalf("Create() error = %v", err) } wrongKey := []byte("abcdef0123456789abcdef0123456789") if _, err := NewFilesystemStore(dataDir, wrongKey); err == nil { t.Fatalf("NewFilesystemStore() with wrong key error = nil, want non-nil") } } func TestConcurrentCreateWithOriginalNameProducesUniqueReadableEntries(t *testing.T) { t.Parallel() store, err := NewFilesystemStore(filepath.Join(t.TempDir(), "data"), testEncryptionKey) if err != nil { t.Fatalf("NewFilesystemStore() error = %v", err) } const writers = 64 errCh := make(chan error, writers) idsCh := make(chan string, writers) var wg sync.WaitGroup for i := 0; i < writers; i++ { wg.Add(1) go func(i int) { defer wg.Done() payload := fmt.Sprintf("storage-writer-%03d-%s", i, strings.Repeat("z", i%13)) meta, createErr := store.CreateWithOriginalName( context.Background(), strings.NewReader(payload), "text/plain; charset=utf-8", fmt.Sprintf("w-%03d.txt", i), time.Hour, ) if createErr != nil { errCh <- fmt.Errorf("CreateWithOriginalName() error: %w", createErr) return } reader, _, openErr := store.Open(meta.ID) if openErr != nil { errCh <- fmt.Errorf("Open() error: %w", openErr) return } defer reader.Close() got, readErr := io.ReadAll(reader) if readErr != nil { errCh <- fmt.Errorf("ReadAll() error: %w", readErr) return } if string(got) != payload { errCh <- fmt.Errorf("content mismatch = %q, want %q", string(got), payload) return } idsCh <- meta.ID }(i) } wg.Wait() close(errCh) close(idsCh) for err := range errCh { t.Error(err) } seen := make(map[string]struct{}, writers) for id := range idsCh { if _, exists := seen[id]; exists { t.Errorf("duplicate id detected: %s", id) continue } seen[id] = struct{}{} } if len(seen) != writers { t.Fatalf("unique IDs = %d, want %d", len(seen), writers) } }