Lots more changes

This commit is contained in:
2026-06-01 01:29:47 -05:00
parent 8bec7f0dda
commit 050ec138f3
20 changed files with 1773 additions and 689 deletions
+212 -278
View File
@@ -3,6 +3,11 @@ package storage
import (
"bytes"
"context"
"crypto/aes"
"crypto/cipher"
"crypto/rand"
"crypto/sha256"
"encoding/hex"
"encoding/json"
"errors"
"fmt"
@@ -15,10 +20,25 @@ import (
"time"
)
var testEncryptionKey = []byte("0123456789abcdef0123456789abcdef")
func mustTestAEAD(t *testing.T) cipher.AEAD {
t.Helper()
block, err := aes.NewCipher(testEncryptionKey)
if err != nil {
t.Fatalf("aes.NewCipher() error = %v", err)
}
aead, err := cipher.NewGCM(block)
if err != nil {
t.Fatalf("cipher.NewGCM() error = %v", err)
}
return aead
}
func TestCreateStoresGzipAndOpenIsTransparent(t *testing.T) {
t.Parallel()
store, err := NewFilesystemStore(filepath.Join(t.TempDir(), "data"))
store, err := NewFilesystemStore(filepath.Join(t.TempDir(), "data"), testEncryptionKey)
if err != nil {
t.Fatalf("NewFilesystemStore() error = %v", err)
}
@@ -36,8 +56,8 @@ func TestCreateStoresGzipAndOpenIsTransparent(t *testing.T) {
if err != nil {
t.Fatalf("ReadFile() error = %v", err)
}
if len(onDisk) < 2 || onDisk[0] != 0x1f || onDisk[1] != 0x8b {
t.Fatalf("stored file does not have gzip header")
if len(onDisk) < 4 || string(onDisk[:4]) != "SBX1" {
t.Fatalf("stored file does not have encrypted scratch header")
}
reader, _, err := store.Open(meta.ID)
@@ -54,10 +74,92 @@ func TestCreateStoresGzipAndOpenIsTransparent(t *testing.T) {
}
}
func TestCreateEncryptedStoreDoesNotExposeGzipHeaderOnDisk(t *testing.T) {
t.Parallel()
key := make([]byte, 32)
if _, err := rand.Read(key); err != nil {
t.Fatalf("rand.Read() error = %v", err)
}
dataDir := filepath.Join(t.TempDir(), "data")
store, err := NewFilesystemStore(dataDir, key)
if err != nil {
t.Fatalf("NewEncryptedFilesystemStore() error = %v", err)
}
original := bytes.Repeat([]byte("secret payload "), 1024)
meta, err := store.Create(context.Background(), bytes.NewReader(original), "text/plain", time.Hour)
if err != nil {
t.Fatalf("Create() error = %v", err)
}
onDisk, err := os.ReadFile(meta.FilePath)
if err != nil {
t.Fatalf("ReadFile() error = %v", err)
}
if len(onDisk) < 2 {
t.Fatalf("encrypted file too short")
}
if onDisk[0] == 0x1f && onDisk[1] == 0x8b {
t.Fatalf("encrypted file unexpectedly starts with gzip header")
}
reader, _, err := store.Open(meta.ID)
if err != nil {
t.Fatalf("Open() error = %v", err)
}
defer reader.Close()
got, err := io.ReadAll(reader)
if err != nil {
t.Fatalf("ReadAll() error = %v", err)
}
if !bytes.Equal(got, original) {
t.Fatalf("Open() content mismatch: got %d bytes, want %d", len(got), len(original))
}
filePayload, err := os.ReadFile(meta.FilePath)
if err != nil {
t.Fatalf("ReadFile(encrypted) error = %v", err)
}
sum := sha256.Sum256(filePayload)
wantID := hex.EncodeToString(sum[:])
if meta.ID != wantID {
t.Fatalf("meta.ID = %q, want SHA-256 of encrypted blob %q", meta.ID, wantID)
}
}
func TestEncryptedStoreFailsToOpenWithWrongKey(t *testing.T) {
t.Parallel()
keyA := make([]byte, 32)
if _, err := rand.Read(keyA); err != nil {
t.Fatalf("rand.Read(keyA) error = %v", err)
}
keyB := make([]byte, 32)
if _, err := rand.Read(keyB); err != nil {
t.Fatalf("rand.Read(keyB) error = %v", err)
}
dataDir := filepath.Join(t.TempDir(), "data")
storeA, err := NewFilesystemStore(dataDir, keyA)
if err != nil {
t.Fatalf("NewEncryptedFilesystemStore(keyA) error = %v", err)
}
_, err = storeA.Create(context.Background(), bytes.NewReader([]byte("hello")), "text/plain", time.Hour)
if err != nil {
t.Fatalf("Create() error = %v", err)
}
if _, err := NewFilesystemStore(dataDir, keyB); err == nil {
t.Fatalf("NewFilesystemStore() with wrong key error = nil, want non-nil")
}
}
func TestCreateWithOriginalNameNormalizesFilename(t *testing.T) {
t.Parallel()
store, err := NewFilesystemStore(filepath.Join(t.TempDir(), "data"))
store, err := NewFilesystemStore(filepath.Join(t.TempDir(), "data"), testEncryptionKey)
if err != nil {
t.Fatalf("NewFilesystemStore() error = %v", err)
}
@@ -70,7 +172,7 @@ func TestCreateWithOriginalNameNormalizesFilename(t *testing.T) {
t.Fatalf("meta.OriginalName = %q, want %q", meta.OriginalName, "demo.txt")
}
reloaded, err := NewFilesystemStore(store.dataDir)
reloaded, err := NewFilesystemStore(store.dataDir, testEncryptionKey)
if err != nil {
t.Fatalf("NewFilesystemStore() reload error = %v", err)
}
@@ -83,201 +185,31 @@ func TestCreateWithOriginalNameNormalizesFilename(t *testing.T) {
}
}
func TestCreateWithOriginalNameRetriesOnIDCollision(t *testing.T) {
func TestCreateWithOriginalNameSameContentGetsDifferentIDs(t *testing.T) {
t.Parallel()
store, err := NewFilesystemStore(filepath.Join(t.TempDir(), "data"))
store, err := NewFilesystemStore(filepath.Join(t.TempDir(), "data"), testEncryptionKey)
if err != nil {
t.Fatalf("NewFilesystemStore() error = %v", err)
}
existingID := "aaaaaaaaaaaaaaaaaaaaaaaa"
existingPath := filepath.Join(store.filesDir, existingID)
if err := os.WriteFile(existingPath, []byte{0x1f, 0x8b, 0x08, 0x00}, 0o644); err != nil {
t.Fatalf("WriteFile(existing collision path) error = %v", err)
}
store.mu.Lock()
store.metadata[existingID] = Metadata{
ID: existingID,
FilePath: existingPath,
CreatedAt: time.Now().UTC(),
ExpiresAt: time.Now().UTC().Add(time.Hour),
ContentType: "text/plain",
Size: 4,
}
store.mu.Unlock()
nextID := "bbbbbbbbbbbbbbbbbbbbbbbb"
calls := 0
store.idGenerator = func() (string, error) {
calls++
if calls == 1 {
return existingID, nil
}
return nextID, nil
}
meta, err := store.CreateWithOriginalName(context.Background(), bytes.NewReader([]byte("hello")), "text/plain", "hello.txt", time.Hour)
first, err := store.CreateWithOriginalName(context.Background(), bytes.NewReader([]byte("same body")), "text/plain", "one.txt", time.Hour)
if err != nil {
t.Fatalf("CreateWithOriginalName() error = %v", err)
t.Fatalf("first CreateWithOriginalName() error = %v", err)
}
if meta.ID != nextID {
t.Fatalf("meta.ID = %q, want %q", meta.ID, nextID)
}
if calls != 2 {
t.Fatalf("idGenerator calls = %d, want 2", calls)
}
}
func TestCreateWithOriginalNameFailsAfterMaxIDRetries(t *testing.T) {
t.Parallel()
store, err := NewFilesystemStore(filepath.Join(t.TempDir(), "data"))
if err != nil {
t.Fatalf("NewFilesystemStore() error = %v", err)
}
existingID := "cccccccccccccccccccccccc"
existingPath := filepath.Join(store.filesDir, existingID)
store.mu.Lock()
store.metadata[existingID] = Metadata{
ID: existingID,
FilePath: existingPath,
CreatedAt: time.Now().UTC(),
ExpiresAt: time.Now().UTC().Add(time.Hour),
ContentType: "text/plain",
}
store.mu.Unlock()
store.idGenerator = func() (string, error) {
return existingID, nil
}
_, err = store.CreateWithOriginalName(context.Background(), bytes.NewReader([]byte("hello")), "text/plain", "hello.txt", time.Hour)
if err == nil {
t.Fatalf("CreateWithOriginalName() error = nil, want retry exhaustion error")
}
if !strings.Contains(err.Error(), "failed to allocate unique scratch id") {
t.Fatalf("unexpected error: %v", err)
}
}
func TestCreateWithOriginalNameRetriesWhenIDIsReservedInFlight(t *testing.T) {
t.Parallel()
store, err := NewFilesystemStore(filepath.Join(t.TempDir(), "data"))
if err != nil {
t.Fatalf("NewFilesystemStore() error = %v", err)
}
const collideID = "abababababababababababab"
const uniqueID = "cdcdcdcdcdcdcdcdcdcdcdcd"
var idMu sync.Mutex
idCalls := 0
store.idGenerator = func() (string, error) {
idMu.Lock()
defer idMu.Unlock()
idCalls++
if idCalls <= 2 {
return collideID, nil
}
return uniqueID, nil
}
release := make(chan struct{})
firstMetaCh := make(chan Metadata, 1)
firstErrCh := make(chan error, 1)
go func() {
meta, createErr := store.CreateWithOriginalName(
context.Background(),
&blockingReader{release: release, payload: []byte("first upload")},
"text/plain",
"first.txt",
time.Hour,
)
firstMetaCh <- meta
firstErrCh <- createErr
}()
deadline := time.Now().Add(2 * time.Second)
for {
store.mu.RLock()
_, reserved := store.reservedIDs[collideID]
store.mu.RUnlock()
if reserved {
break
}
if time.Now().After(deadline) {
t.Fatalf("timed out waiting for first upload reservation")
}
time.Sleep(2 * time.Millisecond)
}
secondMeta, err := store.CreateWithOriginalName(
context.Background(),
strings.NewReader("second upload"),
"text/plain",
"second.txt",
time.Hour,
)
second, err := store.CreateWithOriginalName(context.Background(), bytes.NewReader([]byte("same body")), "text/plain", "two.txt", time.Hour)
if err != nil {
t.Fatalf("second CreateWithOriginalName() error = %v", err)
}
if secondMeta.ID != uniqueID {
t.Fatalf("second meta.ID = %q, want %q after collision retry", secondMeta.ID, uniqueID)
}
if secondMeta.ID == collideID {
t.Fatalf("second create reused in-flight reserved ID")
}
close(release)
firstMeta := <-firstMetaCh
if err := <-firstErrCh; err != nil {
t.Fatalf("first CreateWithOriginalName() error = %v", err)
}
if firstMeta.ID != collideID {
t.Fatalf("first meta.ID = %q, want %q", firstMeta.ID, collideID)
}
firstReader, _, err := store.Open(firstMeta.ID)
if err != nil {
t.Fatalf("Open(first) error = %v", err)
}
defer firstReader.Close()
firstBody, err := io.ReadAll(firstReader)
if err != nil {
t.Fatalf("ReadAll(first) error = %v", err)
}
if string(firstBody) != "first upload" {
t.Fatalf("first content = %q, want %q", string(firstBody), "first upload")
}
secondReader, _, err := store.Open(secondMeta.ID)
if err != nil {
t.Fatalf("Open(second) error = %v", err)
}
defer secondReader.Close()
secondBody, err := io.ReadAll(secondReader)
if err != nil {
t.Fatalf("ReadAll(second) error = %v", err)
}
if string(secondBody) != "second upload" {
t.Fatalf("second content = %q, want %q", string(secondBody), "second upload")
}
idMu.Lock()
totalCalls := idCalls
idMu.Unlock()
if totalCalls < 3 {
t.Fatalf("idGenerator call count = %d, want at least 3", totalCalls)
if first.ID == second.ID {
t.Fatalf("IDs should differ because encrypted blobs include random nonce")
}
}
func TestGetDeleteAndNotFound(t *testing.T) {
t.Parallel()
store, err := NewFilesystemStore(filepath.Join(t.TempDir(), "data"))
store, err := NewFilesystemStore(filepath.Join(t.TempDir(), "data"), testEncryptionKey)
if err != nil {
t.Fatalf("NewFilesystemStore() error = %v", err)
}
@@ -302,7 +234,7 @@ func TestGetDeleteAndNotFound(t *testing.T) {
func TestDeleteExpired(t *testing.T) {
t.Parallel()
store, err := NewFilesystemStore(filepath.Join(t.TempDir(), "data"))
store, err := NewFilesystemStore(filepath.Join(t.TempDir(), "data"), testEncryptionKey)
if err != nil {
t.Fatalf("NewFilesystemStore() error = %v", err)
}
@@ -332,7 +264,7 @@ func TestDeleteExpired(t *testing.T) {
func TestOpenInvalidGzipFails(t *testing.T) {
t.Parallel()
store, err := NewFilesystemStore(filepath.Join(t.TempDir(), "data"))
store, err := NewFilesystemStore(filepath.Join(t.TempDir(), "data"), testEncryptionKey)
if err != nil {
t.Fatalf("NewFilesystemStore() error = %v", err)
}
@@ -368,7 +300,7 @@ func TestNewFilesystemStoreInvalidIndex(t *testing.T) {
if err := os.WriteFile(filepath.Join(root, indexFilename), []byte("{bad"), 0o644); err != nil {
t.Fatalf("WriteFile() error = %v", err)
}
if _, err := NewFilesystemStore(root); err == nil {
if _, err := NewFilesystemStore(root, testEncryptionKey); err == nil {
t.Fatalf("NewFilesystemStore() error = nil, want invalid index error")
}
}
@@ -419,15 +351,21 @@ func TestNewFilesystemStoreReconcileOnStartup(t *testing.T) {
Size: 1,
},
}
payload, err := json.Marshal(meta)
payload, err := json.Marshal(indexDocument{
Scratches: meta,
})
if err != nil {
t.Fatalf("json.Marshal() error = %v", err)
}
if err := os.WriteFile(filepath.Join(root, indexFilename), payload, 0o644); err != nil {
encryptedPayload, err := encryptIndexPayload(payload, mustTestAEAD(t))
if err != nil {
t.Fatalf("encryptIndexPayload() error = %v", err)
}
if err := os.WriteFile(filepath.Join(root, indexFilename), encryptedPayload, 0o644); err != nil {
t.Fatalf("WriteFile index error = %v", err)
}
store, err := NewFilesystemStore(root)
store, err := NewFilesystemStore(root, testEncryptionKey)
if err != nil {
t.Fatalf("NewFilesystemStore() error = %v", err)
}
@@ -442,6 +380,63 @@ func TestNewFilesystemStoreReconcileOnStartup(t *testing.T) {
}
}
func TestNewFilesystemStoreWithDefaultTTLShiftsExpiriesOnTTLChange(t *testing.T) {
t.Parallel()
dataDir := filepath.Join(t.TempDir(), "data")
store, err := NewFilesystemStoreWithDefaultTTL(dataDir, time.Hour, testEncryptionKey)
if err != nil {
t.Fatalf("NewFilesystemStoreWithDefaultTTL() error = %v", err)
}
meta, err := store.Create(context.Background(), bytes.NewReader([]byte("hello")), "text/plain", time.Hour)
if err != nil {
t.Fatalf("Create() error = %v", err)
}
originalExpiry := meta.ExpiresAt
reloaded, err := NewFilesystemStoreWithDefaultTTL(dataDir, 2*time.Hour, testEncryptionKey)
if err != nil {
t.Fatalf("NewFilesystemStoreWithDefaultTTL() reload error = %v", err)
}
got, err := reloaded.Get(meta.ID)
if err != nil {
t.Fatalf("Get() error = %v", err)
}
wantExpiry := originalExpiry.Add(time.Hour)
if !got.ExpiresAt.Equal(wantExpiry) {
t.Fatalf("ExpiresAt = %s, want %s", got.ExpiresAt, wantExpiry)
}
again, err := NewFilesystemStoreWithDefaultTTL(dataDir, 2*time.Hour, testEncryptionKey)
if err != nil {
t.Fatalf("NewFilesystemStoreWithDefaultTTL() second reload error = %v", err)
}
gotAgain, err := again.Get(meta.ID)
if err != nil {
t.Fatalf("Get() second reload error = %v", err)
}
if !gotAgain.ExpiresAt.Equal(wantExpiry) {
t.Fatalf("ExpiresAt after unchanged TTL = %s, want %s", gotAgain.ExpiresAt, wantExpiry)
}
indexRaw, err := os.ReadFile(filepath.Join(dataDir, indexFilename))
if err != nil {
t.Fatalf("ReadFile(index) error = %v", err)
}
if bytes.Contains(indexRaw, []byte(`"default_ttl"`)) {
t.Fatalf("index should not be plaintext JSON")
}
indexPlaintext, err := decryptIndexPayload(indexRaw, mustTestAEAD(t))
if err != nil {
t.Fatalf("decryptIndexPayload() error = %v", err)
}
if !bytes.Contains(indexPlaintext, []byte(`"default_ttl": "2h0m0s"`)) {
t.Fatalf("decrypted index missing default ttl marker: %s", string(indexPlaintext))
}
}
type errCloser struct{}
func (errCloser) Close() error { return errors.New("close failed") }
@@ -465,7 +460,7 @@ func (failingReader) Read(_ []byte) (int, error) { return 0, errors.New("boom")
func TestCreateRollbackOnPersistFailure(t *testing.T) {
t.Parallel()
store, err := NewFilesystemStore(filepath.Join(t.TempDir(), "data"))
store, err := NewFilesystemStore(filepath.Join(t.TempDir(), "data"), testEncryptionKey)
if err != nil {
t.Fatalf("NewFilesystemStore() error = %v", err)
}
@@ -487,7 +482,7 @@ func TestCreateRollbackOnPersistFailure(t *testing.T) {
func TestCreateReaderFailure(t *testing.T) {
t.Parallel()
store, err := NewFilesystemStore(filepath.Join(t.TempDir(), "data"))
store, err := NewFilesystemStore(filepath.Join(t.TempDir(), "data"), testEncryptionKey)
if err != nil {
t.Fatalf("NewFilesystemStore() error = %v", err)
}
@@ -499,7 +494,7 @@ func TestCreateReaderFailure(t *testing.T) {
func TestDeletePersistFailureRestoresEntry(t *testing.T) {
t.Parallel()
store, err := NewFilesystemStore(filepath.Join(t.TempDir(), "data"))
store, err := NewFilesystemStore(filepath.Join(t.TempDir(), "data"), testEncryptionKey)
if err != nil {
t.Fatalf("NewFilesystemStore() error = %v", err)
}
@@ -525,7 +520,7 @@ func TestDeletePersistFailureRestoresEntry(t *testing.T) {
func TestDeleteRemoveFailure(t *testing.T) {
t.Parallel()
store, err := NewFilesystemStore(filepath.Join(t.TempDir(), "data"))
store, err := NewFilesystemStore(filepath.Join(t.TempDir(), "data"), testEncryptionKey)
if err != nil {
t.Fatalf("NewFilesystemStore() error = %v", err)
}
@@ -557,7 +552,7 @@ func TestDeleteRemoveFailure(t *testing.T) {
func TestDeleteExpiredPersistFailureRestoresEntries(t *testing.T) {
t.Parallel()
store, err := NewFilesystemStore(filepath.Join(t.TempDir(), "data"))
store, err := NewFilesystemStore(filepath.Join(t.TempDir(), "data"), testEncryptionKey)
if err != nil {
t.Fatalf("NewFilesystemStore() error = %v", err)
}
@@ -583,7 +578,7 @@ func TestDeleteExpiredPersistFailureRestoresEntries(t *testing.T) {
func TestDeleteExpiredRemoveFailure(t *testing.T) {
t.Parallel()
store, err := NewFilesystemStore(filepath.Join(t.TempDir(), "data"))
store, err := NewFilesystemStore(filepath.Join(t.TempDir(), "data"), testEncryptionKey)
if err != nil {
t.Fatalf("NewFilesystemStore() error = %v", err)
}
@@ -619,106 +614,45 @@ func TestNewFilesystemStoreFailsWhenDataDirIsFile(t *testing.T) {
if err := os.WriteFile(filePath, []byte("x"), 0o644); err != nil {
t.Fatalf("WriteFile() error = %v", err)
}
if _, err := NewFilesystemStore(filePath); err == nil {
if _, err := NewFilesystemStore(filePath, testEncryptionKey); err == nil {
t.Fatalf("NewFilesystemStore() error = nil, want mkdir failure")
}
}
type blockingReader struct {
release <-chan struct{}
payload []byte
sent bool
}
func (r *blockingReader) Read(p []byte) (int, error) {
if r.sent {
return 0, io.EOF
}
<-r.release
r.sent = true
return copy(p, r.payload), nil
}
func TestCreateReservationIsNotVisibleToCleanupOrReads(t *testing.T) {
func TestNewFilesystemStoreRequires32ByteKey(t *testing.T) {
t.Parallel()
store, err := NewFilesystemStore(filepath.Join(t.TempDir(), "data"))
dataDir := filepath.Join(t.TempDir(), "data")
if _, err := NewFilesystemStore(dataDir, nil); err == nil {
t.Fatalf("NewFilesystemStore() with nil key error = nil, want non-nil")
}
if _, err := NewFilesystemStore(dataDir, []byte("short")); err == nil {
t.Fatalf("NewFilesystemStore() with short key error = nil, want non-nil")
}
}
func TestNewFilesystemStoreFailsToLoadEncryptedIndexWithWrongKey(t *testing.T) {
t.Parallel()
dataDir := filepath.Join(t.TempDir(), "data")
store, err := NewFilesystemStore(dataDir, testEncryptionKey)
if err != nil {
t.Fatalf("NewFilesystemStore() error = %v", err)
}
const reservedID = "ffffffffffffffffffffffff"
store.idGenerator = func() (string, error) {
return reservedID, nil
if _, err := store.Create(context.Background(), bytes.NewReader([]byte("hello")), "text/plain", time.Hour); err != nil {
t.Fatalf("Create() error = %v", err)
}
release := make(chan struct{})
createErrCh := make(chan error, 1)
go func() {
_, createErr := store.CreateWithOriginalName(
context.Background(),
&blockingReader{release: release, payload: []byte("in-flight")},
"text/plain",
"",
time.Hour,
)
createErrCh <- createErr
}()
deadline := time.Now().Add(2 * time.Second)
for {
store.mu.RLock()
_, reserved := store.reservedIDs[reservedID]
_, visible := store.metadata[reservedID]
store.mu.RUnlock()
if reserved {
if visible {
t.Fatalf("in-flight ID should not appear in metadata")
}
break
}
if time.Now().After(deadline) {
t.Fatalf("timed out waiting for ID reservation")
}
time.Sleep(2 * time.Millisecond)
}
if _, err := store.Get(reservedID); !errors.Is(err, ErrNotFound) {
t.Fatalf("Get() during in-flight create error = %v, want ErrNotFound", err)
}
deleted, err := store.DeleteExpired(time.Now().UTC())
if err != nil {
t.Fatalf("DeleteExpired() error = %v", err)
}
if deleted != 0 {
t.Fatalf("DeleteExpired() = %d, want 0 for in-flight create", deleted)
}
close(release)
if err := <-createErrCh; err != nil {
t.Fatalf("CreateWithOriginalName() error = %v", err)
}
reader, _, err := store.Open(reservedID)
if err != nil {
t.Fatalf("Open() error = %v", err)
}
defer reader.Close()
body, err := io.ReadAll(reader)
if err != nil {
t.Fatalf("ReadAll() error = %v", err)
}
if string(body) != "in-flight" {
t.Fatalf("Open() content = %q, want %q", string(body), "in-flight")
wrongKey := []byte("abcdef0123456789abcdef0123456789")
if _, err := NewFilesystemStore(dataDir, wrongKey); err == nil {
t.Fatalf("NewFilesystemStore() with wrong key error = nil, want non-nil")
}
}
func TestConcurrentCreateWithOriginalNameProducesUniqueReadableEntries(t *testing.T) {
t.Parallel()
store, err := NewFilesystemStore(filepath.Join(t.TempDir(), "data"))
store, err := NewFilesystemStore(filepath.Join(t.TempDir(), "data"), testEncryptionKey)
if err != nil {
t.Fatalf("NewFilesystemStore() error = %v", err)
}