initial commit

This commit is contained in:
2026-05-29 22:39:50 -05:00
commit 47c767f9e8
23 changed files with 3361 additions and 0 deletions
+293
View File
@@ -0,0 +1,293 @@
package storage
import (
"compress/gzip"
"context"
"crypto/rand"
"encoding/hex"
"encoding/json"
"errors"
"fmt"
"io"
"os"
"path/filepath"
"sync"
"time"
)
const (
indexFilename = "metadata.json"
filesDirName = "scratches"
idBytes = 12
)
var ErrNotFound = errors.New("scratch not found")
type Metadata struct {
ID string `json:"id"`
FilePath string `json:"file_path"`
CreatedAt time.Time `json:"created_at"`
ExpiresAt time.Time `json:"expires_at"`
ContentType string `json:"content_type"`
Size int64 `json:"size"`
}
type FilesystemStore struct {
dataDir string
filesDir string
index string
mu sync.RWMutex
metadata map[string]Metadata
}
func NewFilesystemStore(dataDir string) (*FilesystemStore, error) {
filesDir := filepath.Join(dataDir, filesDirName)
if err := os.MkdirAll(filesDir, 0o755); err != nil {
return nil, fmt.Errorf("create files dir: %w", err)
}
store := &FilesystemStore{
dataDir: dataDir,
filesDir: filesDir,
index: filepath.Join(dataDir, indexFilename),
metadata: map[string]Metadata{},
}
if err := store.loadIndex(); err != nil {
return nil, err
}
if err := store.reconcileOnStartup(time.Now()); err != nil {
return nil, err
}
return store, nil
}
func (s *FilesystemStore) Create(ctx context.Context, body io.Reader, contentType string, ttl time.Duration) (Metadata, error) {
id, err := generateID()
if err != nil {
return Metadata{}, fmt.Errorf("generate id: %w", err)
}
tempFile, err := os.CreateTemp(s.filesDir, id+".tmp-*")
if err != nil {
return Metadata{}, fmt.Errorf("create temp file: %w", err)
}
gzipWriter := gzip.NewWriter(tempFile)
size, copyErr := io.Copy(gzipWriter, body)
gzipCloseErr := gzipWriter.Close()
closeErr := tempFile.Close()
if copyErr != nil {
_ = os.Remove(tempFile.Name())
return Metadata{}, fmt.Errorf("write scratch content: %w", copyErr)
}
if gzipCloseErr != nil {
_ = os.Remove(tempFile.Name())
return Metadata{}, fmt.Errorf("close gzip writer: %w", gzipCloseErr)
}
if closeErr != nil {
_ = os.Remove(tempFile.Name())
return Metadata{}, fmt.Errorf("close temp file: %w", closeErr)
}
finalPath := filepath.Join(s.filesDir, id)
if err := os.Rename(tempFile.Name(), finalPath); err != nil {
_ = os.Remove(tempFile.Name())
return Metadata{}, fmt.Errorf("atomically move scratch file: %w", err)
}
now := time.Now().UTC()
meta := Metadata{
ID: id,
FilePath: finalPath,
CreatedAt: now,
ExpiresAt: now.Add(ttl),
ContentType: contentType,
Size: size,
}
s.mu.Lock()
s.metadata[id] = meta
if err := s.persistLocked(); err != nil {
delete(s.metadata, id)
s.mu.Unlock()
_ = os.Remove(finalPath)
return Metadata{}, err
}
s.mu.Unlock()
return meta, nil
}
func (s *FilesystemStore) Get(id string) (Metadata, error) {
s.mu.RLock()
meta, ok := s.metadata[id]
s.mu.RUnlock()
if !ok {
return Metadata{}, ErrNotFound
}
return meta, nil
}
func (s *FilesystemStore) Open(id string) (io.ReadCloser, Metadata, error) {
meta, err := s.Get(id)
if err != nil {
return nil, Metadata{}, err
}
file, err := os.Open(meta.FilePath)
if err != nil {
if errors.Is(err, os.ErrNotExist) {
return nil, Metadata{}, ErrNotFound
}
return nil, Metadata{}, fmt.Errorf("open scratch file: %w", err)
}
gzipReader, err := gzip.NewReader(file)
if err != nil {
_ = file.Close()
return nil, Metadata{}, fmt.Errorf("create gzip reader: %w", err)
}
return &combinedReadCloser{
reader: gzipReader,
closers: []io.Closer{
gzipReader,
file,
},
}, meta, nil
}
func (s *FilesystemStore) Delete(id string) error {
s.mu.Lock()
meta, ok := s.metadata[id]
if !ok {
s.mu.Unlock()
return ErrNotFound
}
delete(s.metadata, id)
if err := s.persistLocked(); err != nil {
s.metadata[id] = meta
s.mu.Unlock()
return err
}
s.mu.Unlock()
if err := os.Remove(meta.FilePath); err != nil && !errors.Is(err, os.ErrNotExist) {
return fmt.Errorf("remove scratch file: %w", err)
}
return nil
}
func (s *FilesystemStore) DeleteExpired(now time.Time) (int, error) {
s.mu.Lock()
expired := make([]Metadata, 0)
for id, meta := range s.metadata {
if !meta.ExpiresAt.After(now) {
expired = append(expired, meta)
delete(s.metadata, id)
}
}
if len(expired) == 0 {
s.mu.Unlock()
return 0, nil
}
if err := s.persistLocked(); err != nil {
for _, meta := range expired {
s.metadata[meta.ID] = meta
}
s.mu.Unlock()
return 0, err
}
s.mu.Unlock()
for _, meta := range expired {
if err := os.Remove(meta.FilePath); err != nil && !errors.Is(err, os.ErrNotExist) {
return 0, fmt.Errorf("remove expired file %s: %w", meta.ID, err)
}
}
return len(expired), nil
}
func (s *FilesystemStore) loadIndex() error {
b, err := os.ReadFile(s.index)
if err != nil {
if errors.Is(err, os.ErrNotExist) {
return nil
}
return fmt.Errorf("read metadata index: %w", err)
}
loaded := map[string]Metadata{}
if err := json.Unmarshal(b, &loaded); err != nil {
return fmt.Errorf("decode metadata index: %w", err)
}
s.metadata = loaded
return nil
}
func (s *FilesystemStore) persistLocked() error {
tmp := s.index + ".tmp"
payload, err := json.MarshalIndent(s.metadata, "", " ")
if err != nil {
return fmt.Errorf("encode metadata index: %w", err)
}
if err := os.WriteFile(tmp, payload, 0o644); err != nil {
return fmt.Errorf("write metadata temp file: %w", err)
}
if err := os.Rename(tmp, s.index); err != nil {
return fmt.Errorf("replace metadata index: %w", err)
}
return nil
}
func (s *FilesystemStore) reconcileOnStartup(now time.Time) error {
s.mu.Lock()
defer s.mu.Unlock()
changed := false
for id, meta := range s.metadata {
if !meta.ExpiresAt.After(now) {
delete(s.metadata, id)
_ = os.Remove(meta.FilePath)
changed = true
continue
}
if _, err := os.Stat(meta.FilePath); err != nil {
if errors.Is(err, os.ErrNotExist) {
delete(s.metadata, id)
changed = true
continue
}
return fmt.Errorf("stat scratch %s: %w", id, err)
}
}
if !changed {
return nil
}
return s.persistLocked()
}
func generateID() (string, error) {
b := make([]byte, idBytes)
if _, err := rand.Read(b); err != nil {
return "", err
}
return hex.EncodeToString(b), nil
}
type combinedReadCloser struct {
reader io.Reader
closers []io.Closer
}
func (c *combinedReadCloser) Read(p []byte) (int, error) {
return c.reader.Read(p)
}
func (c *combinedReadCloser) Close() error {
var firstErr error
for _, closer := range c.closers {
if err := closer.Close(); err != nil && firstErr == nil {
firstErr = err
}
}
return firstErr
}
+402
View File
@@ -0,0 +1,402 @@
package storage
import (
"bytes"
"context"
"encoding/json"
"errors"
"io"
"os"
"path/filepath"
"testing"
"time"
)
func TestCreateStoresGzipAndOpenIsTransparent(t *testing.T) {
t.Parallel()
store, err := NewFilesystemStore(filepath.Join(t.TempDir(), "data"))
if err != nil {
t.Fatalf("NewFilesystemStore() error = %v", err)
}
original := bytes.Repeat([]byte("hello scratch "), 1024)
meta, err := store.Create(context.Background(), bytes.NewReader(original), "text/plain; charset=utf-8", time.Hour)
if err != nil {
t.Fatalf("Create() error = %v", err)
}
if meta.Size != int64(len(original)) {
t.Fatalf("meta.Size = %d, want %d", meta.Size, len(original))
}
onDisk, err := os.ReadFile(meta.FilePath)
if err != nil {
t.Fatalf("ReadFile() error = %v", err)
}
if len(onDisk) < 2 || onDisk[0] != 0x1f || onDisk[1] != 0x8b {
t.Fatalf("stored file does not have gzip header")
}
reader, _, err := store.Open(meta.ID)
if err != nil {
t.Fatalf("Open() error = %v", err)
}
defer reader.Close()
got, err := io.ReadAll(reader)
if err != nil {
t.Fatalf("ReadAll() error = %v", err)
}
if !bytes.Equal(got, original) {
t.Fatalf("open content mismatch: got %d bytes, want %d", len(got), len(original))
}
}
func TestGetDeleteAndNotFound(t *testing.T) {
t.Parallel()
store, err := NewFilesystemStore(filepath.Join(t.TempDir(), "data"))
if err != nil {
t.Fatalf("NewFilesystemStore() error = %v", err)
}
meta, err := store.Create(context.Background(), bytes.NewReader([]byte("hello")), "text/plain", time.Hour)
if err != nil {
t.Fatalf("Create() error = %v", err)
}
if _, err := store.Get(meta.ID); err != nil {
t.Fatalf("Get() error = %v", err)
}
if err := store.Delete(meta.ID); err != nil {
t.Fatalf("Delete() error = %v", err)
}
if _, err := store.Get(meta.ID); !errors.Is(err, ErrNotFound) {
t.Fatalf("Get() error = %v, want ErrNotFound", err)
}
if err := store.Delete(meta.ID); !errors.Is(err, ErrNotFound) {
t.Fatalf("Delete() error = %v, want ErrNotFound", err)
}
}
func TestDeleteExpired(t *testing.T) {
t.Parallel()
store, err := NewFilesystemStore(filepath.Join(t.TempDir(), "data"))
if err != nil {
t.Fatalf("NewFilesystemStore() error = %v", err)
}
expired, err := store.Create(context.Background(), bytes.NewReader([]byte("expired")), "text/plain", -time.Second)
if err != nil {
t.Fatalf("Create expired() error = %v", err)
}
active, err := store.Create(context.Background(), bytes.NewReader([]byte("active")), "text/plain", time.Hour)
if err != nil {
t.Fatalf("Create active() error = %v", err)
}
deleted, err := store.DeleteExpired(time.Now().UTC())
if err != nil {
t.Fatalf("DeleteExpired() error = %v", err)
}
if deleted != 1 {
t.Fatalf("DeleteExpired() = %d, want 1", deleted)
}
if _, err := store.Get(expired.ID); !errors.Is(err, ErrNotFound) {
t.Fatalf("expired Get() error = %v, want ErrNotFound", err)
}
if _, err := store.Get(active.ID); err != nil {
t.Fatalf("active Get() error = %v", err)
}
}
func TestOpenInvalidGzipFails(t *testing.T) {
t.Parallel()
store, err := NewFilesystemStore(filepath.Join(t.TempDir(), "data"))
if err != nil {
t.Fatalf("NewFilesystemStore() error = %v", err)
}
id := "deadbeefdeadbeefdeadbeef"
badPath := filepath.Join(store.filesDir, id)
if err := os.WriteFile(badPath, []byte("not-gzip"), 0o644); err != nil {
t.Fatalf("WriteFile() error = %v", err)
}
store.metadata[id] = Metadata{
ID: id,
FilePath: badPath,
CreatedAt: time.Now().UTC(),
ExpiresAt: time.Now().UTC().Add(time.Hour),
ContentType: "text/plain",
Size: int64(len("not-gzip")),
}
if err := store.persistLocked(); err != nil {
t.Fatalf("persistLocked() error = %v", err)
}
if _, _, err := store.Open(id); err == nil {
t.Fatalf("Open() error = nil, want gzip reader error")
}
}
func TestNewFilesystemStoreInvalidIndex(t *testing.T) {
t.Parallel()
root := t.TempDir()
scratches := filepath.Join(root, filesDirName)
if err := os.MkdirAll(scratches, 0o755); err != nil {
t.Fatalf("MkdirAll() error = %v", err)
}
if err := os.WriteFile(filepath.Join(root, indexFilename), []byte("{bad"), 0o644); err != nil {
t.Fatalf("WriteFile() error = %v", err)
}
if _, err := NewFilesystemStore(root); err == nil {
t.Fatalf("NewFilesystemStore() error = nil, want invalid index error")
}
}
func TestNewFilesystemStoreReconcileOnStartup(t *testing.T) {
t.Parallel()
root := t.TempDir()
scratches := filepath.Join(root, filesDirName)
if err := os.MkdirAll(scratches, 0o755); err != nil {
t.Fatalf("MkdirAll() error = %v", err)
}
keepID := "aaaaaaaaaaaaaaaaaaaaaaaa"
missingID := "bbbbbbbbbbbbbbbbbbbbbbbb"
expiredID := "cccccccccccccccccccccccc"
keepPath := filepath.Join(scratches, keepID)
expiredPath := filepath.Join(scratches, expiredID)
if err := os.WriteFile(keepPath, []byte{0x1f, 0x8b, 0x08, 0x00}, 0o644); err != nil {
t.Fatalf("WriteFile keep error = %v", err)
}
if err := os.WriteFile(expiredPath, []byte{0x1f, 0x8b, 0x08, 0x00}, 0o644); err != nil {
t.Fatalf("WriteFile expired error = %v", err)
}
meta := map[string]Metadata{
keepID: {
ID: keepID,
FilePath: keepPath,
CreatedAt: time.Now().UTC(),
ExpiresAt: time.Now().UTC().Add(time.Hour),
ContentType: "text/plain",
Size: 1,
},
missingID: {
ID: missingID,
FilePath: filepath.Join(scratches, missingID),
CreatedAt: time.Now().UTC(),
ExpiresAt: time.Now().UTC().Add(time.Hour),
ContentType: "text/plain",
Size: 1,
},
expiredID: {
ID: expiredID,
FilePath: expiredPath,
CreatedAt: time.Now().UTC(),
ExpiresAt: time.Now().UTC().Add(-time.Minute),
ContentType: "text/plain",
Size: 1,
},
}
payload, err := json.Marshal(meta)
if err != nil {
t.Fatalf("json.Marshal() error = %v", err)
}
if err := os.WriteFile(filepath.Join(root, indexFilename), payload, 0o644); err != nil {
t.Fatalf("WriteFile index error = %v", err)
}
store, err := NewFilesystemStore(root)
if err != nil {
t.Fatalf("NewFilesystemStore() error = %v", err)
}
if _, err := store.Get(keepID); err != nil {
t.Fatalf("keep entry missing: %v", err)
}
if _, err := store.Get(missingID); !errors.Is(err, ErrNotFound) {
t.Fatalf("missing entry error = %v, want ErrNotFound", err)
}
if _, err := store.Get(expiredID); !errors.Is(err, ErrNotFound) {
t.Fatalf("expired entry error = %v, want ErrNotFound", err)
}
}
type errCloser struct{}
func (errCloser) Close() error { return errors.New("close failed") }
func TestCombinedReadCloserClosePropagatesFirstError(t *testing.T) {
t.Parallel()
c := &combinedReadCloser{
reader: bytes.NewReader([]byte("ok")),
closers: []io.Closer{errCloser{}, errCloser{}},
}
if err := c.Close(); err == nil {
t.Fatalf("Close() error = nil, want non-nil")
}
}
type failingReader struct{}
func (failingReader) Read(_ []byte) (int, error) { return 0, errors.New("boom") }
func TestCreateRollbackOnPersistFailure(t *testing.T) {
t.Parallel()
store, err := NewFilesystemStore(filepath.Join(t.TempDir(), "data"))
if err != nil {
t.Fatalf("NewFilesystemStore() error = %v", err)
}
badIndexDir := filepath.Join(t.TempDir(), "index-dir")
if err := os.MkdirAll(badIndexDir, 0o755); err != nil {
t.Fatalf("MkdirAll() error = %v", err)
}
store.index = badIndexDir
if _, err := store.Create(context.Background(), bytes.NewReader([]byte("x")), "text/plain", time.Hour); err == nil {
t.Fatalf("Create() error = nil, want persist failure")
}
if len(store.metadata) != 0 {
t.Fatalf("metadata should be rolled back on persist failure")
}
}
func TestCreateReaderFailure(t *testing.T) {
t.Parallel()
store, err := NewFilesystemStore(filepath.Join(t.TempDir(), "data"))
if err != nil {
t.Fatalf("NewFilesystemStore() error = %v", err)
}
if _, err := store.Create(context.Background(), failingReader{}, "text/plain", time.Hour); err == nil {
t.Fatalf("Create() error = nil, want reader failure")
}
}
func TestDeletePersistFailureRestoresEntry(t *testing.T) {
t.Parallel()
store, err := NewFilesystemStore(filepath.Join(t.TempDir(), "data"))
if err != nil {
t.Fatalf("NewFilesystemStore() error = %v", err)
}
meta, err := store.Create(context.Background(), bytes.NewReader([]byte("hello")), "text/plain", time.Hour)
if err != nil {
t.Fatalf("Create() error = %v", err)
}
badIndexDir := filepath.Join(t.TempDir(), "index-dir")
if err := os.MkdirAll(badIndexDir, 0o755); err != nil {
t.Fatalf("MkdirAll() error = %v", err)
}
store.index = badIndexDir
if err := store.Delete(meta.ID); err == nil {
t.Fatalf("Delete() error = nil, want persist failure")
}
if _, err := store.Get(meta.ID); err != nil {
t.Fatalf("entry should be restored after delete persist failure: %v", err)
}
}
func TestDeleteRemoveFailure(t *testing.T) {
t.Parallel()
store, err := NewFilesystemStore(filepath.Join(t.TempDir(), "data"))
if err != nil {
t.Fatalf("NewFilesystemStore() error = %v", err)
}
nonEmptyDir := filepath.Join(t.TempDir(), "non-empty")
if err := os.MkdirAll(nonEmptyDir, 0o755); err != nil {
t.Fatalf("MkdirAll() error = %v", err)
}
if err := os.WriteFile(filepath.Join(nonEmptyDir, "child"), []byte("x"), 0o644); err != nil {
t.Fatalf("WriteFile() error = %v", err)
}
id := "dddddddddddddddddddddddd"
store.metadata[id] = Metadata{
ID: id,
FilePath: nonEmptyDir,
CreatedAt: time.Now().UTC(),
ExpiresAt: time.Now().UTC().Add(time.Hour),
ContentType: "text/plain",
Size: 1,
}
if err := store.persistLocked(); err != nil {
t.Fatalf("persistLocked() error = %v", err)
}
if err := store.Delete(id); err == nil {
t.Fatalf("Delete() error = nil, want remove failure")
}
}
func TestDeleteExpiredPersistFailureRestoresEntries(t *testing.T) {
t.Parallel()
store, err := NewFilesystemStore(filepath.Join(t.TempDir(), "data"))
if err != nil {
t.Fatalf("NewFilesystemStore() error = %v", err)
}
meta, err := store.Create(context.Background(), bytes.NewReader([]byte("expired")), "text/plain", -time.Second)
if err != nil {
t.Fatalf("Create() error = %v", err)
}
badIndexDir := filepath.Join(t.TempDir(), "index-dir")
if err := os.MkdirAll(badIndexDir, 0o755); err != nil {
t.Fatalf("MkdirAll() error = %v", err)
}
store.index = badIndexDir
if _, err := store.DeleteExpired(time.Now().UTC()); err == nil {
t.Fatalf("DeleteExpired() error = nil, want persist failure")
}
if _, err := store.Get(meta.ID); err != nil {
t.Fatalf("entry should be restored after DeleteExpired persist failure: %v", err)
}
}
func TestDeleteExpiredRemoveFailure(t *testing.T) {
t.Parallel()
store, err := NewFilesystemStore(filepath.Join(t.TempDir(), "data"))
if err != nil {
t.Fatalf("NewFilesystemStore() error = %v", err)
}
nonEmptyDir := filepath.Join(t.TempDir(), "expired-non-empty")
if err := os.MkdirAll(nonEmptyDir, 0o755); err != nil {
t.Fatalf("MkdirAll() error = %v", err)
}
if err := os.WriteFile(filepath.Join(nonEmptyDir, "child"), []byte("x"), 0o644); err != nil {
t.Fatalf("WriteFile() error = %v", err)
}
id := "eeeeeeeeeeeeeeeeeeeeeeee"
store.metadata[id] = Metadata{
ID: id,
FilePath: nonEmptyDir,
CreatedAt: time.Now().UTC(),
ExpiresAt: time.Now().UTC().Add(-time.Second),
ContentType: "text/plain",
Size: 1,
}
if err := store.persistLocked(); err != nil {
t.Fatalf("persistLocked() error = %v", err)
}
if _, err := store.DeleteExpired(time.Now().UTC()); err == nil {
t.Fatalf("DeleteExpired() error = nil, want remove failure")
}
}
func TestNewFilesystemStoreFailsWhenDataDirIsFile(t *testing.T) {
t.Parallel()
tmp := t.TempDir()
filePath := filepath.Join(tmp, "not-dir")
if err := os.WriteFile(filePath, []byte("x"), 0o644); err != nil {
t.Fatalf("WriteFile() error = %v", err)
}
if _, err := NewFilesystemStore(filePath); err == nil {
t.Fatalf("NewFilesystemStore() error = nil, want mkdir failure")
}
}