Lots more changes
This commit is contained in:
+388
-134
@@ -3,7 +3,11 @@ package storage
|
||||
import (
|
||||
"compress/gzip"
|
||||
"context"
|
||||
"crypto/aes"
|
||||
"crypto/cipher"
|
||||
"crypto/rand"
|
||||
"crypto/sha256"
|
||||
"encoding/binary"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
@@ -17,14 +21,16 @@ import (
|
||||
)
|
||||
|
||||
const (
|
||||
indexFilename = "metadata.json"
|
||||
filesDirName = "scratches"
|
||||
idBytes = 12
|
||||
maxCreateIDAttempts = 100
|
||||
indexFilename = "metadata"
|
||||
filesDirName = "scratches"
|
||||
encryptedChunkSize = 64 * 1024
|
||||
)
|
||||
|
||||
var ErrNotFound = errors.New("scratch not found")
|
||||
|
||||
var encryptedFileMagic = [4]byte{'S', 'B', 'X', '1'}
|
||||
var encryptedIndexMagic = [4]byte{'S', 'M', 'D', '1'}
|
||||
|
||||
type Metadata struct {
|
||||
ID string `json:"id"`
|
||||
FilePath string `json:"file_path"`
|
||||
@@ -35,35 +41,63 @@ type Metadata struct {
|
||||
Size int64 `json:"size"`
|
||||
}
|
||||
|
||||
type FilesystemStore struct {
|
||||
dataDir string
|
||||
filesDir string
|
||||
index string
|
||||
idGenerator func() (string, error)
|
||||
|
||||
mu sync.RWMutex
|
||||
metadata map[string]Metadata
|
||||
reservedIDs map[string]struct{}
|
||||
type indexDocument struct {
|
||||
DefaultTTL string `json:"default_ttl,omitempty"`
|
||||
Scratches map[string]Metadata `json:"scratches"`
|
||||
}
|
||||
|
||||
func NewFilesystemStore(dataDir string) (*FilesystemStore, error) {
|
||||
type FilesystemStore struct {
|
||||
dataDir string
|
||||
filesDir string
|
||||
index string
|
||||
defaultTTL time.Duration
|
||||
fileCipher cipher.AEAD
|
||||
|
||||
mu sync.RWMutex
|
||||
metadata map[string]Metadata
|
||||
}
|
||||
|
||||
func NewFilesystemStore(dataDir string, encryptionKey []byte) (*FilesystemStore, error) {
|
||||
return newFilesystemStore(dataDir, 0, encryptionKey)
|
||||
}
|
||||
|
||||
func NewFilesystemStoreWithDefaultTTL(dataDir string, defaultTTL time.Duration, encryptionKey []byte) (*FilesystemStore, error) {
|
||||
return newFilesystemStore(dataDir, defaultTTL, encryptionKey)
|
||||
}
|
||||
|
||||
func newFilesystemStore(dataDir string, defaultTTL time.Duration, encryptionKey []byte) (*FilesystemStore, error) {
|
||||
filesDir := filepath.Join(dataDir, filesDirName)
|
||||
if err := os.MkdirAll(filesDir, 0o755); err != nil {
|
||||
return nil, fmt.Errorf("create files dir: %w", err)
|
||||
}
|
||||
|
||||
if len(encryptionKey) != 32 {
|
||||
return nil, fmt.Errorf("storage encryption key must be 32 bytes, got %d", len(encryptionKey))
|
||||
}
|
||||
block, err := aes.NewCipher(encryptionKey)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("init storage cipher: %w", err)
|
||||
}
|
||||
fileCipher, err := cipher.NewGCM(block)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("init storage aead: %w", err)
|
||||
}
|
||||
|
||||
store := &FilesystemStore{
|
||||
dataDir: dataDir,
|
||||
filesDir: filesDir,
|
||||
index: filepath.Join(dataDir, indexFilename),
|
||||
idGenerator: generateID,
|
||||
metadata: map[string]Metadata{},
|
||||
reservedIDs: map[string]struct{}{},
|
||||
dataDir: dataDir,
|
||||
filesDir: filesDir,
|
||||
index: filepath.Join(dataDir, indexFilename),
|
||||
fileCipher: fileCipher,
|
||||
metadata: map[string]Metadata{},
|
||||
}
|
||||
if err := store.loadIndex(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if err := store.reconcileOnStartup(time.Now()); err != nil {
|
||||
now := time.Now().UTC()
|
||||
if err := store.applyDefaultTTLOnStartup(defaultTTL); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if err := store.reconcileOnStartup(now); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return store, nil
|
||||
@@ -74,107 +108,84 @@ func (s *FilesystemStore) Create(ctx context.Context, body io.Reader, contentTyp
|
||||
}
|
||||
|
||||
func (s *FilesystemStore) CreateWithOriginalName(ctx context.Context, body io.Reader, contentType string, originalName string, ttl time.Duration) (Metadata, error) {
|
||||
for attempt := 0; attempt < maxCreateIDAttempts; attempt++ {
|
||||
id, err := s.idGenerator()
|
||||
if err != nil {
|
||||
return Metadata{}, fmt.Errorf("generate id: %w", err)
|
||||
}
|
||||
finalPath := filepath.Join(s.filesDir, id)
|
||||
|
||||
s.mu.RLock()
|
||||
_, exists := s.metadata[id]
|
||||
_, reserved := s.reservedIDs[id]
|
||||
s.mu.RUnlock()
|
||||
if exists || reserved {
|
||||
continue
|
||||
}
|
||||
if _, err := os.Stat(finalPath); err == nil {
|
||||
continue
|
||||
} else if !errors.Is(err, os.ErrNotExist) {
|
||||
return Metadata{}, fmt.Errorf("stat candidate scratch path: %w", err)
|
||||
}
|
||||
|
||||
s.mu.Lock()
|
||||
if _, exists := s.metadata[id]; exists {
|
||||
s.mu.Unlock()
|
||||
continue
|
||||
}
|
||||
if _, reserved := s.reservedIDs[id]; reserved {
|
||||
s.mu.Unlock()
|
||||
continue
|
||||
}
|
||||
// Reserve the ID before consuming the request body to avoid
|
||||
// concurrent writers choosing the same ID.
|
||||
s.reservedIDs[id] = struct{}{}
|
||||
s.mu.Unlock()
|
||||
|
||||
tempFile, err := os.CreateTemp(s.filesDir, id+".tmp-*")
|
||||
if err != nil {
|
||||
s.mu.Lock()
|
||||
delete(s.reservedIDs, id)
|
||||
s.mu.Unlock()
|
||||
return Metadata{}, fmt.Errorf("create temp file: %w", err)
|
||||
}
|
||||
|
||||
gzipWriter := gzip.NewWriter(tempFile)
|
||||
size, copyErr := io.Copy(gzipWriter, body)
|
||||
gzipCloseErr := gzipWriter.Close()
|
||||
closeErr := tempFile.Close()
|
||||
if copyErr != nil {
|
||||
_ = os.Remove(tempFile.Name())
|
||||
s.mu.Lock()
|
||||
delete(s.reservedIDs, id)
|
||||
s.mu.Unlock()
|
||||
return Metadata{}, fmt.Errorf("write scratch content: %w", copyErr)
|
||||
}
|
||||
if gzipCloseErr != nil {
|
||||
_ = os.Remove(tempFile.Name())
|
||||
s.mu.Lock()
|
||||
delete(s.reservedIDs, id)
|
||||
s.mu.Unlock()
|
||||
return Metadata{}, fmt.Errorf("close gzip writer: %w", gzipCloseErr)
|
||||
}
|
||||
if closeErr != nil {
|
||||
_ = os.Remove(tempFile.Name())
|
||||
s.mu.Lock()
|
||||
delete(s.reservedIDs, id)
|
||||
s.mu.Unlock()
|
||||
return Metadata{}, fmt.Errorf("close temp file: %w", closeErr)
|
||||
}
|
||||
|
||||
if err := os.Rename(tempFile.Name(), finalPath); err != nil {
|
||||
_ = os.Remove(tempFile.Name())
|
||||
s.mu.Lock()
|
||||
delete(s.reservedIDs, id)
|
||||
s.mu.Unlock()
|
||||
return Metadata{}, fmt.Errorf("atomically move scratch file: %w", err)
|
||||
}
|
||||
|
||||
now := time.Now().UTC()
|
||||
meta := Metadata{
|
||||
ID: id,
|
||||
FilePath: finalPath,
|
||||
CreatedAt: now,
|
||||
ExpiresAt: now.Add(ttl),
|
||||
ContentType: contentType,
|
||||
OriginalName: normalizeOriginalName(originalName),
|
||||
Size: size,
|
||||
}
|
||||
|
||||
s.mu.Lock()
|
||||
delete(s.reservedIDs, id)
|
||||
s.metadata[id] = meta
|
||||
if err := s.persistLocked(); err != nil {
|
||||
delete(s.metadata, id)
|
||||
s.mu.Unlock()
|
||||
_ = os.Remove(finalPath)
|
||||
return Metadata{}, err
|
||||
}
|
||||
s.mu.Unlock()
|
||||
return meta, nil
|
||||
tempFile, err := os.CreateTemp(s.filesDir, "scratch.tmp-*")
|
||||
if err != nil {
|
||||
return Metadata{}, fmt.Errorf("create temp file: %w", err)
|
||||
}
|
||||
|
||||
return Metadata{}, fmt.Errorf("failed to allocate unique scratch id after %d attempts", maxCreateIDAttempts)
|
||||
blobHash := sha256.New()
|
||||
blobSink := io.MultiWriter(tempFile, blobHash)
|
||||
scratchWriter := io.Writer(blobSink)
|
||||
encryptedWriter, err := newEncryptedFileWriter(blobSink, s.fileCipher)
|
||||
if err != nil {
|
||||
_ = tempFile.Close()
|
||||
_ = os.Remove(tempFile.Name())
|
||||
return Metadata{}, fmt.Errorf("init encrypted scratch writer: %w", err)
|
||||
}
|
||||
scratchWriter = encryptedWriter
|
||||
|
||||
gzipWriter := gzip.NewWriter(scratchWriter)
|
||||
size, copyErr := io.Copy(gzipWriter, body)
|
||||
gzipCloseErr := gzipWriter.Close()
|
||||
encryptCloseErr := encryptedWriter.Close()
|
||||
closeErr := tempFile.Close()
|
||||
if copyErr != nil {
|
||||
_ = os.Remove(tempFile.Name())
|
||||
return Metadata{}, fmt.Errorf("write scratch content: %w", copyErr)
|
||||
}
|
||||
if gzipCloseErr != nil {
|
||||
_ = os.Remove(tempFile.Name())
|
||||
return Metadata{}, fmt.Errorf("close gzip writer: %w", gzipCloseErr)
|
||||
}
|
||||
if encryptCloseErr != nil {
|
||||
_ = os.Remove(tempFile.Name())
|
||||
return Metadata{}, fmt.Errorf("close encrypted writer: %w", encryptCloseErr)
|
||||
}
|
||||
if closeErr != nil {
|
||||
_ = os.Remove(tempFile.Name())
|
||||
return Metadata{}, fmt.Errorf("close temp file: %w", closeErr)
|
||||
}
|
||||
|
||||
id := hex.EncodeToString(blobHash.Sum(nil))
|
||||
finalPath := filepath.Join(s.filesDir, id)
|
||||
if _, statErr := os.Stat(finalPath); statErr == nil {
|
||||
_ = os.Remove(tempFile.Name())
|
||||
return Metadata{}, errors.New("scratch id collision for content hash")
|
||||
} else if !errors.Is(statErr, os.ErrNotExist) {
|
||||
_ = os.Remove(tempFile.Name())
|
||||
return Metadata{}, fmt.Errorf("stat candidate scratch path: %w", statErr)
|
||||
}
|
||||
if err := os.Rename(tempFile.Name(), finalPath); err != nil {
|
||||
_ = os.Remove(tempFile.Name())
|
||||
return Metadata{}, fmt.Errorf("atomically move scratch file: %w", err)
|
||||
}
|
||||
|
||||
now := time.Now().UTC()
|
||||
meta := Metadata{
|
||||
ID: id,
|
||||
FilePath: finalPath,
|
||||
CreatedAt: now,
|
||||
ExpiresAt: now.Add(ttl),
|
||||
ContentType: contentType,
|
||||
OriginalName: normalizeOriginalName(originalName),
|
||||
Size: size,
|
||||
}
|
||||
|
||||
s.mu.Lock()
|
||||
if _, exists := s.metadata[id]; exists {
|
||||
s.mu.Unlock()
|
||||
_ = os.Remove(finalPath)
|
||||
return Metadata{}, errors.New("scratch id collision in metadata index")
|
||||
}
|
||||
s.metadata[id] = meta
|
||||
if err := s.persistLocked(); err != nil {
|
||||
delete(s.metadata, id)
|
||||
s.mu.Unlock()
|
||||
_ = os.Remove(finalPath)
|
||||
return Metadata{}, err
|
||||
}
|
||||
s.mu.Unlock()
|
||||
return meta, nil
|
||||
}
|
||||
|
||||
func normalizeOriginalName(originalName string) string {
|
||||
@@ -212,7 +223,12 @@ func (s *FilesystemStore) Open(id string) (io.ReadCloser, Metadata, error) {
|
||||
}
|
||||
return nil, Metadata{}, fmt.Errorf("open scratch file: %w", err)
|
||||
}
|
||||
gzipReader, err := gzip.NewReader(file)
|
||||
decryptedReader, decryptErr := newEncryptedFileReader(file, s.fileCipher)
|
||||
if decryptErr != nil {
|
||||
_ = file.Close()
|
||||
return nil, Metadata{}, fmt.Errorf("create encrypted reader: %w", decryptErr)
|
||||
}
|
||||
gzipReader, err := gzip.NewReader(decryptedReader)
|
||||
if err != nil {
|
||||
_ = file.Close()
|
||||
return nil, Metadata{}, fmt.Errorf("create gzip reader: %w", err)
|
||||
@@ -286,22 +302,46 @@ func (s *FilesystemStore) loadIndex() error {
|
||||
}
|
||||
return fmt.Errorf("read metadata index: %w", err)
|
||||
}
|
||||
plaintext, err := decryptIndexPayload(b, s.fileCipher)
|
||||
if err != nil {
|
||||
return fmt.Errorf("decrypt metadata index: %w", err)
|
||||
}
|
||||
|
||||
loaded := map[string]Metadata{}
|
||||
if err := json.Unmarshal(b, &loaded); err != nil {
|
||||
doc := indexDocument{}
|
||||
if err := json.Unmarshal(plaintext, &doc); err != nil {
|
||||
return fmt.Errorf("decode metadata index: %w", err)
|
||||
}
|
||||
s.metadata = loaded
|
||||
if doc.Scratches == nil {
|
||||
doc.Scratches = map[string]Metadata{}
|
||||
}
|
||||
s.metadata = doc.Scratches
|
||||
if ttlRaw := strings.TrimSpace(doc.DefaultTTL); ttlRaw != "" {
|
||||
ttl, parseErr := time.ParseDuration(ttlRaw)
|
||||
if parseErr != nil {
|
||||
return fmt.Errorf("decode metadata index default ttl: %w", parseErr)
|
||||
}
|
||||
s.defaultTTL = ttl
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *FilesystemStore) persistLocked() error {
|
||||
tmp := s.index + ".tmp"
|
||||
payload, err := json.MarshalIndent(s.metadata, "", " ")
|
||||
doc := indexDocument{
|
||||
Scratches: s.metadata,
|
||||
}
|
||||
if s.defaultTTL > 0 {
|
||||
doc.DefaultTTL = s.defaultTTL.String()
|
||||
}
|
||||
payload, err := json.MarshalIndent(doc, "", " ")
|
||||
if err != nil {
|
||||
return fmt.Errorf("encode metadata index: %w", err)
|
||||
}
|
||||
if err := os.WriteFile(tmp, payload, 0o644); err != nil {
|
||||
encryptedPayload, err := encryptIndexPayload(payload, s.fileCipher)
|
||||
if err != nil {
|
||||
return fmt.Errorf("encrypt metadata index: %w", err)
|
||||
}
|
||||
if err := os.WriteFile(tmp, encryptedPayload, 0o644); err != nil {
|
||||
return fmt.Errorf("write metadata temp file: %w", err)
|
||||
}
|
||||
if err := os.Rename(tmp, s.index); err != nil {
|
||||
@@ -310,6 +350,28 @@ func (s *FilesystemStore) persistLocked() error {
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *FilesystemStore) applyDefaultTTLOnStartup(defaultTTL time.Duration) error {
|
||||
if defaultTTL <= 0 {
|
||||
return nil
|
||||
}
|
||||
|
||||
if s.defaultTTL <= 0 {
|
||||
s.defaultTTL = defaultTTL
|
||||
return s.persistLocked()
|
||||
}
|
||||
if s.defaultTTL == defaultTTL {
|
||||
return nil
|
||||
}
|
||||
|
||||
delta := defaultTTL - s.defaultTTL
|
||||
for id, meta := range s.metadata {
|
||||
meta.ExpiresAt = meta.ExpiresAt.Add(delta)
|
||||
s.metadata[id] = meta
|
||||
}
|
||||
s.defaultTTL = defaultTTL
|
||||
return s.persistLocked()
|
||||
}
|
||||
|
||||
func (s *FilesystemStore) reconcileOnStartup(now time.Time) error {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
@@ -337,14 +399,6 @@ func (s *FilesystemStore) reconcileOnStartup(now time.Time) error {
|
||||
return s.persistLocked()
|
||||
}
|
||||
|
||||
func generateID() (string, error) {
|
||||
b := make([]byte, idBytes)
|
||||
if _, err := rand.Read(b); err != nil {
|
||||
return "", err
|
||||
}
|
||||
return hex.EncodeToString(b), nil
|
||||
}
|
||||
|
||||
type combinedReadCloser struct {
|
||||
reader io.Reader
|
||||
closers []io.Closer
|
||||
@@ -363,3 +417,203 @@ func (c *combinedReadCloser) Close() error {
|
||||
}
|
||||
return firstErr
|
||||
}
|
||||
|
||||
type encryptedFileWriter struct {
|
||||
dst io.Writer
|
||||
aead cipher.AEAD
|
||||
noncePrefix [4]byte
|
||||
counter uint64
|
||||
pending []byte
|
||||
}
|
||||
|
||||
func newEncryptedFileWriter(dst io.Writer, aead cipher.AEAD) (*encryptedFileWriter, error) {
|
||||
writer := &encryptedFileWriter{
|
||||
dst: dst,
|
||||
aead: aead,
|
||||
pending: make([]byte, 0, encryptedChunkSize),
|
||||
}
|
||||
if _, err := rand.Read(writer.noncePrefix[:]); err != nil {
|
||||
return nil, fmt.Errorf("generate nonce prefix: %w", err)
|
||||
}
|
||||
header := make([]byte, 0, len(encryptedFileMagic)+len(writer.noncePrefix))
|
||||
header = append(header, encryptedFileMagic[:]...)
|
||||
header = append(header, writer.noncePrefix[:]...)
|
||||
if _, err := writer.dst.Write(header); err != nil {
|
||||
return nil, fmt.Errorf("write encrypted file header: %w", err)
|
||||
}
|
||||
return writer, nil
|
||||
}
|
||||
|
||||
func (w *encryptedFileWriter) Write(p []byte) (int, error) {
|
||||
written := len(p)
|
||||
for len(p) > 0 {
|
||||
space := encryptedChunkSize - len(w.pending)
|
||||
if space > len(p) {
|
||||
space = len(p)
|
||||
}
|
||||
w.pending = append(w.pending, p[:space]...)
|
||||
p = p[space:]
|
||||
if len(w.pending) == encryptedChunkSize {
|
||||
if err := w.flushChunk(w.pending); err != nil {
|
||||
return written - len(p), err
|
||||
}
|
||||
w.pending = w.pending[:0]
|
||||
}
|
||||
}
|
||||
return written, nil
|
||||
}
|
||||
|
||||
func (w *encryptedFileWriter) Close() error {
|
||||
if len(w.pending) == 0 {
|
||||
return nil
|
||||
}
|
||||
if err := w.flushChunk(w.pending); err != nil {
|
||||
return err
|
||||
}
|
||||
w.pending = w.pending[:0]
|
||||
return nil
|
||||
}
|
||||
|
||||
func (w *encryptedFileWriter) flushChunk(plaintext []byte) error {
|
||||
nonce := make([]byte, w.aead.NonceSize())
|
||||
copy(nonce, w.noncePrefix[:])
|
||||
binary.BigEndian.PutUint64(nonce[len(w.noncePrefix):], w.counter)
|
||||
ciphertext := w.aead.Seal(nil, nonce, plaintext, nil)
|
||||
|
||||
var sizeBuf [4]byte
|
||||
binary.BigEndian.PutUint32(sizeBuf[:], uint32(len(ciphertext)))
|
||||
if _, err := w.dst.Write(sizeBuf[:]); err != nil {
|
||||
return fmt.Errorf("write encrypted chunk size: %w", err)
|
||||
}
|
||||
if _, err := w.dst.Write(ciphertext); err != nil {
|
||||
return fmt.Errorf("write encrypted chunk payload: %w", err)
|
||||
}
|
||||
w.counter++
|
||||
return nil
|
||||
}
|
||||
|
||||
type encryptedFileReader struct {
|
||||
src io.Reader
|
||||
aead cipher.AEAD
|
||||
noncePrefix [4]byte
|
||||
counter uint64
|
||||
buf []byte
|
||||
offset int
|
||||
eof bool
|
||||
}
|
||||
|
||||
func newEncryptedFileReader(src io.Reader, aead cipher.AEAD) (*encryptedFileReader, error) {
|
||||
reader := &encryptedFileReader{
|
||||
src: src,
|
||||
aead: aead,
|
||||
}
|
||||
header := make([]byte, len(encryptedFileMagic)+len(reader.noncePrefix))
|
||||
if _, err := io.ReadFull(src, header); err != nil {
|
||||
return nil, fmt.Errorf("read encrypted file header: %w", err)
|
||||
}
|
||||
if !equalBytes(header[:len(encryptedFileMagic)], encryptedFileMagic[:]) {
|
||||
return nil, errors.New("invalid encrypted file header")
|
||||
}
|
||||
copy(reader.noncePrefix[:], header[len(encryptedFileMagic):])
|
||||
return reader, nil
|
||||
}
|
||||
|
||||
func (r *encryptedFileReader) Read(p []byte) (int, error) {
|
||||
for r.offset >= len(r.buf) {
|
||||
if r.eof {
|
||||
return 0, io.EOF
|
||||
}
|
||||
if err := r.loadNextChunk(); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
}
|
||||
n := copy(p, r.buf[r.offset:])
|
||||
r.offset += n
|
||||
return n, nil
|
||||
}
|
||||
|
||||
func (r *encryptedFileReader) loadNextChunk() error {
|
||||
var sizeBuf [4]byte
|
||||
_, err := io.ReadFull(r.src, sizeBuf[:])
|
||||
if err != nil {
|
||||
if errors.Is(err, io.EOF) {
|
||||
r.eof = true
|
||||
return io.EOF
|
||||
}
|
||||
if errors.Is(err, io.ErrUnexpectedEOF) {
|
||||
return errors.New("truncated encrypted chunk size")
|
||||
}
|
||||
return fmt.Errorf("read encrypted chunk size: %w", err)
|
||||
}
|
||||
chunkSize := binary.BigEndian.Uint32(sizeBuf[:])
|
||||
if chunkSize == 0 {
|
||||
return errors.New("invalid empty encrypted chunk")
|
||||
}
|
||||
maxChunkSize := uint32(encryptedChunkSize + r.aead.Overhead())
|
||||
if chunkSize > maxChunkSize {
|
||||
return fmt.Errorf("encrypted chunk too large: %d", chunkSize)
|
||||
}
|
||||
ciphertext := make([]byte, chunkSize)
|
||||
if _, err := io.ReadFull(r.src, ciphertext); err != nil {
|
||||
return fmt.Errorf("read encrypted chunk payload: %w", err)
|
||||
}
|
||||
nonce := make([]byte, r.aead.NonceSize())
|
||||
copy(nonce, r.noncePrefix[:])
|
||||
binary.BigEndian.PutUint64(nonce[len(r.noncePrefix):], r.counter)
|
||||
plaintext, err := r.aead.Open(nil, nonce, ciphertext, nil)
|
||||
if err != nil {
|
||||
return fmt.Errorf("decrypt chunk: %w", err)
|
||||
}
|
||||
r.counter++
|
||||
r.buf = plaintext
|
||||
r.offset = 0
|
||||
return nil
|
||||
}
|
||||
|
||||
func equalBytes(a, b []byte) bool {
|
||||
if len(a) != len(b) {
|
||||
return false
|
||||
}
|
||||
for i := range a {
|
||||
if a[i] != b[i] {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
func encryptIndexPayload(plaintext []byte, aead cipher.AEAD) ([]byte, error) {
|
||||
nonce := make([]byte, aead.NonceSize())
|
||||
if _, err := rand.Read(nonce); err != nil {
|
||||
return nil, fmt.Errorf("generate index nonce: %w", err)
|
||||
}
|
||||
ciphertext := aead.Seal(nil, nonce, plaintext, nil)
|
||||
out := make([]byte, 0, len(encryptedIndexMagic)+len(nonce)+len(ciphertext))
|
||||
out = append(out, encryptedIndexMagic[:]...)
|
||||
out = append(out, nonce...)
|
||||
out = append(out, ciphertext...)
|
||||
return out, nil
|
||||
}
|
||||
|
||||
func decryptIndexPayload(raw []byte, aead cipher.AEAD) ([]byte, error) {
|
||||
headerLen := len(encryptedIndexMagic)
|
||||
nonceSize := aead.NonceSize()
|
||||
if len(raw) < headerLen+nonceSize {
|
||||
return nil, errors.New("encrypted index is too short")
|
||||
}
|
||||
if !equalBytes(raw[:headerLen], encryptedIndexMagic[:]) {
|
||||
return nil, errors.New("invalid encrypted index header")
|
||||
}
|
||||
nonceStart := headerLen
|
||||
nonceEnd := nonceStart + nonceSize
|
||||
nonce := raw[nonceStart:nonceEnd]
|
||||
ciphertext := raw[nonceEnd:]
|
||||
if len(ciphertext) == 0 {
|
||||
return nil, errors.New("encrypted index ciphertext is empty")
|
||||
}
|
||||
plaintext, err := aead.Open(nil, nonce, ciphertext, nil)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("decrypt index payload: %w", err)
|
||||
}
|
||||
return plaintext, nil
|
||||
}
|
||||
|
||||
+212
-278
@@ -3,6 +3,11 @@ package storage
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"crypto/aes"
|
||||
"crypto/cipher"
|
||||
"crypto/rand"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
@@ -15,10 +20,25 @@ import (
|
||||
"time"
|
||||
)
|
||||
|
||||
var testEncryptionKey = []byte("0123456789abcdef0123456789abcdef")
|
||||
|
||||
func mustTestAEAD(t *testing.T) cipher.AEAD {
|
||||
t.Helper()
|
||||
block, err := aes.NewCipher(testEncryptionKey)
|
||||
if err != nil {
|
||||
t.Fatalf("aes.NewCipher() error = %v", err)
|
||||
}
|
||||
aead, err := cipher.NewGCM(block)
|
||||
if err != nil {
|
||||
t.Fatalf("cipher.NewGCM() error = %v", err)
|
||||
}
|
||||
return aead
|
||||
}
|
||||
|
||||
func TestCreateStoresGzipAndOpenIsTransparent(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
store, err := NewFilesystemStore(filepath.Join(t.TempDir(), "data"))
|
||||
store, err := NewFilesystemStore(filepath.Join(t.TempDir(), "data"), testEncryptionKey)
|
||||
if err != nil {
|
||||
t.Fatalf("NewFilesystemStore() error = %v", err)
|
||||
}
|
||||
@@ -36,8 +56,8 @@ func TestCreateStoresGzipAndOpenIsTransparent(t *testing.T) {
|
||||
if err != nil {
|
||||
t.Fatalf("ReadFile() error = %v", err)
|
||||
}
|
||||
if len(onDisk) < 2 || onDisk[0] != 0x1f || onDisk[1] != 0x8b {
|
||||
t.Fatalf("stored file does not have gzip header")
|
||||
if len(onDisk) < 4 || string(onDisk[:4]) != "SBX1" {
|
||||
t.Fatalf("stored file does not have encrypted scratch header")
|
||||
}
|
||||
|
||||
reader, _, err := store.Open(meta.ID)
|
||||
@@ -54,10 +74,92 @@ func TestCreateStoresGzipAndOpenIsTransparent(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestCreateEncryptedStoreDoesNotExposeGzipHeaderOnDisk(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
key := make([]byte, 32)
|
||||
if _, err := rand.Read(key); err != nil {
|
||||
t.Fatalf("rand.Read() error = %v", err)
|
||||
}
|
||||
|
||||
dataDir := filepath.Join(t.TempDir(), "data")
|
||||
store, err := NewFilesystemStore(dataDir, key)
|
||||
if err != nil {
|
||||
t.Fatalf("NewEncryptedFilesystemStore() error = %v", err)
|
||||
}
|
||||
|
||||
original := bytes.Repeat([]byte("secret payload "), 1024)
|
||||
meta, err := store.Create(context.Background(), bytes.NewReader(original), "text/plain", time.Hour)
|
||||
if err != nil {
|
||||
t.Fatalf("Create() error = %v", err)
|
||||
}
|
||||
|
||||
onDisk, err := os.ReadFile(meta.FilePath)
|
||||
if err != nil {
|
||||
t.Fatalf("ReadFile() error = %v", err)
|
||||
}
|
||||
if len(onDisk) < 2 {
|
||||
t.Fatalf("encrypted file too short")
|
||||
}
|
||||
if onDisk[0] == 0x1f && onDisk[1] == 0x8b {
|
||||
t.Fatalf("encrypted file unexpectedly starts with gzip header")
|
||||
}
|
||||
|
||||
reader, _, err := store.Open(meta.ID)
|
||||
if err != nil {
|
||||
t.Fatalf("Open() error = %v", err)
|
||||
}
|
||||
defer reader.Close()
|
||||
got, err := io.ReadAll(reader)
|
||||
if err != nil {
|
||||
t.Fatalf("ReadAll() error = %v", err)
|
||||
}
|
||||
if !bytes.Equal(got, original) {
|
||||
t.Fatalf("Open() content mismatch: got %d bytes, want %d", len(got), len(original))
|
||||
}
|
||||
|
||||
filePayload, err := os.ReadFile(meta.FilePath)
|
||||
if err != nil {
|
||||
t.Fatalf("ReadFile(encrypted) error = %v", err)
|
||||
}
|
||||
sum := sha256.Sum256(filePayload)
|
||||
wantID := hex.EncodeToString(sum[:])
|
||||
if meta.ID != wantID {
|
||||
t.Fatalf("meta.ID = %q, want SHA-256 of encrypted blob %q", meta.ID, wantID)
|
||||
}
|
||||
}
|
||||
|
||||
func TestEncryptedStoreFailsToOpenWithWrongKey(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
keyA := make([]byte, 32)
|
||||
if _, err := rand.Read(keyA); err != nil {
|
||||
t.Fatalf("rand.Read(keyA) error = %v", err)
|
||||
}
|
||||
keyB := make([]byte, 32)
|
||||
if _, err := rand.Read(keyB); err != nil {
|
||||
t.Fatalf("rand.Read(keyB) error = %v", err)
|
||||
}
|
||||
|
||||
dataDir := filepath.Join(t.TempDir(), "data")
|
||||
storeA, err := NewFilesystemStore(dataDir, keyA)
|
||||
if err != nil {
|
||||
t.Fatalf("NewEncryptedFilesystemStore(keyA) error = %v", err)
|
||||
}
|
||||
_, err = storeA.Create(context.Background(), bytes.NewReader([]byte("hello")), "text/plain", time.Hour)
|
||||
if err != nil {
|
||||
t.Fatalf("Create() error = %v", err)
|
||||
}
|
||||
|
||||
if _, err := NewFilesystemStore(dataDir, keyB); err == nil {
|
||||
t.Fatalf("NewFilesystemStore() with wrong key error = nil, want non-nil")
|
||||
}
|
||||
}
|
||||
|
||||
func TestCreateWithOriginalNameNormalizesFilename(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
store, err := NewFilesystemStore(filepath.Join(t.TempDir(), "data"))
|
||||
store, err := NewFilesystemStore(filepath.Join(t.TempDir(), "data"), testEncryptionKey)
|
||||
if err != nil {
|
||||
t.Fatalf("NewFilesystemStore() error = %v", err)
|
||||
}
|
||||
@@ -70,7 +172,7 @@ func TestCreateWithOriginalNameNormalizesFilename(t *testing.T) {
|
||||
t.Fatalf("meta.OriginalName = %q, want %q", meta.OriginalName, "demo.txt")
|
||||
}
|
||||
|
||||
reloaded, err := NewFilesystemStore(store.dataDir)
|
||||
reloaded, err := NewFilesystemStore(store.dataDir, testEncryptionKey)
|
||||
if err != nil {
|
||||
t.Fatalf("NewFilesystemStore() reload error = %v", err)
|
||||
}
|
||||
@@ -83,201 +185,31 @@ func TestCreateWithOriginalNameNormalizesFilename(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestCreateWithOriginalNameRetriesOnIDCollision(t *testing.T) {
|
||||
func TestCreateWithOriginalNameSameContentGetsDifferentIDs(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
store, err := NewFilesystemStore(filepath.Join(t.TempDir(), "data"))
|
||||
store, err := NewFilesystemStore(filepath.Join(t.TempDir(), "data"), testEncryptionKey)
|
||||
if err != nil {
|
||||
t.Fatalf("NewFilesystemStore() error = %v", err)
|
||||
}
|
||||
|
||||
existingID := "aaaaaaaaaaaaaaaaaaaaaaaa"
|
||||
existingPath := filepath.Join(store.filesDir, existingID)
|
||||
if err := os.WriteFile(existingPath, []byte{0x1f, 0x8b, 0x08, 0x00}, 0o644); err != nil {
|
||||
t.Fatalf("WriteFile(existing collision path) error = %v", err)
|
||||
}
|
||||
store.mu.Lock()
|
||||
store.metadata[existingID] = Metadata{
|
||||
ID: existingID,
|
||||
FilePath: existingPath,
|
||||
CreatedAt: time.Now().UTC(),
|
||||
ExpiresAt: time.Now().UTC().Add(time.Hour),
|
||||
ContentType: "text/plain",
|
||||
Size: 4,
|
||||
}
|
||||
store.mu.Unlock()
|
||||
|
||||
nextID := "bbbbbbbbbbbbbbbbbbbbbbbb"
|
||||
calls := 0
|
||||
store.idGenerator = func() (string, error) {
|
||||
calls++
|
||||
if calls == 1 {
|
||||
return existingID, nil
|
||||
}
|
||||
return nextID, nil
|
||||
}
|
||||
|
||||
meta, err := store.CreateWithOriginalName(context.Background(), bytes.NewReader([]byte("hello")), "text/plain", "hello.txt", time.Hour)
|
||||
first, err := store.CreateWithOriginalName(context.Background(), bytes.NewReader([]byte("same body")), "text/plain", "one.txt", time.Hour)
|
||||
if err != nil {
|
||||
t.Fatalf("CreateWithOriginalName() error = %v", err)
|
||||
t.Fatalf("first CreateWithOriginalName() error = %v", err)
|
||||
}
|
||||
if meta.ID != nextID {
|
||||
t.Fatalf("meta.ID = %q, want %q", meta.ID, nextID)
|
||||
}
|
||||
if calls != 2 {
|
||||
t.Fatalf("idGenerator calls = %d, want 2", calls)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCreateWithOriginalNameFailsAfterMaxIDRetries(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
store, err := NewFilesystemStore(filepath.Join(t.TempDir(), "data"))
|
||||
if err != nil {
|
||||
t.Fatalf("NewFilesystemStore() error = %v", err)
|
||||
}
|
||||
|
||||
existingID := "cccccccccccccccccccccccc"
|
||||
existingPath := filepath.Join(store.filesDir, existingID)
|
||||
store.mu.Lock()
|
||||
store.metadata[existingID] = Metadata{
|
||||
ID: existingID,
|
||||
FilePath: existingPath,
|
||||
CreatedAt: time.Now().UTC(),
|
||||
ExpiresAt: time.Now().UTC().Add(time.Hour),
|
||||
ContentType: "text/plain",
|
||||
}
|
||||
store.mu.Unlock()
|
||||
|
||||
store.idGenerator = func() (string, error) {
|
||||
return existingID, nil
|
||||
}
|
||||
|
||||
_, err = store.CreateWithOriginalName(context.Background(), bytes.NewReader([]byte("hello")), "text/plain", "hello.txt", time.Hour)
|
||||
if err == nil {
|
||||
t.Fatalf("CreateWithOriginalName() error = nil, want retry exhaustion error")
|
||||
}
|
||||
if !strings.Contains(err.Error(), "failed to allocate unique scratch id") {
|
||||
t.Fatalf("unexpected error: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCreateWithOriginalNameRetriesWhenIDIsReservedInFlight(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
store, err := NewFilesystemStore(filepath.Join(t.TempDir(), "data"))
|
||||
if err != nil {
|
||||
t.Fatalf("NewFilesystemStore() error = %v", err)
|
||||
}
|
||||
|
||||
const collideID = "abababababababababababab"
|
||||
const uniqueID = "cdcdcdcdcdcdcdcdcdcdcdcd"
|
||||
|
||||
var idMu sync.Mutex
|
||||
idCalls := 0
|
||||
store.idGenerator = func() (string, error) {
|
||||
idMu.Lock()
|
||||
defer idMu.Unlock()
|
||||
idCalls++
|
||||
if idCalls <= 2 {
|
||||
return collideID, nil
|
||||
}
|
||||
return uniqueID, nil
|
||||
}
|
||||
|
||||
release := make(chan struct{})
|
||||
firstMetaCh := make(chan Metadata, 1)
|
||||
firstErrCh := make(chan error, 1)
|
||||
go func() {
|
||||
meta, createErr := store.CreateWithOriginalName(
|
||||
context.Background(),
|
||||
&blockingReader{release: release, payload: []byte("first upload")},
|
||||
"text/plain",
|
||||
"first.txt",
|
||||
time.Hour,
|
||||
)
|
||||
firstMetaCh <- meta
|
||||
firstErrCh <- createErr
|
||||
}()
|
||||
|
||||
deadline := time.Now().Add(2 * time.Second)
|
||||
for {
|
||||
store.mu.RLock()
|
||||
_, reserved := store.reservedIDs[collideID]
|
||||
store.mu.RUnlock()
|
||||
if reserved {
|
||||
break
|
||||
}
|
||||
if time.Now().After(deadline) {
|
||||
t.Fatalf("timed out waiting for first upload reservation")
|
||||
}
|
||||
time.Sleep(2 * time.Millisecond)
|
||||
}
|
||||
|
||||
secondMeta, err := store.CreateWithOriginalName(
|
||||
context.Background(),
|
||||
strings.NewReader("second upload"),
|
||||
"text/plain",
|
||||
"second.txt",
|
||||
time.Hour,
|
||||
)
|
||||
second, err := store.CreateWithOriginalName(context.Background(), bytes.NewReader([]byte("same body")), "text/plain", "two.txt", time.Hour)
|
||||
if err != nil {
|
||||
t.Fatalf("second CreateWithOriginalName() error = %v", err)
|
||||
}
|
||||
if secondMeta.ID != uniqueID {
|
||||
t.Fatalf("second meta.ID = %q, want %q after collision retry", secondMeta.ID, uniqueID)
|
||||
}
|
||||
if secondMeta.ID == collideID {
|
||||
t.Fatalf("second create reused in-flight reserved ID")
|
||||
}
|
||||
|
||||
close(release)
|
||||
firstMeta := <-firstMetaCh
|
||||
if err := <-firstErrCh; err != nil {
|
||||
t.Fatalf("first CreateWithOriginalName() error = %v", err)
|
||||
}
|
||||
if firstMeta.ID != collideID {
|
||||
t.Fatalf("first meta.ID = %q, want %q", firstMeta.ID, collideID)
|
||||
}
|
||||
|
||||
firstReader, _, err := store.Open(firstMeta.ID)
|
||||
if err != nil {
|
||||
t.Fatalf("Open(first) error = %v", err)
|
||||
}
|
||||
defer firstReader.Close()
|
||||
firstBody, err := io.ReadAll(firstReader)
|
||||
if err != nil {
|
||||
t.Fatalf("ReadAll(first) error = %v", err)
|
||||
}
|
||||
if string(firstBody) != "first upload" {
|
||||
t.Fatalf("first content = %q, want %q", string(firstBody), "first upload")
|
||||
}
|
||||
|
||||
secondReader, _, err := store.Open(secondMeta.ID)
|
||||
if err != nil {
|
||||
t.Fatalf("Open(second) error = %v", err)
|
||||
}
|
||||
defer secondReader.Close()
|
||||
secondBody, err := io.ReadAll(secondReader)
|
||||
if err != nil {
|
||||
t.Fatalf("ReadAll(second) error = %v", err)
|
||||
}
|
||||
if string(secondBody) != "second upload" {
|
||||
t.Fatalf("second content = %q, want %q", string(secondBody), "second upload")
|
||||
}
|
||||
|
||||
idMu.Lock()
|
||||
totalCalls := idCalls
|
||||
idMu.Unlock()
|
||||
if totalCalls < 3 {
|
||||
t.Fatalf("idGenerator call count = %d, want at least 3", totalCalls)
|
||||
if first.ID == second.ID {
|
||||
t.Fatalf("IDs should differ because encrypted blobs include random nonce")
|
||||
}
|
||||
}
|
||||
|
||||
func TestGetDeleteAndNotFound(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
store, err := NewFilesystemStore(filepath.Join(t.TempDir(), "data"))
|
||||
store, err := NewFilesystemStore(filepath.Join(t.TempDir(), "data"), testEncryptionKey)
|
||||
if err != nil {
|
||||
t.Fatalf("NewFilesystemStore() error = %v", err)
|
||||
}
|
||||
@@ -302,7 +234,7 @@ func TestGetDeleteAndNotFound(t *testing.T) {
|
||||
func TestDeleteExpired(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
store, err := NewFilesystemStore(filepath.Join(t.TempDir(), "data"))
|
||||
store, err := NewFilesystemStore(filepath.Join(t.TempDir(), "data"), testEncryptionKey)
|
||||
if err != nil {
|
||||
t.Fatalf("NewFilesystemStore() error = %v", err)
|
||||
}
|
||||
@@ -332,7 +264,7 @@ func TestDeleteExpired(t *testing.T) {
|
||||
func TestOpenInvalidGzipFails(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
store, err := NewFilesystemStore(filepath.Join(t.TempDir(), "data"))
|
||||
store, err := NewFilesystemStore(filepath.Join(t.TempDir(), "data"), testEncryptionKey)
|
||||
if err != nil {
|
||||
t.Fatalf("NewFilesystemStore() error = %v", err)
|
||||
}
|
||||
@@ -368,7 +300,7 @@ func TestNewFilesystemStoreInvalidIndex(t *testing.T) {
|
||||
if err := os.WriteFile(filepath.Join(root, indexFilename), []byte("{bad"), 0o644); err != nil {
|
||||
t.Fatalf("WriteFile() error = %v", err)
|
||||
}
|
||||
if _, err := NewFilesystemStore(root); err == nil {
|
||||
if _, err := NewFilesystemStore(root, testEncryptionKey); err == nil {
|
||||
t.Fatalf("NewFilesystemStore() error = nil, want invalid index error")
|
||||
}
|
||||
}
|
||||
@@ -419,15 +351,21 @@ func TestNewFilesystemStoreReconcileOnStartup(t *testing.T) {
|
||||
Size: 1,
|
||||
},
|
||||
}
|
||||
payload, err := json.Marshal(meta)
|
||||
payload, err := json.Marshal(indexDocument{
|
||||
Scratches: meta,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("json.Marshal() error = %v", err)
|
||||
}
|
||||
if err := os.WriteFile(filepath.Join(root, indexFilename), payload, 0o644); err != nil {
|
||||
encryptedPayload, err := encryptIndexPayload(payload, mustTestAEAD(t))
|
||||
if err != nil {
|
||||
t.Fatalf("encryptIndexPayload() error = %v", err)
|
||||
}
|
||||
if err := os.WriteFile(filepath.Join(root, indexFilename), encryptedPayload, 0o644); err != nil {
|
||||
t.Fatalf("WriteFile index error = %v", err)
|
||||
}
|
||||
|
||||
store, err := NewFilesystemStore(root)
|
||||
store, err := NewFilesystemStore(root, testEncryptionKey)
|
||||
if err != nil {
|
||||
t.Fatalf("NewFilesystemStore() error = %v", err)
|
||||
}
|
||||
@@ -442,6 +380,63 @@ func TestNewFilesystemStoreReconcileOnStartup(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestNewFilesystemStoreWithDefaultTTLShiftsExpiriesOnTTLChange(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
dataDir := filepath.Join(t.TempDir(), "data")
|
||||
store, err := NewFilesystemStoreWithDefaultTTL(dataDir, time.Hour, testEncryptionKey)
|
||||
if err != nil {
|
||||
t.Fatalf("NewFilesystemStoreWithDefaultTTL() error = %v", err)
|
||||
}
|
||||
|
||||
meta, err := store.Create(context.Background(), bytes.NewReader([]byte("hello")), "text/plain", time.Hour)
|
||||
if err != nil {
|
||||
t.Fatalf("Create() error = %v", err)
|
||||
}
|
||||
originalExpiry := meta.ExpiresAt
|
||||
|
||||
reloaded, err := NewFilesystemStoreWithDefaultTTL(dataDir, 2*time.Hour, testEncryptionKey)
|
||||
if err != nil {
|
||||
t.Fatalf("NewFilesystemStoreWithDefaultTTL() reload error = %v", err)
|
||||
}
|
||||
got, err := reloaded.Get(meta.ID)
|
||||
if err != nil {
|
||||
t.Fatalf("Get() error = %v", err)
|
||||
}
|
||||
|
||||
wantExpiry := originalExpiry.Add(time.Hour)
|
||||
if !got.ExpiresAt.Equal(wantExpiry) {
|
||||
t.Fatalf("ExpiresAt = %s, want %s", got.ExpiresAt, wantExpiry)
|
||||
}
|
||||
|
||||
again, err := NewFilesystemStoreWithDefaultTTL(dataDir, 2*time.Hour, testEncryptionKey)
|
||||
if err != nil {
|
||||
t.Fatalf("NewFilesystemStoreWithDefaultTTL() second reload error = %v", err)
|
||||
}
|
||||
gotAgain, err := again.Get(meta.ID)
|
||||
if err != nil {
|
||||
t.Fatalf("Get() second reload error = %v", err)
|
||||
}
|
||||
if !gotAgain.ExpiresAt.Equal(wantExpiry) {
|
||||
t.Fatalf("ExpiresAt after unchanged TTL = %s, want %s", gotAgain.ExpiresAt, wantExpiry)
|
||||
}
|
||||
|
||||
indexRaw, err := os.ReadFile(filepath.Join(dataDir, indexFilename))
|
||||
if err != nil {
|
||||
t.Fatalf("ReadFile(index) error = %v", err)
|
||||
}
|
||||
if bytes.Contains(indexRaw, []byte(`"default_ttl"`)) {
|
||||
t.Fatalf("index should not be plaintext JSON")
|
||||
}
|
||||
indexPlaintext, err := decryptIndexPayload(indexRaw, mustTestAEAD(t))
|
||||
if err != nil {
|
||||
t.Fatalf("decryptIndexPayload() error = %v", err)
|
||||
}
|
||||
if !bytes.Contains(indexPlaintext, []byte(`"default_ttl": "2h0m0s"`)) {
|
||||
t.Fatalf("decrypted index missing default ttl marker: %s", string(indexPlaintext))
|
||||
}
|
||||
}
|
||||
|
||||
type errCloser struct{}
|
||||
|
||||
func (errCloser) Close() error { return errors.New("close failed") }
|
||||
@@ -465,7 +460,7 @@ func (failingReader) Read(_ []byte) (int, error) { return 0, errors.New("boom")
|
||||
func TestCreateRollbackOnPersistFailure(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
store, err := NewFilesystemStore(filepath.Join(t.TempDir(), "data"))
|
||||
store, err := NewFilesystemStore(filepath.Join(t.TempDir(), "data"), testEncryptionKey)
|
||||
if err != nil {
|
||||
t.Fatalf("NewFilesystemStore() error = %v", err)
|
||||
}
|
||||
@@ -487,7 +482,7 @@ func TestCreateRollbackOnPersistFailure(t *testing.T) {
|
||||
func TestCreateReaderFailure(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
store, err := NewFilesystemStore(filepath.Join(t.TempDir(), "data"))
|
||||
store, err := NewFilesystemStore(filepath.Join(t.TempDir(), "data"), testEncryptionKey)
|
||||
if err != nil {
|
||||
t.Fatalf("NewFilesystemStore() error = %v", err)
|
||||
}
|
||||
@@ -499,7 +494,7 @@ func TestCreateReaderFailure(t *testing.T) {
|
||||
func TestDeletePersistFailureRestoresEntry(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
store, err := NewFilesystemStore(filepath.Join(t.TempDir(), "data"))
|
||||
store, err := NewFilesystemStore(filepath.Join(t.TempDir(), "data"), testEncryptionKey)
|
||||
if err != nil {
|
||||
t.Fatalf("NewFilesystemStore() error = %v", err)
|
||||
}
|
||||
@@ -525,7 +520,7 @@ func TestDeletePersistFailureRestoresEntry(t *testing.T) {
|
||||
func TestDeleteRemoveFailure(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
store, err := NewFilesystemStore(filepath.Join(t.TempDir(), "data"))
|
||||
store, err := NewFilesystemStore(filepath.Join(t.TempDir(), "data"), testEncryptionKey)
|
||||
if err != nil {
|
||||
t.Fatalf("NewFilesystemStore() error = %v", err)
|
||||
}
|
||||
@@ -557,7 +552,7 @@ func TestDeleteRemoveFailure(t *testing.T) {
|
||||
func TestDeleteExpiredPersistFailureRestoresEntries(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
store, err := NewFilesystemStore(filepath.Join(t.TempDir(), "data"))
|
||||
store, err := NewFilesystemStore(filepath.Join(t.TempDir(), "data"), testEncryptionKey)
|
||||
if err != nil {
|
||||
t.Fatalf("NewFilesystemStore() error = %v", err)
|
||||
}
|
||||
@@ -583,7 +578,7 @@ func TestDeleteExpiredPersistFailureRestoresEntries(t *testing.T) {
|
||||
func TestDeleteExpiredRemoveFailure(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
store, err := NewFilesystemStore(filepath.Join(t.TempDir(), "data"))
|
||||
store, err := NewFilesystemStore(filepath.Join(t.TempDir(), "data"), testEncryptionKey)
|
||||
if err != nil {
|
||||
t.Fatalf("NewFilesystemStore() error = %v", err)
|
||||
}
|
||||
@@ -619,106 +614,45 @@ func TestNewFilesystemStoreFailsWhenDataDirIsFile(t *testing.T) {
|
||||
if err := os.WriteFile(filePath, []byte("x"), 0o644); err != nil {
|
||||
t.Fatalf("WriteFile() error = %v", err)
|
||||
}
|
||||
if _, err := NewFilesystemStore(filePath); err == nil {
|
||||
if _, err := NewFilesystemStore(filePath, testEncryptionKey); err == nil {
|
||||
t.Fatalf("NewFilesystemStore() error = nil, want mkdir failure")
|
||||
}
|
||||
}
|
||||
|
||||
type blockingReader struct {
|
||||
release <-chan struct{}
|
||||
payload []byte
|
||||
sent bool
|
||||
}
|
||||
|
||||
func (r *blockingReader) Read(p []byte) (int, error) {
|
||||
if r.sent {
|
||||
return 0, io.EOF
|
||||
}
|
||||
<-r.release
|
||||
r.sent = true
|
||||
return copy(p, r.payload), nil
|
||||
}
|
||||
|
||||
func TestCreateReservationIsNotVisibleToCleanupOrReads(t *testing.T) {
|
||||
func TestNewFilesystemStoreRequires32ByteKey(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
store, err := NewFilesystemStore(filepath.Join(t.TempDir(), "data"))
|
||||
dataDir := filepath.Join(t.TempDir(), "data")
|
||||
if _, err := NewFilesystemStore(dataDir, nil); err == nil {
|
||||
t.Fatalf("NewFilesystemStore() with nil key error = nil, want non-nil")
|
||||
}
|
||||
if _, err := NewFilesystemStore(dataDir, []byte("short")); err == nil {
|
||||
t.Fatalf("NewFilesystemStore() with short key error = nil, want non-nil")
|
||||
}
|
||||
}
|
||||
|
||||
func TestNewFilesystemStoreFailsToLoadEncryptedIndexWithWrongKey(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
dataDir := filepath.Join(t.TempDir(), "data")
|
||||
store, err := NewFilesystemStore(dataDir, testEncryptionKey)
|
||||
if err != nil {
|
||||
t.Fatalf("NewFilesystemStore() error = %v", err)
|
||||
}
|
||||
|
||||
const reservedID = "ffffffffffffffffffffffff"
|
||||
store.idGenerator = func() (string, error) {
|
||||
return reservedID, nil
|
||||
if _, err := store.Create(context.Background(), bytes.NewReader([]byte("hello")), "text/plain", time.Hour); err != nil {
|
||||
t.Fatalf("Create() error = %v", err)
|
||||
}
|
||||
|
||||
release := make(chan struct{})
|
||||
createErrCh := make(chan error, 1)
|
||||
go func() {
|
||||
_, createErr := store.CreateWithOriginalName(
|
||||
context.Background(),
|
||||
&blockingReader{release: release, payload: []byte("in-flight")},
|
||||
"text/plain",
|
||||
"",
|
||||
time.Hour,
|
||||
)
|
||||
createErrCh <- createErr
|
||||
}()
|
||||
|
||||
deadline := time.Now().Add(2 * time.Second)
|
||||
for {
|
||||
store.mu.RLock()
|
||||
_, reserved := store.reservedIDs[reservedID]
|
||||
_, visible := store.metadata[reservedID]
|
||||
store.mu.RUnlock()
|
||||
if reserved {
|
||||
if visible {
|
||||
t.Fatalf("in-flight ID should not appear in metadata")
|
||||
}
|
||||
break
|
||||
}
|
||||
if time.Now().After(deadline) {
|
||||
t.Fatalf("timed out waiting for ID reservation")
|
||||
}
|
||||
time.Sleep(2 * time.Millisecond)
|
||||
}
|
||||
|
||||
if _, err := store.Get(reservedID); !errors.Is(err, ErrNotFound) {
|
||||
t.Fatalf("Get() during in-flight create error = %v, want ErrNotFound", err)
|
||||
}
|
||||
|
||||
deleted, err := store.DeleteExpired(time.Now().UTC())
|
||||
if err != nil {
|
||||
t.Fatalf("DeleteExpired() error = %v", err)
|
||||
}
|
||||
if deleted != 0 {
|
||||
t.Fatalf("DeleteExpired() = %d, want 0 for in-flight create", deleted)
|
||||
}
|
||||
|
||||
close(release)
|
||||
if err := <-createErrCh; err != nil {
|
||||
t.Fatalf("CreateWithOriginalName() error = %v", err)
|
||||
}
|
||||
|
||||
reader, _, err := store.Open(reservedID)
|
||||
if err != nil {
|
||||
t.Fatalf("Open() error = %v", err)
|
||||
}
|
||||
defer reader.Close()
|
||||
|
||||
body, err := io.ReadAll(reader)
|
||||
if err != nil {
|
||||
t.Fatalf("ReadAll() error = %v", err)
|
||||
}
|
||||
if string(body) != "in-flight" {
|
||||
t.Fatalf("Open() content = %q, want %q", string(body), "in-flight")
|
||||
wrongKey := []byte("abcdef0123456789abcdef0123456789")
|
||||
if _, err := NewFilesystemStore(dataDir, wrongKey); err == nil {
|
||||
t.Fatalf("NewFilesystemStore() with wrong key error = nil, want non-nil")
|
||||
}
|
||||
}
|
||||
|
||||
func TestConcurrentCreateWithOriginalNameProducesUniqueReadableEntries(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
store, err := NewFilesystemStore(filepath.Join(t.TempDir(), "data"))
|
||||
store, err := NewFilesystemStore(filepath.Join(t.TempDir(), "data"), testEncryptionKey)
|
||||
if err != nil {
|
||||
t.Fatalf("NewFilesystemStore() error = %v", err)
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user