s1d3sw1ped/steamcache2
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 20 Wiki Activity

Enhance DiskFS initialization and error handling

Browse Source 
- Updated `disk.New` to support asynchronous initialization for large caches, improving responsiveness during startup.
- Introduced an eviction function parameter to `disk.New`, ensuring proper handling of over-capacity scenarios.
- Enhanced error handling in various components, including memory and disk tests, to ensure robustness and clarity.
- Refactored tests to validate new behaviors, including checks for delayed attachment and proper error propagation.
- Removed obsolete error handling code and tests related to the now-deleted errors package, streamlining the codebase.
This commit is contained in:
Justin Harms
2026-05-27 13:15:33 -05:00
parent 4861f93e6f
commit feda55e225
18 changed files with 584 additions and 1380 deletions
Download Patch File Download Diff File Expand all files Collapse all files
vfs/disk/disk.go
+141 -72
View File
@@ -18,7 +18,6 @@ import (
"sync/atomic"
"time"
"github.com/docker/go-units"
"github.com/edsrzf/mmap-go"
)
@@ -39,6 +38,11 @@ type DiskFS struct {
keyLocks []sync.Map // Sharded lock pools for better concurrency
LRU *lru.LRUList[*vfs.FileInfo]
timeUpdater *vfs.BatchedTimeUpdate // Batched time updates for better performance
// initDone is closed once background population of size/info/LRU finishes; Size() receives on it for the barrier.
initDone chan struct{}
// initCloseOnce ensures initDone closed exactly once even on panic in bg populator (panic safety for Issue 1).
initCloseOnce sync.Once
startupEvict func(vfs.VFS, uint) uint // passed to New (via gc.GetGCAlgorithm); invoked as last step of bg init if over cap (no post-ctor race)
}
// shardPath converts a Steam cache key to a sharded directory path to reduce inode pressure
@@ -74,67 +78,78 @@ func (d *DiskFS) pathForKey(key string) string {
return path
}
// filePathToKey reverses a physical on-disk path (under root) back to logical cache key.
// Used by bg init-time scan (from New) to populate info/LRU for correct Size after barrier.
func (d *DiskFS) filePathToKey(fullPath string) string {
rel, err := filepath.Rel(d.root, fullPath)
if err != nil {
return filepath.Base(fullPath)
}
rel = strings.ReplaceAll(rel, "\\", "/")
if strings.HasPrefix(rel, "steam/") {
if hash := filepath.Base(rel); hash != "" && hash != "." {
return "steam/" + hash
}
}
return rel
}
// New creates a new DiskFS.
func New(root string, capacity int64) *DiskFS {
// The evict param (from gc.GetGCAlgorithm, or nil) is stored before launching the bg
// population goroutine, eliminating any post-New handoff window/race for the relocated
// startup over-capacity guard (now the last step inside calculateSizeAndPopulateIndex).
// New returns fast even for millions of files (async bg scan + streaming batch inserts).
// Callers (e.g. steamcache.New) that need populated state or post-guard size must call Size()
// (or ops that do) which blocks on the internal init barrier until population + optional guard complete.
// See README "Large Cache Initialization" for migration/observable behavior during the proxy window.
func New(root string, capacity int64, evict func(vfs.VFS, uint) uint) (*DiskFS, error) {
if capacity <= 0 {
panic("disk capacity must be greater than 0")
return nil, fmt.Errorf("disk capacity must be greater than 0")
}
// Create root directory if it doesn't exist
os.MkdirAll(root, 0755)
// Create root directory if it doesn't exist. Propagate error (ctor now returns err for hygiene).
if err := os.MkdirAll(root, 0755); err != nil {
return nil, fmt.Errorf("failed to create root directory %s: %w", root, err)
}
// Initialize sharded locks
keyLocks := make([]sync.Map, locks.NumLockShards)
d := &DiskFS{
root: root,
info: make(map[string]*vfs.FileInfo),
capacity: capacity,
size: 0,
keyLocks: keyLocks,
LRU: lru.NewLRUList[*vfs.FileInfo](),
timeUpdater: vfs.NewBatchedTimeUpdate(100 * time.Millisecond), // Update time every 100ms
root: root,
info: make(map[string]*vfs.FileInfo),
capacity: capacity,
size: 0,
keyLocks: keyLocks,
LRU: lru.NewLRUList[*vfs.FileInfo](),
timeUpdater: vfs.NewBatchedTimeUpdate(100 * time.Millisecond), // Update time every 100ms
startupEvict: evict,
}
d.init()
return d
d.initDone = make(chan struct{})
// Launch heavy population asynchronously so New returns fast (scans millions of files without blocking ctor or using O(N) temp RAM).
// The initDone barrier ensures first Size() and subsequent ops (including late tier attach) see fully populated + post-eviction state.
go d.calculateSizeAndPopulateIndex()
return d, nil
}
// init loads existing files from disk with ultra-fast lazy initialization
func (d *DiskFS) init() {
// calculateSizeAndPopulateIndex runs in background from New to avoid blocking startup or O(N) RAM for large caches (millions of Steam files).
// It streams batch inserts (bounded by maxEvictBatch) to keep lock times short and eliminate giant temporary slice.
// Startup over-capacity eviction (if needed) runs as the very last step (using the evict func passed to New, selected via gc.GetGCAlgorithm).
// Only then is initDone closed so Size() and waiters see consistent post-eviction state.
// Panic recovery ensures initDone is always closed (unblocks Size callers) even on scan/IO panic; uses Once for safety.
func (d *DiskFS) calculateSizeAndPopulateIndex() {
defer func() {
if r := recover(); r != nil {
logger.Logger.Error().Interface("recovered_panic", r).Msg("calculateSizeAndPopulateIndex panicked; ensuring initDone closed to unblock Size waiters and prevent hang")
}
d.initCloseOnce.Do(func() { close(d.initDone) })
}()
tstart := time.Now()
// Ultra-fast initialization: only scan directory structure, defer file stats
d.scanDirectoriesOnly()
// Start background size calculation in a separate goroutine
go d.calculateSizeInBackground()
logger.Logger.Info().
Str("name", d.Name()).
Str("root", d.root).
Str("capacity", units.HumanSize(float64(d.capacity))).
Str("size", units.HumanSize(float64(d.Size()))).
Str("files", fmt.Sprint(len(d.info))).
Str("duration", time.Since(tstart).String()).
Msg("init")
}
// scanDirectoriesOnly performs ultra-fast directory structure scanning without file stats
func (d *DiskFS) scanDirectoriesOnly() {
// Just ensure the root directory exists and is accessible
// No file scanning during init - files will be discovered on-demand
logger.Logger.Debug().
Str("root", d.root).
Msg("Directory structure scan completed (lazy file discovery enabled)")
}
// calculateSizeInBackground calculates the total size of all files in the background
func (d *DiskFS) calculateSizeInBackground() {
tstart := time.Now()
// Channel for collecting file information
fileChan := make(chan fileSizeInfo, 1000)
// Channel for collecting file information (now includes metadata for info/LRU population)
fileChan := make(chan discoveredFile, 1000)
// Progress tracking
var totalFiles int64
@@ -153,8 +168,10 @@ func (d *DiskFS) calculateSizeInBackground() {
d.scanFilesForSize(d.root, fileChan, &totalFiles)
}()
// Collect results with progress reporting
// Collect results with progress reporting + streaming batch population (no O(N) discovered slice, bounded locks)
var totalSize int64
const batchSize = maxEvictBatch
var batch []discoveredFile
// Use a separate goroutine to collect results
done := make(chan struct{})
@@ -162,12 +179,17 @@ func (d *DiskFS) calculateSizeInBackground() {
defer close(done)
for {
select {
case fi, ok := <-fileChan:
case df, ok := <-fileChan:
if !ok {
return
}
totalSize += fi.size
totalSize += df.size
processedFiles++
batch = append(batch, df)
if len(batch) >= batchSize {
d.insertBatch(batch)
batch = batch[:0]
}
case <-progressTicker.C:
if totalFiles > 0 {
logger.Logger.Debug().
@@ -185,25 +207,60 @@ func (d *DiskFS) calculateSizeInBackground() {
wg.Wait()
<-done
// Update the total size
d.mu.Lock()
d.size = totalSize
d.mu.Unlock()
// Final partial batch + set (no size stomp: inserts do the += for discovered; concurrent Creates are additive via their paths)
if len(batch) > 0 {
d.insertBatch(batch)
}
logger.Logger.Info().
Int64("files_scanned", processedFiles).
Int64("total_size", totalSize).
Str("duration", time.Since(tstart).String()).
Msg("Background size calculation completed")
Msg("Size and index population completed")
// Run over-capacity startup eviction here (LAST step of bg init) using freshly populated index+size.
// The func (passed at New time via gc.GetGCAlgorithm) is guaranteed visible (no post-ctor handoff).
// Snapshot size under RLock to eliminate data race on d.size vs concurrent Create/Evict (fixes -race on guard decision).
d.mu.RLock()
overCapacity := d.size > d.capacity
needed := uint(0)
if overCapacity {
needed = uint(d.size - d.capacity)
}
d.mu.RUnlock()
if overCapacity && d.startupEvict != nil {
d.startupEvict(d, needed)
}
// Signal readiness: Size() and callers (late tier attach + Evict*) now see correct populated + post-eviction state.
// Use Once (recover path also uses it) to guarantee exactly one close even under panic.
d.initCloseOnce.Do(func() { close(d.initDone) })
}
// fileSizeInfo represents a file found during size calculation
type fileSizeInfo struct {
size int64
// insertBatch populates info/LRU under lock for a bounded batch (follows maxEvictBatch pattern for short critical sections).
// Size is incremented here only for files actually added (prevents double-count vs. concurrent Create during window).
func (d *DiskFS) insertBatch(batch []discoveredFile) {
d.mu.Lock()
for _, df := range batch {
if _, exists := d.info[df.key]; !exists {
fi := vfs.NewFileInfoFromOS(df.osInfo, df.key)
d.info[df.key] = fi
d.LRU.Add(df.key, fi)
d.size += df.size
}
}
d.mu.Unlock()
}
// scanFilesForSize performs recursive file scanning for size calculation only
func (d *DiskFS) scanFilesForSize(dirPath string, fileChan chan<- fileSizeInfo, totalFiles *int64) {
// discoveredFile carries metadata for (bg) init-time population of info/LRU.
type discoveredFile struct {
key string
size int64
osInfo os.FileInfo
}
// scanFilesForSize performs recursive file scanning for size + metadata (to populate LRU/info via bg streaming in New).
func (d *DiskFS) scanFilesForSize(dirPath string, fileChan chan<- discoveredFile, totalFiles *int64) {
// Use ReadDir for faster directory listing
entries, err := os.ReadDir(dirPath)
if err != nil {
@@ -236,22 +293,27 @@ func (d *DiskFS) scanFilesForSize(dirPath string, fileChan chan<- fileSizeInfo,
d.scanFilesForSize(path, fileChan, totalFiles)
}(entryPath)
} else {
// Process file for size only
// Process file for size + key (for LRU/info population)
wg.Add(1)
go func(entry os.DirEntry) {
defer wg.Done()
semaphore <- struct{}{} // Acquire semaphore
defer func() { <-semaphore }() // Release semaphore
fullPath := filepath.Join(dirPath, entry.Name())
key := d.filePathToKey(fullPath)
// Get file info for size calculation
info, err := entry.Info()
if err != nil {
return
}
// Send file size info
fileChan <- fileSizeInfo{
size: info.Size(),
// Send discovered file info
fileChan <- discoveredFile{
key: key,
size: info.Size(),
osInfo: info,
}
}(entry)
}
@@ -265,8 +327,14 @@ func (d *DiskFS) Name() string {
return "DiskFS"
}
// Size returns the current size
// Size returns the current size.
// The receive on initDone ensures that after New callers observe the real on-disk total + populated info/LRU
// (barrier unblocks only after bg streaming population + any startup eviction finishes).
// All subsequent calls are non-blocking (closed chan receive is instantaneous).
// During long init for huge caches, this (and callers like GetMetrics, attach logic) will block until ready;
// this is the documented contract enabling "no disk activity until ready" for TieredCache.
func (d *DiskFS) Size() int64 {
<-d.initDone
d.mu.RLock()
defer d.mu.RUnlock()
return d.size
@@ -405,11 +473,12 @@ func (d *DiskFS) Open(key string) (io.ReadCloser, error) {
}
}
// Update access time and LRU
d.mu.Lock()
fi.UpdateAccessBatched(d.timeUpdater)
d.LRU.MoveToFront(key, d.timeUpdater)
d.mu.Unlock()
// Update access time and LRU (use TryLock to avoid serializing all readers on the global mu despite sharding; approximate LRU under load is acceptable)
if d.mu.TryLock() {
fi.UpdateAccessBatched(d.timeUpdater)
d.LRU.MoveToFront(key, d.timeUpdater)
d.mu.Unlock()
}
path := d.pathForKey(key)
@@ -548,8 +617,8 @@ func (d *DiskFS) Stat(key string) (*vfs.FileInfo, error) {
d.info[key] = fi
d.LRU.Add(key, fi)
fi.UpdateAccessBatched(d.timeUpdater)
// Note: Don't add to d.size here as it's being calculated in background
// The background calculation will handle the total size
// Note: size not updated on lazy discovery (preserves prior behavior; initial on-disk accounted via bg populate at New time,
// subsequent files come via Create which accounts size).
d.mu.Unlock()
return fi, nil
vfs/disk/disk_test.go
+168 -9
View File
@@ -4,16 +4,23 @@ import (
"fmt"
"io"
"os"
"path/filepath"
"strings"
"sync"
"sync/atomic"
"testing"
"time"
"s1d3sw1ped/steamcache2/vfs"
)
func TestDiskFS_Basic(t *testing.T) {
t.Parallel()
td := t.TempDir()
d := New(td, 10*1024*1024)
d, err := New(td, 10*1024*1024, nil)
if err != nil {
t.Fatal(err)
}
if d.Name() != "DiskFS" {
t.Error("name")
}
@@ -45,10 +52,83 @@ func TestDiskFS_Basic(t *testing.T) {
}
}
// TestDiskFS_NewInvalidCapacity exercises the new error return (was panic) for ctor hygiene (Item 3 coverage).
func TestDiskFS_NewInvalidCapacity(t *testing.T) {
t.Parallel()
td := t.TempDir()
_, err := New(td, 0, nil)
if err == nil {
t.Fatal("expected error for capacity=0")
}
if !strings.Contains(err.Error(), "must be greater than 0") {
t.Errorf("err %q missing 'must be greater than 0'", err)
}
_, err = New(td, -1, nil)
if err == nil || !strings.Contains(err.Error(), "must be greater than 0") {
t.Errorf("negative capacity should return error containing phrase, got %v", err)
}
}
// TestDiskFS_InitPopulatesIndexOnRestart exercises the Item 1 fix: pre-populate disk dir (simulating restart with existing data),
// call New, immediately verify Size + info/LRU are populated (so post-init Size + eviction see truth).
func TestDiskFS_InitPopulatesIndexOnRestart(t *testing.T) {
t.Parallel()
td := t.TempDir()
// Pre-populate using raw FS ops (as prior run would have; simple keys -> direct paths under root)
// Total 300 bytes > small cap below.
prepare := func(key string, sz int64) {
p := td + "/" + key
if err := os.MkdirAll(td, 0755); err != nil {
t.Fatalf("mkdir: %v", err)
}
if err := os.WriteFile(p, make([]byte, sz), 0644); err != nil {
t.Fatalf("write %s: %v", key, err)
}
}
prepare("f1", 100)
prepare("f2", 200)
// Small cap so we are over; New launches bg populate (Size() blocks until done)
d, err := New(td, 150, nil)
if err != nil {
t.Fatal(err)
}
if d.Size() != 300 {
t.Errorf("Size after restart init = %d, want 300 (populated from disk)", d.Size())
}
if len(d.info) != 2 {
t.Errorf("info len after init = %d, want 2", len(d.info))
}
if d.LRU.Len() != 2 {
t.Errorf("LRU len after init = %d, want 2", d.LRU.Len())
}
// Immediate discoverability (lazy still works but now warm)
if _, err := d.Stat("f1"); err != nil {
t.Error("stat f1 failed immediately after init pop")
}
// Size > cap exercises the path where startup eviction would run at end of disk init (when GC algo provided via Set).
if d.Size() <= d.Capacity() {
t.Error("expected Size > Capacity to exercise over-cap path post-fix")
}
// Exercise eviction now has candidates thanks to population
ev := d.EvictLRU(200)
if ev == 0 {
t.Error("EvictLRU did nothing despite over cap + populated LRU (startup eviction path would have failed before Item 1 fix)")
}
}
func TestDiskFS_EvictAndLazyStat(t *testing.T) {
t.Parallel()
td := t.TempDir()
d := New(td, 400)
d, err := New(td, 400, nil)
if err != nil {
t.Fatal(err)
}
// create files that will be evicted
keys := []string{}
for i := 0; i < 5; i++ {
@@ -85,7 +165,10 @@ func TestDiskFS_Concurrent(t *testing.T) {
}
t.Parallel()
td := t.TempDir()
d := New(td, 50*1024*1024)
d, err := New(td, 50*1024*1024, nil)
if err != nil {
t.Fatal(err)
}
var wg sync.WaitGroup
var ops int64
for i := 0; i < 4; i++ {
@@ -128,7 +211,10 @@ func TestDiskFS_Concurrent(t *testing.T) {
func BenchmarkDiskFS_CreateOpen(b *testing.B) {
td := b.TempDir()
d := New(td, 128*1024*1024)
d, err := New(td, 128*1024*1024, nil)
if err != nil {
b.Fatal(err)
}
data := make([]byte, 8192)
b.ReportAllocs()
b.ResetTimer()
@@ -154,7 +240,10 @@ func BenchmarkDiskFS_CreateOpen(b *testing.B) {
// Uses cycling keys via testKey for stable disk usage; exercises LRU path (other strategies lightly covered via tests + EvictHybrid uses DecayedScore).
func BenchmarkDiskFS_EvictionUnderPressure(b *testing.B) {
td := b.TempDir()
d := New(td, 1*1024*1024)
d, err := New(td, 1*1024*1024, nil)
if err != nil {
b.Fatal(err)
}
b.ReportAllocs()
b.ResetTimer()
for i := 0; i < b.N; i++ {
@@ -175,7 +264,10 @@ func BenchmarkDiskFS_EvictionUnderPressure(b *testing.B) {
func TestDiskFS_EvictVariantsAndInvalid(t *testing.T) {
t.Parallel()
td := t.TempDir()
d := New(td, 600)
d, err := New(td, 600, nil)
if err != nil {
t.Fatal(err)
}
for i := 0; i < 4; i++ {
w, _ := d.Create("dv"+string(rune('0'+i)), 120)
w.Write(make([]byte, 120))
@@ -211,7 +303,10 @@ func TestEvict_ConcurrentCloseDuringEviction(t *testing.T) {
t.Parallel()
td := t.TempDir()
cap := int64(256 * 1024)
d := New(td, cap)
d, err := New(td, cap, nil)
if err != nil {
t.Fatal(err)
}
var wg sync.WaitGroup
const nWriters = 4
const nEvictors = 3
@@ -281,7 +376,10 @@ func TestDiskFS_EvictDiskVisibilityAndRecreateSafety(t *testing.T) {
t.Parallel()
td := t.TempDir()
cap := int64(500)
d := New(td, cap)
d, err := New(td, cap, nil)
if err != nil {
t.Fatal(err)
}
created := []string{"v1", "v2", "v3", "s1"}
for _, k := range created {
sz := int64(150)
@@ -352,7 +450,10 @@ func TestDiskFS_EvictBoundedLargeN(t *testing.T) {
t.Parallel()
td := t.TempDir()
cap := int64(128 * 1024) // slightly larger for practicality
d := New(td, cap)
d, err := New(td, cap, nil)
if err != nil {
t.Fatal(err)
}
const nFiles = 3000 // > maxEvictBatch to exercise early-break on multiple rounds
const fSize = 128
for i := 0; i < nFiles; i++ {
@@ -397,3 +498,61 @@ func TestDiskFS_EvictBoundedLargeN(t *testing.T) {
}
_ = totalEvicted
}
// TestDiskFS_StartupEvictionFuncInvokedDuringInit covers the relocated guard path:
// pre-populate over capacity, New with non-nil evict func (selected via Get), wait for init,
// verify the func was invoked inside calculate (before close(initDone)) and size reduced.
func TestDiskFS_StartupEvictionFuncInvokedDuringInit(t *testing.T) {
t.Parallel()
td := t.TempDir()
prepare := func(key string, sz int64) {
p := td + "/" + key
if err := os.MkdirAll(td, 0755); err != nil {
t.Fatalf("mkdir: %v", err)
}
if err := os.WriteFile(p, make([]byte, sz), 0644); err != nil {
t.Fatalf("write %s: %v", key, err)
}
}
prepare("f1", 100)
prepare("f2", 200)
// Use real eviction func (delegates to EvictLRU impl, as GC algos do) + pre-pop > cap.
// Assert post-Size() (post-guard) that size was reduced to <= cap + index updated (Issue 4 coverage).
evictFn := func(v vfs.VFS, b uint) uint {
// real path: same as hybrid/lru would via the VFS methods (exercises lock, LRU remove, size adjust, os.Remove)
if dd, ok := v.(*DiskFS); ok {
return dd.EvictLRU(b)
}
return 0
}
d, err := New(td, 150, evictFn)
if err != nil {
t.Fatal(err)
}
_ = d.Size() // wait for bg init + guard (last step) + close
if d.Size() > d.Capacity() {
t.Errorf("startup guard with real evictFn did not reduce size: got %d > cap %d", d.Size(), d.Capacity())
}
// LRU/info updated by real evict; at least one file gone (original 2 files, 300B)
if len(d.info) == 2 {
t.Error("expected real eviction to have removed at least one over-cap file from index")
}
}
// TestDiskFS_NewMkdirError covers propagation of MkdirAll error from New (ctor now returns err; Issue 6).
func TestDiskFS_NewMkdirError(t *testing.T) {
t.Parallel()
// Create a regular file at the path we will pass as "root dir"; MkdirAll will fail with "file exists" or perm.
td := t.TempDir()
badPath := filepath.Join(td, "notadir")
if err := os.WriteFile(badPath, []byte("x"), 0644); err != nil {
t.Fatal(err)
}
_, err := New(badPath, 1024, nil)
if err == nil || !strings.Contains(err.Error(), "failed to create root directory") {
t.Errorf("expected mkdir failure error for file-as-dir, got: %v", err)
}
}