package disk import ( "fmt" "io" "os" "path/filepath" "strings" "sync" "sync/atomic" "testing" "time" "s1d3sw1ped/steamcache2/vfs" ) func TestDiskFS_Basic(t *testing.T) { t.Parallel() td := t.TempDir() d, err := New(td, 10*1024*1024, nil) if err != nil { t.Fatal(err) } if d.Name() != "DiskFS" { t.Error("name") } w, err := d.Create("k1", 50) if err != nil { t.Fatal(err) } w.Write([]byte("hello disk cache test data here")) w.Close() if d.Size() < 30 { // actual may differ slightly from declared t.Errorf("size too small %d", d.Size()) } r, err := d.Open("k1") if err != nil { t.Fatal(err) } data, _ := io.ReadAll(r) r.Close() if len(data) < 10 { t.Error("read small") } d.Delete("k1") if _, err := d.Open("k1"); err == nil { t.Error("deleted still readable") } } // TestDiskFS_NewInvalidCapacity exercises the new error return (was panic) for ctor hygiene (Item 3 coverage). func TestDiskFS_NewInvalidCapacity(t *testing.T) { t.Parallel() td := t.TempDir() _, err := New(td, 0, nil) if err == nil { t.Fatal("expected error for capacity=0") } if !strings.Contains(err.Error(), "must be greater than 0") { t.Errorf("err %q missing 'must be greater than 0'", err) } _, err = New(td, -1, nil) if err == nil || !strings.Contains(err.Error(), "must be greater than 0") { t.Errorf("negative capacity should return error containing phrase, got %v", err) } } // TestDiskFS_InitPopulatesIndexOnRestart exercises the Item 1 fix: pre-populate disk dir (simulating restart with existing data), // call New, immediately verify Size + info/LRU are populated (so post-init Size + eviction see truth). func TestDiskFS_InitPopulatesIndexOnRestart(t *testing.T) { t.Parallel() td := t.TempDir() // Pre-populate using raw FS ops (as prior run would have; simple keys -> direct paths under root) // Total 300 bytes > small cap below. prepare := func(key string, sz int64) { p := td + "/" + key if err := os.MkdirAll(td, 0755); err != nil { t.Fatalf("mkdir: %v", err) } if err := os.WriteFile(p, make([]byte, sz), 0644); err != nil { t.Fatalf("write %s: %v", key, err) } } prepare("f1", 100) prepare("f2", 200) // Small cap so we are over; New launches bg populate (Size() blocks until done) d, err := New(td, 150, nil) if err != nil { t.Fatal(err) } if d.Size() != 300 { t.Errorf("Size after restart init = %d, want 300 (populated from disk)", d.Size()) } if len(d.info) != 2 { t.Errorf("info len after init = %d, want 2", len(d.info)) } if d.LRU.Len() != 2 { t.Errorf("LRU len after init = %d, want 2", d.LRU.Len()) } // Immediate discoverability (lazy still works but now warm) if _, err := d.Stat("f1"); err != nil { t.Error("stat f1 failed immediately after init pop") } // Size > cap exercises the path where startup eviction would run at end of disk init (when GC algo provided via Set). if d.Size() <= d.Capacity() { t.Error("expected Size > Capacity to exercise over-cap path post-fix") } // Exercise eviction now has candidates thanks to population ev := d.EvictLRU(200) if ev == 0 { t.Error("EvictLRU did nothing despite over cap + populated LRU (startup eviction path would have failed before Item 1 fix)") } } func TestDiskFS_EvictAndLazyStat(t *testing.T) { t.Parallel() td := t.TempDir() d, err := New(td, 400, nil) if err != nil { t.Fatal(err) } // create files that will be evicted keys := []string{} for i := 0; i < 5; i++ { k := "f" + string(rune('0'+i)) keys = append(keys, k) w, _ := d.Create(k, 120) w.Write(make([]byte, 120)) w.Close() } ev := d.EvictLRU(200) if ev == 0 { t.Log("no evict (size calc async or snapshot tolerance?)") } // Explicit post-evict consistency checks: for any key no longer visible via Stat, its on-disk // file must be absent (verifies coordinated unlink + no resurrection via lazy discovery). // Keys still present after this small evict are allowed (accounting tolerance in raw DiskFS). for _, k := range keys { if _, err := d.Stat(k); err != nil { p := d.pathForKey(k) if _, err2 := os.Stat(p); !os.IsNotExist(err2) { t.Errorf("key %s absent in Stat but stray file remains on disk at %s: %v", k, p, err2) } } } // lazy stat should still work for remaining; batch eviction may be approximate under heavy pressure if d.Size() > d.Capacity()*2 { // generous for async bg size t.Errorf("disk size %d >> cap after evict", d.Size()) } } func TestDiskFS_Concurrent(t *testing.T) { if testing.Short() { t.Skip() } t.Parallel() td := t.TempDir() d, err := New(td, 50*1024*1024, nil) if err != nil { t.Fatal(err) } var wg sync.WaitGroup var ops int64 for i := 0; i < 4; i++ { wg.Add(1) go func(id int) { defer wg.Done() for j := 0; j < 30; j++ { key := "d" + string(rune(id+'a')) + string(rune(j)) w, e := d.Create(key, 256) if e == nil { w.Write(make([]byte, 256)) w.Close() atomic.AddInt64(&ops, 1) } if r, e := d.Open(key); e == nil { io.Copy(io.Discard, r) r.Close() atomic.AddInt64(&ops, 1) } d.Delete(key) if j%7 == 0 { d.EvictLRU(1024) } } }(i) } wg.Wait() // Bounded poll instead of fixed sleep for bg size calc goroutine settlement (robust to variance). deadline := time.Now().Add(300 * time.Millisecond) for time.Now().Before(deadline) { if d.Size() <= d.Capacity() { break } time.Sleep(5 * time.Millisecond) } if d.Size() > d.Capacity() { t.Errorf("concurrent disk size exceeded: %d", d.Size()) } } func BenchmarkDiskFS_CreateOpen(b *testing.B) { td := b.TempDir() d, err := New(td, 128*1024*1024, nil) if err != nil { b.Fatal(err) } data := make([]byte, 8192) b.ReportAllocs() b.ResetTimer() for i := 0; i < b.N; i++ { key := testKey(i % 500) w, err := d.Create(key, 8192) if err != nil { b.Fatal(err) } w.Write(data) w.Close() r, err := d.Open(key) if err != nil { b.Fatal(err) } io.Copy(io.Discard, r) r.Close() d.Delete(key) } } // BenchmarkDiskFS_EvictionUnderPressure exercises disk eviction under synthetic pressure (mirrors memory version for parity). // Uses cycling keys via testKey for stable disk usage; exercises LRU path (other strategies lightly covered via tests + EvictHybrid uses DecayedScore). func BenchmarkDiskFS_EvictionUnderPressure(b *testing.B) { td := b.TempDir() d, err := New(td, 1*1024*1024, nil) if err != nil { b.Fatal(err) } b.ReportAllocs() b.ResetTimer() for i := 0; i < b.N; i++ { // fill then evict (setup fill not timed separately to keep bench focused on pressure+evict cycle) for j := 0; j < 20; j++ { w, err := d.Create(testKey(j), 64*1024) if err != nil { b.Fatal(err) } w.Write(make([]byte, 64*1024)) w.Close() } d.EvictLRU(512 * 1024) } _ = d // keep } func TestDiskFS_EvictVariantsAndInvalid(t *testing.T) { t.Parallel() td := t.TempDir() d, err := New(td, 600, nil) if err != nil { t.Fatal(err) } for i := 0; i < 4; i++ { w, _ := d.Create("dv"+string(rune('0'+i)), 120) w.Write(make([]byte, 120)) w.Close() } _ = d.EvictBySize(80, false) // largest _ = d.EvictFIFO(50) _ = d.EvictLFU(30) _ = d.EvictHybrid(30) // invalids (sanitized in Create/Open) if _, err := d.Create("", 1); err == nil { t.Error("empty") } if _, err := d.Create("/abs/bad", 1); err == nil { t.Error("abs") } if _, err := d.Open("missing"); err == nil { t.Error("missing open") } _ = d.Delete("missing") _, _ = d.Stat("missing") } // TestEvict_ConcurrentCloseDuringEviction exercises Creates, Opens, and Closes (which mutate *FileInfo and size under lock) // concurrently with all Evict* (LRU + non-LRU scalar snapshot paths) on DiskFS under pressure. // Sufficient goroutines/iterations to exercise snapshot + re-fetch + close-during-evict paths. Asserts size invariant with // documented epsilon tolerance for raw DiskFS (background size calc + snapshot tolerance during batch eviction). -race must pass. func TestEvict_ConcurrentCloseDuringEviction(t *testing.T) { if testing.Short() { t.Skip() } t.Parallel() td := t.TempDir() cap := int64(256 * 1024) d, err := New(td, cap, nil) if err != nil { t.Fatal(err) } var wg sync.WaitGroup const nWriters = 4 const nEvictors = 3 const iters = 25 for i := 0; i < nWriters; i++ { wg.Add(1) go func(id int) { defer wg.Done() for j := 0; j < iters; j++ { key := "r" + string(rune('0'+id%5)) + "/" + string(rune('0'+j%10)) w, err := d.Create(key, 8192) if err == nil { w.Write(make([]byte, 4096)) w.Close() // exercises Close size mutation path concurrent with evicts } if r, err := d.Open(key); err == nil { io.Copy(io.Discard, r) r.Close() } if j%4 == 0 { d.Delete(key) } } }(i) } for i := 0; i < nEvictors; i++ { wg.Add(1) go func(id int) { defer wg.Done() for j := 0; j < iters*2; j++ { // Cycle through strategies to cover all snapshot + re-fetch + LRU-Lock paths switch j % 6 { case 0: d.EvictLRU(4096) case 1: d.EvictBySize(4096, true) case 2: d.EvictBySize(4096, false) case 3: d.EvictFIFO(4096) case 4: d.EvictLFU(4096) default: d.EvictHybrid(4096) } } }(i) } wg.Wait() // Final size <= cap with epsilon (raw DiskFS allows small over per bg size + snapshot design; see TestDiskFS_Concurrent and memory +50 pattern) if sz := d.Size(); sz > cap+2048 { t.Errorf("final size %d exceeded cap %d + epsilon tolerance after concurrent close+evict", sz, cap) } } // testKey helper for stable key generation across tests. func testKey(i int) string { return fmt.Sprintf("test/key/%04d", i) } // TestDiskFS_EvictDiskVisibilityAndRecreateSafety verifies that after eviction the on-disk // artifacts for victims are immediately gone (no resurrection via lazy discovery in Stat/Open), // and that recreating the same key produces independent content that is not subject to any // stale eviction unlinks. This exercises the coordinated WLock remove path for DiskFS. // Uses tolerant checks suitable for raw DiskFS lazy discovery + bg size. func TestDiskFS_EvictDiskVisibilityAndRecreateSafety(t *testing.T) { t.Parallel() td := t.TempDir() cap := int64(500) d, err := New(td, cap, nil) if err != nil { t.Fatal(err) } created := []string{"v1", "v2", "v3", "s1"} for _, k := range created { sz := int64(150) if k == "s1" { sz = 50 } w, err := d.Create(k, sz) if err != nil { t.Fatal(err) } w.Write(make([]byte, sz)) w.Close() } // Force eviction pressure with large request; repeat to handle batching + approx accounting. for i := 0; i < 5; i++ { _ = d.EvictLRU(1024 * 1024) _ = d.EvictBySize(1024*1024, true) } // Consistency check: never have a key absent from Stat but with a file on disk (would indicate // either resurrection risk or orphan). If Stat succeeds, file should exist. // A few retries tolerate the documented lazy discovery + eviction coordination windows under // artificial "force massive eviction then immediate audit" load (especially visible under -race). for attempt := 0; attempt < 3; attempt++ { bad := false for _, k := range created { p := d.pathForKey(k) _, statErr := d.Stat(k) _, diskErr := os.Stat(p) if statErr != nil { if !os.IsNotExist(diskErr) { bad = true } } else { if diskErr != nil { bad = true } } } if !bad { break } if attempt < 2 { time.Sleep(10 * time.Millisecond) } else { // On final attempt, report the last observed state for the keys for _, k := range created { p := d.pathForKey(k) _, statErr := d.Stat(k) _, diskErr := os.Stat(p) if statErr != nil { if !os.IsNotExist(diskErr) { t.Errorf("key %s absent via Stat but file lingers on disk at %s (resurrection risk)", k, p) } } else { if diskErr != nil { t.Errorf("key %s present via Stat but missing on disk: %v", k, diskErr) } } } } } // Recreate one that is currently absent (or any): must work, and new content must not be // subject to stale unlinks (guaranteed by inside-WLock removes on evict + keyMu on Create). k := "v1" w2, err := d.Create(k, 40) if err != nil { t.Fatalf("recreate %s failed: %v", k, err) } w2.Write([]byte("fresh-after-evict")) w2.Close() p := d.pathForKey(k) if st, err := os.Stat(p); err != nil || st.Size() < 10 { t.Errorf("recreated %s disk state bad: size=%v err=%v", k, st, err) } if r, err := d.Open(k); err != nil { t.Errorf("recreated %s not readable: %v", k, err) } else { r.Close() } } // TestDiskFS_EvictBoundedLargeN exercises the maxEvictBatch early-break logic (Idea #2) // under a map size >> batch limit. Forces repeated eviction rounds via GC-style pressure // and asserts progress + consistency (no resurrection/orphans). Covers bounded collection // for the non-LRU (and LRU) paths. Tolerant of raw DiskFS bg size + approx accounting. func TestDiskFS_EvictBoundedLargeN(t *testing.T) { if testing.Short() { t.Skip() } t.Parallel() td := t.TempDir() cap := int64(128 * 1024) // slightly larger for practicality d, err := New(td, cap, nil) if err != nil { t.Fatal(err) } const nFiles = 3000 // > maxEvictBatch to exercise early-break on multiple rounds const fSize = 128 for i := 0; i < nFiles; i++ { k := fmt.Sprintf("big/%05d", i) w, err := d.Create(k, fSize) if err != nil { t.Fatal(err) } w.Write(make([]byte, fSize)) w.Close() if i%800 == 0 { d.EvictLRU(4096) } } // Drive reclamation with larger per-call request (to exercise meaningful batches quickly). rounds := 0 totalEvicted := uint(0) for d.Size() > d.Capacity() && rounds < 100 { ev := d.EvictLRU(64 * 1024) totalEvicted += ev rounds++ if ev == 0 && rounds > 5 { break } } // Progress + no-hang is the goal; final size check tolerant for DiskFS bg/snapshot design. finalSize := d.Size() if rounds < 2 { t.Logf("large-N disk: completed with %d rounds (evicted=%d final=%d)", rounds, totalEvicted, finalSize) } // Spot-check consistency (if Stat ok => disk ok; if Stat not => disk absent). Catches resurrection. for i := 0; i < 5; i++ { k := fmt.Sprintf("big/%05d", i*600) p := d.pathForKey(k) if _, err := d.Stat(k); err == nil { if _, err2 := os.Stat(p); err2 != nil { t.Errorf("in-index %s missing on disk: %v", k, err2) } } else if _, err2 := os.Stat(p); !os.IsNotExist(err2) { t.Errorf("absent %s has stray disk file: %v", k, err2) } } _ = totalEvicted } // TestDiskFS_StartupEvictionFuncInvokedDuringInit covers the relocated guard path: // pre-populate over capacity, New with non-nil evict func (selected via Get), wait for init, // verify the func was invoked inside calculate (before close(initDone)) and size reduced. func TestDiskFS_StartupEvictionFuncInvokedDuringInit(t *testing.T) { t.Parallel() td := t.TempDir() prepare := func(key string, sz int64) { p := td + "/" + key if err := os.MkdirAll(td, 0755); err != nil { t.Fatalf("mkdir: %v", err) } if err := os.WriteFile(p, make([]byte, sz), 0644); err != nil { t.Fatalf("write %s: %v", key, err) } } prepare("f1", 100) prepare("f2", 200) // Use real eviction func (delegates to EvictLRU impl, as GC algos do) + pre-pop > cap. // Assert post-Size() (post-guard) that size was reduced to <= cap + index updated (Issue 4 coverage). evictFn := func(v vfs.VFS, b uint) uint { // real path: same as hybrid/lru would via the VFS methods (exercises lock, LRU remove, size adjust, os.Remove) if dd, ok := v.(*DiskFS); ok { return dd.EvictLRU(b) } return 0 } d, err := New(td, 150, evictFn) if err != nil { t.Fatal(err) } _ = d.Size() // wait for bg init + guard (last step) + close if d.Size() > d.Capacity() { t.Errorf("startup guard with real evictFn did not reduce size: got %d > cap %d", d.Size(), d.Capacity()) } // LRU/info updated by real evict; at least one file gone (original 2 files, 300B) if len(d.info) == 2 { t.Error("expected real eviction to have removed at least one over-cap file from index") } } // TestDiskFS_NewMkdirError covers propagation of MkdirAll error from New (ctor now returns err; Issue 6). func TestDiskFS_NewMkdirError(t *testing.T) { t.Parallel() // Create a regular file at the path we will pass as "root dir"; MkdirAll will fail with "file exists" or perm. td := t.TempDir() badPath := filepath.Join(td, "notadir") if err := os.WriteFile(badPath, []byte("x"), 0644); err != nil { t.Fatal(err) } _, err := New(badPath, 1024, nil) if err == nil || !strings.Contains(err.Error(), "failed to create root directory") { t.Errorf("expected mkdir failure error for file-as-dir, got: %v", err) } }