Refactor Makefile and enhance disk/memory eviction tests

- Updated the 'bench' target in the Makefile to run all benchmarks for MemoryFS and DiskFS, improving clarity and coverage. - Added explicit post-eviction consistency checks in DiskFS tests to ensure on-disk files are removed after eviction. - Introduced new benchmarks for memory eviction strategies under pressure, enhancing test coverage for memory management. - Improved error handling in benchmark tests for both disk and memory file systems, ensuring robustness during performance evaluations. - Refactored key generation in tests for consistency and clarity.
2026-05-27 03:02:34 -05:00
parent 6f28362790
commit ffa9aa04f7
5 changed files with 317 additions and 26 deletions
@@ -1,7 +1,9 @@
 package disk

 import (
+	"fmt"
 	"io"
+	"os"
 	"sync"
 	"sync/atomic"
 	"testing"
@@ -48,8 +50,11 @@ func TestDiskFS_EvictAndLazyStat(t *testing.T) {
 	td := t.TempDir()
 	d := New(td, 400)
 	// create files that will be evicted
+	keys := []string{}
 	for i := 0; i < 5; i++ {
-		w, _ := d.Create("f"+string(rune('0'+i)), 120)
+		k := "f" + string(rune('0'+i))
+		keys = append(keys, k)
+		w, _ := d.Create(k, 120)
 		w.Write(make([]byte, 120))
 		w.Close()
 	}
@@ -57,6 +62,17 @@ func TestDiskFS_EvictAndLazyStat(t *testing.T) {
 	if ev == 0 {
 		t.Log("no evict (size calc async or snapshot tolerance?)")
 	}
+	// Explicit post-evict consistency checks: for any key no longer visible via Stat, its on-disk
+	// file must be absent (verifies coordinated unlink + no resurrection via lazy discovery).
+	// Keys still present after this small evict are allowed (accounting tolerance in raw DiskFS).
+	for _, k := range keys {
+		if _, err := d.Stat(k); err != nil {
+			p := d.pathForKey(k)
+			if _, err2 := os.Stat(p); !os.IsNotExist(err2) {
+				t.Errorf("key %s absent in Stat but stray file remains on disk at %s: %v", k, p, err2)
+			}
+		}
+	}
 	// lazy stat should still work for remaining; batch eviction may be approximate under heavy pressure
 	if d.Size() > d.Capacity()*2 { // generous for async bg size
 		t.Errorf("disk size %d >> cap after evict", d.Size())
@@ -97,7 +113,7 @@ func TestDiskFS_Concurrent(t *testing.T) {
 		}(i)
 	}
 	wg.Wait()
-	// Bounded poll instead of fixed sleep for bg size calc goroutine settlement (robust to variance; issue7)
+	// Bounded poll instead of fixed sleep for bg size calc goroutine settlement (robust to variance).
 	deadline := time.Now().Add(300 * time.Millisecond)
 	for time.Now().Before(deadline) {
 		if d.Size() <= d.Capacity() {
@@ -117,17 +133,45 @@ func BenchmarkDiskFS_CreateOpen(b *testing.B) {
 	b.ReportAllocs()
 	b.ResetTimer()
 	for i := 0; i < b.N; i++ {
-		key := "bd" + string(rune(i%500))
-		w, _ := d.Create(key, 8192)
+		key := testKey(i % 500)
+		w, err := d.Create(key, 8192)
+		if err != nil {
+			b.Fatal(err)
+		}
 		w.Write(data)
 		w.Close()
-		r, _ := d.Open(key)
+		r, err := d.Open(key)
+		if err != nil {
+			b.Fatal(err)
+		}
 		io.Copy(io.Discard, r)
 		r.Close()
 		d.Delete(key)
 	}
 }

+// BenchmarkDiskFS_EvictionUnderPressure exercises disk eviction under synthetic pressure (mirrors memory version for parity).
+// Uses cycling keys via testKey for stable disk usage; exercises LRU path (other strategies lightly covered via tests + EvictHybrid uses DecayedScore).
+func BenchmarkDiskFS_EvictionUnderPressure(b *testing.B) {
+	td := b.TempDir()
+	d := New(td, 1*1024*1024)
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		// fill then evict (setup fill not timed separately to keep bench focused on pressure+evict cycle)
+		for j := 0; j < 20; j++ {
+			w, err := d.Create(testKey(j), 64*1024)
+			if err != nil {
+				b.Fatal(err)
+			}
+			w.Write(make([]byte, 64*1024))
+			w.Close()
+		}
+		d.EvictLRU(512 * 1024)
+	}
+	_ = d // keep
+}
+
 func TestDiskFS_EvictVariantsAndInvalid(t *testing.T) {
 	t.Parallel()
 	td := t.TempDir()
@@ -158,8 +202,8 @@ func TestDiskFS_EvictVariantsAndInvalid(t *testing.T) {

 // TestEvict_ConcurrentCloseDuringEviction exercises Creates, Opens, and Closes (which mutate *FileInfo and size under lock)
 // concurrently with all Evict* (LRU + non-LRU scalar snapshot paths) on DiskFS under pressure.
-// Sufficient goroutines/iterations to hit prior race windows for Issues 1-3. Asserts size invariant with
-// Documented epsilon tolerance for raw DiskFS (background size calc + snapshot tolerance during batch eviction). -race must pass.
+// Sufficient goroutines/iterations to exercise snapshot + re-fetch + close-during-evict paths. Asserts size invariant with
+// documented epsilon tolerance for raw DiskFS (background size calc + snapshot tolerance during batch eviction). -race must pass.
 func TestEvict_ConcurrentCloseDuringEviction(t *testing.T) {
 	if testing.Short() {
 		t.Skip()
@@ -222,3 +266,134 @@ func TestEvict_ConcurrentCloseDuringEviction(t *testing.T) {
 		t.Errorf("final size %d exceeded cap %d + epsilon tolerance after concurrent close+evict", sz, cap)
 	}
 }
+
+// testKey helper for stable key generation across tests.
+func testKey(i int) string {
+	return fmt.Sprintf("test/key/%04d", i)
+}
+
+// TestDiskFS_EvictDiskVisibilityAndRecreateSafety verifies that after eviction the on-disk
+// artifacts for victims are immediately gone (no resurrection via lazy discovery in Stat/Open),
+// and that recreating the same key produces independent content that is not subject to any
+// stale eviction unlinks. This exercises the coordinated WLock remove path for DiskFS.
+// Uses tolerant checks suitable for raw DiskFS lazy discovery + bg size.
+func TestDiskFS_EvictDiskVisibilityAndRecreateSafety(t *testing.T) {
+	t.Parallel()
+	td := t.TempDir()
+	cap := int64(500)
+	d := New(td, cap)
+	created := []string{"v1", "v2", "v3", "s1"}
+	for _, k := range created {
+		sz := int64(150)
+		if k == "s1" {
+			sz = 50
+		}
+		w, err := d.Create(k, sz)
+		if err != nil {
+			t.Fatal(err)
+		}
+		w.Write(make([]byte, sz))
+		w.Close()
+	}
+
+	// Force eviction pressure with large request; repeat to handle batching + approx accounting.
+	for i := 0; i < 5; i++ {
+		_ = d.EvictLRU(1024 * 1024)
+		_ = d.EvictBySize(1024*1024, true)
+	}
+
+	// Consistency check: never have a key absent from Stat but with a file on disk (would indicate
+	// either resurrection risk or orphan). If Stat succeeds, file should exist.
+	for _, k := range created {
+		p := d.pathForKey(k)
+		_, statErr := d.Stat(k)
+		_, diskErr := os.Stat(p)
+		if statErr != nil {
+			// Absent logically: disk must not have the file (no resurrection).
+			if !os.IsNotExist(diskErr) {
+				t.Errorf("key %s absent via Stat but file lingers on disk at %s (resurrection risk)", k, p)
+			}
+		} else {
+			// Present logically: disk file should exist.
+			if diskErr != nil {
+				t.Errorf("key %s present via Stat but missing on disk: %v", k, diskErr)
+			}
+		}
+	}
+
+	// Recreate one that is currently absent (or any): must work, and new content must not be
+	// subject to stale unlinks (guaranteed by inside-WLock removes on evict + keyMu on Create).
+	k := "v1"
+	w2, err := d.Create(k, 40)
+	if err != nil {
+		t.Fatalf("recreate %s failed: %v", k, err)
+	}
+	w2.Write([]byte("fresh-after-evict"))
+	w2.Close()
+	p := d.pathForKey(k)
+	if st, err := os.Stat(p); err != nil || st.Size() < 10 {
+		t.Errorf("recreated %s disk state bad: size=%v err=%v", k, st, err)
+	}
+	if r, err := d.Open(k); err != nil {
+		t.Errorf("recreated %s not readable: %v", k, err)
+	} else {
+		r.Close()
+	}
+}
+
+// TestDiskFS_EvictBoundedLargeN exercises the maxEvictBatch early-break logic (Idea #2)
+// under a map size >> batch limit. Forces repeated eviction rounds via GC-style pressure
+// and asserts progress + consistency (no resurrection/orphans). Covers bounded collection
+// for the non-LRU (and LRU) paths. Tolerant of raw DiskFS bg size + approx accounting.
+func TestDiskFS_EvictBoundedLargeN(t *testing.T) {
+	if testing.Short() {
+		t.Skip()
+	}
+	t.Parallel()
+	td := t.TempDir()
+	cap := int64(128 * 1024) // slightly larger for practicality
+	d := New(td, cap)
+	const nFiles = 3000 // > maxEvictBatch to exercise early-break on multiple rounds
+	const fSize = 128
+	for i := 0; i < nFiles; i++ {
+		k := fmt.Sprintf("big/%05d", i)
+		w, err := d.Create(k, fSize)
+		if err != nil {
+			t.Fatal(err)
+		}
+		w.Write(make([]byte, fSize))
+		w.Close()
+		if i%800 == 0 {
+			d.EvictLRU(4096)
+		}
+	}
+	// Drive reclamation with larger per-call request (to exercise meaningful batches quickly).
+	rounds := 0
+	totalEvicted := uint(0)
+	for d.Size() > d.Capacity() && rounds < 100 {
+		ev := d.EvictLRU(64 * 1024)
+		totalEvicted += ev
+		rounds++
+		if ev == 0 && rounds > 5 {
+			break
+		}
+	}
+	// Progress + no-hang is the goal; final size check tolerant for DiskFS bg/snapshot design.
+	finalSize := d.Size()
+	if rounds < 2 {
+		t.Logf("large-N disk: completed with %d rounds (evicted=%d final=%d)", rounds, totalEvicted, finalSize)
+	}
+	// Spot-check consistency (if Stat ok => disk ok; if Stat not => disk absent). Catches resurrection.
+	for i := 0; i < 5; i++ {
+		k := fmt.Sprintf("big/%05d", i*600)
+		p := d.pathForKey(k)
+		if _, err := d.Stat(k); err == nil {
+			if _, err2 := os.Stat(p); err2 != nil {
+				t.Errorf("in-index %s missing on disk: %v", k, err2)
+			}
+		} else if _, err2 := os.Stat(p); !os.IsNotExist(err2) {
+			t.Errorf("absent %s has stray disk file: %v", k, err2)
+		}
+	}
+	_ = totalEvicted
+}