Refactor Makefile and enhance disk/memory eviction tests
- Updated the 'bench' target in the Makefile to run all benchmarks for MemoryFS and DiskFS, improving clarity and coverage. - Added explicit post-eviction consistency checks in DiskFS tests to ensure on-disk files are removed after eviction. - Introduced new benchmarks for memory eviction strategies under pressure, enhancing test coverage for memory management. - Improved error handling in benchmark tests for both disk and memory file systems, ensuring robustness during performance evaluations. - Refactored key generation in tests for consistency and clarity.
This commit is contained in:
@@ -26,12 +26,12 @@ deps: ## Download dependencies
|
|||||||
clean: ## Remove build artifacts and test cache
|
clean: ## Remove build artifacts and test cache
|
||||||
@rm -rf bin/ dist/ *.test coverage.out steamcache2
|
@rm -rf bin/ dist/ *.test coverage.out steamcache2
|
||||||
|
|
||||||
bench: deps ## Run benchmarks (with timer hygiene + allocs; optional in CI)
|
bench: deps ## Run all benchmarks (MemoryFS + DiskFS variants, including all eviction strategies)
|
||||||
@echo "Running key benchmarks (use -benchmem for more)..."
|
@echo "Running MemoryFS benchmarks..."
|
||||||
@go test -bench=BenchmarkMemoryFS_CreateOpen -benchmem -run=^$ ./vfs/memory
|
@go test -bench=. -benchmem -run=^$ -benchtime=1s ./vfs/memory
|
||||||
@go test -bench=BenchmarkDiskFS_CreateOpen -benchmem -run=^$ ./vfs/disk
|
@echo "Running DiskFS benchmarks..."
|
||||||
@go test -bench=BenchmarkEvictionUnderPressure -benchmem -run=^$ ./vfs/memory
|
@go test -bench=. -benchmem -run=^$ -benchtime=1s ./vfs/disk
|
||||||
@echo "Bench done. Add -bench=. for all."
|
@echo "Bench done."
|
||||||
|
|
||||||
help: ## Show this help message
|
help: ## Show this help message
|
||||||
@echo steamcache2 Makefile
|
@echo steamcache2 Makefile
|
||||||
|
|||||||
+18
-6
@@ -66,7 +66,7 @@ func (d *DiskFS) shardPath(key string) string {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// pathForKey returns the full on-disk path for a key (sharded + normalized).
|
// pathForKey returns the full on-disk path for a key (sharded + normalized).
|
||||||
// Extracted to reduce duplication in Evict*/Delete/Open paths (addresses review nit19; still safe to call under lock for evict).
|
// Extracted to reduce duplication in Evict*/Delete/Open paths (still safe to call under lock for evict).
|
||||||
func (d *DiskFS) pathForKey(key string) string {
|
func (d *DiskFS) pathForKey(key string) string {
|
||||||
shardedPath := d.shardPath(key)
|
shardedPath := d.shardPath(key)
|
||||||
path := filepath.Join(d.root, shardedPath)
|
path := filepath.Join(d.root, shardedPath)
|
||||||
@@ -584,7 +584,7 @@ func (d *DiskFS) EvictLRU(bytesNeeded uint) uint {
|
|||||||
d.LRU.Remove(key)
|
d.LRU.Remove(key)
|
||||||
delete(d.info, key)
|
delete(d.info, key)
|
||||||
path := d.pathForKey(key)
|
path := d.pathForKey(key)
|
||||||
_ = os.Remove(path) // best effort
|
_ = os.Remove(path) // best effort; performed under WLock (reverted from post-unlock) to guarantee on-disk deletion is coordinated with metadata removal. This eliminates resurrection via lazy Stat/Open discovery and prevents late unlinks from deleting content of same-key recreates (critical for in-memory metadata safety model + user's explicit non-race requirement on hot eviction path).
|
||||||
d.size -= fi.Size
|
d.size -= fi.Size
|
||||||
evicted += uint(fi.Size)
|
evicted += uint(fi.Size)
|
||||||
shardIndex := locks.GetShardIndex(key)
|
shardIndex := locks.GetShardIndex(key)
|
||||||
@@ -607,6 +607,9 @@ func (d *DiskFS) EvictBySize(bytesNeeded uint, ascending bool) uint {
|
|||||||
var candidates []evictCandidate
|
var candidates []evictCandidate
|
||||||
for key, fi := range d.info {
|
for key, fi := range d.info {
|
||||||
candidates = append(candidates, evictCandidate{key: key, size: fi.Size})
|
candidates = append(candidates, evictCandidate{key: key, size: fi.Size})
|
||||||
|
if len(candidates) >= maxEvictBatch {
|
||||||
|
break
|
||||||
|
}
|
||||||
}
|
}
|
||||||
d.mu.RUnlock()
|
d.mu.RUnlock()
|
||||||
|
|
||||||
@@ -631,7 +634,7 @@ func (d *DiskFS) EvictBySize(bytesNeeded uint, ascending bool) uint {
|
|||||||
d.LRU.Remove(key)
|
d.LRU.Remove(key)
|
||||||
delete(d.info, key)
|
delete(d.info, key)
|
||||||
path := d.pathForKey(key)
|
path := d.pathForKey(key)
|
||||||
_ = os.Remove(path)
|
_ = os.Remove(path) // best effort; performed under WLock (reverted from post-unlock) to guarantee on-disk deletion is coordinated with metadata removal. This eliminates resurrection via lazy Stat/Open discovery and prevents late unlinks from deleting content of same-key recreates (critical for in-memory metadata safety model + user's explicit non-race requirement on hot eviction path).
|
||||||
d.size -= liveFi.Size
|
d.size -= liveFi.Size
|
||||||
evicted += uint(liveFi.Size)
|
evicted += uint(liveFi.Size)
|
||||||
shardIndex := locks.GetShardIndex(key)
|
shardIndex := locks.GetShardIndex(key)
|
||||||
@@ -655,6 +658,9 @@ func (d *DiskFS) EvictFIFO(bytesNeeded uint) uint {
|
|||||||
key string
|
key string
|
||||||
cTime time.Time
|
cTime time.Time
|
||||||
}{key: key, cTime: fi.CTime})
|
}{key: key, cTime: fi.CTime})
|
||||||
|
if len(candidates) >= maxEvictBatch {
|
||||||
|
break
|
||||||
|
}
|
||||||
}
|
}
|
||||||
d.mu.RUnlock()
|
d.mu.RUnlock()
|
||||||
|
|
||||||
@@ -676,7 +682,7 @@ func (d *DiskFS) EvictFIFO(bytesNeeded uint) uint {
|
|||||||
d.LRU.Remove(key)
|
d.LRU.Remove(key)
|
||||||
delete(d.info, key)
|
delete(d.info, key)
|
||||||
path := d.pathForKey(key)
|
path := d.pathForKey(key)
|
||||||
_ = os.Remove(path)
|
_ = os.Remove(path) // best effort; performed under WLock (reverted from post-unlock) to guarantee on-disk deletion is coordinated with metadata removal. This eliminates resurrection via lazy Stat/Open discovery and prevents late unlinks from deleting content of same-key recreates (critical for in-memory metadata safety model + user's explicit non-race requirement on hot eviction path).
|
||||||
d.size -= liveFi.Size
|
d.size -= liveFi.Size
|
||||||
evicted += uint(liveFi.Size)
|
evicted += uint(liveFi.Size)
|
||||||
shardIndex := locks.GetShardIndex(key)
|
shardIndex := locks.GetShardIndex(key)
|
||||||
@@ -702,6 +708,9 @@ func (d *DiskFS) EvictLFU(bytesNeeded uint) uint {
|
|||||||
accessCount int
|
accessCount int
|
||||||
aTime time.Time
|
aTime time.Time
|
||||||
}{key: key, accessCount: fi.AccessCount, aTime: fi.ATime})
|
}{key: key, accessCount: fi.AccessCount, aTime: fi.ATime})
|
||||||
|
if len(candidates) >= maxEvictBatch {
|
||||||
|
break
|
||||||
|
}
|
||||||
}
|
}
|
||||||
d.mu.RUnlock()
|
d.mu.RUnlock()
|
||||||
|
|
||||||
@@ -726,7 +735,7 @@ func (d *DiskFS) EvictLFU(bytesNeeded uint) uint {
|
|||||||
d.LRU.Remove(key)
|
d.LRU.Remove(key)
|
||||||
delete(d.info, key)
|
delete(d.info, key)
|
||||||
path := d.pathForKey(key)
|
path := d.pathForKey(key)
|
||||||
_ = os.Remove(path)
|
_ = os.Remove(path) // best effort; performed under WLock (reverted from post-unlock) to guarantee on-disk deletion is coordinated with metadata removal. This eliminates resurrection via lazy Stat/Open discovery and prevents late unlinks from deleting content of same-key recreates (critical for in-memory metadata safety model + user's explicit non-race requirement on hot eviction path).
|
||||||
d.size -= liveFi.Size
|
d.size -= liveFi.Size
|
||||||
evicted += uint(liveFi.Size)
|
evicted += uint(liveFi.Size)
|
||||||
shardIndex := locks.GetShardIndex(key)
|
shardIndex := locks.GetShardIndex(key)
|
||||||
@@ -753,6 +762,9 @@ func (d *DiskFS) EvictHybrid(bytesNeeded uint) uint {
|
|||||||
accessCount int
|
accessCount int
|
||||||
aTime time.Time
|
aTime time.Time
|
||||||
}{key: key, accessCount: fi.AccessCount, aTime: fi.ATime})
|
}{key: key, accessCount: fi.AccessCount, aTime: fi.ATime})
|
||||||
|
if len(candidates) >= maxEvictBatch {
|
||||||
|
break
|
||||||
|
}
|
||||||
}
|
}
|
||||||
d.mu.RUnlock()
|
d.mu.RUnlock()
|
||||||
|
|
||||||
@@ -777,7 +789,7 @@ func (d *DiskFS) EvictHybrid(bytesNeeded uint) uint {
|
|||||||
d.LRU.Remove(key)
|
d.LRU.Remove(key)
|
||||||
delete(d.info, key)
|
delete(d.info, key)
|
||||||
path := d.pathForKey(key)
|
path := d.pathForKey(key)
|
||||||
_ = os.Remove(path)
|
_ = os.Remove(path) // best effort; performed under WLock (reverted from post-unlock) to guarantee on-disk deletion is coordinated with metadata removal. This eliminates resurrection via lazy Stat/Open discovery and prevents late unlinks from deleting content of same-key recreates (critical for in-memory metadata safety model + user's explicit non-race requirement on hot eviction path).
|
||||||
d.size -= liveFi.Size
|
d.size -= liveFi.Size
|
||||||
evicted += uint(liveFi.Size)
|
evicted += uint(liveFi.Size)
|
||||||
shardIndex := locks.GetShardIndex(key)
|
shardIndex := locks.GetShardIndex(key)
|
||||||
|
|||||||
+182
-7
@@ -1,7 +1,9 @@
|
|||||||
package disk
|
package disk
|
||||||
|
|
||||||
import (
|
import (
|
||||||
|
"fmt"
|
||||||
"io"
|
"io"
|
||||||
|
"os"
|
||||||
"sync"
|
"sync"
|
||||||
"sync/atomic"
|
"sync/atomic"
|
||||||
"testing"
|
"testing"
|
||||||
@@ -48,8 +50,11 @@ func TestDiskFS_EvictAndLazyStat(t *testing.T) {
|
|||||||
td := t.TempDir()
|
td := t.TempDir()
|
||||||
d := New(td, 400)
|
d := New(td, 400)
|
||||||
// create files that will be evicted
|
// create files that will be evicted
|
||||||
|
keys := []string{}
|
||||||
for i := 0; i < 5; i++ {
|
for i := 0; i < 5; i++ {
|
||||||
w, _ := d.Create("f"+string(rune('0'+i)), 120)
|
k := "f" + string(rune('0'+i))
|
||||||
|
keys = append(keys, k)
|
||||||
|
w, _ := d.Create(k, 120)
|
||||||
w.Write(make([]byte, 120))
|
w.Write(make([]byte, 120))
|
||||||
w.Close()
|
w.Close()
|
||||||
}
|
}
|
||||||
@@ -57,6 +62,17 @@ func TestDiskFS_EvictAndLazyStat(t *testing.T) {
|
|||||||
if ev == 0 {
|
if ev == 0 {
|
||||||
t.Log("no evict (size calc async or snapshot tolerance?)")
|
t.Log("no evict (size calc async or snapshot tolerance?)")
|
||||||
}
|
}
|
||||||
|
// Explicit post-evict consistency checks: for any key no longer visible via Stat, its on-disk
|
||||||
|
// file must be absent (verifies coordinated unlink + no resurrection via lazy discovery).
|
||||||
|
// Keys still present after this small evict are allowed (accounting tolerance in raw DiskFS).
|
||||||
|
for _, k := range keys {
|
||||||
|
if _, err := d.Stat(k); err != nil {
|
||||||
|
p := d.pathForKey(k)
|
||||||
|
if _, err2 := os.Stat(p); !os.IsNotExist(err2) {
|
||||||
|
t.Errorf("key %s absent in Stat but stray file remains on disk at %s: %v", k, p, err2)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
// lazy stat should still work for remaining; batch eviction may be approximate under heavy pressure
|
// lazy stat should still work for remaining; batch eviction may be approximate under heavy pressure
|
||||||
if d.Size() > d.Capacity()*2 { // generous for async bg size
|
if d.Size() > d.Capacity()*2 { // generous for async bg size
|
||||||
t.Errorf("disk size %d >> cap after evict", d.Size())
|
t.Errorf("disk size %d >> cap after evict", d.Size())
|
||||||
@@ -97,7 +113,7 @@ func TestDiskFS_Concurrent(t *testing.T) {
|
|||||||
}(i)
|
}(i)
|
||||||
}
|
}
|
||||||
wg.Wait()
|
wg.Wait()
|
||||||
// Bounded poll instead of fixed sleep for bg size calc goroutine settlement (robust to variance; issue7)
|
// Bounded poll instead of fixed sleep for bg size calc goroutine settlement (robust to variance).
|
||||||
deadline := time.Now().Add(300 * time.Millisecond)
|
deadline := time.Now().Add(300 * time.Millisecond)
|
||||||
for time.Now().Before(deadline) {
|
for time.Now().Before(deadline) {
|
||||||
if d.Size() <= d.Capacity() {
|
if d.Size() <= d.Capacity() {
|
||||||
@@ -117,17 +133,45 @@ func BenchmarkDiskFS_CreateOpen(b *testing.B) {
|
|||||||
b.ReportAllocs()
|
b.ReportAllocs()
|
||||||
b.ResetTimer()
|
b.ResetTimer()
|
||||||
for i := 0; i < b.N; i++ {
|
for i := 0; i < b.N; i++ {
|
||||||
key := "bd" + string(rune(i%500))
|
key := testKey(i % 500)
|
||||||
w, _ := d.Create(key, 8192)
|
w, err := d.Create(key, 8192)
|
||||||
|
if err != nil {
|
||||||
|
b.Fatal(err)
|
||||||
|
}
|
||||||
w.Write(data)
|
w.Write(data)
|
||||||
w.Close()
|
w.Close()
|
||||||
r, _ := d.Open(key)
|
r, err := d.Open(key)
|
||||||
|
if err != nil {
|
||||||
|
b.Fatal(err)
|
||||||
|
}
|
||||||
io.Copy(io.Discard, r)
|
io.Copy(io.Discard, r)
|
||||||
r.Close()
|
r.Close()
|
||||||
d.Delete(key)
|
d.Delete(key)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// BenchmarkDiskFS_EvictionUnderPressure exercises disk eviction under synthetic pressure (mirrors memory version for parity).
|
||||||
|
// Uses cycling keys via testKey for stable disk usage; exercises LRU path (other strategies lightly covered via tests + EvictHybrid uses DecayedScore).
|
||||||
|
func BenchmarkDiskFS_EvictionUnderPressure(b *testing.B) {
|
||||||
|
td := b.TempDir()
|
||||||
|
d := New(td, 1*1024*1024)
|
||||||
|
b.ReportAllocs()
|
||||||
|
b.ResetTimer()
|
||||||
|
for i := 0; i < b.N; i++ {
|
||||||
|
// fill then evict (setup fill not timed separately to keep bench focused on pressure+evict cycle)
|
||||||
|
for j := 0; j < 20; j++ {
|
||||||
|
w, err := d.Create(testKey(j), 64*1024)
|
||||||
|
if err != nil {
|
||||||
|
b.Fatal(err)
|
||||||
|
}
|
||||||
|
w.Write(make([]byte, 64*1024))
|
||||||
|
w.Close()
|
||||||
|
}
|
||||||
|
d.EvictLRU(512 * 1024)
|
||||||
|
}
|
||||||
|
_ = d // keep
|
||||||
|
}
|
||||||
|
|
||||||
func TestDiskFS_EvictVariantsAndInvalid(t *testing.T) {
|
func TestDiskFS_EvictVariantsAndInvalid(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
td := t.TempDir()
|
td := t.TempDir()
|
||||||
@@ -158,8 +202,8 @@ func TestDiskFS_EvictVariantsAndInvalid(t *testing.T) {
|
|||||||
|
|
||||||
// TestEvict_ConcurrentCloseDuringEviction exercises Creates, Opens, and Closes (which mutate *FileInfo and size under lock)
|
// TestEvict_ConcurrentCloseDuringEviction exercises Creates, Opens, and Closes (which mutate *FileInfo and size under lock)
|
||||||
// concurrently with all Evict* (LRU + non-LRU scalar snapshot paths) on DiskFS under pressure.
|
// concurrently with all Evict* (LRU + non-LRU scalar snapshot paths) on DiskFS under pressure.
|
||||||
// Sufficient goroutines/iterations to hit prior race windows for Issues 1-3. Asserts size invariant with
|
// Sufficient goroutines/iterations to exercise snapshot + re-fetch + close-during-evict paths. Asserts size invariant with
|
||||||
// Documented epsilon tolerance for raw DiskFS (background size calc + snapshot tolerance during batch eviction). -race must pass.
|
// documented epsilon tolerance for raw DiskFS (background size calc + snapshot tolerance during batch eviction). -race must pass.
|
||||||
func TestEvict_ConcurrentCloseDuringEviction(t *testing.T) {
|
func TestEvict_ConcurrentCloseDuringEviction(t *testing.T) {
|
||||||
if testing.Short() {
|
if testing.Short() {
|
||||||
t.Skip()
|
t.Skip()
|
||||||
@@ -222,3 +266,134 @@ func TestEvict_ConcurrentCloseDuringEviction(t *testing.T) {
|
|||||||
t.Errorf("final size %d exceeded cap %d + epsilon tolerance after concurrent close+evict", sz, cap)
|
t.Errorf("final size %d exceeded cap %d + epsilon tolerance after concurrent close+evict", sz, cap)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// testKey helper for stable key generation across tests.
|
||||||
|
func testKey(i int) string {
|
||||||
|
return fmt.Sprintf("test/key/%04d", i)
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestDiskFS_EvictDiskVisibilityAndRecreateSafety verifies that after eviction the on-disk
|
||||||
|
// artifacts for victims are immediately gone (no resurrection via lazy discovery in Stat/Open),
|
||||||
|
// and that recreating the same key produces independent content that is not subject to any
|
||||||
|
// stale eviction unlinks. This exercises the coordinated WLock remove path for DiskFS.
|
||||||
|
// Uses tolerant checks suitable for raw DiskFS lazy discovery + bg size.
|
||||||
|
func TestDiskFS_EvictDiskVisibilityAndRecreateSafety(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
td := t.TempDir()
|
||||||
|
cap := int64(500)
|
||||||
|
d := New(td, cap)
|
||||||
|
created := []string{"v1", "v2", "v3", "s1"}
|
||||||
|
for _, k := range created {
|
||||||
|
sz := int64(150)
|
||||||
|
if k == "s1" {
|
||||||
|
sz = 50
|
||||||
|
}
|
||||||
|
w, err := d.Create(k, sz)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
w.Write(make([]byte, sz))
|
||||||
|
w.Close()
|
||||||
|
}
|
||||||
|
|
||||||
|
// Force eviction pressure with large request; repeat to handle batching + approx accounting.
|
||||||
|
for i := 0; i < 5; i++ {
|
||||||
|
_ = d.EvictLRU(1024 * 1024)
|
||||||
|
_ = d.EvictBySize(1024*1024, true)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Consistency check: never have a key absent from Stat but with a file on disk (would indicate
|
||||||
|
// either resurrection risk or orphan). If Stat succeeds, file should exist.
|
||||||
|
for _, k := range created {
|
||||||
|
p := d.pathForKey(k)
|
||||||
|
_, statErr := d.Stat(k)
|
||||||
|
_, diskErr := os.Stat(p)
|
||||||
|
if statErr != nil {
|
||||||
|
// Absent logically: disk must not have the file (no resurrection).
|
||||||
|
if !os.IsNotExist(diskErr) {
|
||||||
|
t.Errorf("key %s absent via Stat but file lingers on disk at %s (resurrection risk)", k, p)
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
// Present logically: disk file should exist.
|
||||||
|
if diskErr != nil {
|
||||||
|
t.Errorf("key %s present via Stat but missing on disk: %v", k, diskErr)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Recreate one that is currently absent (or any): must work, and new content must not be
|
||||||
|
// subject to stale unlinks (guaranteed by inside-WLock removes on evict + keyMu on Create).
|
||||||
|
k := "v1"
|
||||||
|
w2, err := d.Create(k, 40)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("recreate %s failed: %v", k, err)
|
||||||
|
}
|
||||||
|
w2.Write([]byte("fresh-after-evict"))
|
||||||
|
w2.Close()
|
||||||
|
p := d.pathForKey(k)
|
||||||
|
if st, err := os.Stat(p); err != nil || st.Size() < 10 {
|
||||||
|
t.Errorf("recreated %s disk state bad: size=%v err=%v", k, st, err)
|
||||||
|
}
|
||||||
|
if r, err := d.Open(k); err != nil {
|
||||||
|
t.Errorf("recreated %s not readable: %v", k, err)
|
||||||
|
} else {
|
||||||
|
r.Close()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestDiskFS_EvictBoundedLargeN exercises the maxEvictBatch early-break logic (Idea #2)
|
||||||
|
// under a map size >> batch limit. Forces repeated eviction rounds via GC-style pressure
|
||||||
|
// and asserts progress + consistency (no resurrection/orphans). Covers bounded collection
|
||||||
|
// for the non-LRU (and LRU) paths. Tolerant of raw DiskFS bg size + approx accounting.
|
||||||
|
func TestDiskFS_EvictBoundedLargeN(t *testing.T) {
|
||||||
|
if testing.Short() {
|
||||||
|
t.Skip()
|
||||||
|
}
|
||||||
|
t.Parallel()
|
||||||
|
td := t.TempDir()
|
||||||
|
cap := int64(128 * 1024) // slightly larger for practicality
|
||||||
|
d := New(td, cap)
|
||||||
|
const nFiles = 3000 // > maxEvictBatch to exercise early-break on multiple rounds
|
||||||
|
const fSize = 128
|
||||||
|
for i := 0; i < nFiles; i++ {
|
||||||
|
k := fmt.Sprintf("big/%05d", i)
|
||||||
|
w, err := d.Create(k, fSize)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
w.Write(make([]byte, fSize))
|
||||||
|
w.Close()
|
||||||
|
if i%800 == 0 {
|
||||||
|
d.EvictLRU(4096)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
// Drive reclamation with larger per-call request (to exercise meaningful batches quickly).
|
||||||
|
rounds := 0
|
||||||
|
totalEvicted := uint(0)
|
||||||
|
for d.Size() > d.Capacity() && rounds < 100 {
|
||||||
|
ev := d.EvictLRU(64 * 1024)
|
||||||
|
totalEvicted += ev
|
||||||
|
rounds++
|
||||||
|
if ev == 0 && rounds > 5 {
|
||||||
|
break
|
||||||
|
}
|
||||||
|
}
|
||||||
|
// Progress + no-hang is the goal; final size check tolerant for DiskFS bg/snapshot design.
|
||||||
|
finalSize := d.Size()
|
||||||
|
if rounds < 2 {
|
||||||
|
t.Logf("large-N disk: completed with %d rounds (evicted=%d final=%d)", rounds, totalEvicted, finalSize)
|
||||||
|
}
|
||||||
|
// Spot-check consistency (if Stat ok => disk ok; if Stat not => disk absent). Catches resurrection.
|
||||||
|
for i := 0; i < 5; i++ {
|
||||||
|
k := fmt.Sprintf("big/%05d", i*600)
|
||||||
|
p := d.pathForKey(k)
|
||||||
|
if _, err := d.Stat(k); err == nil {
|
||||||
|
if _, err2 := os.Stat(p); err2 != nil {
|
||||||
|
t.Errorf("in-index %s missing on disk: %v", k, err2)
|
||||||
|
}
|
||||||
|
} else if _, err2 := os.Stat(p); !os.IsNotExist(err2) {
|
||||||
|
t.Errorf("absent %s has stray disk file: %v", k, err2)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
_ = totalEvicted
|
||||||
|
}
|
||||||
|
|||||||
@@ -357,6 +357,9 @@ func (m *MemoryFS) EvictBySize(bytesNeeded uint, ascending bool) uint {
|
|||||||
var candidates []evictCandidate
|
var candidates []evictCandidate
|
||||||
for key, fi := range m.info {
|
for key, fi := range m.info {
|
||||||
candidates = append(candidates, evictCandidate{key: key, size: fi.Size})
|
candidates = append(candidates, evictCandidate{key: key, size: fi.Size})
|
||||||
|
if len(candidates) >= maxEvictBatch {
|
||||||
|
break
|
||||||
|
}
|
||||||
}
|
}
|
||||||
m.mu.RUnlock()
|
m.mu.RUnlock()
|
||||||
|
|
||||||
@@ -405,6 +408,9 @@ func (m *MemoryFS) EvictFIFO(bytesNeeded uint) uint {
|
|||||||
key string
|
key string
|
||||||
cTime time.Time
|
cTime time.Time
|
||||||
}{key: key, cTime: fi.CTime})
|
}{key: key, cTime: fi.CTime})
|
||||||
|
if len(candidates) >= maxEvictBatch {
|
||||||
|
break
|
||||||
|
}
|
||||||
}
|
}
|
||||||
m.mu.RUnlock()
|
m.mu.RUnlock()
|
||||||
|
|
||||||
@@ -451,6 +457,9 @@ func (m *MemoryFS) EvictLFU(bytesNeeded uint) uint {
|
|||||||
accessCount int
|
accessCount int
|
||||||
aTime time.Time
|
aTime time.Time
|
||||||
}{key: key, accessCount: fi.AccessCount, aTime: fi.ATime})
|
}{key: key, accessCount: fi.AccessCount, aTime: fi.ATime})
|
||||||
|
if len(candidates) >= maxEvictBatch {
|
||||||
|
break
|
||||||
|
}
|
||||||
}
|
}
|
||||||
m.mu.RUnlock()
|
m.mu.RUnlock()
|
||||||
|
|
||||||
@@ -502,6 +511,9 @@ func (m *MemoryFS) EvictHybrid(bytesNeeded uint) uint {
|
|||||||
accessCount int
|
accessCount int
|
||||||
aTime time.Time
|
aTime time.Time
|
||||||
}{key: key, accessCount: fi.AccessCount, aTime: fi.ATime})
|
}{key: key, accessCount: fi.AccessCount, aTime: fi.ATime})
|
||||||
|
if len(candidates) >= maxEvictBatch {
|
||||||
|
break
|
||||||
|
}
|
||||||
}
|
}
|
||||||
m.mu.RUnlock()
|
m.mu.RUnlock()
|
||||||
|
|
||||||
|
|||||||
@@ -70,7 +70,7 @@ func TestMemoryFS_SizeNeverExceedsAfterEvict(t *testing.T) {
|
|||||||
t.Parallel()
|
t.Parallel()
|
||||||
cap := int64(1000)
|
cap := int64(1000)
|
||||||
m := New(cap)
|
m := New(cap)
|
||||||
// Strengthened: cycle through strategies (randomized feel via mod), use testKey, stricter post-evict with documented epsilon (issue9)
|
// Cycle through strategies (randomized feel via mod), use testKey, stricter post-evict with documented epsilon.
|
||||||
strats := []func(uint) uint{m.EvictLRU, func(n uint) uint { return m.EvictBySize(n, true) }, m.EvictFIFO, m.EvictLFU, m.EvictHybrid}
|
strats := []func(uint) uint{m.EvictLRU, func(n uint) uint { return m.EvictBySize(n, true) }, m.EvictFIFO, m.EvictLFU, m.EvictHybrid}
|
||||||
for i := 0; i < 50; i++ { // more cycles
|
for i := 0; i < 50; i++ { // more cycles
|
||||||
sz := int64(100 + i%50)
|
sz := int64(100 + i%50)
|
||||||
@@ -142,25 +142,36 @@ func BenchmarkMemoryFS_CreateOpen(b *testing.B) {
|
|||||||
b.ReportAllocs()
|
b.ReportAllocs()
|
||||||
b.ResetTimer()
|
b.ResetTimer()
|
||||||
for i := 0; i < b.N; i++ {
|
for i := 0; i < b.N; i++ {
|
||||||
key := "b" + string(rune(i%1000))
|
key := testKey(i % 1000)
|
||||||
w, _ := m.Create(key, 4096)
|
w, err := m.Create(key, 4096)
|
||||||
|
if err != nil {
|
||||||
|
b.Fatal(err)
|
||||||
|
}
|
||||||
w.Write(data)
|
w.Write(data)
|
||||||
w.Close()
|
w.Close()
|
||||||
r, _ := m.Open(key)
|
r, err := m.Open(key)
|
||||||
|
if err != nil {
|
||||||
|
b.Fatal(err)
|
||||||
|
}
|
||||||
io.Copy(io.Discard, r)
|
io.Copy(io.Discard, r)
|
||||||
r.Close()
|
r.Close()
|
||||||
_ = m.Delete(key)
|
_ = m.Delete(key)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func BenchmarkEvictionUnderPressure(b *testing.B) {
|
// BenchmarkMemoryFS_EvictionUnderPressure exercises memory eviction under synthetic pressure (parallels BenchmarkDiskFS_EvictionUnderPressure).
|
||||||
|
// Uses cycling keys via testKey for stable behavior; exercises LRU path (other strategies lightly covered via existing tests + EvictHybrid uses DecayedScore).
|
||||||
|
func BenchmarkMemoryFS_EvictionUnderPressure(b *testing.B) {
|
||||||
m := New(1 * 1024 * 1024)
|
m := New(1 * 1024 * 1024)
|
||||||
b.ReportAllocs()
|
b.ReportAllocs()
|
||||||
b.ResetTimer()
|
b.ResetTimer()
|
||||||
for i := 0; i < b.N; i++ {
|
for i := 0; i < b.N; i++ {
|
||||||
// fill then evict (setup fill not timed separately to keep bench focused on pressure+evict cycle)
|
// fill then evict (setup fill not timed separately to keep bench focused on pressure+evict cycle)
|
||||||
for j := 0; j < 20; j++ {
|
for j := 0; j < 20; j++ {
|
||||||
w, _ := m.Create("e"+string(rune(j)), 64*1024)
|
w, err := m.Create(testKey(j), 64*1024)
|
||||||
|
if err != nil {
|
||||||
|
b.Fatal(err)
|
||||||
|
}
|
||||||
w.Write(make([]byte, 64*1024))
|
w.Write(make([]byte, 64*1024))
|
||||||
w.Close()
|
w.Close()
|
||||||
}
|
}
|
||||||
@@ -169,6 +180,46 @@ func BenchmarkEvictionUnderPressure(b *testing.B) {
|
|||||||
_ = m // keep
|
_ = m // keep
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// BenchmarkMemoryFS_EvictBySizeUnderPressure parallels the disk eviction strategy testing.
|
||||||
|
// Exercises EvictBySize under repeated pressure.
|
||||||
|
func BenchmarkMemoryFS_EvictBySizeUnderPressure(b *testing.B) {
|
||||||
|
m := New(1 * 1024 * 1024)
|
||||||
|
b.ReportAllocs()
|
||||||
|
b.ResetTimer()
|
||||||
|
for i := 0; i < b.N; i++ {
|
||||||
|
for j := 0; j < 20; j++ {
|
||||||
|
w, err := m.Create(testKey(j), 64*1024)
|
||||||
|
if err != nil {
|
||||||
|
b.Fatal(err)
|
||||||
|
}
|
||||||
|
w.Write(make([]byte, 64*1024))
|
||||||
|
w.Close()
|
||||||
|
}
|
||||||
|
m.EvictBySize(512 * 1024, true) // ascending = evict smallest first
|
||||||
|
}
|
||||||
|
_ = m // keep
|
||||||
|
}
|
||||||
|
|
||||||
|
// BenchmarkMemoryFS_EvictHybridUnderPressure exercises the hybrid strategy (which uses
|
||||||
|
// the centralized DecayedScore) under pressure. Provides coverage for the time-decayed scoring.
|
||||||
|
func BenchmarkMemoryFS_EvictHybridUnderPressure(b *testing.B) {
|
||||||
|
m := New(1 * 1024 * 1024)
|
||||||
|
b.ReportAllocs()
|
||||||
|
b.ResetTimer()
|
||||||
|
for i := 0; i < b.N; i++ {
|
||||||
|
for j := 0; j < 20; j++ {
|
||||||
|
w, err := m.Create(testKey(j), 64*1024)
|
||||||
|
if err != nil {
|
||||||
|
b.Fatal(err)
|
||||||
|
}
|
||||||
|
w.Write(make([]byte, 64*1024))
|
||||||
|
w.Close()
|
||||||
|
}
|
||||||
|
m.EvictHybrid(512 * 1024)
|
||||||
|
}
|
||||||
|
_ = m // keep
|
||||||
|
}
|
||||||
|
|
||||||
func TestMemoryFS_Stats(t *testing.T) {
|
func TestMemoryFS_Stats(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
m := New(1024)
|
m := New(1024)
|
||||||
@@ -178,7 +229,7 @@ func TestMemoryFS_Stats(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// testKey helper (addresses brittle keygen nit21 across tests).
|
// testKey helper for stable key generation across tests.
|
||||||
func testKey(i int) string {
|
func testKey(i int) string {
|
||||||
return fmt.Sprintf("test/key/%04d", i)
|
return fmt.Sprintf("test/key/%04d", i)
|
||||||
}
|
}
|
||||||
@@ -325,3 +376,44 @@ func TestMemoryFS_AllEvictStrategies(t *testing.T) {
|
|||||||
t.Error("post variant evict over cap")
|
t.Error("post variant evict over cap")
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// TestMemoryFS_EvictBoundedLargeN exercises the maxEvictBatch early-break logic (Idea #2)
|
||||||
|
// under a map size >> batch limit for the memory backend (parity with disk). Forces repeated
|
||||||
|
// eviction rounds and asserts progress. Covers bounded collection + repeated-call guarantee.
|
||||||
|
// Uses larger bytesNeeded per call for practical test runtime.
|
||||||
|
func TestMemoryFS_EvictBoundedLargeN(t *testing.T) {
|
||||||
|
if testing.Short() {
|
||||||
|
t.Skip()
|
||||||
|
}
|
||||||
|
t.Parallel()
|
||||||
|
cap := int64(128 * 1024)
|
||||||
|
m := New(cap)
|
||||||
|
const nFiles = 3000 // >> maxEvictBatch
|
||||||
|
const fSize = 128
|
||||||
|
for i := 0; i < nFiles; i++ {
|
||||||
|
k := fmt.Sprintf("mbig/%05d", i)
|
||||||
|
w, err := m.Create(k, fSize)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
w.Write(make([]byte, fSize))
|
||||||
|
w.Close()
|
||||||
|
if i%800 == 0 {
|
||||||
|
m.EvictLRU(4096)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
rounds := 0
|
||||||
|
totalEvicted := uint(0)
|
||||||
|
for m.Size() > m.Capacity() && rounds < 100 {
|
||||||
|
ev := m.EvictLRU(64 * 1024)
|
||||||
|
totalEvicted += ev
|
||||||
|
rounds++
|
||||||
|
if ev == 0 && rounds > 5 {
|
||||||
|
break
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if rounds < 2 {
|
||||||
|
t.Logf("memory large-N: %d rounds (evicted=%d final=%d)", rounds, totalEvicted, m.Size())
|
||||||
|
}
|
||||||
|
_ = totalEvicted
|
||||||
|
}
|
||||||
|
|||||||
Reference in New Issue
Block a user