steamcache2/vfs/disk/disk_test.go

package disk

import (
	"fmt"
	"io"
	"os"
	"path/filepath"
	"strings"
	"sync"
	"sync/atomic"
	"testing"
	"time"

	"s1d3sw1ped/steamcache2/steamcache/metrics"
	"s1d3sw1ped/steamcache2/vfs"
)

func TestDiskFS_Basic(t *testing.T) {
	t.Parallel()
	td := t.TempDir()
	d, err := New(td, 10*1024*1024, nil)
	if err != nil {
		t.Fatal(err)
	}
	if d.Name() != "DiskFS" {
		t.Error("name")
	}

	w, err := d.Create("k1", 50)
	if err != nil {
		t.Fatal(err)
	}
	w.Write([]byte("hello disk cache test data here"))
	w.Close()

	if d.Size() < 30 { // actual may differ slightly from declared
		t.Errorf("size too small %d", d.Size())
	}

	r, err := d.Open("k1")
	if err != nil {
		t.Fatal(err)
	}
	data, _ := io.ReadAll(r)
	r.Close()
	if len(data) < 10 {
		t.Error("read small")
	}

	d.Delete("k1")
	if _, err := d.Open("k1"); err == nil {
		t.Error("deleted still readable")
	}
}

// TestDiskFS_NewInvalidCapacity exercises the new error return (was panic) for ctor hygiene (Item 3 coverage).
func TestDiskFS_NewInvalidCapacity(t *testing.T) {
	t.Parallel()
	td := t.TempDir()
	_, err := New(td, 0, nil)
	if err == nil {
		t.Fatal("expected error for capacity=0")
	}
	if !strings.Contains(err.Error(), "must be greater than 0") {
		t.Errorf("err %q missing 'must be greater than 0'", err)
	}

	_, err = New(td, -1, nil)
	if err == nil || !strings.Contains(err.Error(), "must be greater than 0") {
		t.Errorf("negative capacity should return error containing phrase, got %v", err)
	}
}

// TestDiskFS_InitPopulatesIndexOnRestart exercises the Item 1 fix: pre-populate disk dir (simulating restart with existing data),
// call New, immediately verify Size + info/LRU are populated (so post-init Size + eviction see truth).
func TestDiskFS_InitPopulatesIndexOnRestart(t *testing.T) {
	t.Parallel()
	td := t.TempDir()

	// Pre-populate using raw FS ops (as prior run would have; simple keys -> direct paths under root)
	// Total 300 bytes > small cap below.
	prepare := func(key string, sz int64) {
		p := td + "/" + key
		if err := os.MkdirAll(td, 0755); err != nil {
			t.Fatalf("mkdir: %v", err)
		}
		if err := os.WriteFile(p, make([]byte, sz), 0644); err != nil {
			t.Fatalf("write %s: %v", key, err)
		}
	}
	prepare("f1", 100)
	prepare("f2", 200)

	// Small cap so we are over; New launches bg populate (Size() blocks until done)
	d, err := New(td, 150, nil)
	if err != nil {
		t.Fatal(err)
	}

	if d.Size() != 300 {
		t.Errorf("Size after restart init = %d, want 300 (populated from disk)", d.Size())
	}
	if len(d.info) != 2 {
		t.Errorf("info len after init = %d, want 2", len(d.info))
	}
	if d.LRU.Len() != 2 {
		t.Errorf("LRU len after init = %d, want 2", d.LRU.Len())
	}

	// Immediate discoverability (lazy still works but now warm)
	if _, err := d.Stat("f1"); err != nil {
		t.Error("stat f1 failed immediately after init pop")
	}

	// Size > cap exercises the path where startup eviction would run at end of disk init (when GC algo provided via Set).
	if d.Size() <= d.Capacity() {
		t.Error("expected Size > Capacity to exercise over-cap path post-fix")
	}
	// Exercise eviction now has candidates thanks to population
	ev := d.EvictLRU(200)
	if ev == 0 {
		t.Error("EvictLRU did nothing despite over cap + populated LRU (startup eviction path would have failed before Item 1 fix)")
	}
}

func TestDiskFS_CapacityPressureOnEvict(t *testing.T) {
	t.Parallel()
	td := t.TempDir()
	d, err := New(td, 500, nil)
	if err != nil {
		t.Fatal(err)
	}
	_ = d.Size()
	met := metrics.NewMetrics()
	d.SetMetrics(met)
	for i := 0; i < 3; i++ {
		k := "f" + string(rune('0'+i))
		w, cerr := d.Create(k, 200)
		if cerr != nil {
			t.Fatal(cerr)
		}
		if _, werr := w.Write(make([]byte, 200)); werr != nil {
			t.Fatal(werr)
		}
		if cerr := w.Close(); cerr != nil {
			t.Fatal(cerr)
		}
	}
	evicted := d.EvictLRU(100)
	if evicted == 0 {
		t.Fatalf("expected eviction under cap, size=%d cap=%d", d.Size(), d.Capacity())
	}
	st := met.GetStats()
	if st.Evictions == 0 {
		t.Error("evictions counter not incremented under disk cap pressure")
	}
	if st.CapacityPressureEvents == 0 {
		t.Error("capacity_pressure_events not incremented under disk cap pressure")
	}
}

func TestDiskFS_EvictAndLazyStat(t *testing.T) {
	t.Parallel()
	td := t.TempDir()
	d, err := New(td, 400, nil)
	if err != nil {
		t.Fatal(err)
	}
	// create files that will be evicted
	keys := []string{}
	for i := 0; i < 5; i++ {
		k := "f" + string(rune('0'+i))
		keys = append(keys, k)
		w, _ := d.Create(k, 120)
		w.Write(make([]byte, 120))
		w.Close()
	}
	met := metrics.NewMetrics()
	d.SetMetrics(met)
	ev := d.EvictLRU(200)
	if ev == 0 {
		t.Log("no evict (size calc async or snapshot tolerance?)")
	}
	if ev > 0 {
		st := met.GetStats()
		if st.Evictions == 0 {
			t.Error("evictions counter not incremented after disk EvictLRU freed bytes")
		}
		if st.CapacityPressureEvents == 0 {
			t.Error("capacity_pressure_events not incremented after disk EvictLRU freed bytes")
		}
	}
	// Explicit post-evict consistency checks: for any key no longer visible via Stat, its on-disk
	// file must be absent (verifies coordinated unlink + no resurrection via lazy discovery).
	// Keys still present after this small evict are allowed (accounting tolerance in raw DiskFS).
	for _, k := range keys {
		if _, err := d.Stat(k); err != nil {
			p := d.pathForKey(k)
			if _, err2 := os.Stat(p); !os.IsNotExist(err2) {
				t.Errorf("key %s absent in Stat but stray file remains on disk at %s: %v", k, p, err2)
			}
		}
	}
	// lazy stat should still work for remaining; batch eviction may be approximate under heavy pressure
	if d.Size() > d.Capacity()*2 { // generous for async bg size
		t.Errorf("disk size %d >> cap after evict", d.Size())
	}
}

func TestDiskFS_Concurrent(t *testing.T) {
	if testing.Short() {
		t.Skip()
	}
	t.Parallel()
	td := t.TempDir()
	d, err := New(td, 50*1024*1024, nil)
	if err != nil {
		t.Fatal(err)
	}
	var wg sync.WaitGroup
	var ops int64
	for i := 0; i < 4; i++ {
		wg.Add(1)
		go func(id int) {
			defer wg.Done()
			for j := 0; j < 30; j++ {
				key := "d" + string(rune(id+'a')) + string(rune(j))
				w, e := d.Create(key, 256)
				if e == nil {
					w.Write(make([]byte, 256))
					w.Close()
					atomic.AddInt64(&ops, 1)
				}
				if r, e := d.Open(key); e == nil {
					io.Copy(io.Discard, r)
					r.Close()
					atomic.AddInt64(&ops, 1)
				}
				d.Delete(key)
				if j%7 == 0 {
					d.EvictLRU(1024)
				}
			}
		}(i)
	}
	wg.Wait()
	// Bounded poll instead of fixed sleep for bg size calc goroutine settlement (robust to variance).
	deadline := time.Now().Add(300 * time.Millisecond)
	for time.Now().Before(deadline) {
		if d.Size() <= d.Capacity() {
			break
		}
		time.Sleep(5 * time.Millisecond)
	}
	if d.Size() > d.Capacity() {
		t.Errorf("concurrent disk size exceeded: %d", d.Size())
	}
}

func BenchmarkDiskFS_CreateOpen(b *testing.B) {
	td := b.TempDir()
	d, err := New(td, 128*1024*1024, nil)
	if err != nil {
		b.Fatal(err)
	}
	data := make([]byte, 8192)
	b.ReportAllocs()
	b.ResetTimer()
	for i := 0; i < b.N; i++ {
		key := testKey(i % 500)
		w, err := d.Create(key, 8192)
		if err != nil {
			b.Fatal(err)
		}
		w.Write(data)
		w.Close()
		r, err := d.Open(key)
		if err != nil {
			b.Fatal(err)
		}
		io.Copy(io.Discard, r)
		r.Close()
		d.Delete(key)
	}
}

// BenchmarkDiskFS_EvictionUnderPressure exercises disk eviction under synthetic pressure (mirrors memory version for parity).
// Uses cycling keys via testKey for stable disk usage; exercises LRU path (other strategies lightly covered via tests + EvictHybrid uses DecayedScore).
func BenchmarkDiskFS_EvictionUnderPressure(b *testing.B) {
	td := b.TempDir()
	d, err := New(td, 1*1024*1024, nil)
	if err != nil {
		b.Fatal(err)
	}
	b.ReportAllocs()
	b.ResetTimer()
	for i := 0; i < b.N; i++ {
		// fill then evict (setup fill not timed separately to keep bench focused on pressure+evict cycle)
		for j := 0; j < 20; j++ {
			w, err := d.Create(testKey(j), 64*1024)
			if err != nil {
				b.Fatal(err)
			}
			w.Write(make([]byte, 64*1024))
			w.Close()
		}
		d.EvictLRU(512 * 1024)
	}
	_ = d // keep
}

func TestDiskFS_EvictVariantsAndInvalid(t *testing.T) {
	t.Parallel()
	td := t.TempDir()
	d, err := New(td, 600, nil)
	if err != nil {
		t.Fatal(err)
	}
	for i := 0; i < 4; i++ {
		w, _ := d.Create("dv"+string(rune('0'+i)), 120)
		w.Write(make([]byte, 120))
		w.Close()
	}
	_ = d.EvictBySize(80, false) // largest
	_ = d.EvictFIFO(50)
	_ = d.EvictLFU(30)
	_ = d.EvictHybrid(30)

	// invalids (sanitized in Create/Open)
	if _, err := d.Create("", 1); err == nil {
		t.Error("empty")
	}
	if _, err := d.Create("/abs/bad", 1); err == nil {
		t.Error("abs")
	}
	if _, err := d.Open("missing"); err == nil {
		t.Error("missing open")
	}
	_ = d.Delete("missing")
	_, _ = d.Stat("missing")
}

// TestEvict_ConcurrentCloseDuringEviction exercises Creates, Opens, and Closes (which mutate *FileInfo and size under lock)
// concurrently with all Evict* (LRU + non-LRU scalar snapshot paths) on DiskFS under pressure.
// Sufficient goroutines/iterations to exercise snapshot + re-fetch + close-during-evict paths. Asserts size invariant with
// documented epsilon tolerance for raw DiskFS (background size calc + snapshot tolerance during batch eviction). -race must pass.
func TestEvict_ConcurrentCloseDuringEviction(t *testing.T) {
	if testing.Short() {
		t.Skip()
	}
	t.Parallel()
	td := t.TempDir()
	cap := int64(256 * 1024)
	d, err := New(td, cap, nil)
	if err != nil {
		t.Fatal(err)
	}
	var wg sync.WaitGroup
	const nWriters = 4
	const nEvictors = 3
	const iters = 25
	for i := 0; i < nWriters; i++ {
		wg.Add(1)
		go func(id int) {
			defer wg.Done()
			for j := 0; j < iters; j++ {
				key := "r" + string(rune('0'+id%5)) + "/" + string(rune('0'+j%10))
				w, err := d.Create(key, 8192)
				if err == nil {
					w.Write(make([]byte, 4096))
					w.Close() // exercises Close size mutation path concurrent with evicts
				}
				if r, err := d.Open(key); err == nil {
					io.Copy(io.Discard, r)
					r.Close()
				}
				if j%4 == 0 {
					d.Delete(key)
				}
			}
		}(i)
	}
	for i := 0; i < nEvictors; i++ {
		wg.Add(1)
		go func(id int) {
			defer wg.Done()
			for j := 0; j < iters*2; j++ {
				// Cycle through strategies to cover all snapshot + re-fetch + LRU-Lock paths
				switch j % 6 {
				case 0:
					d.EvictLRU(4096)
				case 1:
					d.EvictBySize(4096, true)
				case 2:
					d.EvictBySize(4096, false)
				case 3:
					d.EvictFIFO(4096)
				case 4:
					d.EvictLFU(4096)
				default:
					d.EvictHybrid(4096)
				}
			}
		}(i)
	}
	wg.Wait()
	// Final size <= cap with epsilon (raw DiskFS allows small over per bg size + snapshot design; see TestDiskFS_Concurrent and memory +50 pattern)
	if sz := d.Size(); sz > cap+2048 {
		t.Errorf("final size %d exceeded cap %d + epsilon tolerance after concurrent close+evict", sz, cap)
	}
}

// testKey helper for stable key generation across tests.
func testKey(i int) string {
	return fmt.Sprintf("test/key/%04d", i)
}

// TestDiskFS_EvictDiskVisibilityAndRecreateSafety verifies that after eviction the on-disk
// artifacts for victims are immediately gone (no resurrection via lazy discovery in Stat/Open),
// and that recreating the same key produces independent content that is not subject to any
// stale eviction unlinks. This exercises the coordinated WLock remove path for DiskFS.
// Create does not wait on initDone, so this also covers insertBatch racing with eviction:
// gone files must not be re-indexed (Stat present / disk missing).
func TestDiskFS_EvictDiskVisibilityAndRecreateSafety(t *testing.T) {
	t.Parallel()
	td := t.TempDir()
	cap := int64(500)
	d, err := New(td, cap, nil)
	if err != nil {
		t.Fatal(err)
	}
	created := []string{"v1", "v2", "v3", "s1"}
	for _, k := range created {
		sz := int64(150)
		if k == "s1" {
			sz = 50
		}
		w, err := d.Create(k, sz)
		if err != nil {
			t.Fatal(err)
		}
		w.Write(make([]byte, sz))
		w.Close()
	}

	// Force eviction pressure with large request; repeat to handle batching + approx accounting.
	for i := 0; i < 5; i++ {
		_ = d.EvictLRU(1024 * 1024)
		_ = d.EvictBySize(1024*1024, true)
	}

	// Drain bg population so insertBatch cannot still be in flight when we audit.
	_ = d.Size()

	// Consistency: Stat success iff the file exists on disk. insertBatch must not resurrect
	// keys whose backing files were already evicted.
	for _, k := range created {
		p := d.pathForKey(k)
		_, statErr := d.Stat(k)
		_, diskErr := os.Stat(p)
		if statErr != nil {
			if !os.IsNotExist(diskErr) {
				t.Errorf("key %s absent via Stat but file lingers on disk at %s (resurrection risk)", k, p)
			}
		} else if diskErr != nil {
			t.Errorf("key %s present via Stat but missing on disk: %v", k, diskErr)
		}
	}

	// Recreate one that is currently absent (or any): must work, and new content must not be
	// subject to stale unlinks (guaranteed by inside-WLock removes on evict + keyMu on Create).
	k := "v1"
	w2, err := d.Create(k, 40)
	if err != nil {
		t.Fatalf("recreate %s failed: %v", k, err)
	}
	w2.Write([]byte("fresh-after-evict"))
	w2.Close()
	p := d.pathForKey(k)
	if st, err := os.Stat(p); err != nil || st.Size() < 10 {
		t.Errorf("recreated %s disk state bad: size=%v err=%v", k, st, err)
	}
	if r, err := d.Open(k); err != nil {
		t.Errorf("recreated %s not readable: %v", k, err)
	} else {
		r.Close()
	}
}

// TestDiskFS_InsertBatchSkipsGoneFiles is the fail-closed contract for bg/lazy index
// insert: a discoveredFile whose path was removed (evicted) must not be re-inserted
// into d.info. That resurrection is what made Stat succeed while os.Stat failed.
func TestDiskFS_InsertBatchSkipsGoneFiles(t *testing.T) {
	t.Parallel()
	td := t.TempDir()
	d, err := New(td, 10*1024*1024, nil)
	if err != nil {
		t.Fatal(err)
	}
	_ = d.Size() // finish constructor scan so it cannot also index these keys

	liveKey := "live"
	goneKey := "gone"
	writeKey := func(key, body string) os.FileInfo {
		t.Helper()
		p := d.pathForKey(key)
		if err := os.MkdirAll(filepath.Dir(p), 0700); err != nil {
			t.Fatal(err)
		}
		if err := os.WriteFile(p, []byte(body), 0600); err != nil {
			t.Fatal(err)
		}
		st, err := os.Stat(p)
		if err != nil {
			t.Fatal(err)
		}
		return st
	}
	liveInfo := writeKey(liveKey, "still-here")
	goneInfo := writeKey(goneKey, "about-to-vanish")
	if err := os.Remove(d.pathForKey(goneKey)); err != nil {
		t.Fatal(err)
	}

	d.insertBatch([]discoveredFile{
		{key: liveKey, size: liveInfo.Size(), osInfo: liveInfo},
		{key: goneKey, size: goneInfo.Size(), osInfo: goneInfo},
	})

	d.mu.RLock()
	_, liveExists := d.info[liveKey]
	_, goneExists := d.info[goneKey]
	d.mu.RUnlock()
	if !liveExists {
		t.Errorf("insertBatch skipped live key %s", liveKey)
	}
	if goneExists {
		t.Errorf("insertBatch resurrected gone key %s", goneKey)
	}
	if _, err := d.Stat(goneKey); err == nil {
		t.Errorf("Stat succeeded for gone key %s", goneKey)
	}
	if _, err := os.Stat(d.pathForKey(liveKey)); err != nil {
		t.Errorf("live key %s missing on disk: %v", liveKey, err)
	}
}

// TestDiskFS_EvictBoundedLargeN exercises the maxEvictBatch early-break logic (Idea #2)
// under a map size >> batch limit. Forces repeated eviction rounds via GC-style pressure
// and asserts progress + consistency (no resurrection/orphans). Covers bounded collection
// for the non-LRU (and LRU) paths. Tolerant of raw DiskFS bg size + approx accounting.
func TestDiskFS_EvictBoundedLargeN(t *testing.T) {
	if testing.Short() {
		t.Skip()
	}
	t.Parallel()
	td := t.TempDir()
	cap := int64(128 * 1024) // slightly larger for practicality
	d, err := New(td, cap, nil)
	if err != nil {
		t.Fatal(err)
	}
	const nFiles = 3000 // > maxEvictBatch to exercise early-break on multiple rounds
	const fSize = 128
	for i := 0; i < nFiles; i++ {
		k := fmt.Sprintf("big/%05d", i)
		w, err := d.Create(k, fSize)
		if err != nil {
			t.Fatal(err)
		}
		w.Write(make([]byte, fSize))
		w.Close()
		if i%800 == 0 {
			d.EvictLRU(4096)
		}
	}
	// Drive reclamation with larger per-call request (to exercise meaningful batches quickly).
	rounds := 0
	totalEvicted := uint(0)
	for d.Size() > d.Capacity() && rounds < 100 {
		ev := d.EvictLRU(64 * 1024)
		totalEvicted += ev
		rounds++
		if ev == 0 && rounds > 5 {
			break
		}
	}
	// Progress + no-hang is the goal; final size check tolerant for DiskFS bg/snapshot design.
	finalSize := d.Size()
	if rounds < 2 {
		t.Logf("large-N disk: completed with %d rounds (evicted=%d final=%d)", rounds, totalEvicted, finalSize)
	}
	// Spot-check consistency (if Stat ok => disk ok; if Stat not => disk absent). Catches resurrection.
	for i := 0; i < 5; i++ {
		k := fmt.Sprintf("big/%05d", i*600)
		p := d.pathForKey(k)
		if _, err := d.Stat(k); err == nil {
			if _, err2 := os.Stat(p); err2 != nil {
				t.Errorf("in-index %s missing on disk: %v", k, err2)
			}
		} else if _, err2 := os.Stat(p); !os.IsNotExist(err2) {
			t.Errorf("absent %s has stray disk file: %v", k, err2)
		}
	}
	_ = totalEvicted
}

// TestDiskFS_StartupEvictionFuncInvokedDuringInit covers the relocated guard path:
// pre-populate over capacity, New with non-nil evict func (selected via Get), wait for init,
// verify the func was invoked inside calculate (before close(initDone)) and size reduced.
func TestDiskFS_StartupEvictionFuncInvokedDuringInit(t *testing.T) {
	t.Parallel()
	td := t.TempDir()

	prepare := func(key string, sz int64) {
		p := td + "/" + key
		if err := os.MkdirAll(td, 0755); err != nil {
			t.Fatalf("mkdir: %v", err)
		}
		if err := os.WriteFile(p, make([]byte, sz), 0644); err != nil {
			t.Fatalf("write %s: %v", key, err)
		}
	}
	prepare("f1", 100)
	prepare("f2", 200)

	// Use real eviction func (delegates to EvictLRU impl, as GC algos do) + pre-pop > cap.
	// Assert post-Size() (post-guard) that size was reduced to <= cap + index updated (Issue 4 coverage).
	evictFn := func(v vfs.VFS, b uint) uint {
		// real path: same as hybrid/lru would via the VFS methods (exercises lock, LRU remove, size adjust, os.Remove)
		if dd, ok := v.(*DiskFS); ok {
			return dd.EvictLRU(b)
		}
		return 0
	}
	d, err := New(td, 150, evictFn)
	if err != nil {
		t.Fatal(err)
	}

	_ = d.Size() // wait for bg init + guard (last step) + close
	if d.Size() > d.Capacity() {
		t.Errorf("startup guard with real evictFn did not reduce size: got %d > cap %d", d.Size(), d.Capacity())
	}
	// LRU/info updated by real evict; at least one file gone (original 2 files, 300B)
	if len(d.info) == 2 {
		t.Error("expected real eviction to have removed at least one over-cap file from index")
	}
}

// TestDiskFS_NewMkdirError covers propagation of MkdirAll error from New (ctor now returns err; Issue 6).
func TestDiskFS_NewMkdirError(t *testing.T) {
	t.Parallel()
	// Create a regular file at the path we will pass as "root dir"; MkdirAll will fail with "file exists" or perm.
	td := t.TempDir()
	badPath := filepath.Join(td, "notadir")
	if err := os.WriteFile(badPath, []byte("x"), 0644); err != nil {
		t.Fatal(err)
	}
	_, err := New(badPath, 1024, nil)
	if err == nil || !strings.Contains(err.Error(), "failed to create root directory") {
		t.Errorf("expected mkdir failure error for file-as-dir, got: %v", err)
	}
}

// TestDiskFS_InitHoldBlocksOnlyRegisteredRoot covers the per-root init hold:
// Size() stays blocked while the hold is open, and a DiskFS on a different root
// does not wait on that hold.
func TestDiskFS_InitHoldBlocksOnlyRegisteredRoot(t *testing.T) {
	td := t.TempDir()
	hold := make(chan struct{})
	var holdOnce sync.Once
	closeHold := func() { holdOnce.Do(func() { close(hold) }) }
	RegisterInitHold(td, hold)
	t.Cleanup(func() {
		closeHold()
		ClearInitHold(td)
	})

	d, err := New(td, 10*1024*1024, nil)
	if err != nil {
		t.Fatal(err)
	}

	blocked := make(chan struct{})
	go func() {
		_ = d.Size()
		close(blocked)
	}()
	select {
	case <-blocked:
		t.Fatal("Size returned while init hold still open")
	case <-time.After(50 * time.Millisecond):
	}

	td2 := t.TempDir()
	d2, err := New(td2, 10*1024*1024, nil)
	if err != nil {
		t.Fatal(err)
	}
	other := make(chan struct{})
	go func() {
		_ = d2.Size()
		close(other)
	}()
	select {
	case <-other:
	case <-time.After(2 * time.Second):
		t.Fatal("unrelated DiskFS Size hung; init hold leaked across roots")
	}

	closeHold()
	select {
	case <-blocked:
	case <-time.After(2 * time.Second):
		t.Fatal("Size did not return after init hold released")
	}
}