Enhance DiskFS initialization and error handling
- Updated `disk.New` to support asynchronous initialization for large caches, improving responsiveness during startup. - Introduced an eviction function parameter to `disk.New`, ensuring proper handling of over-capacity scenarios. - Enhanced error handling in various components, including memory and disk tests, to ensure robustness and clarity. - Refactored tests to validate new behaviors, including checks for delayed attachment and proper error propagation. - Removed obsolete error handling code and tests related to the now-deleted errors package, streamlining the codebase.
This commit is contained in:
+168
-9
@@ -4,16 +4,23 @@ import (
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"s1d3sw1ped/steamcache2/vfs"
|
||||
)
|
||||
|
||||
func TestDiskFS_Basic(t *testing.T) {
|
||||
t.Parallel()
|
||||
td := t.TempDir()
|
||||
d := New(td, 10*1024*1024)
|
||||
d, err := New(td, 10*1024*1024, nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if d.Name() != "DiskFS" {
|
||||
t.Error("name")
|
||||
}
|
||||
@@ -45,10 +52,83 @@ func TestDiskFS_Basic(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// TestDiskFS_NewInvalidCapacity exercises the new error return (was panic) for ctor hygiene (Item 3 coverage).
|
||||
func TestDiskFS_NewInvalidCapacity(t *testing.T) {
|
||||
t.Parallel()
|
||||
td := t.TempDir()
|
||||
_, err := New(td, 0, nil)
|
||||
if err == nil {
|
||||
t.Fatal("expected error for capacity=0")
|
||||
}
|
||||
if !strings.Contains(err.Error(), "must be greater than 0") {
|
||||
t.Errorf("err %q missing 'must be greater than 0'", err)
|
||||
}
|
||||
|
||||
_, err = New(td, -1, nil)
|
||||
if err == nil || !strings.Contains(err.Error(), "must be greater than 0") {
|
||||
t.Errorf("negative capacity should return error containing phrase, got %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// TestDiskFS_InitPopulatesIndexOnRestart exercises the Item 1 fix: pre-populate disk dir (simulating restart with existing data),
|
||||
// call New, immediately verify Size + info/LRU are populated (so post-init Size + eviction see truth).
|
||||
func TestDiskFS_InitPopulatesIndexOnRestart(t *testing.T) {
|
||||
t.Parallel()
|
||||
td := t.TempDir()
|
||||
|
||||
// Pre-populate using raw FS ops (as prior run would have; simple keys -> direct paths under root)
|
||||
// Total 300 bytes > small cap below.
|
||||
prepare := func(key string, sz int64) {
|
||||
p := td + "/" + key
|
||||
if err := os.MkdirAll(td, 0755); err != nil {
|
||||
t.Fatalf("mkdir: %v", err)
|
||||
}
|
||||
if err := os.WriteFile(p, make([]byte, sz), 0644); err != nil {
|
||||
t.Fatalf("write %s: %v", key, err)
|
||||
}
|
||||
}
|
||||
prepare("f1", 100)
|
||||
prepare("f2", 200)
|
||||
|
||||
// Small cap so we are over; New launches bg populate (Size() blocks until done)
|
||||
d, err := New(td, 150, nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
if d.Size() != 300 {
|
||||
t.Errorf("Size after restart init = %d, want 300 (populated from disk)", d.Size())
|
||||
}
|
||||
if len(d.info) != 2 {
|
||||
t.Errorf("info len after init = %d, want 2", len(d.info))
|
||||
}
|
||||
if d.LRU.Len() != 2 {
|
||||
t.Errorf("LRU len after init = %d, want 2", d.LRU.Len())
|
||||
}
|
||||
|
||||
// Immediate discoverability (lazy still works but now warm)
|
||||
if _, err := d.Stat("f1"); err != nil {
|
||||
t.Error("stat f1 failed immediately after init pop")
|
||||
}
|
||||
|
||||
// Size > cap exercises the path where startup eviction would run at end of disk init (when GC algo provided via Set).
|
||||
if d.Size() <= d.Capacity() {
|
||||
t.Error("expected Size > Capacity to exercise over-cap path post-fix")
|
||||
}
|
||||
// Exercise eviction now has candidates thanks to population
|
||||
ev := d.EvictLRU(200)
|
||||
if ev == 0 {
|
||||
t.Error("EvictLRU did nothing despite over cap + populated LRU (startup eviction path would have failed before Item 1 fix)")
|
||||
}
|
||||
}
|
||||
|
||||
func TestDiskFS_EvictAndLazyStat(t *testing.T) {
|
||||
t.Parallel()
|
||||
td := t.TempDir()
|
||||
d := New(td, 400)
|
||||
d, err := New(td, 400, nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// create files that will be evicted
|
||||
keys := []string{}
|
||||
for i := 0; i < 5; i++ {
|
||||
@@ -85,7 +165,10 @@ func TestDiskFS_Concurrent(t *testing.T) {
|
||||
}
|
||||
t.Parallel()
|
||||
td := t.TempDir()
|
||||
d := New(td, 50*1024*1024)
|
||||
d, err := New(td, 50*1024*1024, nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var wg sync.WaitGroup
|
||||
var ops int64
|
||||
for i := 0; i < 4; i++ {
|
||||
@@ -128,7 +211,10 @@ func TestDiskFS_Concurrent(t *testing.T) {
|
||||
|
||||
func BenchmarkDiskFS_CreateOpen(b *testing.B) {
|
||||
td := b.TempDir()
|
||||
d := New(td, 128*1024*1024)
|
||||
d, err := New(td, 128*1024*1024, nil)
|
||||
if err != nil {
|
||||
b.Fatal(err)
|
||||
}
|
||||
data := make([]byte, 8192)
|
||||
b.ReportAllocs()
|
||||
b.ResetTimer()
|
||||
@@ -154,7 +240,10 @@ func BenchmarkDiskFS_CreateOpen(b *testing.B) {
|
||||
// Uses cycling keys via testKey for stable disk usage; exercises LRU path (other strategies lightly covered via tests + EvictHybrid uses DecayedScore).
|
||||
func BenchmarkDiskFS_EvictionUnderPressure(b *testing.B) {
|
||||
td := b.TempDir()
|
||||
d := New(td, 1*1024*1024)
|
||||
d, err := New(td, 1*1024*1024, nil)
|
||||
if err != nil {
|
||||
b.Fatal(err)
|
||||
}
|
||||
b.ReportAllocs()
|
||||
b.ResetTimer()
|
||||
for i := 0; i < b.N; i++ {
|
||||
@@ -175,7 +264,10 @@ func BenchmarkDiskFS_EvictionUnderPressure(b *testing.B) {
|
||||
func TestDiskFS_EvictVariantsAndInvalid(t *testing.T) {
|
||||
t.Parallel()
|
||||
td := t.TempDir()
|
||||
d := New(td, 600)
|
||||
d, err := New(td, 600, nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for i := 0; i < 4; i++ {
|
||||
w, _ := d.Create("dv"+string(rune('0'+i)), 120)
|
||||
w.Write(make([]byte, 120))
|
||||
@@ -211,7 +303,10 @@ func TestEvict_ConcurrentCloseDuringEviction(t *testing.T) {
|
||||
t.Parallel()
|
||||
td := t.TempDir()
|
||||
cap := int64(256 * 1024)
|
||||
d := New(td, cap)
|
||||
d, err := New(td, cap, nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var wg sync.WaitGroup
|
||||
const nWriters = 4
|
||||
const nEvictors = 3
|
||||
@@ -281,7 +376,10 @@ func TestDiskFS_EvictDiskVisibilityAndRecreateSafety(t *testing.T) {
|
||||
t.Parallel()
|
||||
td := t.TempDir()
|
||||
cap := int64(500)
|
||||
d := New(td, cap)
|
||||
d, err := New(td, cap, nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
created := []string{"v1", "v2", "v3", "s1"}
|
||||
for _, k := range created {
|
||||
sz := int64(150)
|
||||
@@ -352,7 +450,10 @@ func TestDiskFS_EvictBoundedLargeN(t *testing.T) {
|
||||
t.Parallel()
|
||||
td := t.TempDir()
|
||||
cap := int64(128 * 1024) // slightly larger for practicality
|
||||
d := New(td, cap)
|
||||
d, err := New(td, cap, nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
const nFiles = 3000 // > maxEvictBatch to exercise early-break on multiple rounds
|
||||
const fSize = 128
|
||||
for i := 0; i < nFiles; i++ {
|
||||
@@ -397,3 +498,61 @@ func TestDiskFS_EvictBoundedLargeN(t *testing.T) {
|
||||
}
|
||||
_ = totalEvicted
|
||||
}
|
||||
|
||||
// TestDiskFS_StartupEvictionFuncInvokedDuringInit covers the relocated guard path:
|
||||
// pre-populate over capacity, New with non-nil evict func (selected via Get), wait for init,
|
||||
// verify the func was invoked inside calculate (before close(initDone)) and size reduced.
|
||||
func TestDiskFS_StartupEvictionFuncInvokedDuringInit(t *testing.T) {
|
||||
t.Parallel()
|
||||
td := t.TempDir()
|
||||
|
||||
prepare := func(key string, sz int64) {
|
||||
p := td + "/" + key
|
||||
if err := os.MkdirAll(td, 0755); err != nil {
|
||||
t.Fatalf("mkdir: %v", err)
|
||||
}
|
||||
if err := os.WriteFile(p, make([]byte, sz), 0644); err != nil {
|
||||
t.Fatalf("write %s: %v", key, err)
|
||||
}
|
||||
}
|
||||
prepare("f1", 100)
|
||||
prepare("f2", 200)
|
||||
|
||||
// Use real eviction func (delegates to EvictLRU impl, as GC algos do) + pre-pop > cap.
|
||||
// Assert post-Size() (post-guard) that size was reduced to <= cap + index updated (Issue 4 coverage).
|
||||
evictFn := func(v vfs.VFS, b uint) uint {
|
||||
// real path: same as hybrid/lru would via the VFS methods (exercises lock, LRU remove, size adjust, os.Remove)
|
||||
if dd, ok := v.(*DiskFS); ok {
|
||||
return dd.EvictLRU(b)
|
||||
}
|
||||
return 0
|
||||
}
|
||||
d, err := New(td, 150, evictFn)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
_ = d.Size() // wait for bg init + guard (last step) + close
|
||||
if d.Size() > d.Capacity() {
|
||||
t.Errorf("startup guard with real evictFn did not reduce size: got %d > cap %d", d.Size(), d.Capacity())
|
||||
}
|
||||
// LRU/info updated by real evict; at least one file gone (original 2 files, 300B)
|
||||
if len(d.info) == 2 {
|
||||
t.Error("expected real eviction to have removed at least one over-cap file from index")
|
||||
}
|
||||
}
|
||||
|
||||
// TestDiskFS_NewMkdirError covers propagation of MkdirAll error from New (ctor now returns err; Issue 6).
|
||||
func TestDiskFS_NewMkdirError(t *testing.T) {
|
||||
t.Parallel()
|
||||
// Create a regular file at the path we will pass as "root dir"; MkdirAll will fail with "file exists" or perm.
|
||||
td := t.TempDir()
|
||||
badPath := filepath.Join(td, "notadir")
|
||||
if err := os.WriteFile(badPath, []byte("x"), 0644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
_, err := New(badPath, 1024, nil)
|
||||
if err == nil || !strings.Contains(err.Error(), "failed to create root directory") {
|
||||
t.Errorf("expected mkdir failure error for file-as-dir, got: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user