Enhance DiskFS initialization and error handling

- Updated `disk.New` to support asynchronous initialization for large caches, improving responsiveness during startup.
- Introduced an eviction function parameter to `disk.New`, ensuring proper handling of over-capacity scenarios.
- Enhanced error handling in various components, including memory and disk tests, to ensure robustness and clarity.
- Refactored tests to validate new behaviors, including checks for delayed attachment and proper error propagation.
- Removed obsolete error handling code and tests related to the now-deleted errors package, streamlining the codebase.
This commit is contained in:
2026-05-27 13:15:33 -05:00
parent 4861f93e6f
commit feda55e225
18 changed files with 584 additions and 1380 deletions
+62 -21
View File
@@ -7,8 +7,9 @@ import (
"io"
"net/http"
"net/http/httptest"
"os"
"path/filepath"
"runtime"
"s1d3sw1ped/steamcache2/steamcache/errors"
"s1d3sw1ped/steamcache2/vfs/eviction"
"s1d3sw1ped/steamcache2/vfs/memory"
"s1d3sw1ped/steamcache2/vfs/vfserror"
@@ -126,7 +127,7 @@ func TestCaching(t *testing.T) {
}
func TestCacheMissAndHit(t *testing.T) {
sc, err := New("localhost:8080", "0", "1G", t.TempDir(), "", "lru", "lru", 200, 5, "0", nil)
sc, err := New("localhost:8080", "1MB", "1G", t.TempDir(), "", "lru", "lru", 200, 5, "0", nil)
if err != nil {
t.Fatalf("failed to create SteamCache: %v", err)
}
@@ -369,7 +370,7 @@ func TestServiceManagerExpandability(t *testing.T) {
// Removed hash calculation tests since we switched to lightweight validation
func TestSteamKeySharding(t *testing.T) {
sc, err := New("localhost:8080", "0", "1G", t.TempDir(), "", "lru", "lru", 200, 5, "0", nil)
sc, err := New("localhost:8080", "1MB", "1G", t.TempDir(), "", "lru", "lru", 200, 5, "0", nil)
if err != nil {
t.Fatalf("failed to create SteamCache: %v", err)
}
@@ -471,23 +472,6 @@ func TestErrorTypes(t *testing.T) {
if vfsErr.Unwrap() != vfserror.ErrNotFound {
t.Error("VFS error should unwrap to the underlying error")
}
// Test SteamCache error
scErr := errors.NewSteamCacheError("test", "/test/url", "127.0.0.1", errors.ErrInvalidURL)
if scErr.Error() == "" {
t.Error("SteamCache error should have a message")
}
if scErr.Unwrap() != errors.ErrInvalidURL {
t.Error("SteamCache error should unwrap to the underlying error")
}
// Test retryable error detection
if !errors.IsRetryableError(errors.ErrUpstreamUnavailable) {
t.Error("Upstream unavailable should be retryable")
}
if errors.IsRetryableError(errors.ErrInvalidURL) {
t.Error("Invalid URL should not be retryable")
}
}
// TestMetrics tests the metrics functionality
@@ -868,6 +852,8 @@ func TestNewInvalidSizes(t *testing.T) {
{"0", "bad", "0", "invalid disk size"},
// maxObjectSize limit (zero default + basic coverage)
{"1MB", "0", "notasize", "invalid max object size"}, // bad value
// Covers the "no memory or disk" error path (was os.Exit, now clean error return per Item 3)
{"0", "0", "0", "no memory or disk cache configured"},
}
for _, c := range cases {
t.Run(c.mem+"_"+c.disk, func(t *testing.T) {
@@ -994,7 +980,10 @@ func TestP1_03_EvictionAlgorithmsDistinct(t *testing.T) {
t.Skip("LFU vs LRU vs Hybrid distinct behavior test; run explicitly when needed.")
// Create controlled candidates in a fresh memory FS for each strategy.
createAndEvict := func(algo string, bytesNeeded uint) (int, error) { // returns #evicted items approx via size delta
mfs := memory.New(250) // small cap < 300 to force evict on needed
mfs, err := memory.New(250) // small cap < 300 to force evict on needed
if err != nil {
return 0, err
}
// create 3 files of 100 bytes each via VFS Create (AccessCount=1 init)
for i := 0; i < 3; i++ {
w, err := mfs.Create(fmt.Sprintf("f%d", i), 100)
@@ -1025,3 +1014,55 @@ func TestP1_03_EvictionAlgorithmsDistinct(t *testing.T) {
// Size deltas may vary due to internal LRU during Create + exact thresholds; main goal is no crash + distinct code exercised (verified by coverage).
t.Logf("distinct eviction counts under controlled access: LRU=%d, LFU=%d, HYB=%d", evLRU, evLFU, evHYB)
}
// TestDiskOnlyDelayedAttach covers pure disk-only mode (mem=0 + disk>0) hitting the exact delayed attach path.
// During init window (pre Size barrier), TieredCache has no slow tier so Create returns ErrNotFound (proxy semantics, no disk caching).
// Post-barrier + attach, Create succeeds. Uses real temp dir (Issue 5).
func TestDiskOnlyDelayedAttach(t *testing.T) {
t.Parallel()
td := t.TempDir()
diskPath := filepath.Join(td, "disk")
if err := os.MkdirAll(diskPath, 0755); err != nil {
t.Fatal(err)
}
// mem=0, disk>0 -> pure disk delayed path (go func)
sc, err := New("localhost:0", "0", "10MB", diskPath, "", "lru", "lru", 10, 1, "0", nil)
if err != nil {
t.Fatalf("New disk-only: %v", err)
}
t.Cleanup(func() { sc.Shutdown() })
// Immediately in window: no slow tier attached yet -> Create must ErrNotFound (proxy, no disk write)
_, err = sc.vfs.Create("during-init-key", 100)
if err != vfserror.ErrNotFound {
t.Errorf("during init window, expected ErrNotFound from disk-only tiered Create (no slow), got %v", err)
}
// Wait the barrier (exercises the attach go's Size wait)
_ = sc.disk.Size()
// Now attached; Create should succeed (slow tier active). Retry briefly for go scheduler (attach go does Size then SetSlow).
var w io.WriteCloser
for i := 0; i < 100; i++ {
var cerr error
w, cerr = sc.vfs.Create("post-attach-key", 50)
if cerr == nil {
err = nil
break
}
err = cerr
time.Sleep(1 * time.Millisecond)
}
if err != nil {
t.Fatalf("post attach Create failed (slow tier not set after barrier?): %v", err)
}
w.Write([]byte("ok"))
w.Close()
// verify visible
if rc, err := sc.vfs.Open("post-attach-key"); err != nil || rc == nil {
t.Error("post-attach open failed")
} else {
rc.Close()
}
}