s1d3sw1ped
feda55e225
- Updated `disk.New` to support asynchronous initialization for large caches, improving responsiveness during startup. - Introduced an eviction function parameter to `disk.New`, ensuring proper handling of over-capacity scenarios. - Enhanced error handling in various components, including memory and disk tests, to ensure robustness and clarity. - Refactored tests to validate new behaviors, including checks for delayed attachment and proper error propagation. - Removed obsolete error handling code and tests related to the now-deleted errors package, streamlining the codebase.
477 lines
11 KiB
Go
477 lines
11 KiB
Go
package memory
|
|
|
|
import (
|
|
"fmt"
|
|
"io"
|
|
"strings"
|
|
"sync"
|
|
"sync/atomic"
|
|
"testing"
|
|
"time"
|
|
)
|
|
|
|
func TestMemoryFS_Basic(t *testing.T) {
|
|
t.Parallel()
|
|
m, err := New(1024 * 1024)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if m.Name() != "MemoryFS" {
|
|
t.Error("bad name")
|
|
}
|
|
if m.Capacity() != 1024*1024 {
|
|
t.Error("bad cap")
|
|
}
|
|
|
|
w, err := m.Create("k1", 100)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
n, _ := w.Write(make([]byte, 100))
|
|
w.Close()
|
|
if n != 100 {
|
|
t.Error("write len")
|
|
}
|
|
if m.Size() != 100 {
|
|
t.Errorf("size=%d want 100", m.Size())
|
|
}
|
|
|
|
r, err := m.Open("k1")
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
data, _ := io.ReadAll(r)
|
|
r.Close()
|
|
if len(data) != 100 {
|
|
t.Error("read mismatch")
|
|
}
|
|
|
|
if err := m.Delete("k1"); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if _, err := m.Open("k1"); err == nil {
|
|
t.Error("deleted key still openable")
|
|
}
|
|
}
|
|
|
|
func TestMemoryFS_EvictUnderPressure(t *testing.T) {
|
|
t.Parallel()
|
|
m, err := New(500)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
// create 3x200 = 600 >500, should trigger internal? but direct evict call
|
|
for i := 0; i < 3; i++ {
|
|
w, _ := m.Create("f"+string(rune('0'+i)), 200)
|
|
w.Write(make([]byte, 200))
|
|
w.Close()
|
|
}
|
|
// force evict
|
|
evicted := m.EvictLRU(100)
|
|
if evicted == 0 || m.Size() > 500 {
|
|
t.Errorf("evict failed: evicted=%d size=%d", evicted, m.Size())
|
|
}
|
|
}
|
|
|
|
func TestMemoryFS_SizeNeverExceedsAfterEvict(t *testing.T) {
|
|
t.Parallel()
|
|
cap := int64(1000)
|
|
m, err := New(cap)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
// Cycle through strategies (randomized feel via mod), use testKey, stricter post-evict with documented epsilon.
|
|
strats := []func(uint) uint{m.EvictLRU, func(n uint) uint { return m.EvictBySize(n, true) }, m.EvictFIFO, m.EvictLFU, m.EvictHybrid}
|
|
for i := 0; i < 50; i++ { // more cycles
|
|
sz := int64(100 + i%50)
|
|
w, err := m.Create(testKey(i), sz)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
w.Write(make([]byte, sz))
|
|
w.Close()
|
|
// Raw MemoryFS allows temporary over (enforced by GCFS wrapper in real use).
|
|
// Force evict under pressure and verify post-evict invariant.
|
|
if m.Size() > cap-50 {
|
|
fn := strats[i%len(strats)]
|
|
fn(200)
|
|
if m.Size() > cap+50 { // RLock snapshot + batch may temporarily exceed; GC layer enforces strict limit
|
|
t.Fatalf("size %d >> cap %d after evict", m.Size(), cap)
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
func TestMemoryFS_ConcurrentCreateOpenDelete(t *testing.T) {
|
|
if testing.Short() {
|
|
t.Skip()
|
|
}
|
|
t.Parallel()
|
|
m, err := New(10 * 1024 * 1024)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
var wg sync.WaitGroup
|
|
const N = 50
|
|
var ops int64
|
|
for i := 0; i < 8; i++ {
|
|
wg.Add(1)
|
|
go func(id int) {
|
|
defer wg.Done()
|
|
for j := 0; j < N; j++ {
|
|
key := "c" + string(rune('a'+id)) + string(rune(j%10))
|
|
w, err := m.Create(key, 128)
|
|
if err == nil {
|
|
w.Write(make([]byte, 128))
|
|
w.Close()
|
|
atomic.AddInt64(&ops, 1)
|
|
}
|
|
if r, err := m.Open(key); err == nil {
|
|
io.Copy(io.Discard, r)
|
|
r.Close()
|
|
atomic.AddInt64(&ops, 1)
|
|
}
|
|
_ = m.Delete(key)
|
|
atomic.AddInt64(&ops, 1)
|
|
if j%10 == 0 {
|
|
m.EvictLRU(256)
|
|
}
|
|
}
|
|
}(i)
|
|
}
|
|
wg.Wait()
|
|
if ops < 100 {
|
|
t.Errorf("too few concurrent ops: %d", ops)
|
|
}
|
|
// size should be bounded
|
|
if m.Size() > m.Capacity() {
|
|
t.Errorf("final size %d > cap", m.Size())
|
|
}
|
|
}
|
|
|
|
func BenchmarkMemoryFS_CreateOpen(b *testing.B) {
|
|
m, err := New(64 * 1024 * 1024)
|
|
if err != nil {
|
|
b.Fatal(err)
|
|
}
|
|
data := make([]byte, 4096)
|
|
b.ReportAllocs()
|
|
b.ResetTimer()
|
|
for i := 0; i < b.N; i++ {
|
|
key := testKey(i % 1000)
|
|
w, err := m.Create(key, 4096)
|
|
if err != nil {
|
|
b.Fatal(err)
|
|
}
|
|
w.Write(data)
|
|
w.Close()
|
|
r, err := m.Open(key)
|
|
if err != nil {
|
|
b.Fatal(err)
|
|
}
|
|
io.Copy(io.Discard, r)
|
|
r.Close()
|
|
_ = m.Delete(key)
|
|
}
|
|
}
|
|
|
|
// BenchmarkMemoryFS_EvictionUnderPressure exercises memory eviction under synthetic pressure (parallels BenchmarkDiskFS_EvictionUnderPressure).
|
|
// Uses cycling keys via testKey for stable behavior; exercises LRU path (other strategies lightly covered via existing tests + EvictHybrid uses DecayedScore).
|
|
func BenchmarkMemoryFS_EvictionUnderPressure(b *testing.B) {
|
|
m, err := New(1 * 1024 * 1024)
|
|
if err != nil {
|
|
b.Fatal(err)
|
|
}
|
|
b.ReportAllocs()
|
|
b.ResetTimer()
|
|
for i := 0; i < b.N; i++ {
|
|
// fill then evict (setup fill not timed separately to keep bench focused on pressure+evict cycle)
|
|
for j := 0; j < 20; j++ {
|
|
w, err := m.Create(testKey(j), 64*1024)
|
|
if err != nil {
|
|
b.Fatal(err)
|
|
}
|
|
w.Write(make([]byte, 64*1024))
|
|
w.Close()
|
|
}
|
|
m.EvictLRU(512 * 1024)
|
|
}
|
|
_ = m // keep
|
|
}
|
|
|
|
// BenchmarkMemoryFS_EvictBySizeUnderPressure parallels the disk eviction strategy testing.
|
|
// Exercises EvictBySize under repeated pressure.
|
|
func BenchmarkMemoryFS_EvictBySizeUnderPressure(b *testing.B) {
|
|
m, err := New(1 * 1024 * 1024)
|
|
if err != nil {
|
|
b.Fatal(err)
|
|
}
|
|
b.ReportAllocs()
|
|
b.ResetTimer()
|
|
for i := 0; i < b.N; i++ {
|
|
for j := 0; j < 20; j++ {
|
|
w, err := m.Create(testKey(j), 64*1024)
|
|
if err != nil {
|
|
b.Fatal(err)
|
|
}
|
|
w.Write(make([]byte, 64*1024))
|
|
w.Close()
|
|
}
|
|
m.EvictBySize(512*1024, true) // ascending = evict smallest first
|
|
}
|
|
_ = m // keep
|
|
}
|
|
|
|
// BenchmarkMemoryFS_EvictHybridUnderPressure exercises the hybrid strategy (which uses
|
|
// the centralized DecayedScore) under pressure. Provides coverage for the time-decayed scoring.
|
|
func BenchmarkMemoryFS_EvictHybridUnderPressure(b *testing.B) {
|
|
m, err := New(1 * 1024 * 1024)
|
|
if err != nil {
|
|
b.Fatal(err)
|
|
}
|
|
b.ReportAllocs()
|
|
b.ResetTimer()
|
|
for i := 0; i < b.N; i++ {
|
|
for j := 0; j < 20; j++ {
|
|
w, err := m.Create(testKey(j), 64*1024)
|
|
if err != nil {
|
|
b.Fatal(err)
|
|
}
|
|
w.Write(make([]byte, 64*1024))
|
|
w.Close()
|
|
}
|
|
m.EvictHybrid(512 * 1024)
|
|
}
|
|
_ = m // keep
|
|
}
|
|
|
|
func TestMemoryFS_Stats(t *testing.T) {
|
|
t.Parallel()
|
|
m, err := New(1024)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
stats := m.GetFragmentationStats()
|
|
if stats["buffer_count"] != 0 {
|
|
t.Error("initial buffers >0?")
|
|
}
|
|
}
|
|
|
|
// testKey helper for stable key generation across tests.
|
|
func testKey(i int) string {
|
|
return fmt.Sprintf("test/key/%04d", i)
|
|
}
|
|
|
|
// TestMemoryFS_ConcurrentCloseAndEvict_RaceFree is a synthetic load test exercising concurrent Close during eviction (validates the R/W split fixes).
|
|
// Exercises overlapping writer Close() (mutates fi.Size under W) + all Evict* strategies under load.
|
|
// Must be -race clean; also strengthens property coverage.
|
|
func TestMemoryFS_ConcurrentCloseAndEvict_RaceFree(t *testing.T) {
|
|
if testing.Short() {
|
|
t.Skip()
|
|
}
|
|
t.Parallel()
|
|
m, err := New(2 * 1024 * 1024) // 2MB
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
var wg sync.WaitGroup
|
|
stopCh := make(chan struct{})
|
|
const writers = 3
|
|
const evictors = 3
|
|
|
|
// Writers: create + write + close (triggers size mutation in Close)
|
|
for i := 0; i < writers; i++ {
|
|
wg.Add(1)
|
|
go func(id int) {
|
|
defer wg.Done()
|
|
for j := 0; ; j++ {
|
|
select {
|
|
case <-stopCh:
|
|
return
|
|
default:
|
|
}
|
|
key := testKey(id*10000 + j)
|
|
w, err := m.Create(key, 4096)
|
|
if err == nil {
|
|
w.Write(make([]byte, 4096))
|
|
w.Close() // mutates live *FileInfo.Size + global size (race target)
|
|
}
|
|
if j%5 == 0 {
|
|
m.Delete(key)
|
|
}
|
|
if j > 100 {
|
|
break // bound per writer
|
|
}
|
|
}
|
|
}(i)
|
|
}
|
|
|
|
// Evictors: hammer all 5 strategies + LRU (exercises snapshot copy + live re-fetch + short LRU Lock)
|
|
strats := []func(uint) uint{
|
|
m.EvictLRU,
|
|
func(n uint) uint { return m.EvictBySize(n, true) },
|
|
func(n uint) uint { return m.EvictBySize(n, false) },
|
|
m.EvictFIFO,
|
|
m.EvictLFU,
|
|
m.EvictHybrid,
|
|
}
|
|
for i := 0; i < evictors; i++ {
|
|
wg.Add(1)
|
|
go func(id int) {
|
|
defer wg.Done()
|
|
for j := 0; ; j++ {
|
|
select {
|
|
case <-stopCh:
|
|
return
|
|
default:
|
|
}
|
|
s := strats[j%len(strats)]
|
|
s(1024)
|
|
if j > 50 {
|
|
break
|
|
}
|
|
}
|
|
}(i)
|
|
}
|
|
|
|
time.Sleep(150 * time.Millisecond) // load duration; bounded
|
|
close(stopCh)
|
|
wg.Wait()
|
|
|
|
// Post-run invariants (loose due to raw MemoryFS overcommit design; GCFS enforces)
|
|
if m.Size() < 0 {
|
|
t.Error("negative size after concurrent close+evict")
|
|
}
|
|
// LRU len reasonable
|
|
_ = m.LRU.Len()
|
|
}
|
|
|
|
func TestMemoryFS_EvictVariantsAndErrors(t *testing.T) {
|
|
t.Parallel()
|
|
m, err := New(800)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
// populate
|
|
for i := 0; i < 4; i++ {
|
|
w, _ := m.Create("ev"+string(rune('0'+i)), 150)
|
|
w.Write(make([]byte, 150))
|
|
w.Close()
|
|
}
|
|
_ = m.EvictBySize(100, true) // smallest
|
|
_ = m.EvictFIFO(50)
|
|
_ = m.EvictLFU(50)
|
|
_ = m.EvictHybrid(50)
|
|
|
|
// invalid keys
|
|
if _, err := m.Create("", 1); err == nil {
|
|
t.Error("empty key allowed")
|
|
}
|
|
if _, err := m.Create("/abs", 1); err == nil {
|
|
t.Error("abs key allowed")
|
|
}
|
|
if _, err := m.Create("..bad", 1); err == nil {
|
|
t.Error("traversal key allowed")
|
|
}
|
|
if _, err := m.Open("nope"); err == nil {
|
|
t.Error("open missing")
|
|
}
|
|
if err := m.Delete("nope"); err == nil {
|
|
t.Error("delete missing")
|
|
}
|
|
if _, err := m.Stat("nope"); err == nil {
|
|
t.Error("stat missing")
|
|
}
|
|
// overwrite path + actual size update via closer
|
|
w2, _ := m.Create("ow", 10)
|
|
w2.Write([]byte{1, 2, 3})
|
|
w2.Close() // updates to real 3
|
|
if fi, _ := m.Stat("ow"); fi.Size != 3 {
|
|
t.Errorf("overwrite size %d !=3", fi.Size)
|
|
}
|
|
// hit fragmentation stats after activity
|
|
_ = m.GetFragmentationStats()
|
|
}
|
|
|
|
func TestMemoryFS_AllEvictStrategies(t *testing.T) {
|
|
t.Parallel()
|
|
m, err := New(300)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
for i := 0; i < 3; i++ {
|
|
w, _ := m.Create("s"+string(rune(i)), 120)
|
|
w.Write(make([]byte, 120))
|
|
w.Close()
|
|
}
|
|
_ = m.EvictBySize(50, true)
|
|
_ = m.EvictBySize(50, false)
|
|
_ = m.EvictFIFO(20)
|
|
_ = m.EvictLFU(20)
|
|
_ = m.EvictHybrid(20)
|
|
if m.Size() > m.Capacity() {
|
|
t.Error("post variant evict over cap")
|
|
}
|
|
}
|
|
|
|
// TestMemoryFS_EvictBoundedLargeN exercises the maxEvictBatch early-break logic (Idea #2)
|
|
// under a map size >> batch limit for the memory backend (parity with disk). Forces repeated
|
|
// eviction rounds and asserts progress. Covers bounded collection + repeated-call guarantee.
|
|
// Uses larger bytesNeeded per call for practical test runtime.
|
|
func TestMemoryFS_EvictBoundedLargeN(t *testing.T) {
|
|
if testing.Short() {
|
|
t.Skip()
|
|
}
|
|
t.Parallel()
|
|
cap := int64(128 * 1024)
|
|
m, err := New(cap)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
const nFiles = 3000 // >> maxEvictBatch
|
|
const fSize = 128
|
|
for i := 0; i < nFiles; i++ {
|
|
k := fmt.Sprintf("mbig/%05d", i)
|
|
w, err := m.Create(k, fSize)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
w.Write(make([]byte, fSize))
|
|
w.Close()
|
|
if i%800 == 0 {
|
|
m.EvictLRU(4096)
|
|
}
|
|
}
|
|
rounds := 0
|
|
totalEvicted := uint(0)
|
|
for m.Size() > m.Capacity() && rounds < 100 {
|
|
ev := m.EvictLRU(64 * 1024)
|
|
totalEvicted += ev
|
|
rounds++
|
|
if ev == 0 && rounds > 5 {
|
|
break
|
|
}
|
|
}
|
|
if rounds < 2 {
|
|
t.Logf("memory large-N: %d rounds (evicted=%d final=%d)", rounds, totalEvicted, m.Size())
|
|
}
|
|
_ = totalEvicted
|
|
}
|
|
|
|
// TestMemoryFS_NewInvalidCapacity exercises the new error return (was panic) for ctor hygiene (Item 3 coverage).
|
|
func TestMemoryFS_NewInvalidCapacity(t *testing.T) {
|
|
t.Parallel()
|
|
_, err := New(0)
|
|
if err == nil {
|
|
t.Fatal("expected error for capacity=0")
|
|
}
|
|
if !strings.Contains(err.Error(), "must be greater than 0") {
|
|
t.Errorf("err %q missing 'must be greater than 0'", err)
|
|
}
|
|
|
|
_, err = New(-1)
|
|
if err == nil || !strings.Contains(err.Error(), "must be greater than 0") {
|
|
t.Errorf("negative capacity should return error containing phrase, got %v", err)
|
|
}
|
|
}
|