Enhance DiskFS initialization and error handling
- Updated `disk.New` to support asynchronous initialization for large caches, improving responsiveness during startup. - Introduced an eviction function parameter to `disk.New`, ensuring proper handling of over-capacity scenarios. - Enhanced error handling in various components, including memory and disk tests, to ensure robustness and clarity. - Refactored tests to validate new behaviors, including checks for delayed attachment and proper error propagation. - Removed obsolete error handling code and tests related to the now-deleted errors package, streamlining the codebase.
This commit is contained in:
@@ -3,6 +3,7 @@ package memory
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"fmt"
|
||||
"io"
|
||||
"s1d3sw1ped/steamcache2/vfs"
|
||||
"s1d3sw1ped/steamcache2/vfs/locks"
|
||||
@@ -35,9 +36,9 @@ type MemoryFS struct {
|
||||
}
|
||||
|
||||
// New creates a new MemoryFS
|
||||
func New(capacity int64) *MemoryFS {
|
||||
func New(capacity int64) (*MemoryFS, error) {
|
||||
if capacity <= 0 {
|
||||
panic("memory capacity must be greater than 0")
|
||||
return nil, fmt.Errorf("memory capacity must be greater than 0")
|
||||
}
|
||||
|
||||
// Initialize sharded locks
|
||||
@@ -51,7 +52,7 @@ func New(capacity int64) *MemoryFS {
|
||||
keyLocks: keyLocks,
|
||||
LRU: lru.NewLRUList[*types.FileInfo](),
|
||||
timeUpdater: types.NewBatchedTimeUpdate(100 * time.Millisecond), // Update time every 100ms
|
||||
}
|
||||
}, nil
|
||||
}
|
||||
|
||||
// Name returns the name of this VFS
|
||||
|
||||
+71
-14
@@ -3,6 +3,7 @@ package memory
|
||||
import (
|
||||
"fmt"
|
||||
"io"
|
||||
"strings"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
@@ -11,7 +12,10 @@ import (
|
||||
|
||||
func TestMemoryFS_Basic(t *testing.T) {
|
||||
t.Parallel()
|
||||
m := New(1024 * 1024)
|
||||
m, err := New(1024 * 1024)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if m.Name() != "MemoryFS" {
|
||||
t.Error("bad name")
|
||||
}
|
||||
@@ -52,7 +56,10 @@ func TestMemoryFS_Basic(t *testing.T) {
|
||||
|
||||
func TestMemoryFS_EvictUnderPressure(t *testing.T) {
|
||||
t.Parallel()
|
||||
m := New(500)
|
||||
m, err := New(500)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// create 3x200 = 600 >500, should trigger internal? but direct evict call
|
||||
for i := 0; i < 3; i++ {
|
||||
w, _ := m.Create("f"+string(rune('0'+i)), 200)
|
||||
@@ -69,7 +76,10 @@ func TestMemoryFS_EvictUnderPressure(t *testing.T) {
|
||||
func TestMemoryFS_SizeNeverExceedsAfterEvict(t *testing.T) {
|
||||
t.Parallel()
|
||||
cap := int64(1000)
|
||||
m := New(cap)
|
||||
m, err := New(cap)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// Cycle through strategies (randomized feel via mod), use testKey, stricter post-evict with documented epsilon.
|
||||
strats := []func(uint) uint{m.EvictLRU, func(n uint) uint { return m.EvictBySize(n, true) }, m.EvictFIFO, m.EvictLFU, m.EvictHybrid}
|
||||
for i := 0; i < 50; i++ { // more cycles
|
||||
@@ -97,7 +107,10 @@ func TestMemoryFS_ConcurrentCreateOpenDelete(t *testing.T) {
|
||||
t.Skip()
|
||||
}
|
||||
t.Parallel()
|
||||
m := New(10 * 1024 * 1024)
|
||||
m, err := New(10 * 1024 * 1024)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var wg sync.WaitGroup
|
||||
const N = 50
|
||||
var ops int64
|
||||
@@ -137,7 +150,10 @@ func TestMemoryFS_ConcurrentCreateOpenDelete(t *testing.T) {
|
||||
}
|
||||
|
||||
func BenchmarkMemoryFS_CreateOpen(b *testing.B) {
|
||||
m := New(64 * 1024 * 1024)
|
||||
m, err := New(64 * 1024 * 1024)
|
||||
if err != nil {
|
||||
b.Fatal(err)
|
||||
}
|
||||
data := make([]byte, 4096)
|
||||
b.ReportAllocs()
|
||||
b.ResetTimer()
|
||||
@@ -162,7 +178,10 @@ func BenchmarkMemoryFS_CreateOpen(b *testing.B) {
|
||||
// BenchmarkMemoryFS_EvictionUnderPressure exercises memory eviction under synthetic pressure (parallels BenchmarkDiskFS_EvictionUnderPressure).
|
||||
// Uses cycling keys via testKey for stable behavior; exercises LRU path (other strategies lightly covered via existing tests + EvictHybrid uses DecayedScore).
|
||||
func BenchmarkMemoryFS_EvictionUnderPressure(b *testing.B) {
|
||||
m := New(1 * 1024 * 1024)
|
||||
m, err := New(1 * 1024 * 1024)
|
||||
if err != nil {
|
||||
b.Fatal(err)
|
||||
}
|
||||
b.ReportAllocs()
|
||||
b.ResetTimer()
|
||||
for i := 0; i < b.N; i++ {
|
||||
@@ -183,7 +202,10 @@ func BenchmarkMemoryFS_EvictionUnderPressure(b *testing.B) {
|
||||
// BenchmarkMemoryFS_EvictBySizeUnderPressure parallels the disk eviction strategy testing.
|
||||
// Exercises EvictBySize under repeated pressure.
|
||||
func BenchmarkMemoryFS_EvictBySizeUnderPressure(b *testing.B) {
|
||||
m := New(1 * 1024 * 1024)
|
||||
m, err := New(1 * 1024 * 1024)
|
||||
if err != nil {
|
||||
b.Fatal(err)
|
||||
}
|
||||
b.ReportAllocs()
|
||||
b.ResetTimer()
|
||||
for i := 0; i < b.N; i++ {
|
||||
@@ -195,7 +217,7 @@ func BenchmarkMemoryFS_EvictBySizeUnderPressure(b *testing.B) {
|
||||
w.Write(make([]byte, 64*1024))
|
||||
w.Close()
|
||||
}
|
||||
m.EvictBySize(512 * 1024, true) // ascending = evict smallest first
|
||||
m.EvictBySize(512*1024, true) // ascending = evict smallest first
|
||||
}
|
||||
_ = m // keep
|
||||
}
|
||||
@@ -203,7 +225,10 @@ func BenchmarkMemoryFS_EvictBySizeUnderPressure(b *testing.B) {
|
||||
// BenchmarkMemoryFS_EvictHybridUnderPressure exercises the hybrid strategy (which uses
|
||||
// the centralized DecayedScore) under pressure. Provides coverage for the time-decayed scoring.
|
||||
func BenchmarkMemoryFS_EvictHybridUnderPressure(b *testing.B) {
|
||||
m := New(1 * 1024 * 1024)
|
||||
m, err := New(1 * 1024 * 1024)
|
||||
if err != nil {
|
||||
b.Fatal(err)
|
||||
}
|
||||
b.ReportAllocs()
|
||||
b.ResetTimer()
|
||||
for i := 0; i < b.N; i++ {
|
||||
@@ -222,7 +247,10 @@ func BenchmarkMemoryFS_EvictHybridUnderPressure(b *testing.B) {
|
||||
|
||||
func TestMemoryFS_Stats(t *testing.T) {
|
||||
t.Parallel()
|
||||
m := New(1024)
|
||||
m, err := New(1024)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
stats := m.GetFragmentationStats()
|
||||
if stats["buffer_count"] != 0 {
|
||||
t.Error("initial buffers >0?")
|
||||
@@ -242,7 +270,10 @@ func TestMemoryFS_ConcurrentCloseAndEvict_RaceFree(t *testing.T) {
|
||||
t.Skip()
|
||||
}
|
||||
t.Parallel()
|
||||
m := New(2 * 1024 * 1024) // 2MB
|
||||
m, err := New(2 * 1024 * 1024) // 2MB
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var wg sync.WaitGroup
|
||||
stopCh := make(chan struct{})
|
||||
const writers = 3
|
||||
@@ -317,7 +348,10 @@ func TestMemoryFS_ConcurrentCloseAndEvict_RaceFree(t *testing.T) {
|
||||
|
||||
func TestMemoryFS_EvictVariantsAndErrors(t *testing.T) {
|
||||
t.Parallel()
|
||||
m := New(800)
|
||||
m, err := New(800)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// populate
|
||||
for i := 0; i < 4; i++ {
|
||||
w, _ := m.Create("ev"+string(rune('0'+i)), 150)
|
||||
@@ -361,7 +395,10 @@ func TestMemoryFS_EvictVariantsAndErrors(t *testing.T) {
|
||||
|
||||
func TestMemoryFS_AllEvictStrategies(t *testing.T) {
|
||||
t.Parallel()
|
||||
m := New(300)
|
||||
m, err := New(300)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for i := 0; i < 3; i++ {
|
||||
w, _ := m.Create("s"+string(rune(i)), 120)
|
||||
w.Write(make([]byte, 120))
|
||||
@@ -387,7 +424,10 @@ func TestMemoryFS_EvictBoundedLargeN(t *testing.T) {
|
||||
}
|
||||
t.Parallel()
|
||||
cap := int64(128 * 1024)
|
||||
m := New(cap)
|
||||
m, err := New(cap)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
const nFiles = 3000 // >> maxEvictBatch
|
||||
const fSize = 128
|
||||
for i := 0; i < nFiles; i++ {
|
||||
@@ -417,3 +457,20 @@ func TestMemoryFS_EvictBoundedLargeN(t *testing.T) {
|
||||
}
|
||||
_ = totalEvicted
|
||||
}
|
||||
|
||||
// TestMemoryFS_NewInvalidCapacity exercises the new error return (was panic) for ctor hygiene (Item 3 coverage).
|
||||
func TestMemoryFS_NewInvalidCapacity(t *testing.T) {
|
||||
t.Parallel()
|
||||
_, err := New(0)
|
||||
if err == nil {
|
||||
t.Fatal("expected error for capacity=0")
|
||||
}
|
||||
if !strings.Contains(err.Error(), "must be greater than 0") {
|
||||
t.Errorf("err %q missing 'must be greater than 0'", err)
|
||||
}
|
||||
|
||||
_, err = New(-1)
|
||||
if err == nil || !strings.Contains(err.Error(), "must be greater than 0") {
|
||||
t.Errorf("negative capacity should return error containing phrase, got %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,274 +0,0 @@
|
||||
package memory
|
||||
|
||||
import (
|
||||
"runtime"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"time"
|
||||
)
|
||||
|
||||
// MemoryMonitor tracks system memory usage and provides dynamic sizing recommendations
|
||||
type MemoryMonitor struct {
|
||||
targetMemoryUsage uint64 // Target total memory usage in bytes
|
||||
currentMemoryUsage uint64 // Current total memory usage in bytes
|
||||
monitoringInterval time.Duration
|
||||
adjustmentThreshold float64 // Threshold for cache size adjustments (e.g., 0.1 = 10%)
|
||||
mu sync.RWMutex
|
||||
ctx chan struct{}
|
||||
stopChan chan struct{}
|
||||
isMonitoring int32
|
||||
|
||||
// Dynamic cache management fields
|
||||
originalCacheSize uint64
|
||||
currentCacheSize uint64
|
||||
cache interface{} // Generic cache interface
|
||||
adjustmentInterval time.Duration
|
||||
lastAdjustment time.Time
|
||||
adjustmentCount int64
|
||||
isAdjusting int32
|
||||
}
|
||||
|
||||
// NewMemoryMonitor creates a new memory monitor
|
||||
func NewMemoryMonitor(targetMemoryUsage uint64, monitoringInterval time.Duration, adjustmentThreshold float64) *MemoryMonitor {
|
||||
return &MemoryMonitor{
|
||||
targetMemoryUsage: targetMemoryUsage,
|
||||
monitoringInterval: monitoringInterval,
|
||||
adjustmentThreshold: adjustmentThreshold,
|
||||
ctx: make(chan struct{}),
|
||||
stopChan: make(chan struct{}),
|
||||
adjustmentInterval: 30 * time.Second, // Default adjustment interval
|
||||
}
|
||||
}
|
||||
|
||||
// NewMemoryMonitorWithCache creates a new memory monitor with cache management
|
||||
func NewMemoryMonitorWithCache(targetMemoryUsage uint64, monitoringInterval time.Duration, adjustmentThreshold float64, cache interface{}, originalCacheSize uint64) *MemoryMonitor {
|
||||
mm := NewMemoryMonitor(targetMemoryUsage, monitoringInterval, adjustmentThreshold)
|
||||
mm.cache = cache
|
||||
mm.originalCacheSize = originalCacheSize
|
||||
mm.currentCacheSize = originalCacheSize
|
||||
return mm
|
||||
}
|
||||
|
||||
// Start begins monitoring memory usage
|
||||
func (mm *MemoryMonitor) Start() {
|
||||
if atomic.CompareAndSwapInt32(&mm.isMonitoring, 0, 1) {
|
||||
go mm.monitor()
|
||||
}
|
||||
}
|
||||
|
||||
// Stop stops monitoring memory usage
|
||||
func (mm *MemoryMonitor) Stop() {
|
||||
if atomic.CompareAndSwapInt32(&mm.isMonitoring, 1, 0) {
|
||||
close(mm.stopChan)
|
||||
}
|
||||
}
|
||||
|
||||
// GetCurrentMemoryUsage returns the current total memory usage
|
||||
func (mm *MemoryMonitor) GetCurrentMemoryUsage() uint64 {
|
||||
mm.mu.RLock()
|
||||
defer mm.mu.RUnlock()
|
||||
return atomic.LoadUint64(&mm.currentMemoryUsage)
|
||||
}
|
||||
|
||||
// GetTargetMemoryUsage returns the target memory usage
|
||||
func (mm *MemoryMonitor) GetTargetMemoryUsage() uint64 {
|
||||
mm.mu.RLock()
|
||||
defer mm.mu.RUnlock()
|
||||
return mm.targetMemoryUsage
|
||||
}
|
||||
|
||||
// GetMemoryUtilization returns the current memory utilization as a percentage
|
||||
func (mm *MemoryMonitor) GetMemoryUtilization() float64 {
|
||||
mm.mu.RLock()
|
||||
defer mm.mu.RUnlock()
|
||||
current := atomic.LoadUint64(&mm.currentMemoryUsage)
|
||||
return float64(current) / float64(mm.targetMemoryUsage)
|
||||
}
|
||||
|
||||
// GetRecommendedCacheSize calculates the recommended cache size based on current memory usage
|
||||
func (mm *MemoryMonitor) GetRecommendedCacheSize(originalCacheSize uint64) uint64 {
|
||||
mm.mu.RLock()
|
||||
defer mm.mu.RUnlock()
|
||||
|
||||
current := atomic.LoadUint64(&mm.currentMemoryUsage)
|
||||
target := mm.targetMemoryUsage
|
||||
|
||||
// If we're under target, we can use the full cache size
|
||||
if current <= target {
|
||||
return originalCacheSize
|
||||
}
|
||||
|
||||
// Calculate how much we're over target
|
||||
overage := current - target
|
||||
|
||||
// If overage is significant, reduce cache size
|
||||
if overage > uint64(float64(target)*mm.adjustmentThreshold) {
|
||||
// Reduce cache size by the overage amount, but don't go below 10% of original
|
||||
minCacheSize := uint64(float64(originalCacheSize) * 0.1)
|
||||
recommendedSize := originalCacheSize - overage
|
||||
|
||||
if recommendedSize < minCacheSize {
|
||||
recommendedSize = minCacheSize
|
||||
}
|
||||
|
||||
return recommendedSize
|
||||
}
|
||||
|
||||
return originalCacheSize
|
||||
}
|
||||
|
||||
// monitor runs the memory monitoring loop
|
||||
func (mm *MemoryMonitor) monitor() {
|
||||
ticker := time.NewTicker(mm.monitoringInterval)
|
||||
defer ticker.Stop()
|
||||
|
||||
for {
|
||||
select {
|
||||
case <-mm.stopChan:
|
||||
return
|
||||
case <-ticker.C:
|
||||
mm.updateMemoryUsage()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// updateMemoryUsage updates the current memory usage
|
||||
func (mm *MemoryMonitor) updateMemoryUsage() {
|
||||
var m runtime.MemStats
|
||||
runtime.ReadMemStats(&m)
|
||||
|
||||
// Use Alloc (currently allocated memory) as our metric
|
||||
atomic.StoreUint64(&mm.currentMemoryUsage, m.Alloc)
|
||||
}
|
||||
|
||||
// SetTargetMemoryUsage updates the target memory usage
|
||||
func (mm *MemoryMonitor) SetTargetMemoryUsage(target uint64) {
|
||||
mm.mu.Lock()
|
||||
defer mm.mu.Unlock()
|
||||
mm.targetMemoryUsage = target
|
||||
}
|
||||
|
||||
// GetMemoryStats returns detailed memory statistics
|
||||
func (mm *MemoryMonitor) GetMemoryStats() map[string]interface{} {
|
||||
var m runtime.MemStats
|
||||
runtime.ReadMemStats(&m)
|
||||
|
||||
mm.mu.RLock()
|
||||
defer mm.mu.RUnlock()
|
||||
|
||||
return map[string]interface{}{
|
||||
"current_usage": atomic.LoadUint64(&mm.currentMemoryUsage),
|
||||
"target_usage": mm.targetMemoryUsage,
|
||||
"utilization": mm.GetMemoryUtilization(),
|
||||
"heap_alloc": m.HeapAlloc,
|
||||
"heap_sys": m.HeapSys,
|
||||
"heap_idle": m.HeapIdle,
|
||||
"heap_inuse": m.HeapInuse,
|
||||
"stack_inuse": m.StackInuse,
|
||||
"stack_sys": m.StackSys,
|
||||
"gc_cycles": m.NumGC,
|
||||
"gc_pause_total": m.PauseTotalNs,
|
||||
}
|
||||
}
|
||||
|
||||
// Dynamic Cache Management Methods
|
||||
|
||||
// StartDynamicAdjustment begins the dynamic cache size adjustment process
|
||||
func (mm *MemoryMonitor) StartDynamicAdjustment() {
|
||||
if mm.cache != nil {
|
||||
go mm.adjustmentLoop()
|
||||
}
|
||||
}
|
||||
|
||||
// GetCurrentCacheSize returns the current cache size
|
||||
func (mm *MemoryMonitor) GetCurrentCacheSize() uint64 {
|
||||
mm.mu.RLock()
|
||||
defer mm.mu.RUnlock()
|
||||
return atomic.LoadUint64(&mm.currentCacheSize)
|
||||
}
|
||||
|
||||
// GetOriginalCacheSize returns the original cache size
|
||||
func (mm *MemoryMonitor) GetOriginalCacheSize() uint64 {
|
||||
mm.mu.RLock()
|
||||
defer mm.mu.RUnlock()
|
||||
return mm.originalCacheSize
|
||||
}
|
||||
|
||||
// GetAdjustmentCount returns the number of adjustments made
|
||||
func (mm *MemoryMonitor) GetAdjustmentCount() int64 {
|
||||
return atomic.LoadInt64(&mm.adjustmentCount)
|
||||
}
|
||||
|
||||
// adjustmentLoop runs the cache size adjustment loop
|
||||
func (mm *MemoryMonitor) adjustmentLoop() {
|
||||
ticker := time.NewTicker(mm.adjustmentInterval)
|
||||
defer ticker.Stop()
|
||||
|
||||
for range ticker.C {
|
||||
mm.performAdjustment()
|
||||
}
|
||||
}
|
||||
|
||||
// performAdjustment performs a cache size adjustment if needed
|
||||
func (mm *MemoryMonitor) performAdjustment() {
|
||||
// Prevent concurrent adjustments
|
||||
if !atomic.CompareAndSwapInt32(&mm.isAdjusting, 0, 1) {
|
||||
return
|
||||
}
|
||||
defer atomic.StoreInt32(&mm.isAdjusting, 0)
|
||||
|
||||
// Check if enough time has passed since last adjustment
|
||||
if time.Since(mm.lastAdjustment) < mm.adjustmentInterval {
|
||||
return
|
||||
}
|
||||
|
||||
// Get recommended cache size
|
||||
recommendedSize := mm.GetRecommendedCacheSize(mm.originalCacheSize)
|
||||
currentSize := atomic.LoadUint64(&mm.currentCacheSize)
|
||||
|
||||
// Only adjust if there's a significant difference (more than 5%)
|
||||
sizeDiff := float64(recommendedSize) / float64(currentSize)
|
||||
if sizeDiff < 0.95 || sizeDiff > 1.05 {
|
||||
mm.adjustCacheSize(recommendedSize)
|
||||
mm.lastAdjustment = time.Now()
|
||||
atomic.AddInt64(&mm.adjustmentCount, 1)
|
||||
}
|
||||
}
|
||||
|
||||
// adjustCacheSize adjusts the cache size to the recommended size
|
||||
func (mm *MemoryMonitor) adjustCacheSize(newSize uint64) {
|
||||
mm.mu.Lock()
|
||||
defer mm.mu.Unlock()
|
||||
|
||||
oldSize := atomic.LoadUint64(&mm.currentCacheSize)
|
||||
atomic.StoreUint64(&mm.currentCacheSize, newSize)
|
||||
|
||||
// If we're reducing the cache size, trigger GC to free up memory
|
||||
if newSize < oldSize {
|
||||
// Calculate how much to free
|
||||
bytesToFree := oldSize - newSize
|
||||
|
||||
// Trigger GC on the cache to free up the excess memory
|
||||
// This is a simplified approach - in practice, you'd want to integrate
|
||||
// with the actual GC system to free the right amount
|
||||
if gcCache, ok := mm.cache.(interface{ ForceGC(uint) }); ok {
|
||||
gcCache.ForceGC(uint(bytesToFree))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// GetDynamicStats returns statistics about the dynamic cache manager
|
||||
func (mm *MemoryMonitor) GetDynamicStats() map[string]interface{} {
|
||||
mm.mu.RLock()
|
||||
defer mm.mu.RUnlock()
|
||||
|
||||
return map[string]interface{}{
|
||||
"original_cache_size": mm.originalCacheSize,
|
||||
"current_cache_size": atomic.LoadUint64(&mm.currentCacheSize),
|
||||
"adjustment_count": atomic.LoadInt64(&mm.adjustmentCount),
|
||||
"last_adjustment": mm.lastAdjustment,
|
||||
"memory_utilization": mm.GetMemoryUtilization(),
|
||||
"target_memory_usage": mm.GetTargetMemoryUsage(),
|
||||
"current_memory_usage": mm.GetCurrentMemoryUsage(),
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user