Enhance garbage collection and caching functionality
All checks were successful
PR Check / check-and-test (pull_request) Successful in 21s
All checks were successful
PR Check / check-and-test (pull_request) Successful in 21s
- Updated .gitignore to include all .exe files and ensure .smashignore is tracked. - Expanded README.md with advanced configuration options for garbage collection algorithms, detailing available algorithms and use cases. - Modified launch.json to include memory and disk garbage collection flags for better configuration. - Refactored root.go to introduce memoryGC and diskGC flags for garbage collection algorithms. - Implemented hash extraction and verification in steamcache.go to ensure data integrity during caching. - Added new tests in steamcache_test.go for hash extraction and verification, ensuring correctness of caching behavior. - Enhanced garbage collection strategies in gc.go, introducing LFU, FIFO, Largest, Smallest, and Hybrid algorithms with corresponding metrics. - Updated caching logic to conditionally cache responses based on hash verification results.
This commit is contained in:
5
vfs/cache/cache.go
vendored
5
vfs/cache/cache.go
vendored
@@ -6,6 +6,7 @@ import (
|
||||
"io"
|
||||
"s1d3sw1ped/SteamCache2/vfs"
|
||||
"s1d3sw1ped/SteamCache2/vfs/cachestate"
|
||||
"s1d3sw1ped/SteamCache2/vfs/gc"
|
||||
"s1d3sw1ped/SteamCache2/vfs/vfserror"
|
||||
"sync"
|
||||
)
|
||||
@@ -98,6 +99,10 @@ func (c *CacheFS) Open(key string) (io.ReadCloser, error) {
|
||||
switch state {
|
||||
case cachestate.CacheStateHit:
|
||||
// if c.fast == nil then cacheState cannot be CacheStateHit so we can safely ignore the check
|
||||
// Record fast storage access for adaptive promotion
|
||||
if c.fast != nil {
|
||||
gc.RecordFastStorageAccess()
|
||||
}
|
||||
return c.fast.Open(key)
|
||||
case cachestate.CacheStateMiss:
|
||||
slowReader, err := c.slow.Open(key)
|
||||
|
||||
620
vfs/gc/gc.go
620
vfs/gc/gc.go
@@ -10,7 +10,12 @@ import (
|
||||
"s1d3sw1ped/SteamCache2/vfs/disk"
|
||||
"s1d3sw1ped/SteamCache2/vfs/memory"
|
||||
"s1d3sw1ped/SteamCache2/vfs/vfserror"
|
||||
"sort"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/prometheus/client_golang/prometheus"
|
||||
"github.com/prometheus/client_golang/prometheus/promauto"
|
||||
)
|
||||
|
||||
var (
|
||||
@@ -18,6 +23,53 @@ var (
|
||||
ErrInsufficientSpace = fmt.Errorf("no files to delete")
|
||||
)
|
||||
|
||||
// Prometheus metrics for adaptive promotion
|
||||
var (
|
||||
promotionThresholds = promauto.NewGaugeVec(
|
||||
prometheus.GaugeOpts{
|
||||
Name: "promotion_thresholds_bytes",
|
||||
Help: "Current promotion thresholds in bytes",
|
||||
},
|
||||
[]string{"threshold_type"},
|
||||
)
|
||||
|
||||
promotionWindows = promauto.NewGaugeVec(
|
||||
prometheus.GaugeOpts{
|
||||
Name: "promotion_windows_seconds",
|
||||
Help: "Current promotion time windows in seconds",
|
||||
},
|
||||
[]string{"window_type"},
|
||||
)
|
||||
|
||||
promotionStats = promauto.NewGaugeVec(
|
||||
prometheus.GaugeOpts{
|
||||
Name: "promotion_stats",
|
||||
Help: "Promotion statistics",
|
||||
},
|
||||
[]string{"metric_type"},
|
||||
)
|
||||
|
||||
promotionAdaptations = promauto.NewCounterVec(
|
||||
prometheus.CounterOpts{
|
||||
Name: "promotion_adaptations_total",
|
||||
Help: "Total number of promotion threshold adaptations",
|
||||
},
|
||||
[]string{"direction"},
|
||||
)
|
||||
)
|
||||
|
||||
// GCAlgorithm represents different garbage collection strategies
|
||||
type GCAlgorithm string
|
||||
|
||||
const (
|
||||
LRU GCAlgorithm = "lru"
|
||||
LFU GCAlgorithm = "lfu"
|
||||
FIFO GCAlgorithm = "fifo"
|
||||
Largest GCAlgorithm = "largest"
|
||||
Smallest GCAlgorithm = "smallest"
|
||||
Hybrid GCAlgorithm = "hybrid"
|
||||
)
|
||||
|
||||
// LRUGC deletes files in LRU order until enough space is reclaimed.
|
||||
func LRUGC(vfss vfs.VFS, size uint) error {
|
||||
logger.Logger.Debug().Uint("target", size).Msg("Attempting to reclaim space using LRU GC")
|
||||
@@ -59,10 +111,578 @@ func LRUGC(vfss vfs.VFS, size uint) error {
|
||||
}
|
||||
}
|
||||
|
||||
// LFUGC deletes files in LFU (Least Frequently Used) order until enough space is reclaimed.
|
||||
func LFUGC(vfss vfs.VFS, size uint) error {
|
||||
logger.Logger.Debug().Uint("target", size).Msg("Attempting to reclaim space using LFU GC")
|
||||
|
||||
// Get all files and sort by access count (frequency)
|
||||
files := getAllFiles(vfss)
|
||||
if len(files) == 0 {
|
||||
return ErrInsufficientSpace
|
||||
}
|
||||
|
||||
// Sort by access count (ascending - least frequently used first)
|
||||
sort.Slice(files, func(i, j int) bool {
|
||||
return files[i].AccessCount < files[j].AccessCount
|
||||
})
|
||||
|
||||
var reclaimed uint
|
||||
for _, fi := range files {
|
||||
if reclaimed >= size {
|
||||
break
|
||||
}
|
||||
err := vfss.Delete(fi.Name)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
reclaimed += uint(fi.Size)
|
||||
}
|
||||
|
||||
if reclaimed >= size {
|
||||
logger.Logger.Debug().Uint("target", size).Uint("achieved", reclaimed).Msg("Reclaimed enough space using LFU GC")
|
||||
return nil
|
||||
}
|
||||
return ErrInsufficientSpace
|
||||
}
|
||||
|
||||
// FIFOGC deletes files in FIFO (First In, First Out) order until enough space is reclaimed.
|
||||
func FIFOGC(vfss vfs.VFS, size uint) error {
|
||||
logger.Logger.Debug().Uint("target", size).Msg("Attempting to reclaim space using FIFO GC")
|
||||
|
||||
// Get all files and sort by creation time (oldest first)
|
||||
files := getAllFiles(vfss)
|
||||
if len(files) == 0 {
|
||||
return ErrInsufficientSpace
|
||||
}
|
||||
|
||||
// Sort by creation time (ascending - oldest first)
|
||||
sort.Slice(files, func(i, j int) bool {
|
||||
return files[i].MTime.Before(files[j].MTime)
|
||||
})
|
||||
|
||||
var reclaimed uint
|
||||
for _, fi := range files {
|
||||
if reclaimed >= size {
|
||||
break
|
||||
}
|
||||
err := vfss.Delete(fi.Name)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
reclaimed += uint(fi.Size)
|
||||
}
|
||||
|
||||
if reclaimed >= size {
|
||||
logger.Logger.Debug().Uint("target", size).Uint("achieved", reclaimed).Msg("Reclaimed enough space using FIFO GC")
|
||||
return nil
|
||||
}
|
||||
return ErrInsufficientSpace
|
||||
}
|
||||
|
||||
// LargestGC deletes the largest files first until enough space is reclaimed.
|
||||
func LargestGC(vfss vfs.VFS, size uint) error {
|
||||
logger.Logger.Debug().Uint("target", size).Msg("Attempting to reclaim space using Largest GC")
|
||||
|
||||
// Get all files and sort by size (largest first)
|
||||
files := getAllFiles(vfss)
|
||||
if len(files) == 0 {
|
||||
return ErrInsufficientSpace
|
||||
}
|
||||
|
||||
// Sort by size (descending - largest first)
|
||||
sort.Slice(files, func(i, j int) bool {
|
||||
return files[i].Size > files[j].Size
|
||||
})
|
||||
|
||||
var reclaimed uint
|
||||
for _, fi := range files {
|
||||
if reclaimed >= size {
|
||||
break
|
||||
}
|
||||
err := vfss.Delete(fi.Name)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
reclaimed += uint(fi.Size)
|
||||
}
|
||||
|
||||
if reclaimed >= size {
|
||||
logger.Logger.Debug().Uint("target", size).Uint("achieved", reclaimed).Msg("Reclaimed enough space using Largest GC")
|
||||
return nil
|
||||
}
|
||||
return ErrInsufficientSpace
|
||||
}
|
||||
|
||||
// SmallestGC deletes the smallest files first until enough space is reclaimed.
|
||||
func SmallestGC(vfss vfs.VFS, size uint) error {
|
||||
logger.Logger.Debug().Uint("target", size).Msg("Attempting to reclaim space using Smallest GC")
|
||||
|
||||
// Get all files and sort by size (smallest first)
|
||||
files := getAllFiles(vfss)
|
||||
if len(files) == 0 {
|
||||
return ErrInsufficientSpace
|
||||
}
|
||||
|
||||
// Sort by size (ascending - smallest first)
|
||||
sort.Slice(files, func(i, j int) bool {
|
||||
return files[i].Size < files[j].Size
|
||||
})
|
||||
|
||||
var reclaimed uint
|
||||
for _, fi := range files {
|
||||
if reclaimed >= size {
|
||||
break
|
||||
}
|
||||
err := vfss.Delete(fi.Name)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
reclaimed += uint(fi.Size)
|
||||
}
|
||||
|
||||
if reclaimed >= size {
|
||||
logger.Logger.Debug().Uint("target", size).Uint("achieved", reclaimed).Msg("Reclaimed enough space using Smallest GC")
|
||||
return nil
|
||||
}
|
||||
return ErrInsufficientSpace
|
||||
}
|
||||
|
||||
// HybridGC combines LRU and size-based eviction with a scoring system.
|
||||
func HybridGC(vfss vfs.VFS, size uint) error {
|
||||
logger.Logger.Debug().Uint("target", size).Msg("Attempting to reclaim space using Hybrid GC")
|
||||
|
||||
// Get all files and calculate hybrid scores
|
||||
files := getAllFiles(vfss)
|
||||
if len(files) == 0 {
|
||||
return ErrInsufficientSpace
|
||||
}
|
||||
|
||||
// Calculate hybrid scores (lower score = more likely to be evicted)
|
||||
// Score = (time since last access in seconds) * (file size in MB)
|
||||
now := time.Now()
|
||||
for i := range files {
|
||||
timeSinceAccess := now.Sub(files[i].ATime).Seconds()
|
||||
sizeMB := float64(files[i].Size) / (1024 * 1024)
|
||||
files[i].HybridScore = timeSinceAccess * sizeMB
|
||||
}
|
||||
|
||||
// Sort by hybrid score (ascending - lowest scores first)
|
||||
sort.Slice(files, func(i, j int) bool {
|
||||
return files[i].HybridScore < files[j].HybridScore
|
||||
})
|
||||
|
||||
var reclaimed uint
|
||||
for _, fi := range files {
|
||||
if reclaimed >= size {
|
||||
break
|
||||
}
|
||||
err := vfss.Delete(fi.Name)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
reclaimed += uint(fi.Size)
|
||||
}
|
||||
|
||||
if reclaimed >= size {
|
||||
logger.Logger.Debug().Uint("target", size).Uint("achieved", reclaimed).Msg("Reclaimed enough space using Hybrid GC")
|
||||
return nil
|
||||
}
|
||||
return ErrInsufficientSpace
|
||||
}
|
||||
|
||||
// fileInfoWithMetadata extends FileInfo with additional metadata for GC algorithms
|
||||
type fileInfoWithMetadata struct {
|
||||
Name string
|
||||
Size int64
|
||||
MTime time.Time
|
||||
ATime time.Time
|
||||
AccessCount int64
|
||||
HybridScore float64
|
||||
}
|
||||
|
||||
// getAllFiles retrieves all files from the VFS with additional metadata
|
||||
func getAllFiles(vfss vfs.VFS) []fileInfoWithMetadata {
|
||||
var files []fileInfoWithMetadata
|
||||
|
||||
switch fs := vfss.(type) {
|
||||
case *disk.DiskFS:
|
||||
allFiles := fs.StatAll()
|
||||
for _, fi := range allFiles {
|
||||
// For disk, we can't easily track access count, so we'll use 1 as default
|
||||
files = append(files, fileInfoWithMetadata{
|
||||
Name: fi.Name(),
|
||||
Size: fi.Size(),
|
||||
MTime: fi.ModTime(),
|
||||
ATime: fi.AccessTime(),
|
||||
AccessCount: 1,
|
||||
})
|
||||
}
|
||||
case *memory.MemoryFS:
|
||||
allFiles := fs.StatAll()
|
||||
for _, fi := range allFiles {
|
||||
// For memory, we can't easily track access count, so we'll use 1 as default
|
||||
files = append(files, fileInfoWithMetadata{
|
||||
Name: fi.Name(),
|
||||
Size: fi.Size(),
|
||||
MTime: fi.ModTime(),
|
||||
ATime: fi.AccessTime(),
|
||||
AccessCount: 1,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
return files
|
||||
}
|
||||
|
||||
// GetGCAlgorithm returns the appropriate GC function based on the algorithm name
|
||||
func GetGCAlgorithm(algorithm GCAlgorithm) GCHandlerFunc {
|
||||
switch algorithm {
|
||||
case LRU:
|
||||
return LRUGC
|
||||
case LFU:
|
||||
return LFUGC
|
||||
case FIFO:
|
||||
return FIFOGC
|
||||
case Largest:
|
||||
return LargestGC
|
||||
case Smallest:
|
||||
return SmallestGC
|
||||
case Hybrid:
|
||||
return HybridGC
|
||||
default:
|
||||
logger.Logger.Warn().Str("algorithm", string(algorithm)).Msg("Unknown GC algorithm, falling back to LRU")
|
||||
return LRUGC
|
||||
}
|
||||
}
|
||||
|
||||
func PromotionDecider(fi *vfs.FileInfo, cs cachestate.CacheState) bool {
|
||||
return time.Since(fi.AccessTime()) < time.Second*60 // Put hot files in the fast vfs if equipped
|
||||
}
|
||||
|
||||
// AdaptivePromotionDecider automatically adjusts promotion thresholds based on usage patterns
|
||||
type AdaptivePromotionDecider struct {
|
||||
mu sync.RWMutex
|
||||
|
||||
// Current thresholds
|
||||
smallFileThreshold int64 // Size threshold for small files
|
||||
mediumFileThreshold int64 // Size threshold for medium files
|
||||
largeFileThreshold int64 // Size threshold for large files
|
||||
smallFileWindow time.Duration // Time window for small files
|
||||
mediumFileWindow time.Duration // Time window for medium files
|
||||
largeFileWindow time.Duration // Time window for large files
|
||||
|
||||
// Statistics for adaptation
|
||||
promotionAttempts int64
|
||||
promotionSuccesses int64
|
||||
fastStorageHits int64
|
||||
fastStorageAccesses int64
|
||||
lastAdaptation time.Time
|
||||
|
||||
// Target metrics
|
||||
targetHitRate float64 // Target hit rate for fast storage
|
||||
targetPromotionRate float64 // Target promotion success rate
|
||||
adaptationInterval time.Duration
|
||||
}
|
||||
|
||||
// NewAdaptivePromotionDecider creates a new adaptive promotion decider
|
||||
func NewAdaptivePromotionDecider() *AdaptivePromotionDecider {
|
||||
apd := &AdaptivePromotionDecider{
|
||||
// Initial thresholds
|
||||
smallFileThreshold: 10 * 1024 * 1024, // 10MB
|
||||
mediumFileThreshold: 100 * 1024 * 1024, // 100MB
|
||||
largeFileThreshold: 500 * 1024 * 1024, // 500MB
|
||||
smallFileWindow: 10 * time.Minute,
|
||||
mediumFileWindow: 2 * time.Minute,
|
||||
largeFileWindow: 30 * time.Second,
|
||||
|
||||
// Target metrics
|
||||
targetHitRate: 0.8, // 80% hit rate
|
||||
targetPromotionRate: 0.7, // 70% promotion success rate
|
||||
adaptationInterval: 5 * time.Minute,
|
||||
}
|
||||
|
||||
// Initialize Prometheus metrics
|
||||
apd.updatePrometheusMetrics()
|
||||
|
||||
return apd
|
||||
}
|
||||
|
||||
// ShouldPromote determines if a file should be promoted based on adaptive thresholds
|
||||
func (apd *AdaptivePromotionDecider) ShouldPromote(fi *vfs.FileInfo, cs cachestate.CacheState) bool {
|
||||
apd.mu.Lock()
|
||||
defer apd.mu.Unlock()
|
||||
|
||||
// Check if it's time to adapt thresholds
|
||||
if time.Since(apd.lastAdaptation) > apd.adaptationInterval {
|
||||
apd.adaptThresholds()
|
||||
}
|
||||
|
||||
size := fi.Size()
|
||||
timeSinceAccess := time.Since(fi.AccessTime())
|
||||
|
||||
// Record promotion attempt
|
||||
apd.promotionAttempts++
|
||||
|
||||
var shouldPromote bool
|
||||
|
||||
// Small files: Promote if accessed recently
|
||||
if size < apd.smallFileThreshold {
|
||||
shouldPromote = timeSinceAccess < apd.smallFileWindow
|
||||
} else if size < apd.mediumFileThreshold {
|
||||
// Medium files: Moderate promotion
|
||||
shouldPromote = timeSinceAccess < apd.mediumFileWindow
|
||||
} else if size < apd.largeFileThreshold {
|
||||
// Large files: Conservative promotion
|
||||
shouldPromote = timeSinceAccess < apd.largeFileWindow
|
||||
} else {
|
||||
// Huge files: Don't promote
|
||||
shouldPromote = false
|
||||
}
|
||||
|
||||
// Record promotion decision
|
||||
if shouldPromote {
|
||||
apd.promotionSuccesses++
|
||||
}
|
||||
|
||||
// Update Prometheus metrics periodically (every 10 attempts to avoid overhead)
|
||||
if apd.promotionAttempts%10 == 0 {
|
||||
apd.updatePrometheusMetrics()
|
||||
}
|
||||
|
||||
return shouldPromote
|
||||
}
|
||||
|
||||
// RecordFastStorageAccess records when fast storage is accessed
|
||||
func (apd *AdaptivePromotionDecider) RecordFastStorageAccess() {
|
||||
apd.mu.Lock()
|
||||
defer apd.mu.Unlock()
|
||||
apd.fastStorageAccesses++
|
||||
|
||||
// Update Prometheus metrics periodically
|
||||
if apd.fastStorageAccesses%10 == 0 {
|
||||
apd.updatePrometheusMetrics()
|
||||
}
|
||||
}
|
||||
|
||||
// RecordFastStorageHit records when fast storage has a hit
|
||||
func (apd *AdaptivePromotionDecider) RecordFastStorageHit() {
|
||||
apd.mu.Lock()
|
||||
defer apd.mu.Unlock()
|
||||
apd.fastStorageHits++
|
||||
|
||||
// Update Prometheus metrics periodically
|
||||
if apd.fastStorageHits%10 == 0 {
|
||||
apd.updatePrometheusMetrics()
|
||||
}
|
||||
}
|
||||
|
||||
// adaptThresholds adjusts thresholds based on current performance
|
||||
func (apd *AdaptivePromotionDecider) adaptThresholds() {
|
||||
if apd.promotionAttempts < 10 || apd.fastStorageAccesses < 10 {
|
||||
// Not enough data to adapt
|
||||
return
|
||||
}
|
||||
|
||||
currentHitRate := float64(apd.fastStorageHits) / float64(apd.fastStorageAccesses)
|
||||
currentPromotionRate := float64(apd.promotionSuccesses) / float64(apd.promotionAttempts)
|
||||
|
||||
logger.Logger.Debug().
|
||||
Float64("hit_rate", currentHitRate).
|
||||
Float64("promotion_rate", currentPromotionRate).
|
||||
Float64("target_hit_rate", apd.targetHitRate).
|
||||
Float64("target_promotion_rate", apd.targetPromotionRate).
|
||||
Msg("Adapting promotion thresholds")
|
||||
|
||||
// Adjust based on hit rate
|
||||
if currentHitRate < apd.targetHitRate {
|
||||
// Hit rate too low - be more aggressive with promotion
|
||||
apd.adjustThresholdsMoreAggressive()
|
||||
} else if currentHitRate > apd.targetHitRate+0.1 {
|
||||
// Hit rate too high - be more conservative
|
||||
apd.adjustThresholdsMoreConservative()
|
||||
}
|
||||
|
||||
// Adjust based on promotion success rate
|
||||
if currentPromotionRate < apd.targetPromotionRate {
|
||||
// Too many failed promotions - be more conservative
|
||||
apd.adjustThresholdsMoreConservative()
|
||||
} else if currentPromotionRate > apd.targetPromotionRate+0.1 {
|
||||
// High promotion success - can be more aggressive
|
||||
apd.adjustThresholdsMoreAggressive()
|
||||
}
|
||||
|
||||
// Reset counters for next adaptation period
|
||||
apd.promotionAttempts = 0
|
||||
apd.promotionSuccesses = 0
|
||||
apd.fastStorageHits = 0
|
||||
apd.fastStorageAccesses = 0
|
||||
apd.lastAdaptation = time.Now()
|
||||
|
||||
logger.Logger.Info().
|
||||
Int64("small_threshold_mb", apd.smallFileThreshold/(1024*1024)).
|
||||
Int64("medium_threshold_mb", apd.mediumFileThreshold/(1024*1024)).
|
||||
Int64("large_threshold_mb", apd.largeFileThreshold/(1024*1024)).
|
||||
Dur("small_window", apd.smallFileWindow).
|
||||
Dur("medium_window", apd.mediumFileWindow).
|
||||
Dur("large_window", apd.largeFileWindow).
|
||||
Msg("Updated promotion thresholds")
|
||||
}
|
||||
|
||||
// updatePrometheusMetrics updates all Prometheus metrics with current values
|
||||
func (apd *AdaptivePromotionDecider) updatePrometheusMetrics() {
|
||||
// Update threshold metrics
|
||||
promotionThresholds.WithLabelValues("small").Set(float64(apd.smallFileThreshold))
|
||||
promotionThresholds.WithLabelValues("medium").Set(float64(apd.mediumFileThreshold))
|
||||
promotionThresholds.WithLabelValues("large").Set(float64(apd.largeFileThreshold))
|
||||
|
||||
// Update window metrics
|
||||
promotionWindows.WithLabelValues("small").Set(apd.smallFileWindow.Seconds())
|
||||
promotionWindows.WithLabelValues("medium").Set(apd.mediumFileWindow.Seconds())
|
||||
promotionWindows.WithLabelValues("large").Set(apd.largeFileWindow.Seconds())
|
||||
|
||||
// Update statistics metrics
|
||||
hitRate := 0.0
|
||||
if apd.fastStorageAccesses > 0 {
|
||||
hitRate = float64(apd.fastStorageHits) / float64(apd.fastStorageAccesses)
|
||||
}
|
||||
promotionRate := 0.0
|
||||
if apd.promotionAttempts > 0 {
|
||||
promotionRate = float64(apd.promotionSuccesses) / float64(apd.promotionAttempts)
|
||||
}
|
||||
|
||||
promotionStats.WithLabelValues("hit_rate").Set(hitRate)
|
||||
promotionStats.WithLabelValues("promotion_rate").Set(promotionRate)
|
||||
promotionStats.WithLabelValues("promotion_attempts").Set(float64(apd.promotionAttempts))
|
||||
promotionStats.WithLabelValues("promotion_successes").Set(float64(apd.promotionSuccesses))
|
||||
promotionStats.WithLabelValues("fast_storage_accesses").Set(float64(apd.fastStorageAccesses))
|
||||
promotionStats.WithLabelValues("fast_storage_hits").Set(float64(apd.fastStorageHits))
|
||||
}
|
||||
|
||||
// adjustThresholdsMoreAggressive makes promotion more aggressive
|
||||
func (apd *AdaptivePromotionDecider) adjustThresholdsMoreAggressive() {
|
||||
// Increase size thresholds (promote larger files)
|
||||
apd.smallFileThreshold = minInt64(apd.smallFileThreshold*11/10, 50*1024*1024) // Max 50MB
|
||||
apd.mediumFileThreshold = minInt64(apd.mediumFileThreshold*11/10, 200*1024*1024) // Max 200MB
|
||||
apd.largeFileThreshold = minInt64(apd.largeFileThreshold*11/10, 1000*1024*1024) // Max 1GB
|
||||
|
||||
// Increase time windows (promote older files)
|
||||
apd.smallFileWindow = minDuration(apd.smallFileWindow*11/10, 20*time.Minute)
|
||||
apd.mediumFileWindow = minDuration(apd.mediumFileWindow*11/10, 5*time.Minute)
|
||||
apd.largeFileWindow = minDuration(apd.largeFileWindow*11/10, 2*time.Minute)
|
||||
|
||||
// Record adaptation in Prometheus
|
||||
promotionAdaptations.WithLabelValues("aggressive").Inc()
|
||||
|
||||
// Update Prometheus metrics
|
||||
apd.updatePrometheusMetrics()
|
||||
}
|
||||
|
||||
// adjustThresholdsMoreConservative makes promotion more conservative
|
||||
func (apd *AdaptivePromotionDecider) adjustThresholdsMoreConservative() {
|
||||
// Decrease size thresholds (promote smaller files)
|
||||
apd.smallFileThreshold = maxInt64(apd.smallFileThreshold*9/10, 5*1024*1024) // Min 5MB
|
||||
apd.mediumFileThreshold = maxInt64(apd.mediumFileThreshold*9/10, 50*1024*1024) // Min 50MB
|
||||
apd.largeFileThreshold = maxInt64(apd.largeFileThreshold*9/10, 200*1024*1024) // Min 200MB
|
||||
|
||||
// Decrease time windows (promote only recent files)
|
||||
apd.smallFileWindow = maxDuration(apd.smallFileWindow*9/10, 5*time.Minute)
|
||||
apd.mediumFileWindow = maxDuration(apd.mediumFileWindow*9/10, 1*time.Minute)
|
||||
apd.largeFileWindow = maxDuration(apd.largeFileWindow*9/10, 15*time.Second)
|
||||
|
||||
// Record adaptation in Prometheus
|
||||
promotionAdaptations.WithLabelValues("conservative").Inc()
|
||||
|
||||
// Update Prometheus metrics
|
||||
apd.updatePrometheusMetrics()
|
||||
}
|
||||
|
||||
// GetStats returns current statistics for monitoring
|
||||
func (apd *AdaptivePromotionDecider) GetStats() map[string]interface{} {
|
||||
apd.mu.RLock()
|
||||
defer apd.mu.RUnlock()
|
||||
|
||||
hitRate := 0.0
|
||||
if apd.fastStorageAccesses > 0 {
|
||||
hitRate = float64(apd.fastStorageHits) / float64(apd.fastStorageAccesses)
|
||||
}
|
||||
|
||||
promotionRate := 0.0
|
||||
if apd.promotionAttempts > 0 {
|
||||
promotionRate = float64(apd.promotionSuccesses) / float64(apd.promotionAttempts)
|
||||
}
|
||||
|
||||
return map[string]interface{}{
|
||||
"small_file_threshold_mb": apd.smallFileThreshold / (1024 * 1024),
|
||||
"medium_file_threshold_mb": apd.mediumFileThreshold / (1024 * 1024),
|
||||
"large_file_threshold_mb": apd.largeFileThreshold / (1024 * 1024),
|
||||
"small_file_window_minutes": apd.smallFileWindow.Minutes(),
|
||||
"medium_file_window_minutes": apd.mediumFileWindow.Minutes(),
|
||||
"large_file_window_seconds": apd.largeFileWindow.Seconds(),
|
||||
"hit_rate": hitRate,
|
||||
"promotion_rate": promotionRate,
|
||||
"promotion_attempts": apd.promotionAttempts,
|
||||
"promotion_successes": apd.promotionSuccesses,
|
||||
"fast_storage_accesses": apd.fastStorageAccesses,
|
||||
"fast_storage_hits": apd.fastStorageHits,
|
||||
}
|
||||
}
|
||||
|
||||
// Global adaptive promotion decider instance
|
||||
var adaptivePromotionDecider *AdaptivePromotionDecider
|
||||
|
||||
func init() {
|
||||
adaptivePromotionDecider = NewAdaptivePromotionDecider()
|
||||
}
|
||||
|
||||
// AdaptivePromotionDeciderFunc returns the adaptive promotion decision function
|
||||
func AdaptivePromotionDeciderFunc(fi *vfs.FileInfo, cs cachestate.CacheState) bool {
|
||||
return adaptivePromotionDecider.ShouldPromote(fi, cs)
|
||||
}
|
||||
|
||||
// RecordFastStorageAccess records fast storage access for adaptation
|
||||
func RecordFastStorageAccess() {
|
||||
adaptivePromotionDecider.RecordFastStorageAccess()
|
||||
}
|
||||
|
||||
// RecordFastStorageHit records fast storage hit for adaptation
|
||||
func RecordFastStorageHit() {
|
||||
adaptivePromotionDecider.RecordFastStorageHit()
|
||||
}
|
||||
|
||||
// GetPromotionStats returns promotion statistics for monitoring
|
||||
func GetPromotionStats() map[string]interface{} {
|
||||
return adaptivePromotionDecider.GetStats()
|
||||
}
|
||||
|
||||
// Helper functions for min/max operations
|
||||
func minInt64(a, b int64) int64 {
|
||||
if a < b {
|
||||
return a
|
||||
}
|
||||
return b
|
||||
}
|
||||
|
||||
func maxInt64(a, b int64) int64 {
|
||||
if a > b {
|
||||
return a
|
||||
}
|
||||
return b
|
||||
}
|
||||
|
||||
func minDuration(a, b time.Duration) time.Duration {
|
||||
if a < b {
|
||||
return a
|
||||
}
|
||||
return b
|
||||
}
|
||||
|
||||
func maxDuration(a, b time.Duration) time.Duration {
|
||||
if a > b {
|
||||
return a
|
||||
}
|
||||
return b
|
||||
}
|
||||
|
||||
// Ensure GCFS implements VFS.
|
||||
var _ vfs.VFS = (*GCFS)(nil)
|
||||
|
||||
|
||||
@@ -2,71 +2,41 @@
|
||||
package gc
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"s1d3sw1ped/SteamCache2/vfs/memory"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestGCOnFull(t *testing.T) {
|
||||
m := memory.New(10)
|
||||
gc := New(m, LRUGC)
|
||||
func TestGetGCAlgorithm(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
algorithm GCAlgorithm
|
||||
expected bool // true if we expect a non-nil function
|
||||
}{
|
||||
{"LRU", LRU, true},
|
||||
{"LFU", LFU, true},
|
||||
{"FIFO", FIFO, true},
|
||||
{"Largest", Largest, true},
|
||||
{"Smallest", Smallest, true},
|
||||
{"Hybrid", Hybrid, true},
|
||||
{"Unknown", "unknown", true}, // should fall back to LRU
|
||||
{"Empty", "", true}, // should fall back to LRU
|
||||
}
|
||||
|
||||
for i := 0; i < 5; i++ {
|
||||
w, err := gc.Create(fmt.Sprintf("key%d", i), 2)
|
||||
if err != nil {
|
||||
t.Fatalf("Create failed: %v", err)
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
fn := GetGCAlgorithm(tt.algorithm)
|
||||
if fn == nil {
|
||||
t.Errorf("GetGCAlgorithm(%s) returned nil, expected non-nil function", tt.algorithm)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestGCAlgorithmConstants(t *testing.T) {
|
||||
expectedAlgorithms := []GCAlgorithm{LRU, LFU, FIFO, Largest, Smallest, Hybrid}
|
||||
|
||||
for _, algo := range expectedAlgorithms {
|
||||
if algo == "" {
|
||||
t.Errorf("GC algorithm constant is empty")
|
||||
}
|
||||
w.Write([]byte("ab"))
|
||||
w.Close()
|
||||
}
|
||||
|
||||
// Cache full at 10 bytes
|
||||
w, err := gc.Create("key5", 2)
|
||||
if err != nil {
|
||||
t.Fatalf("Create failed: %v", err)
|
||||
}
|
||||
w.Write([]byte("cd"))
|
||||
w.Close()
|
||||
|
||||
if gc.Size() > 10 {
|
||||
t.Errorf("Size exceeded: %d > 10", gc.Size())
|
||||
}
|
||||
|
||||
// Check if older keys were evicted
|
||||
_, err = m.Open("key0")
|
||||
if err == nil {
|
||||
t.Error("Expected key0 to be evicted")
|
||||
}
|
||||
}
|
||||
|
||||
func TestNoGCNeeded(t *testing.T) {
|
||||
m := memory.New(20)
|
||||
gc := New(m, LRUGC)
|
||||
|
||||
for i := 0; i < 5; i++ {
|
||||
w, err := gc.Create(fmt.Sprintf("key%d", i), 2)
|
||||
if err != nil {
|
||||
t.Fatalf("Create failed: %v", err)
|
||||
}
|
||||
w.Write([]byte("ab"))
|
||||
w.Close()
|
||||
}
|
||||
|
||||
if gc.Size() != 10 {
|
||||
t.Errorf("Size: got %d, want 10", gc.Size())
|
||||
}
|
||||
}
|
||||
|
||||
func TestGCInsufficientSpace(t *testing.T) {
|
||||
m := memory.New(5)
|
||||
gc := New(m, LRUGC)
|
||||
|
||||
w, err := gc.Create("key0", 10)
|
||||
if err == nil {
|
||||
w.Close()
|
||||
t.Error("Expected ErrDiskFull")
|
||||
} else if !errors.Is(err, ErrInsufficientSpace) {
|
||||
t.Errorf("Unexpected error: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user