revert Enhance FileInfo structure and DiskFS functionality - Added CTime (creation time) and AccessCount fields to FileInfo struct for better file metadata tracking. - Updated NewFileInfo and NewFileInfoFromOS functions to initialize new fields. - Enhanced DiskFS to maintain access counts and file metadata, including flushing to JSON files. - Modified Open and Create methods to increment access counts and set creation times appropriately. - Updated garbage collection logic to utilize real access counts for files.
731 lines
22 KiB
Go
731 lines
22 KiB
Go
// vfs/gc/gc.go
|
|
package gc
|
|
|
|
import (
|
|
"fmt"
|
|
"io"
|
|
"s1d3sw1ped/SteamCache2/steamcache/logger"
|
|
"s1d3sw1ped/SteamCache2/vfs"
|
|
"s1d3sw1ped/SteamCache2/vfs/cachestate"
|
|
"s1d3sw1ped/SteamCache2/vfs/disk"
|
|
"s1d3sw1ped/SteamCache2/vfs/memory"
|
|
"s1d3sw1ped/SteamCache2/vfs/vfserror"
|
|
"sort"
|
|
"sync"
|
|
"time"
|
|
|
|
"github.com/prometheus/client_golang/prometheus"
|
|
"github.com/prometheus/client_golang/prometheus/promauto"
|
|
)
|
|
|
|
var (
|
|
// ErrInsufficientSpace is returned when there are no files to delete in the VFS.
|
|
ErrInsufficientSpace = fmt.Errorf("no files to delete")
|
|
)
|
|
|
|
// Prometheus metrics for adaptive promotion
|
|
var (
|
|
promotionThresholds = promauto.NewGaugeVec(
|
|
prometheus.GaugeOpts{
|
|
Name: "promotion_thresholds_bytes",
|
|
Help: "Current promotion thresholds in bytes",
|
|
},
|
|
[]string{"threshold_type"},
|
|
)
|
|
|
|
promotionWindows = promauto.NewGaugeVec(
|
|
prometheus.GaugeOpts{
|
|
Name: "promotion_windows_seconds",
|
|
Help: "Current promotion time windows in seconds",
|
|
},
|
|
[]string{"window_type"},
|
|
)
|
|
|
|
promotionStats = promauto.NewGaugeVec(
|
|
prometheus.GaugeOpts{
|
|
Name: "promotion_stats",
|
|
Help: "Promotion statistics",
|
|
},
|
|
[]string{"metric_type"},
|
|
)
|
|
|
|
promotionAdaptations = promauto.NewCounterVec(
|
|
prometheus.CounterOpts{
|
|
Name: "promotion_adaptations_total",
|
|
Help: "Total number of promotion threshold adaptations",
|
|
},
|
|
[]string{"direction"},
|
|
)
|
|
)
|
|
|
|
// GCAlgorithm represents different garbage collection strategies
|
|
type GCAlgorithm string
|
|
|
|
const (
|
|
LRU GCAlgorithm = "lru"
|
|
LFU GCAlgorithm = "lfu"
|
|
FIFO GCAlgorithm = "fifo"
|
|
Largest GCAlgorithm = "largest"
|
|
Smallest GCAlgorithm = "smallest"
|
|
Hybrid GCAlgorithm = "hybrid"
|
|
)
|
|
|
|
// LRUGC deletes files in LRU order until enough space is reclaimed.
|
|
func LRUGC(vfss vfs.VFS, size uint) error {
|
|
logger.Logger.Debug().Uint("target", size).Msg("Attempting to reclaim space using LRU GC")
|
|
|
|
var reclaimed uint // reclaimed space in bytes
|
|
|
|
for {
|
|
switch fs := vfss.(type) {
|
|
case *disk.DiskFS:
|
|
fi := fs.LRU.Back()
|
|
if fi == nil {
|
|
return ErrInsufficientSpace // No files to delete
|
|
}
|
|
sz := uint(fi.Size())
|
|
err := fs.Delete(fi.Name())
|
|
if err != nil {
|
|
continue // If delete fails, try the next file
|
|
}
|
|
reclaimed += sz
|
|
case *memory.MemoryFS:
|
|
fi := fs.LRU.Back()
|
|
if fi == nil {
|
|
return ErrInsufficientSpace // No files to delete
|
|
}
|
|
sz := uint(fi.Size())
|
|
err := fs.Delete(fi.Name())
|
|
if err != nil {
|
|
continue // If delete fails, try the next file
|
|
}
|
|
reclaimed += sz
|
|
default:
|
|
panic("unreachable: unsupported VFS type for LRU GC") // panic if the VFS is not disk or memory
|
|
}
|
|
|
|
if reclaimed >= size {
|
|
logger.Logger.Debug().Uint("target", size).Uint("achieved", reclaimed).Msg("Reclaimed enough space using LRU GC")
|
|
return nil // stop if enough space is reclaimed
|
|
}
|
|
}
|
|
}
|
|
|
|
// LFUGC deletes files in LFU (Least Frequently Used) order until enough space is reclaimed.
|
|
func LFUGC(vfss vfs.VFS, size uint) error {
|
|
logger.Logger.Debug().Uint("target", size).Msg("Attempting to reclaim space using LFU GC")
|
|
|
|
// Get all files and sort by access count (frequency)
|
|
files := getAllFiles(vfss)
|
|
if len(files) == 0 {
|
|
return ErrInsufficientSpace
|
|
}
|
|
|
|
// Sort by access count (ascending - least frequently used first)
|
|
sort.Slice(files, func(i, j int) bool {
|
|
return files[i].AccessCount < files[j].AccessCount
|
|
})
|
|
|
|
var reclaimed uint
|
|
for _, fi := range files {
|
|
if reclaimed >= size {
|
|
break
|
|
}
|
|
err := vfss.Delete(fi.Name)
|
|
if err != nil {
|
|
continue
|
|
}
|
|
reclaimed += uint(fi.Size)
|
|
}
|
|
|
|
if reclaimed >= size {
|
|
logger.Logger.Debug().Uint("target", size).Uint("achieved", reclaimed).Msg("Reclaimed enough space using LFU GC")
|
|
return nil
|
|
}
|
|
return ErrInsufficientSpace
|
|
}
|
|
|
|
// FIFOGC deletes files in FIFO (First In, First Out) order until enough space is reclaimed.
|
|
func FIFOGC(vfss vfs.VFS, size uint) error {
|
|
logger.Logger.Debug().Uint("target", size).Msg("Attempting to reclaim space using FIFO GC")
|
|
|
|
// Get all files and sort by creation time (oldest first)
|
|
files := getAllFiles(vfss)
|
|
if len(files) == 0 {
|
|
return ErrInsufficientSpace
|
|
}
|
|
|
|
// Sort by creation time (ascending - oldest first)
|
|
sort.Slice(files, func(i, j int) bool {
|
|
return files[i].MTime.Before(files[j].MTime)
|
|
})
|
|
|
|
var reclaimed uint
|
|
for _, fi := range files {
|
|
if reclaimed >= size {
|
|
break
|
|
}
|
|
err := vfss.Delete(fi.Name)
|
|
if err != nil {
|
|
continue
|
|
}
|
|
reclaimed += uint(fi.Size)
|
|
}
|
|
|
|
if reclaimed >= size {
|
|
logger.Logger.Debug().Uint("target", size).Uint("achieved", reclaimed).Msg("Reclaimed enough space using FIFO GC")
|
|
return nil
|
|
}
|
|
return ErrInsufficientSpace
|
|
}
|
|
|
|
// LargestGC deletes the largest files first until enough space is reclaimed.
|
|
func LargestGC(vfss vfs.VFS, size uint) error {
|
|
logger.Logger.Debug().Uint("target", size).Msg("Attempting to reclaim space using Largest GC")
|
|
|
|
// Get all files and sort by size (largest first)
|
|
files := getAllFiles(vfss)
|
|
if len(files) == 0 {
|
|
return ErrInsufficientSpace
|
|
}
|
|
|
|
// Sort by size (descending - largest first)
|
|
sort.Slice(files, func(i, j int) bool {
|
|
return files[i].Size > files[j].Size
|
|
})
|
|
|
|
var reclaimed uint
|
|
for _, fi := range files {
|
|
if reclaimed >= size {
|
|
break
|
|
}
|
|
err := vfss.Delete(fi.Name)
|
|
if err != nil {
|
|
continue
|
|
}
|
|
reclaimed += uint(fi.Size)
|
|
}
|
|
|
|
if reclaimed >= size {
|
|
logger.Logger.Debug().Uint("target", size).Uint("achieved", reclaimed).Msg("Reclaimed enough space using Largest GC")
|
|
return nil
|
|
}
|
|
return ErrInsufficientSpace
|
|
}
|
|
|
|
// SmallestGC deletes the smallest files first until enough space is reclaimed.
|
|
func SmallestGC(vfss vfs.VFS, size uint) error {
|
|
logger.Logger.Debug().Uint("target", size).Msg("Attempting to reclaim space using Smallest GC")
|
|
|
|
// Get all files and sort by size (smallest first)
|
|
files := getAllFiles(vfss)
|
|
if len(files) == 0 {
|
|
return ErrInsufficientSpace
|
|
}
|
|
|
|
// Sort by size (ascending - smallest first)
|
|
sort.Slice(files, func(i, j int) bool {
|
|
return files[i].Size < files[j].Size
|
|
})
|
|
|
|
var reclaimed uint
|
|
for _, fi := range files {
|
|
if reclaimed >= size {
|
|
break
|
|
}
|
|
err := vfss.Delete(fi.Name)
|
|
if err != nil {
|
|
continue
|
|
}
|
|
reclaimed += uint(fi.Size)
|
|
}
|
|
|
|
if reclaimed >= size {
|
|
logger.Logger.Debug().Uint("target", size).Uint("achieved", reclaimed).Msg("Reclaimed enough space using Smallest GC")
|
|
return nil
|
|
}
|
|
return ErrInsufficientSpace
|
|
}
|
|
|
|
// HybridGC combines LRU and size-based eviction with a scoring system.
|
|
func HybridGC(vfss vfs.VFS, size uint) error {
|
|
logger.Logger.Debug().Uint("target", size).Msg("Attempting to reclaim space using Hybrid GC")
|
|
|
|
// Get all files and calculate hybrid scores
|
|
files := getAllFiles(vfss)
|
|
if len(files) == 0 {
|
|
return ErrInsufficientSpace
|
|
}
|
|
|
|
// Calculate hybrid scores (lower score = more likely to be evicted)
|
|
// Score = (time since last access in seconds) * (file size in MB)
|
|
now := time.Now()
|
|
for i := range files {
|
|
timeSinceAccess := now.Sub(files[i].ATime).Seconds()
|
|
sizeMB := float64(files[i].Size) / (1024 * 1024)
|
|
files[i].HybridScore = timeSinceAccess * sizeMB
|
|
}
|
|
|
|
// Sort by hybrid score (ascending - lowest scores first)
|
|
sort.Slice(files, func(i, j int) bool {
|
|
return files[i].HybridScore < files[j].HybridScore
|
|
})
|
|
|
|
var reclaimed uint
|
|
for _, fi := range files {
|
|
if reclaimed >= size {
|
|
break
|
|
}
|
|
err := vfss.Delete(fi.Name)
|
|
if err != nil {
|
|
continue
|
|
}
|
|
reclaimed += uint(fi.Size)
|
|
}
|
|
|
|
if reclaimed >= size {
|
|
logger.Logger.Debug().Uint("target", size).Uint("achieved", reclaimed).Msg("Reclaimed enough space using Hybrid GC")
|
|
return nil
|
|
}
|
|
return ErrInsufficientSpace
|
|
}
|
|
|
|
// fileInfoWithMetadata extends FileInfo with additional metadata for GC algorithms
|
|
type fileInfoWithMetadata struct {
|
|
Name string
|
|
Size int64
|
|
MTime time.Time
|
|
ATime time.Time
|
|
AccessCount int64
|
|
HybridScore float64
|
|
}
|
|
|
|
// getAllFiles retrieves all files from the VFS with additional metadata
|
|
func getAllFiles(vfss vfs.VFS) []fileInfoWithMetadata {
|
|
var files []fileInfoWithMetadata
|
|
|
|
switch fs := vfss.(type) {
|
|
case *disk.DiskFS:
|
|
allFiles := fs.StatAll()
|
|
for _, fi := range allFiles {
|
|
// For disk, we can't easily track access count, so we'll use 1 as default
|
|
files = append(files, fileInfoWithMetadata{
|
|
Name: fi.Name(),
|
|
Size: fi.Size(),
|
|
MTime: fi.ModTime(),
|
|
ATime: fi.AccessTime(),
|
|
AccessCount: 1,
|
|
})
|
|
}
|
|
case *memory.MemoryFS:
|
|
allFiles := fs.StatAll()
|
|
for _, fi := range allFiles {
|
|
// For memory, we can't easily track access count, so we'll use 1 as default
|
|
files = append(files, fileInfoWithMetadata{
|
|
Name: fi.Name(),
|
|
Size: fi.Size(),
|
|
MTime: fi.ModTime(),
|
|
ATime: fi.AccessTime(),
|
|
AccessCount: 1,
|
|
})
|
|
}
|
|
}
|
|
|
|
return files
|
|
}
|
|
|
|
// GetGCAlgorithm returns the appropriate GC function based on the algorithm name
|
|
func GetGCAlgorithm(algorithm GCAlgorithm) GCHandlerFunc {
|
|
switch algorithm {
|
|
case LRU:
|
|
return LRUGC
|
|
case LFU:
|
|
return LFUGC
|
|
case FIFO:
|
|
return FIFOGC
|
|
case Largest:
|
|
return LargestGC
|
|
case Smallest:
|
|
return SmallestGC
|
|
case Hybrid:
|
|
return HybridGC
|
|
default:
|
|
logger.Logger.Warn().Str("algorithm", string(algorithm)).Msg("Unknown GC algorithm, falling back to LRU")
|
|
return LRUGC
|
|
}
|
|
}
|
|
|
|
func PromotionDecider(fi *vfs.FileInfo, cs cachestate.CacheState) bool {
|
|
return time.Since(fi.AccessTime()) < time.Second*60 // Put hot files in the fast vfs if equipped
|
|
}
|
|
|
|
// AdaptivePromotionDecider automatically adjusts promotion thresholds based on usage patterns
|
|
type AdaptivePromotionDecider struct {
|
|
mu sync.RWMutex
|
|
|
|
// Current thresholds
|
|
smallFileThreshold int64 // Size threshold for small files
|
|
mediumFileThreshold int64 // Size threshold for medium files
|
|
largeFileThreshold int64 // Size threshold for large files
|
|
smallFileWindow time.Duration // Time window for small files
|
|
mediumFileWindow time.Duration // Time window for medium files
|
|
largeFileWindow time.Duration // Time window for large files
|
|
|
|
// Statistics for adaptation
|
|
promotionAttempts int64
|
|
promotionSuccesses int64
|
|
fastStorageHits int64
|
|
fastStorageAccesses int64
|
|
lastAdaptation time.Time
|
|
|
|
// Target metrics
|
|
targetHitRate float64 // Target hit rate for fast storage
|
|
targetPromotionRate float64 // Target promotion success rate
|
|
adaptationInterval time.Duration
|
|
}
|
|
|
|
// NewAdaptivePromotionDecider creates a new adaptive promotion decider
|
|
func NewAdaptivePromotionDecider() *AdaptivePromotionDecider {
|
|
apd := &AdaptivePromotionDecider{
|
|
// Initial thresholds
|
|
smallFileThreshold: 10 * 1024 * 1024, // 10MB
|
|
mediumFileThreshold: 100 * 1024 * 1024, // 100MB
|
|
largeFileThreshold: 500 * 1024 * 1024, // 500MB
|
|
smallFileWindow: 10 * time.Minute,
|
|
mediumFileWindow: 2 * time.Minute,
|
|
largeFileWindow: 30 * time.Second,
|
|
|
|
// Target metrics
|
|
targetHitRate: 0.8, // 80% hit rate
|
|
targetPromotionRate: 0.7, // 70% promotion success rate
|
|
adaptationInterval: 5 * time.Minute,
|
|
}
|
|
|
|
// Initialize Prometheus metrics
|
|
apd.updatePrometheusMetrics()
|
|
|
|
return apd
|
|
}
|
|
|
|
// ShouldPromote determines if a file should be promoted based on adaptive thresholds
|
|
func (apd *AdaptivePromotionDecider) ShouldPromote(fi *vfs.FileInfo, cs cachestate.CacheState) bool {
|
|
apd.mu.Lock()
|
|
defer apd.mu.Unlock()
|
|
|
|
// Check if it's time to adapt thresholds
|
|
if time.Since(apd.lastAdaptation) > apd.adaptationInterval {
|
|
apd.adaptThresholds()
|
|
}
|
|
|
|
size := fi.Size()
|
|
timeSinceAccess := time.Since(fi.AccessTime())
|
|
|
|
// Record promotion attempt
|
|
apd.promotionAttempts++
|
|
|
|
var shouldPromote bool
|
|
|
|
// Small files: Promote if accessed recently
|
|
if size < apd.smallFileThreshold {
|
|
shouldPromote = timeSinceAccess < apd.smallFileWindow
|
|
} else if size < apd.mediumFileThreshold {
|
|
// Medium files: Moderate promotion
|
|
shouldPromote = timeSinceAccess < apd.mediumFileWindow
|
|
} else if size < apd.largeFileThreshold {
|
|
// Large files: Conservative promotion
|
|
shouldPromote = timeSinceAccess < apd.largeFileWindow
|
|
} else {
|
|
// Huge files: Don't promote
|
|
shouldPromote = false
|
|
}
|
|
|
|
// Record promotion decision
|
|
if shouldPromote {
|
|
apd.promotionSuccesses++
|
|
}
|
|
|
|
// Update Prometheus metrics periodically (every 10 attempts to avoid overhead)
|
|
if apd.promotionAttempts%10 == 0 {
|
|
apd.updatePrometheusMetrics()
|
|
}
|
|
|
|
return shouldPromote
|
|
}
|
|
|
|
// RecordFastStorageAccess records when fast storage is accessed
|
|
func (apd *AdaptivePromotionDecider) RecordFastStorageAccess() {
|
|
apd.mu.Lock()
|
|
defer apd.mu.Unlock()
|
|
apd.fastStorageAccesses++
|
|
|
|
// Update Prometheus metrics periodically
|
|
if apd.fastStorageAccesses%10 == 0 {
|
|
apd.updatePrometheusMetrics()
|
|
}
|
|
}
|
|
|
|
// RecordFastStorageHit records when fast storage has a hit
|
|
func (apd *AdaptivePromotionDecider) RecordFastStorageHit() {
|
|
apd.mu.Lock()
|
|
defer apd.mu.Unlock()
|
|
apd.fastStorageHits++
|
|
|
|
// Update Prometheus metrics periodically
|
|
if apd.fastStorageHits%10 == 0 {
|
|
apd.updatePrometheusMetrics()
|
|
}
|
|
}
|
|
|
|
// adaptThresholds adjusts thresholds based on current performance
|
|
func (apd *AdaptivePromotionDecider) adaptThresholds() {
|
|
if apd.promotionAttempts < 10 || apd.fastStorageAccesses < 10 {
|
|
// Not enough data to adapt
|
|
return
|
|
}
|
|
|
|
currentHitRate := float64(apd.fastStorageHits) / float64(apd.fastStorageAccesses)
|
|
currentPromotionRate := float64(apd.promotionSuccesses) / float64(apd.promotionAttempts)
|
|
|
|
logger.Logger.Debug().
|
|
Float64("hit_rate", currentHitRate).
|
|
Float64("promotion_rate", currentPromotionRate).
|
|
Float64("target_hit_rate", apd.targetHitRate).
|
|
Float64("target_promotion_rate", apd.targetPromotionRate).
|
|
Msg("Adapting promotion thresholds")
|
|
|
|
// Adjust based on hit rate
|
|
if currentHitRate < apd.targetHitRate {
|
|
// Hit rate too low - be more aggressive with promotion
|
|
apd.adjustThresholdsMoreAggressive()
|
|
} else if currentHitRate > apd.targetHitRate+0.1 {
|
|
// Hit rate too high - be more conservative
|
|
apd.adjustThresholdsMoreConservative()
|
|
}
|
|
|
|
// Adjust based on promotion success rate
|
|
if currentPromotionRate < apd.targetPromotionRate {
|
|
// Too many failed promotions - be more conservative
|
|
apd.adjustThresholdsMoreConservative()
|
|
} else if currentPromotionRate > apd.targetPromotionRate+0.1 {
|
|
// High promotion success - can be more aggressive
|
|
apd.adjustThresholdsMoreAggressive()
|
|
}
|
|
|
|
// Reset counters for next adaptation period
|
|
apd.promotionAttempts = 0
|
|
apd.promotionSuccesses = 0
|
|
apd.fastStorageHits = 0
|
|
apd.fastStorageAccesses = 0
|
|
apd.lastAdaptation = time.Now()
|
|
|
|
logger.Logger.Info().
|
|
Int64("small_threshold_mb", apd.smallFileThreshold/(1024*1024)).
|
|
Int64("medium_threshold_mb", apd.mediumFileThreshold/(1024*1024)).
|
|
Int64("large_threshold_mb", apd.largeFileThreshold/(1024*1024)).
|
|
Dur("small_window", apd.smallFileWindow).
|
|
Dur("medium_window", apd.mediumFileWindow).
|
|
Dur("large_window", apd.largeFileWindow).
|
|
Msg("Updated promotion thresholds")
|
|
}
|
|
|
|
// updatePrometheusMetrics updates all Prometheus metrics with current values
|
|
func (apd *AdaptivePromotionDecider) updatePrometheusMetrics() {
|
|
// Update threshold metrics
|
|
promotionThresholds.WithLabelValues("small").Set(float64(apd.smallFileThreshold))
|
|
promotionThresholds.WithLabelValues("medium").Set(float64(apd.mediumFileThreshold))
|
|
promotionThresholds.WithLabelValues("large").Set(float64(apd.largeFileThreshold))
|
|
|
|
// Update window metrics
|
|
promotionWindows.WithLabelValues("small").Set(apd.smallFileWindow.Seconds())
|
|
promotionWindows.WithLabelValues("medium").Set(apd.mediumFileWindow.Seconds())
|
|
promotionWindows.WithLabelValues("large").Set(apd.largeFileWindow.Seconds())
|
|
|
|
// Update statistics metrics
|
|
hitRate := 0.0
|
|
if apd.fastStorageAccesses > 0 {
|
|
hitRate = float64(apd.fastStorageHits) / float64(apd.fastStorageAccesses)
|
|
}
|
|
promotionRate := 0.0
|
|
if apd.promotionAttempts > 0 {
|
|
promotionRate = float64(apd.promotionSuccesses) / float64(apd.promotionAttempts)
|
|
}
|
|
|
|
promotionStats.WithLabelValues("hit_rate").Set(hitRate)
|
|
promotionStats.WithLabelValues("promotion_rate").Set(promotionRate)
|
|
promotionStats.WithLabelValues("promotion_attempts").Set(float64(apd.promotionAttempts))
|
|
promotionStats.WithLabelValues("promotion_successes").Set(float64(apd.promotionSuccesses))
|
|
promotionStats.WithLabelValues("fast_storage_accesses").Set(float64(apd.fastStorageAccesses))
|
|
promotionStats.WithLabelValues("fast_storage_hits").Set(float64(apd.fastStorageHits))
|
|
}
|
|
|
|
// adjustThresholdsMoreAggressive makes promotion more aggressive
|
|
func (apd *AdaptivePromotionDecider) adjustThresholdsMoreAggressive() {
|
|
// Increase size thresholds (promote larger files)
|
|
apd.smallFileThreshold = minInt64(apd.smallFileThreshold*11/10, 50*1024*1024) // Max 50MB
|
|
apd.mediumFileThreshold = minInt64(apd.mediumFileThreshold*11/10, 200*1024*1024) // Max 200MB
|
|
apd.largeFileThreshold = minInt64(apd.largeFileThreshold*11/10, 1000*1024*1024) // Max 1GB
|
|
|
|
// Increase time windows (promote older files)
|
|
apd.smallFileWindow = minDuration(apd.smallFileWindow*11/10, 20*time.Minute)
|
|
apd.mediumFileWindow = minDuration(apd.mediumFileWindow*11/10, 5*time.Minute)
|
|
apd.largeFileWindow = minDuration(apd.largeFileWindow*11/10, 2*time.Minute)
|
|
|
|
// Record adaptation in Prometheus
|
|
promotionAdaptations.WithLabelValues("aggressive").Inc()
|
|
|
|
// Update Prometheus metrics
|
|
apd.updatePrometheusMetrics()
|
|
}
|
|
|
|
// adjustThresholdsMoreConservative makes promotion more conservative
|
|
func (apd *AdaptivePromotionDecider) adjustThresholdsMoreConservative() {
|
|
// Decrease size thresholds (promote smaller files)
|
|
apd.smallFileThreshold = maxInt64(apd.smallFileThreshold*9/10, 5*1024*1024) // Min 5MB
|
|
apd.mediumFileThreshold = maxInt64(apd.mediumFileThreshold*9/10, 50*1024*1024) // Min 50MB
|
|
apd.largeFileThreshold = maxInt64(apd.largeFileThreshold*9/10, 200*1024*1024) // Min 200MB
|
|
|
|
// Decrease time windows (promote only recent files)
|
|
apd.smallFileWindow = maxDuration(apd.smallFileWindow*9/10, 5*time.Minute)
|
|
apd.mediumFileWindow = maxDuration(apd.mediumFileWindow*9/10, 1*time.Minute)
|
|
apd.largeFileWindow = maxDuration(apd.largeFileWindow*9/10, 15*time.Second)
|
|
|
|
// Record adaptation in Prometheus
|
|
promotionAdaptations.WithLabelValues("conservative").Inc()
|
|
|
|
// Update Prometheus metrics
|
|
apd.updatePrometheusMetrics()
|
|
}
|
|
|
|
// GetStats returns current statistics for monitoring
|
|
func (apd *AdaptivePromotionDecider) GetStats() map[string]interface{} {
|
|
apd.mu.RLock()
|
|
defer apd.mu.RUnlock()
|
|
|
|
hitRate := 0.0
|
|
if apd.fastStorageAccesses > 0 {
|
|
hitRate = float64(apd.fastStorageHits) / float64(apd.fastStorageAccesses)
|
|
}
|
|
|
|
promotionRate := 0.0
|
|
if apd.promotionAttempts > 0 {
|
|
promotionRate = float64(apd.promotionSuccesses) / float64(apd.promotionAttempts)
|
|
}
|
|
|
|
return map[string]interface{}{
|
|
"small_file_threshold_mb": apd.smallFileThreshold / (1024 * 1024),
|
|
"medium_file_threshold_mb": apd.mediumFileThreshold / (1024 * 1024),
|
|
"large_file_threshold_mb": apd.largeFileThreshold / (1024 * 1024),
|
|
"small_file_window_minutes": apd.smallFileWindow.Minutes(),
|
|
"medium_file_window_minutes": apd.mediumFileWindow.Minutes(),
|
|
"large_file_window_seconds": apd.largeFileWindow.Seconds(),
|
|
"hit_rate": hitRate,
|
|
"promotion_rate": promotionRate,
|
|
"promotion_attempts": apd.promotionAttempts,
|
|
"promotion_successes": apd.promotionSuccesses,
|
|
"fast_storage_accesses": apd.fastStorageAccesses,
|
|
"fast_storage_hits": apd.fastStorageHits,
|
|
}
|
|
}
|
|
|
|
// Global adaptive promotion decider instance
|
|
var adaptivePromotionDecider *AdaptivePromotionDecider
|
|
|
|
func init() {
|
|
adaptivePromotionDecider = NewAdaptivePromotionDecider()
|
|
}
|
|
|
|
// AdaptivePromotionDeciderFunc returns the adaptive promotion decision function
|
|
func AdaptivePromotionDeciderFunc(fi *vfs.FileInfo, cs cachestate.CacheState) bool {
|
|
return adaptivePromotionDecider.ShouldPromote(fi, cs)
|
|
}
|
|
|
|
// RecordFastStorageAccess records fast storage access for adaptation
|
|
func RecordFastStorageAccess() {
|
|
adaptivePromotionDecider.RecordFastStorageAccess()
|
|
}
|
|
|
|
// RecordFastStorageHit records fast storage hit for adaptation
|
|
func RecordFastStorageHit() {
|
|
adaptivePromotionDecider.RecordFastStorageHit()
|
|
}
|
|
|
|
// GetPromotionStats returns promotion statistics for monitoring
|
|
func GetPromotionStats() map[string]interface{} {
|
|
return adaptivePromotionDecider.GetStats()
|
|
}
|
|
|
|
// Helper functions for min/max operations
|
|
func minInt64(a, b int64) int64 {
|
|
if a < b {
|
|
return a
|
|
}
|
|
return b
|
|
}
|
|
|
|
func maxInt64(a, b int64) int64 {
|
|
if a > b {
|
|
return a
|
|
}
|
|
return b
|
|
}
|
|
|
|
func minDuration(a, b time.Duration) time.Duration {
|
|
if a < b {
|
|
return a
|
|
}
|
|
return b
|
|
}
|
|
|
|
func maxDuration(a, b time.Duration) time.Duration {
|
|
if a > b {
|
|
return a
|
|
}
|
|
return b
|
|
}
|
|
|
|
// Ensure GCFS implements VFS.
|
|
var _ vfs.VFS = (*GCFS)(nil)
|
|
|
|
// GCFS is a virtual file system that calls a GC handler when the disk is full. The GC handler is responsible for freeing up space on the disk. The GCFS is a wrapper around another VFS.
|
|
type GCFS struct {
|
|
vfs.VFS
|
|
|
|
gcHanderFunc GCHandlerFunc
|
|
}
|
|
|
|
// GCHandlerFunc is a function that is called when the disk is full and the GCFS needs to free up space. It is passed the VFS and the size of the file that needs to be written. Its up to the implementation to free up space. How much space is freed is also up to the implementation.
|
|
type GCHandlerFunc func(vfs vfs.VFS, size uint) error
|
|
|
|
func New(vfs vfs.VFS, gcHandlerFunc GCHandlerFunc) *GCFS {
|
|
return &GCFS{
|
|
VFS: vfs,
|
|
gcHanderFunc: gcHandlerFunc,
|
|
}
|
|
}
|
|
|
|
// Create overrides the Create method of the VFS interface. It tries to create the key, if it fails due to disk full error, it calls the GC handler and tries again. If it still fails it returns the error.
|
|
func (g *GCFS) Create(key string, size int64) (io.WriteCloser, error) {
|
|
w, err := g.VFS.Create(key, size) // try to create the key
|
|
for err == vfserror.ErrDiskFull && g.gcHanderFunc != nil { // if the error is disk full and there is a GC handler
|
|
errr := g.gcHanderFunc(g.VFS, uint(size)) // call the GC handler
|
|
if errr == ErrInsufficientSpace {
|
|
return nil, errr // if the GC handler returns no files to delete, return the error
|
|
}
|
|
w, err = g.VFS.Create(key, size)
|
|
}
|
|
|
|
if err != nil {
|
|
if err == vfserror.ErrDiskFull {
|
|
logger.Logger.Error().Str("key", key).Int64("size", size).Msg("Failed to create file due to disk full, even after GC")
|
|
} else {
|
|
logger.Logger.Error().Str("key", key).Int64("size", size).Err(err).Msg("Failed to create file")
|
|
}
|
|
}
|
|
|
|
return w, err
|
|
}
|
|
|
|
func (g *GCFS) Name() string {
|
|
return fmt.Sprintf("GCFS(%s)", g.VFS.Name()) // wrap the name of the VFS with GCFS so we can see that its a GCFS
|
|
}
|