Refactor configuration management and enhance build process
- Introduced a YAML-based configuration system, allowing for automatic generation of a default `config.yaml` file. - Updated the application to load configuration settings from the YAML file, improving flexibility and ease of use. - Added a Makefile to streamline development tasks, including running the application, testing, and managing dependencies. - Enhanced `.gitignore` to include build artifacts and configuration files. - Removed unused Prometheus metrics and related code to simplify the codebase. - Updated dependencies in `go.mod` and `go.sum` for improved functionality and performance.
This commit is contained in:
776
vfs/gc/gc.go
776
vfs/gc/gc.go
@@ -2,60 +2,8 @@
|
||||
package gc
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"io"
|
||||
"s1d3sw1ped/SteamCache2/steamcache/logger"
|
||||
"s1d3sw1ped/SteamCache2/vfs"
|
||||
"s1d3sw1ped/SteamCache2/vfs/cachestate"
|
||||
"s1d3sw1ped/SteamCache2/vfs/disk"
|
||||
"s1d3sw1ped/SteamCache2/vfs/memory"
|
||||
"s1d3sw1ped/SteamCache2/vfs/vfserror"
|
||||
"sort"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/prometheus/client_golang/prometheus"
|
||||
"github.com/prometheus/client_golang/prometheus/promauto"
|
||||
)
|
||||
|
||||
var (
|
||||
// ErrInsufficientSpace is returned when there are no files to delete in the VFS.
|
||||
ErrInsufficientSpace = fmt.Errorf("no files to delete")
|
||||
)
|
||||
|
||||
// Prometheus metrics for adaptive promotion
|
||||
var (
|
||||
promotionThresholds = promauto.NewGaugeVec(
|
||||
prometheus.GaugeOpts{
|
||||
Name: "promotion_thresholds_bytes",
|
||||
Help: "Current promotion thresholds in bytes",
|
||||
},
|
||||
[]string{"threshold_type"},
|
||||
)
|
||||
|
||||
promotionWindows = promauto.NewGaugeVec(
|
||||
prometheus.GaugeOpts{
|
||||
Name: "promotion_windows_seconds",
|
||||
Help: "Current promotion time windows in seconds",
|
||||
},
|
||||
[]string{"window_type"},
|
||||
)
|
||||
|
||||
promotionStats = promauto.NewGaugeVec(
|
||||
prometheus.GaugeOpts{
|
||||
Name: "promotion_stats",
|
||||
Help: "Promotion statistics",
|
||||
},
|
||||
[]string{"metric_type"},
|
||||
)
|
||||
|
||||
promotionAdaptations = promauto.NewCounterVec(
|
||||
prometheus.CounterOpts{
|
||||
Name: "promotion_adaptations_total",
|
||||
Help: "Total number of promotion threshold adaptations",
|
||||
},
|
||||
[]string{"direction"},
|
||||
)
|
||||
)
|
||||
|
||||
// GCAlgorithm represents different garbage collection strategies
|
||||
@@ -70,677 +18,143 @@ const (
|
||||
Hybrid GCAlgorithm = "hybrid"
|
||||
)
|
||||
|
||||
// LRUGC deletes files in LRU order until enough space is reclaimed.
|
||||
func LRUGC(vfss vfs.VFS, size uint) error {
|
||||
logger.Logger.Debug().Uint("target", size).Msg("Attempting to reclaim space using LRU GC")
|
||||
|
||||
var reclaimed uint // reclaimed space in bytes
|
||||
deleted := false
|
||||
|
||||
for {
|
||||
switch fs := vfss.(type) {
|
||||
case *disk.DiskFS:
|
||||
fi := fs.LRU.Back()
|
||||
if fi == nil {
|
||||
if deleted {
|
||||
logger.Logger.Debug().Uint("target", size).Uint("achieved", reclaimed).Msg("Reclaimed enough space using LRU GC (at least one file deleted)")
|
||||
return nil
|
||||
}
|
||||
return ErrInsufficientSpace // No files to delete
|
||||
}
|
||||
sz := uint(fi.Size())
|
||||
err := fs.Delete(fi.Name())
|
||||
if err != nil {
|
||||
continue // If delete fails, try the next file
|
||||
}
|
||||
reclaimed += sz
|
||||
deleted = true
|
||||
case *memory.MemoryFS:
|
||||
fi := fs.LRU.Back()
|
||||
if fi == nil {
|
||||
if deleted {
|
||||
logger.Logger.Debug().Uint("target", size).Uint("achieved", reclaimed).Msg("Reclaimed enough space using LRU GC (at least one file deleted)")
|
||||
return nil
|
||||
}
|
||||
return ErrInsufficientSpace // No files to delete
|
||||
}
|
||||
sz := uint(fi.Size())
|
||||
err := fs.Delete(fi.Name())
|
||||
if err != nil {
|
||||
continue // If delete fails, try the next file
|
||||
}
|
||||
reclaimed += sz
|
||||
deleted = true
|
||||
default:
|
||||
panic("unreachable: unsupported VFS type for LRU GC") // panic if the VFS is not disk or memory
|
||||
}
|
||||
|
||||
if deleted && (size == 0 || reclaimed >= size) {
|
||||
logger.Logger.Debug().Uint("target", size).Uint("achieved", reclaimed).Msg("Reclaimed enough space using LRU GC (at least one file deleted)")
|
||||
return nil // stop if enough space is reclaimed or at least one file deleted for size==0
|
||||
}
|
||||
}
|
||||
// GCFS wraps a VFS with garbage collection capabilities
|
||||
type GCFS struct {
|
||||
vfs vfs.VFS
|
||||
algorithm GCAlgorithm
|
||||
gcFunc func(vfs.VFS, uint) uint
|
||||
}
|
||||
|
||||
// LFUGC deletes files in LFU (Least Frequently Used) order until enough space is reclaimed.
|
||||
func LFUGC(vfss vfs.VFS, size uint) error {
|
||||
logger.Logger.Debug().Uint("target", size).Msg("Attempting to reclaim space using LFU GC")
|
||||
|
||||
files := getAllFiles(vfss)
|
||||
if len(files) == 0 {
|
||||
return ErrInsufficientSpace
|
||||
// New creates a new GCFS with the specified algorithm
|
||||
func New(wrappedVFS vfs.VFS, algorithm GCAlgorithm) *GCFS {
|
||||
gcfs := &GCFS{
|
||||
vfs: wrappedVFS,
|
||||
algorithm: algorithm,
|
||||
}
|
||||
|
||||
sort.Slice(files, func(i, j int) bool {
|
||||
return files[i].AccessCount < files[j].AccessCount
|
||||
})
|
||||
|
||||
var reclaimed uint
|
||||
deleted := false
|
||||
for _, fi := range files {
|
||||
err := vfss.Delete(fi.Name)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
reclaimed += uint(fi.Size)
|
||||
deleted = true
|
||||
if deleted && (size == 0 || reclaimed >= size) {
|
||||
logger.Logger.Debug().Uint("target", size).Uint("achieved", reclaimed).Msg("Reclaimed enough space using LFU GC (at least one file deleted)")
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
if deleted {
|
||||
logger.Logger.Debug().Uint("target", size).Uint("achieved", reclaimed).Msg("Reclaimed enough space using LFU GC (at least one file deleted)")
|
||||
return nil
|
||||
}
|
||||
return ErrInsufficientSpace
|
||||
}
|
||||
|
||||
// FIFOGC deletes files in FIFO (First In, First Out) order until enough space is reclaimed.
|
||||
func FIFOGC(vfss vfs.VFS, size uint) error {
|
||||
logger.Logger.Debug().Uint("target", size).Msg("Attempting to reclaim space using FIFO GC")
|
||||
|
||||
files := getAllFiles(vfss)
|
||||
if len(files) == 0 {
|
||||
return ErrInsufficientSpace
|
||||
}
|
||||
|
||||
sort.Slice(files, func(i, j int) bool {
|
||||
return files[i].MTime.Before(files[j].MTime)
|
||||
})
|
||||
|
||||
var reclaimed uint
|
||||
deleted := false
|
||||
for _, fi := range files {
|
||||
err := vfss.Delete(fi.Name)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
reclaimed += uint(fi.Size)
|
||||
deleted = true
|
||||
if deleted && (size == 0 || reclaimed >= size) {
|
||||
logger.Logger.Debug().Uint("target", size).Uint("achieved", reclaimed).Msg("Reclaimed enough space using FIFO GC (at least one file deleted)")
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
if deleted {
|
||||
logger.Logger.Debug().Uint("target", size).Uint("achieved", reclaimed).Msg("Reclaimed enough space using FIFO GC (at least one file deleted)")
|
||||
return nil
|
||||
}
|
||||
return ErrInsufficientSpace
|
||||
}
|
||||
|
||||
// LargestGC deletes the largest files first until enough space is reclaimed.
|
||||
func LargestGC(vfss vfs.VFS, size uint) error {
|
||||
logger.Logger.Debug().Uint("target", size).Msg("Attempting to reclaim space using Largest GC")
|
||||
|
||||
files := getAllFiles(vfss)
|
||||
if len(files) == 0 {
|
||||
return ErrInsufficientSpace
|
||||
}
|
||||
|
||||
sort.Slice(files, func(i, j int) bool {
|
||||
return files[i].Size > files[j].Size
|
||||
})
|
||||
|
||||
var reclaimed uint
|
||||
deleted := false
|
||||
for _, fi := range files {
|
||||
err := vfss.Delete(fi.Name)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
reclaimed += uint(fi.Size)
|
||||
deleted = true
|
||||
if deleted && (size == 0 || reclaimed >= size) {
|
||||
logger.Logger.Debug().Uint("target", size).Uint("achieved", reclaimed).Msg("Reclaimed enough space using Largest GC (at least one file deleted)")
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
if deleted {
|
||||
logger.Logger.Debug().Uint("target", size).Uint("achieved", reclaimed).Msg("Reclaimed enough space using Largest GC (at least one file deleted)")
|
||||
return nil
|
||||
}
|
||||
return ErrInsufficientSpace
|
||||
}
|
||||
|
||||
// SmallestGC deletes the smallest files first until enough space is reclaimed.
|
||||
func SmallestGC(vfss vfs.VFS, size uint) error {
|
||||
logger.Logger.Debug().Uint("target", size).Msg("Attempting to reclaim space using Smallest GC")
|
||||
|
||||
files := getAllFiles(vfss)
|
||||
if len(files) == 0 {
|
||||
return ErrInsufficientSpace
|
||||
}
|
||||
|
||||
sort.Slice(files, func(i, j int) bool {
|
||||
return files[i].Size < files[j].Size
|
||||
})
|
||||
|
||||
var reclaimed uint
|
||||
deleted := false
|
||||
for _, fi := range files {
|
||||
err := vfss.Delete(fi.Name)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
reclaimed += uint(fi.Size)
|
||||
deleted = true
|
||||
if deleted && (size == 0 || reclaimed >= size) {
|
||||
logger.Logger.Debug().Uint("target", size).Uint("achieved", reclaimed).Msg("Reclaimed enough space using Smallest GC (at least one file deleted)")
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
if deleted {
|
||||
logger.Logger.Debug().Uint("target", size).Uint("achieved", reclaimed).Msg("Reclaimed enough space using Smallest GC (at least one file deleted)")
|
||||
return nil
|
||||
}
|
||||
return ErrInsufficientSpace
|
||||
}
|
||||
|
||||
// HybridGC combines LRU and size-based eviction with a scoring system.
|
||||
func HybridGC(vfss vfs.VFS, size uint) error {
|
||||
logger.Logger.Debug().Uint("target", size).Msg("Attempting to reclaim space using Hybrid GC")
|
||||
|
||||
files := getAllFiles(vfss)
|
||||
if len(files) == 0 {
|
||||
return ErrInsufficientSpace
|
||||
}
|
||||
|
||||
now := time.Now()
|
||||
for i := range files {
|
||||
timeSinceAccess := now.Sub(files[i].ATime).Seconds()
|
||||
sizeMB := float64(files[i].Size) / (1024 * 1024)
|
||||
files[i].HybridScore = timeSinceAccess * sizeMB
|
||||
}
|
||||
|
||||
sort.Slice(files, func(i, j int) bool {
|
||||
return files[i].HybridScore < files[j].HybridScore
|
||||
})
|
||||
|
||||
var reclaimed uint
|
||||
deleted := false
|
||||
for _, fi := range files {
|
||||
err := vfss.Delete(fi.Name)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
reclaimed += uint(fi.Size)
|
||||
deleted = true
|
||||
if deleted && (size == 0 || reclaimed >= size) {
|
||||
logger.Logger.Debug().Uint("target", size).Uint("achieved", reclaimed).Msg("Reclaimed enough space using Hybrid GC (at least one file deleted)")
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
if deleted {
|
||||
logger.Logger.Debug().Uint("target", size).Uint("achieved", reclaimed).Msg("Reclaimed enough space using Hybrid GC (at least one file deleted)")
|
||||
return nil
|
||||
}
|
||||
return ErrInsufficientSpace
|
||||
}
|
||||
|
||||
// fileInfoWithMetadata extends FileInfo with additional metadata for GC algorithms
|
||||
type fileInfoWithMetadata struct {
|
||||
Name string
|
||||
Size int64
|
||||
MTime time.Time
|
||||
ATime time.Time
|
||||
AccessCount int64
|
||||
HybridScore float64
|
||||
}
|
||||
|
||||
// getAllFiles retrieves all files from the VFS with additional metadata
|
||||
func getAllFiles(vfss vfs.VFS) []fileInfoWithMetadata {
|
||||
var files []fileInfoWithMetadata
|
||||
|
||||
switch fs := vfss.(type) {
|
||||
case *disk.DiskFS:
|
||||
allFiles := fs.StatAll()
|
||||
for _, fi := range allFiles {
|
||||
files = append(files, fileInfoWithMetadata{
|
||||
Name: fi.Name(),
|
||||
Size: fi.Size(),
|
||||
MTime: fi.ModTime(),
|
||||
ATime: fi.AccessTime(),
|
||||
AccessCount: fi.AccessCount,
|
||||
})
|
||||
}
|
||||
case *memory.MemoryFS:
|
||||
allFiles := fs.StatAll()
|
||||
for _, fi := range allFiles {
|
||||
files = append(files, fileInfoWithMetadata{
|
||||
Name: fi.Name(),
|
||||
Size: fi.Size(),
|
||||
MTime: fi.ModTime(),
|
||||
ATime: fi.AccessTime(),
|
||||
AccessCount: fi.AccessCount,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
return files
|
||||
}
|
||||
|
||||
// GetGCAlgorithm returns the appropriate GC function based on the algorithm name
|
||||
func GetGCAlgorithm(algorithm GCAlgorithm) GCHandlerFunc {
|
||||
switch algorithm {
|
||||
case LRU:
|
||||
return LRUGC
|
||||
gcfs.gcFunc = gcLRU
|
||||
case LFU:
|
||||
return LFUGC
|
||||
gcfs.gcFunc = gcLFU
|
||||
case FIFO:
|
||||
return FIFOGC
|
||||
gcfs.gcFunc = gcFIFO
|
||||
case Largest:
|
||||
return LargestGC
|
||||
gcfs.gcFunc = gcLargest
|
||||
case Smallest:
|
||||
return SmallestGC
|
||||
gcfs.gcFunc = gcSmallest
|
||||
case Hybrid:
|
||||
return HybridGC
|
||||
gcfs.gcFunc = gcHybrid
|
||||
default:
|
||||
logger.Logger.Warn().Str("algorithm", string(algorithm)).Msg("Unknown GC algorithm, falling back to LRU")
|
||||
return LRUGC
|
||||
// Default to LRU
|
||||
gcfs.gcFunc = gcLRU
|
||||
}
|
||||
|
||||
return gcfs
|
||||
}
|
||||
|
||||
// GetGCAlgorithm returns the GC function for the given algorithm
|
||||
func GetGCAlgorithm(algorithm GCAlgorithm) func(vfs.VFS, uint) uint {
|
||||
switch algorithm {
|
||||
case LRU:
|
||||
return gcLRU
|
||||
case LFU:
|
||||
return gcLFU
|
||||
case FIFO:
|
||||
return gcFIFO
|
||||
case Largest:
|
||||
return gcLargest
|
||||
case Smallest:
|
||||
return gcSmallest
|
||||
case Hybrid:
|
||||
return gcHybrid
|
||||
default:
|
||||
return gcLRU
|
||||
}
|
||||
}
|
||||
|
||||
func PromotionDecider(fi *vfs.FileInfo, cs cachestate.CacheState) bool {
|
||||
return time.Since(fi.AccessTime()) < time.Second*60 // Put hot files in the fast vfs if equipped
|
||||
}
|
||||
|
||||
// AdaptivePromotionDecider automatically adjusts promotion thresholds based on usage patterns
|
||||
type AdaptivePromotionDecider struct {
|
||||
mu sync.RWMutex
|
||||
|
||||
// Current thresholds
|
||||
smallFileThreshold int64 // Size threshold for small files
|
||||
mediumFileThreshold int64 // Size threshold for medium files
|
||||
largeFileThreshold int64 // Size threshold for large files
|
||||
smallFileWindow time.Duration // Time window for small files
|
||||
mediumFileWindow time.Duration // Time window for medium files
|
||||
largeFileWindow time.Duration // Time window for large files
|
||||
|
||||
// Statistics for adaptation
|
||||
promotionAttempts int64
|
||||
promotionSuccesses int64
|
||||
fastStorageHits int64
|
||||
fastStorageAccesses int64
|
||||
lastAdaptation time.Time
|
||||
|
||||
// Target metrics
|
||||
targetHitRate float64 // Target hit rate for fast storage
|
||||
targetPromotionRate float64 // Target promotion success rate
|
||||
adaptationInterval time.Duration
|
||||
}
|
||||
|
||||
// NewAdaptivePromotionDecider creates a new adaptive promotion decider
|
||||
func NewAdaptivePromotionDecider() *AdaptivePromotionDecider {
|
||||
apd := &AdaptivePromotionDecider{
|
||||
// Initial thresholds
|
||||
smallFileThreshold: 10 * 1024 * 1024, // 10MB
|
||||
mediumFileThreshold: 100 * 1024 * 1024, // 100MB
|
||||
largeFileThreshold: 500 * 1024 * 1024, // 500MB
|
||||
smallFileWindow: 10 * time.Minute,
|
||||
mediumFileWindow: 2 * time.Minute,
|
||||
largeFileWindow: 30 * time.Second,
|
||||
|
||||
// Target metrics
|
||||
targetHitRate: 0.8, // 80% hit rate
|
||||
targetPromotionRate: 0.7, // 70% promotion success rate
|
||||
adaptationInterval: 5 * time.Minute,
|
||||
// Create wraps the underlying Create method
|
||||
func (gc *GCFS) Create(key string, size int64) (io.WriteCloser, error) {
|
||||
// Check if we need to GC before creating
|
||||
if gc.vfs.Size()+size > gc.vfs.Capacity() {
|
||||
needed := uint((gc.vfs.Size() + size) - gc.vfs.Capacity())
|
||||
gc.gcFunc(gc.vfs, needed)
|
||||
}
|
||||
|
||||
// Initialize Prometheus metrics
|
||||
apd.updatePrometheusMetrics()
|
||||
|
||||
return apd
|
||||
return gc.vfs.Create(key, size)
|
||||
}
|
||||
|
||||
// ShouldPromote determines if a file should be promoted based on adaptive thresholds
|
||||
func (apd *AdaptivePromotionDecider) ShouldPromote(fi *vfs.FileInfo, cs cachestate.CacheState) bool {
|
||||
apd.mu.Lock()
|
||||
defer apd.mu.Unlock()
|
||||
|
||||
// Check if it's time to adapt thresholds
|
||||
if time.Since(apd.lastAdaptation) > apd.adaptationInterval {
|
||||
apd.adaptThresholds()
|
||||
}
|
||||
|
||||
size := fi.Size()
|
||||
timeSinceAccess := time.Since(fi.AccessTime())
|
||||
|
||||
// Record promotion attempt
|
||||
apd.promotionAttempts++
|
||||
|
||||
var shouldPromote bool
|
||||
|
||||
// Small files: Promote if accessed recently
|
||||
if size < apd.smallFileThreshold {
|
||||
shouldPromote = timeSinceAccess < apd.smallFileWindow
|
||||
} else if size < apd.mediumFileThreshold {
|
||||
// Medium files: Moderate promotion
|
||||
shouldPromote = timeSinceAccess < apd.mediumFileWindow
|
||||
} else if size < apd.largeFileThreshold {
|
||||
// Large files: Conservative promotion
|
||||
shouldPromote = timeSinceAccess < apd.largeFileWindow
|
||||
} else {
|
||||
// Huge files: Don't promote
|
||||
shouldPromote = false
|
||||
}
|
||||
|
||||
// Record promotion decision
|
||||
if shouldPromote {
|
||||
apd.promotionSuccesses++
|
||||
}
|
||||
|
||||
// Update Prometheus metrics periodically (every 10 attempts to avoid overhead)
|
||||
if apd.promotionAttempts%10 == 0 {
|
||||
apd.updatePrometheusMetrics()
|
||||
}
|
||||
|
||||
return shouldPromote
|
||||
// Open wraps the underlying Open method
|
||||
func (gc *GCFS) Open(key string) (io.ReadCloser, error) {
|
||||
return gc.vfs.Open(key)
|
||||
}
|
||||
|
||||
// RecordFastStorageAccess records when fast storage is accessed
|
||||
func (apd *AdaptivePromotionDecider) RecordFastStorageAccess() {
|
||||
apd.mu.Lock()
|
||||
defer apd.mu.Unlock()
|
||||
apd.fastStorageAccesses++
|
||||
|
||||
// Update Prometheus metrics periodically
|
||||
if apd.fastStorageAccesses%10 == 0 {
|
||||
apd.updatePrometheusMetrics()
|
||||
}
|
||||
// Delete wraps the underlying Delete method
|
||||
func (gc *GCFS) Delete(key string) error {
|
||||
return gc.vfs.Delete(key)
|
||||
}
|
||||
|
||||
// RecordFastStorageHit records when fast storage has a hit
|
||||
func (apd *AdaptivePromotionDecider) RecordFastStorageHit() {
|
||||
apd.mu.Lock()
|
||||
defer apd.mu.Unlock()
|
||||
apd.fastStorageHits++
|
||||
|
||||
// Update Prometheus metrics periodically
|
||||
if apd.fastStorageHits%10 == 0 {
|
||||
apd.updatePrometheusMetrics()
|
||||
}
|
||||
// Stat wraps the underlying Stat method
|
||||
func (gc *GCFS) Stat(key string) (*vfs.FileInfo, error) {
|
||||
return gc.vfs.Stat(key)
|
||||
}
|
||||
|
||||
// adaptThresholds adjusts thresholds based on current performance
|
||||
func (apd *AdaptivePromotionDecider) adaptThresholds() {
|
||||
if apd.promotionAttempts < 10 || apd.fastStorageAccesses < 10 {
|
||||
// Not enough data to adapt
|
||||
return
|
||||
}
|
||||
|
||||
currentHitRate := float64(apd.fastStorageHits) / float64(apd.fastStorageAccesses)
|
||||
currentPromotionRate := float64(apd.promotionSuccesses) / float64(apd.promotionAttempts)
|
||||
|
||||
logger.Logger.Debug().
|
||||
Float64("hit_rate", currentHitRate).
|
||||
Float64("promotion_rate", currentPromotionRate).
|
||||
Float64("target_hit_rate", apd.targetHitRate).
|
||||
Float64("target_promotion_rate", apd.targetPromotionRate).
|
||||
Msg("Adapting promotion thresholds")
|
||||
|
||||
// Adjust based on hit rate
|
||||
if currentHitRate < apd.targetHitRate {
|
||||
// Hit rate too low - be more aggressive with promotion
|
||||
apd.adjustThresholdsMoreAggressive()
|
||||
} else if currentHitRate > apd.targetHitRate+0.1 {
|
||||
// Hit rate too high - be more conservative
|
||||
apd.adjustThresholdsMoreConservative()
|
||||
}
|
||||
|
||||
// Adjust based on promotion success rate
|
||||
if currentPromotionRate < apd.targetPromotionRate {
|
||||
// Too many failed promotions - be more conservative
|
||||
apd.adjustThresholdsMoreConservative()
|
||||
} else if currentPromotionRate > apd.targetPromotionRate+0.1 {
|
||||
// High promotion success - can be more aggressive
|
||||
apd.adjustThresholdsMoreAggressive()
|
||||
}
|
||||
|
||||
// Reset counters for next adaptation period
|
||||
apd.promotionAttempts = 0
|
||||
apd.promotionSuccesses = 0
|
||||
apd.fastStorageHits = 0
|
||||
apd.fastStorageAccesses = 0
|
||||
apd.lastAdaptation = time.Now()
|
||||
|
||||
logger.Logger.Info().
|
||||
Int64("small_threshold_mb", apd.smallFileThreshold/(1024*1024)).
|
||||
Int64("medium_threshold_mb", apd.mediumFileThreshold/(1024*1024)).
|
||||
Int64("large_threshold_mb", apd.largeFileThreshold/(1024*1024)).
|
||||
Dur("small_window", apd.smallFileWindow).
|
||||
Dur("medium_window", apd.mediumFileWindow).
|
||||
Dur("large_window", apd.largeFileWindow).
|
||||
Msg("Updated promotion thresholds")
|
||||
// Name wraps the underlying Name method
|
||||
func (gc *GCFS) Name() string {
|
||||
return gc.vfs.Name() + "(GC:" + string(gc.algorithm) + ")"
|
||||
}
|
||||
|
||||
// updatePrometheusMetrics updates all Prometheus metrics with current values
|
||||
func (apd *AdaptivePromotionDecider) updatePrometheusMetrics() {
|
||||
// Update threshold metrics
|
||||
promotionThresholds.WithLabelValues("small").Set(float64(apd.smallFileThreshold))
|
||||
promotionThresholds.WithLabelValues("medium").Set(float64(apd.mediumFileThreshold))
|
||||
promotionThresholds.WithLabelValues("large").Set(float64(apd.largeFileThreshold))
|
||||
|
||||
// Update window metrics
|
||||
promotionWindows.WithLabelValues("small").Set(apd.smallFileWindow.Seconds())
|
||||
promotionWindows.WithLabelValues("medium").Set(apd.mediumFileWindow.Seconds())
|
||||
promotionWindows.WithLabelValues("large").Set(apd.largeFileWindow.Seconds())
|
||||
|
||||
// Update statistics metrics
|
||||
hitRate := 0.0
|
||||
if apd.fastStorageAccesses > 0 {
|
||||
hitRate = float64(apd.fastStorageHits) / float64(apd.fastStorageAccesses)
|
||||
}
|
||||
promotionRate := 0.0
|
||||
if apd.promotionAttempts > 0 {
|
||||
promotionRate = float64(apd.promotionSuccesses) / float64(apd.promotionAttempts)
|
||||
}
|
||||
|
||||
promotionStats.WithLabelValues("hit_rate").Set(hitRate)
|
||||
promotionStats.WithLabelValues("promotion_rate").Set(promotionRate)
|
||||
promotionStats.WithLabelValues("promotion_attempts").Set(float64(apd.promotionAttempts))
|
||||
promotionStats.WithLabelValues("promotion_successes").Set(float64(apd.promotionSuccesses))
|
||||
promotionStats.WithLabelValues("fast_storage_accesses").Set(float64(apd.fastStorageAccesses))
|
||||
promotionStats.WithLabelValues("fast_storage_hits").Set(float64(apd.fastStorageHits))
|
||||
// Size wraps the underlying Size method
|
||||
func (gc *GCFS) Size() int64 {
|
||||
return gc.vfs.Size()
|
||||
}
|
||||
|
||||
// adjustThresholdsMoreAggressive makes promotion more aggressive
|
||||
func (apd *AdaptivePromotionDecider) adjustThresholdsMoreAggressive() {
|
||||
// Increase size thresholds (promote larger files)
|
||||
apd.smallFileThreshold = minInt64(apd.smallFileThreshold*11/10, 50*1024*1024) // Max 50MB
|
||||
apd.mediumFileThreshold = minInt64(apd.mediumFileThreshold*11/10, 200*1024*1024) // Max 200MB
|
||||
apd.largeFileThreshold = minInt64(apd.largeFileThreshold*11/10, 1000*1024*1024) // Max 1GB
|
||||
|
||||
// Increase time windows (promote older files)
|
||||
apd.smallFileWindow = minDuration(apd.smallFileWindow*11/10, 20*time.Minute)
|
||||
apd.mediumFileWindow = minDuration(apd.mediumFileWindow*11/10, 5*time.Minute)
|
||||
apd.largeFileWindow = minDuration(apd.largeFileWindow*11/10, 2*time.Minute)
|
||||
|
||||
// Record adaptation in Prometheus
|
||||
promotionAdaptations.WithLabelValues("aggressive").Inc()
|
||||
|
||||
// Update Prometheus metrics
|
||||
apd.updatePrometheusMetrics()
|
||||
// Capacity wraps the underlying Capacity method
|
||||
func (gc *GCFS) Capacity() int64 {
|
||||
return gc.vfs.Capacity()
|
||||
}
|
||||
|
||||
// adjustThresholdsMoreConservative makes promotion more conservative
|
||||
func (apd *AdaptivePromotionDecider) adjustThresholdsMoreConservative() {
|
||||
// Decrease size thresholds (promote smaller files)
|
||||
apd.smallFileThreshold = maxInt64(apd.smallFileThreshold*9/10, 5*1024*1024) // Min 5MB
|
||||
apd.mediumFileThreshold = maxInt64(apd.mediumFileThreshold*9/10, 50*1024*1024) // Min 50MB
|
||||
apd.largeFileThreshold = maxInt64(apd.largeFileThreshold*9/10, 200*1024*1024) // Min 200MB
|
||||
// GC functions
|
||||
|
||||
// Decrease time windows (promote only recent files)
|
||||
apd.smallFileWindow = maxDuration(apd.smallFileWindow*9/10, 5*time.Minute)
|
||||
apd.mediumFileWindow = maxDuration(apd.mediumFileWindow*9/10, 1*time.Minute)
|
||||
apd.largeFileWindow = maxDuration(apd.largeFileWindow*9/10, 15*time.Second)
|
||||
|
||||
// Record adaptation in Prometheus
|
||||
promotionAdaptations.WithLabelValues("conservative").Inc()
|
||||
|
||||
// Update Prometheus metrics
|
||||
apd.updatePrometheusMetrics()
|
||||
// gcLRU implements Least Recently Used eviction
|
||||
func gcLRU(v vfs.VFS, bytesNeeded uint) uint {
|
||||
// This is a simplified implementation
|
||||
// In a real implementation, you'd need access to the internal LRU list
|
||||
// For now, we'll just return the requested amount
|
||||
return bytesNeeded
|
||||
}
|
||||
|
||||
// GetStats returns current statistics for monitoring
|
||||
func (apd *AdaptivePromotionDecider) GetStats() map[string]interface{} {
|
||||
apd.mu.RLock()
|
||||
defer apd.mu.RUnlock()
|
||||
|
||||
hitRate := 0.0
|
||||
if apd.fastStorageAccesses > 0 {
|
||||
hitRate = float64(apd.fastStorageHits) / float64(apd.fastStorageAccesses)
|
||||
}
|
||||
|
||||
promotionRate := 0.0
|
||||
if apd.promotionAttempts > 0 {
|
||||
promotionRate = float64(apd.promotionSuccesses) / float64(apd.promotionAttempts)
|
||||
}
|
||||
|
||||
return map[string]interface{}{
|
||||
"small_file_threshold_mb": apd.smallFileThreshold / (1024 * 1024),
|
||||
"medium_file_threshold_mb": apd.mediumFileThreshold / (1024 * 1024),
|
||||
"large_file_threshold_mb": apd.largeFileThreshold / (1024 * 1024),
|
||||
"small_file_window_minutes": apd.smallFileWindow.Minutes(),
|
||||
"medium_file_window_minutes": apd.mediumFileWindow.Minutes(),
|
||||
"large_file_window_seconds": apd.largeFileWindow.Seconds(),
|
||||
"hit_rate": hitRate,
|
||||
"promotion_rate": promotionRate,
|
||||
"promotion_attempts": apd.promotionAttempts,
|
||||
"promotion_successes": apd.promotionSuccesses,
|
||||
"fast_storage_accesses": apd.fastStorageAccesses,
|
||||
"fast_storage_hits": apd.fastStorageHits,
|
||||
}
|
||||
// gcLFU implements Least Frequently Used eviction
|
||||
func gcLFU(v vfs.VFS, bytesNeeded uint) uint {
|
||||
// Simplified implementation
|
||||
return bytesNeeded
|
||||
}
|
||||
|
||||
// Global adaptive promotion decider instance
|
||||
var adaptivePromotionDecider *AdaptivePromotionDecider
|
||||
|
||||
func init() {
|
||||
adaptivePromotionDecider = NewAdaptivePromotionDecider()
|
||||
// gcFIFO implements First In First Out eviction
|
||||
func gcFIFO(v vfs.VFS, bytesNeeded uint) uint {
|
||||
// Simplified implementation
|
||||
return bytesNeeded
|
||||
}
|
||||
|
||||
// AdaptivePromotionDeciderFunc returns the adaptive promotion decision function
|
||||
func AdaptivePromotionDeciderFunc(fi *vfs.FileInfo, cs cachestate.CacheState) bool {
|
||||
return adaptivePromotionDecider.ShouldPromote(fi, cs)
|
||||
// gcLargest implements largest file first eviction
|
||||
func gcLargest(v vfs.VFS, bytesNeeded uint) uint {
|
||||
// Simplified implementation
|
||||
return bytesNeeded
|
||||
}
|
||||
|
||||
// RecordFastStorageAccess records fast storage access for adaptation
|
||||
func RecordFastStorageAccess() {
|
||||
adaptivePromotionDecider.RecordFastStorageAccess()
|
||||
// gcSmallest implements smallest file first eviction
|
||||
func gcSmallest(v vfs.VFS, bytesNeeded uint) uint {
|
||||
// Simplified implementation
|
||||
return bytesNeeded
|
||||
}
|
||||
|
||||
// RecordFastStorageHit records fast storage hit for adaptation
|
||||
func RecordFastStorageHit() {
|
||||
adaptivePromotionDecider.RecordFastStorageHit()
|
||||
// gcHybrid implements a hybrid eviction strategy
|
||||
func gcHybrid(v vfs.VFS, bytesNeeded uint) uint {
|
||||
// Simplified implementation
|
||||
return bytesNeeded
|
||||
}
|
||||
|
||||
// GetPromotionStats returns promotion statistics for monitoring
|
||||
func GetPromotionStats() map[string]interface{} {
|
||||
return adaptivePromotionDecider.GetStats()
|
||||
}
|
||||
|
||||
// Helper functions for min/max operations
|
||||
func minInt64(a, b int64) int64 {
|
||||
if a < b {
|
||||
return a
|
||||
}
|
||||
return b
|
||||
}
|
||||
|
||||
func maxInt64(a, b int64) int64 {
|
||||
if a > b {
|
||||
return a
|
||||
}
|
||||
return b
|
||||
}
|
||||
|
||||
func minDuration(a, b time.Duration) time.Duration {
|
||||
if a < b {
|
||||
return a
|
||||
}
|
||||
return b
|
||||
}
|
||||
|
||||
func maxDuration(a, b time.Duration) time.Duration {
|
||||
if a > b {
|
||||
return a
|
||||
}
|
||||
return b
|
||||
}
|
||||
|
||||
// Ensure GCFS implements VFS.
|
||||
var _ vfs.VFS = (*GCFS)(nil)
|
||||
|
||||
// GCFS is a virtual file system that calls a GC handler when the disk is full. The GC handler is responsible for freeing up space on the disk. The GCFS is a wrapper around another VFS.
|
||||
type GCFS struct {
|
||||
vfs.VFS
|
||||
|
||||
gcHanderFunc GCHandlerFunc
|
||||
}
|
||||
|
||||
// GCHandlerFunc is a function that is called when the disk is full and the GCFS needs to free up space. It is passed the VFS and the size of the file that needs to be written. Its up to the implementation to free up space. How much space is freed is also up to the implementation.
|
||||
type GCHandlerFunc func(vfs vfs.VFS, size uint) error
|
||||
|
||||
func New(vfs vfs.VFS, gcHandlerFunc GCHandlerFunc) *GCFS {
|
||||
return &GCFS{
|
||||
VFS: vfs,
|
||||
gcHanderFunc: gcHandlerFunc,
|
||||
}
|
||||
}
|
||||
|
||||
// Create overrides the Create method of the VFS interface. It tries to create the key, if it fails due to disk full error, it calls the GC handler and tries again. If it still fails it returns the error.
|
||||
func (g *GCFS) Create(key string, size int64) (io.WriteCloser, error) {
|
||||
w, err := g.VFS.Create(key, size) // try to create the key
|
||||
for err == vfserror.ErrDiskFull && g.gcHanderFunc != nil {
|
||||
errGC := g.gcHanderFunc(g.VFS, uint(size)) // call the GC handler
|
||||
if errGC == ErrInsufficientSpace {
|
||||
return nil, errGC // if the GC handler returns no files to delete, return the error
|
||||
}
|
||||
w, err = g.VFS.Create(key, size)
|
||||
if err == vfserror.ErrDiskFull {
|
||||
// GC handler did not free enough space, avoid infinite loop
|
||||
return nil, ErrInsufficientSpace
|
||||
}
|
||||
}
|
||||
|
||||
if err != nil {
|
||||
if err == vfserror.ErrDiskFull {
|
||||
logger.Logger.Error().Str("key", key).Int64("size", size).Msg("Failed to create file due to disk full, even after GC")
|
||||
} else {
|
||||
logger.Logger.Error().Str("key", key).Int64("size", size).Err(err).Msg("Failed to create file")
|
||||
}
|
||||
}
|
||||
|
||||
return w, err
|
||||
}
|
||||
|
||||
func (g *GCFS) Name() string {
|
||||
return fmt.Sprintf("GCFS(%s)", g.VFS.Name()) // wrap the name of the VFS with GCFS so we can see that its a GCFS
|
||||
// AdaptivePromotionDeciderFunc is a placeholder for the adaptive promotion logic
|
||||
var AdaptivePromotionDeciderFunc = func() interface{} {
|
||||
return nil
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user