s1d3sw1ped/steamcache2
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 19 Wiki Activity

feat: implement enhanced garbage collection statistics logging
All checks were successful
Release Tag / release (push) Successful in 12s
Details

Browse Source
This commit is contained in:
Justin Harms
2025-01-22 20:27:12 -06:00
parent 4a23eecae0
commit 550948951e
4 changed files with 97 additions and 91 deletions
Download Patch File Download Diff File Expand all files Collapse all files

61
steamcache/gc.go
View File

@@ -1,73 +1,42 @@
package steamcache
import (
"s1d3sw1ped/SteamCache2/steamcache/logger"
"s1d3sw1ped/SteamCache2/vfs"
"s1d3sw1ped/SteamCache2/vfs/cachestate"
"time"
"github.com/docker/go-units"
"golang.org/x/exp/rand"
)
func randomgc(vfss vfs.VFS, stats []*vfs.FileInfo) int64 {
// Pick a random file to delete
randfile := stats[rand.Intn(len(stats))]
sz := randfile.Size()
err := vfss.Delete(randfile.Name())
if err != nil {
// If we failed to delete the file, log it and return 0
// logger.Logger.Error().Err(err).Msgf("Failed to delete %s", randfile.Name())
return 0
// RandomGC randomly deletes files until we've reclaimed enough space.
func randomgc(vfss vfs.VFS, size uint) (uint, uint) {
// Randomly delete files until we've reclaimed enough space.
random := func(vfss vfs.VFS, stats []*vfs.FileInfo) int64 {
randfile := stats[rand.Intn(len(stats))]
sz := randfile.Size()
err := vfss.Delete(randfile.Name())
if err != nil {
return 0
}
return sz
}
return sz
}
func memorygc(vfss vfs.VFS, size int) {
tstart := time.Now()
deletions := 0
targetreclaim := int64(size)
var reclaimed int64
stats := vfss.StatAll()
for {
reclaimed += randomgc(vfss, stats)
reclaimed += random(vfss, stats)
deletions++
if reclaimed >= targetreclaim {
break
}
}
logger.Logger.Info().
Str("name", vfss.Name()).
Str("duration", time.Since(tstart).String()).
Str("reclaimed", units.HumanSize(float64(reclaimed))).
Int("deletions", deletions).
Msgf("GC")
}
func diskgc(vfss vfs.VFS, size int) {
tstart := time.Now()
deletions := 0
targetreclaim := int64(size)
var reclaimed int64
stats := vfss.StatAll()
for {
reclaimed += randomgc(vfss, stats)
deletions++
if reclaimed >= targetreclaim {
break
}
}
logger.Logger.Info().
Str("name", vfss.Name()).
Str("duration", time.Since(tstart).String()).
Str("reclaimed", units.HumanSize(float64(reclaimed))).
Int("deletions", deletions).
Msgf("GC")
return uint(reclaimed), uint(deletions)
}
func cachehandler(fi *vfs.FileInfo, cs cachestate.CacheState) bool {

55
steamcache/steamcache.go
View File

@@ -30,6 +30,9 @@ type SteamCache struct {
memory *memory.MemoryFS
disk *disk.DiskFS
memorygc *gc.GCFS
diskgc *gc.GCFS
hits *avgcachestate.AvgCacheState
dirty bool
@@ -52,49 +55,36 @@ func New(address string, memorySize string, memoryMultiplier int, diskSize strin
)
var m *memory.MemoryFS
var mgc *gc.GCFS
if memorysize > 0 {
m = memory.New(memorysize)
mgc = gc.New(m, memoryMultiplier, randomgc)
}
var d *disk.DiskFS
var dgc *gc.GCFS
if disksize > 0 {
d = disk.New(diskPath, disksize)
dgc = gc.New(d, diskMultiplier, randomgc)
}
// configure the cache to match the specified mode (memory only, disk only, or memory and disk) based on the provided sizes
if disksize == 0 && memorysize != 0 {
//memory only mode - no disk
c.SetSlow(mgc)
logger.Logger.Info().Bool("memory", true).Bool("disk", false).Msg("configuration")
c.SetSlow(gc.New(
m,
memoryMultiplier,
memorygc,
))
} else if disksize != 0 && memorysize == 0 {
// disk only mode
c.SetSlow(dgc)
logger.Logger.Info().Bool("memory", false).Bool("disk", true).Msg("configuration")
c.SetSlow(gc.New(
d,
diskMultiplier,
diskgc,
))
} else if disksize != 0 && memorysize != 0 {
// memory and disk mode
c.SetFast(mgc)
c.SetSlow(dgc)
logger.Logger.Info().Bool("memory", true).Bool("disk", true).Msg("configuration")
c.SetFast(gc.New(
m,
memoryMultiplier,
memorygc,
))
c.SetSlow(gc.New(
d,
diskMultiplier,
diskgc,
))
} else {
// no memory or disk isn't a valid configuration
logger.Logger.Error().Bool("memory", false).Bool("disk", false).Msg("configuration invalid :( exiting")
@@ -108,12 +98,15 @@ func New(address string, memorySize string, memoryMultiplier int, diskSize strin
memory: m,
disk: d,
memorygc: mgc,
diskgc: dgc,
hits: avgcachestate.New(10000),
}
if d != nil {
if d.Size() > d.Capacity() {
diskgc(d, int(d.Size()-d.Capacity()))
randomgc(d, uint(d.Size()-d.Capacity()))
}
}
@@ -152,19 +145,37 @@ func (sc *SteamCache) LogStats() {
if sc.dirty {
if sc.memory != nil { // only log memory if memory is enabled
lifetimeBytes, lifetimeFiles, reclaimedBytes, deletedFiles := sc.memorygc.Stats()
logger.Logger.Info().
Str("size", units.HumanSize(float64(sc.memory.Size()))).
Str("capacity", units.HumanSize(float64(sc.memory.Capacity()))).
Str("files", fmt.Sprintf("%d", len(sc.memory.StatAll()))).
Msg("memory")
logger.Logger.Info().
Str("lifetime", units.HumanSize(float64(lifetimeBytes))).
Uint("lifetime_files", lifetimeFiles).
Str("reclaimed", units.HumanSize(float64(reclaimedBytes))).
Uint("deleted_files", deletedFiles).
Msg("memory_gc")
}
if sc.disk != nil { // only log disk if disk is enabled
lifetimeBytes, lifetimeFiles, reclaimedBytes, deletedFiles := sc.diskgc.Stats()
logger.Logger.Info().
Str("size", units.HumanSize(float64(sc.disk.Size()))).
Str("capacity", units.HumanSize(float64(sc.disk.Capacity()))).
Str("files", fmt.Sprintf("%d", len(sc.disk.StatAll()))).
Msg("disk")
logger.Logger.Info().
Str("lifetime", units.HumanSize(float64(lifetimeBytes))).
Uint("lifetime_files", lifetimeFiles).
Str("reclaimed", units.HumanSize(float64(reclaimedBytes))).
Uint("deleted_files", deletedFiles).
Msg("disk_gc")
}
logger.Logger.Info().

42
vfs/gc/gc.go
View File

@@ -4,6 +4,7 @@ import (
"fmt"
"s1d3sw1ped/SteamCache2/vfs"
"s1d3sw1ped/SteamCache2/vfs/vfserror"
"sync"
)
// Ensure GCFS implements VFS.
@@ -12,12 +13,17 @@ var _ vfs.VFS = (*GCFS)(nil)
// GCFS is a virtual file system that calls a GC handler when the disk is full. The GC handler is responsible for freeing up space on the disk. The GCFS is a wrapper around another VFS.
type GCFS struct {
vfs.VFS
multiplier int
gcHanderFunc GCHandlerFunc
multiplier int
// protected by mu
gcHanderFunc GCHandlerFunc
lifetimeBytes, lifetimeFiles uint
reclaimedBytes, deletedFiles uint
mu sync.Mutex
}
// GCHandlerFunc is a function that is called when the disk is full and the GCFS needs to free up space. It is passed the VFS and the size of the file that needs to be written. Its up to the implementation to free up space. How much space is freed is also up to the implementation.
type GCHandlerFunc func(vfs vfs.VFS, size int)
type GCHandlerFunc func(vfs vfs.VFS, size uint) (reclaimedBytes uint, deletedFiles uint)
func New(vfs vfs.VFS, multiplier int, gcHandlerFunc GCHandlerFunc) *GCFS {
if multiplier <= 0 {
@@ -30,13 +36,39 @@ func New(vfs vfs.VFS, multiplier int, gcHandlerFunc GCHandlerFunc) *GCFS {
}
}
// Stats returns the lifetime bytes, lifetime files, reclaimed bytes and deleted files.
// The lifetime bytes and lifetime files are the total bytes and files that have been freed up by the GC handler.
// The reclaimed bytes and deleted files are the bytes and files that have been freed up by the GC handler since the last execution.
// The reclaimed bytes and deleted files are reset to 0 after the call to Stats.
func (g *GCFS) Stats() (lifetimeBytes, lifetimeFiles, reclaimedBytes, deletedFiles uint) {
g.mu.Lock()
defer g.mu.Unlock()
g.lifetimeBytes += g.reclaimedBytes
g.lifetimeFiles += g.deletedFiles
lifetimeBytes = g.lifetimeBytes
lifetimeFiles = g.lifetimeFiles
reclaimedBytes = g.reclaimedBytes
deletedFiles = g.deletedFiles
g.reclaimedBytes = 0
g.deletedFiles = 0
return
}
// Set overrides the Set method of the VFS interface. It tries to set the key and src, if it fails due to disk full error, it calls the GC handler and tries again. If it still fails it returns the error.
func (g *GCFS) Set(key string, src []byte) error {
g.mu.Lock()
defer g.mu.Unlock()
err := g.VFS.Set(key, src) // try to set the key and src
if err == vfserror.ErrDiskFull && g.gcHanderFunc != nil { // if the error is disk full and there is a GC handler
g.gcHanderFunc(g.VFS, len(src)*g.multiplier) // call the GC handler
err = g.VFS.Set(key, src) // try again after GC if it still fails return the error
reclaimedBytes, deletedFiles := g.gcHanderFunc(g.VFS, uint(len(src)*g.multiplier)) // call the GC handler
g.reclaimedBytes += reclaimedBytes
g.deletedFiles += deletedFiles
err = g.VFS.Set(key, src) // try again after GC if it still fails return the error
}
return err

30
vfs/gc/gc_test.go
View File

@@ -6,7 +6,6 @@ import (
"s1d3sw1ped/SteamCache2/vfs/memory"
"sort"
"testing"
"time"
"golang.org/x/exp/rand"
)
@@ -15,13 +14,11 @@ func TestGCSmallRandom(t *testing.T) {
t.Parallel()
m := memory.New(1024 * 1024 * 16)
gc := New(m, 10, func(vfs vfs.VFS, size int) {
tstart := time.Now()
gc := New(m, 10, func(vfs vfs.VFS, size uint) (uint, uint) {
deletions := 0
targetreclaim := int64(size)
var reclaimed int64
var reclaimed uint
t.Logf("GC starting to reclaim %d bytes", targetreclaim)
t.Logf("GC starting to reclaim %d bytes", size)
stats := vfs.StatAll()
sort.Slice(stats, func(i, j int) bool {
@@ -31,7 +28,7 @@ func TestGCSmallRandom(t *testing.T) {
// Delete the oldest files until we've reclaimed enough space.
for _, s := range stats {
sz := s.Size() // Get the size of the file
sz := uint(s.Size()) // Get the size of the file
err := vfs.Delete(s.Name())
if err != nil {
panic(err)
@@ -41,12 +38,11 @@ func TestGCSmallRandom(t *testing.T) {
// t.Logf("GC deleting %s, %v", s.Name(), s.AccessTime().Format(time.RFC3339Nano))
if reclaimed >= targetreclaim { // We've reclaimed enough space
if reclaimed >= size { // We've reclaimed enough space
break
}
}
t.Logf("GC took %v to reclaim %d bytes by deleting %d files", time.Since(tstart), reclaimed, deletions)
return uint(reclaimed), uint(deletions)
})
for i := 0; i < 10000; i++ {
@@ -70,13 +66,11 @@ func TestGCLargeRandom(t *testing.T) {
t.Parallel()
m := memory.New(1024 * 1024 * 16) // 16MB
gc := New(m, 10, func(vfs vfs.VFS, size int) {
tstart := time.Now()
gc := New(m, 10, func(vfs vfs.VFS, size uint) (uint, uint) {
deletions := 0
targetreclaim := int64(size)
var reclaimed int64
var reclaimed uint
t.Logf("GC starting to reclaim %d bytes", targetreclaim)
t.Logf("GC starting to reclaim %d bytes", size)
stats := vfs.StatAll()
sort.Slice(stats, func(i, j int) bool {
@@ -86,17 +80,17 @@ func TestGCLargeRandom(t *testing.T) {
// Delete the oldest files until we've reclaimed enough space.
for _, s := range stats {
sz := s.Size() // Get the size of the file
sz := uint(s.Size()) // Get the size of the file
vfs.Delete(s.Name())
reclaimed += sz // Track how much space we've reclaimed
deletions++ // Track how many files we've deleted
if reclaimed >= targetreclaim { // We've reclaimed enough space
if reclaimed >= size { // We've reclaimed enough space
break
}
}
t.Logf("GC took %v to reclaim %d bytes by deleting %d files", time.Since(tstart), reclaimed, deletions)
return uint(reclaimed), uint(deletions)
})
for i := 0; i < 10000; i++ {