feat: implement enhanced garbage collection statistics logging

steamcache/gc.go

@@ -1,73 +1,42 @@
 package steamcache
 
 import (
-	"s1d3sw1ped/SteamCache2/steamcache/logger"
 	"s1d3sw1ped/SteamCache2/vfs"
 	"s1d3sw1ped/SteamCache2/vfs/cachestate"
 	"time"
 
-	"github.com/docker/go-units"
 	"golang.org/x/exp/rand"
 )
 
-func randomgc(vfss vfs.VFS, stats []*vfs.FileInfo) int64 {
+// RandomGC randomly deletes files until we've reclaimed enough space.
-	// Pick a random file to delete
+func randomgc(vfss vfs.VFS, size uint) (uint, uint) {
-	randfile := stats[rand.Intn(len(stats))]
+
-	sz := randfile.Size()
+	// Randomly delete files until we've reclaimed enough space.
-	err := vfss.Delete(randfile.Name())
+	random := func(vfss vfs.VFS, stats []*vfs.FileInfo) int64 {
-	if err != nil {
+		randfile := stats[rand.Intn(len(stats))]
-		// If we failed to delete the file, log it and return 0
+		sz := randfile.Size()
-		// logger.Logger.Error().Err(err).Msgf("Failed to delete %s", randfile.Name())
+		err := vfss.Delete(randfile.Name())
-		return 0
+		if err != nil {
+			return 0
+		}
+
+		return sz
 	}
 
-	return sz
-}
-
-func memorygc(vfss vfs.VFS, size int) {
-	tstart := time.Now()
 	deletions := 0
 	targetreclaim := int64(size)
 	var reclaimed int64
 
 	stats := vfss.StatAll()
 	for {
-		reclaimed += randomgc(vfss, stats)
+		reclaimed += random(vfss, stats)
 		deletions++
 		if reclaimed >= targetreclaim {
 			break
 		}
 	}
 
-	logger.Logger.Info().
+	return uint(reclaimed), uint(deletions)
-		Str("name", vfss.Name()).
-		Str("duration", time.Since(tstart).String()).
-		Str("reclaimed", units.HumanSize(float64(reclaimed))).
-		Int("deletions", deletions).
-		Msgf("GC")
-}
-
-func diskgc(vfss vfs.VFS, size int) {
-	tstart := time.Now()
-	deletions := 0
-	targetreclaim := int64(size)
-	var reclaimed int64
-
-	stats := vfss.StatAll()
-	for {
-		reclaimed += randomgc(vfss, stats)
-		deletions++
-		if reclaimed >= targetreclaim {
-			break
-		}
-	}
-
-	logger.Logger.Info().
-		Str("name", vfss.Name()).
-		Str("duration", time.Since(tstart).String()).
-		Str("reclaimed", units.HumanSize(float64(reclaimed))).
-		Int("deletions", deletions).
-		Msgf("GC")
 }
 
 func cachehandler(fi *vfs.FileInfo, cs cachestate.CacheState) bool {

steamcache/steamcache.go

@@ -30,6 +30,9 @@ type SteamCache struct {
 	memory *memory.MemoryFS
 	disk   *disk.DiskFS
 
+	memorygc *gc.GCFS
+	diskgc   *gc.GCFS
+
 	hits *avgcachestate.AvgCacheState
 
 	dirty bool
@@ -52,49 +55,36 @@ func New(address string, memorySize string, memoryMultiplier int, diskSize strin
 	)
 
 	var m *memory.MemoryFS
+	var mgc *gc.GCFS
 	if memorysize > 0 {
 		m = memory.New(memorysize)
+		mgc = gc.New(m, memoryMultiplier, randomgc)
 	}
 
 	var d *disk.DiskFS
+	var dgc *gc.GCFS
 	if disksize > 0 {
 		d = disk.New(diskPath, disksize)
+		dgc = gc.New(d, diskMultiplier, randomgc)
 	}
 
 	// configure the cache to match the specified mode (memory only, disk only, or memory and disk) based on the provided sizes
 	if disksize == 0 && memorysize != 0 {
 		//memory only mode - no disk
 
+		c.SetSlow(mgc)
 		logger.Logger.Info().Bool("memory", true).Bool("disk", false).Msg("configuration")
-		c.SetSlow(gc.New(
-			m,
-			memoryMultiplier,
-			memorygc,
-		))
 	} else if disksize != 0 && memorysize == 0 {
 		// disk only mode
 
+		c.SetSlow(dgc)
 		logger.Logger.Info().Bool("memory", false).Bool("disk", true).Msg("configuration")
-		c.SetSlow(gc.New(
-			d,
-			diskMultiplier,
-			diskgc,
-		))
 	} else if disksize != 0 && memorysize != 0 {
 		// memory and disk mode
 
+		c.SetFast(mgc)
+		c.SetSlow(dgc)
 		logger.Logger.Info().Bool("memory", true).Bool("disk", true).Msg("configuration")
-		c.SetFast(gc.New(
-			m,
-			memoryMultiplier,
-			memorygc,
-		))
-
-		c.SetSlow(gc.New(
-			d,
-			diskMultiplier,
-			diskgc,
-		))
 	} else {
 		// no memory or disk isn't a valid configuration
 		logger.Logger.Error().Bool("memory", false).Bool("disk", false).Msg("configuration invalid :( exiting")
@@ -108,12 +98,15 @@ func New(address string, memorySize string, memoryMultiplier int, diskSize strin
 		memory: m,
 		disk:   d,
 
+		memorygc: mgc,
+		diskgc:   dgc,
+
 		hits: avgcachestate.New(10000),
 	}
 
 	if d != nil {
 		if d.Size() > d.Capacity() {
-			diskgc(d, int(d.Size()-d.Capacity()))
+			randomgc(d, uint(d.Size()-d.Capacity()))
 		}
 	}
 
@@ -152,19 +145,37 @@ func (sc *SteamCache) LogStats() {
 	if sc.dirty {
 
 		if sc.memory != nil { // only log memory if memory is enabled
+			lifetimeBytes, lifetimeFiles, reclaimedBytes, deletedFiles := sc.memorygc.Stats()
+
 			logger.Logger.Info().
 				Str("size", units.HumanSize(float64(sc.memory.Size()))).
 				Str("capacity", units.HumanSize(float64(sc.memory.Capacity()))).
 				Str("files", fmt.Sprintf("%d", len(sc.memory.StatAll()))).
 				Msg("memory")
+
+			logger.Logger.Info().
+				Str("lifetime", units.HumanSize(float64(lifetimeBytes))).
+				Uint("lifetime_files", lifetimeFiles).
+				Str("reclaimed", units.HumanSize(float64(reclaimedBytes))).
+				Uint("deleted_files", deletedFiles).
+				Msg("memory_gc")
 		}
 
 		if sc.disk != nil { // only log disk if disk is enabled
+			lifetimeBytes, lifetimeFiles, reclaimedBytes, deletedFiles := sc.diskgc.Stats()
+
 			logger.Logger.Info().
 				Str("size", units.HumanSize(float64(sc.disk.Size()))).
 				Str("capacity", units.HumanSize(float64(sc.disk.Capacity()))).
 				Str("files", fmt.Sprintf("%d", len(sc.disk.StatAll()))).
 				Msg("disk")
+
+			logger.Logger.Info().
+				Str("lifetime", units.HumanSize(float64(lifetimeBytes))).
+				Uint("lifetime_files", lifetimeFiles).
+				Str("reclaimed", units.HumanSize(float64(reclaimedBytes))).
+				Uint("deleted_files", deletedFiles).
+				Msg("disk_gc")
 		}
 
 		logger.Logger.Info().

vfs/gc/gc.go

@@ -4,6 +4,7 @@ import (
 	"fmt"
 	"s1d3sw1ped/SteamCache2/vfs"
 	"s1d3sw1ped/SteamCache2/vfs/vfserror"
+	"sync"
 )
 
 // Ensure GCFS implements VFS.
@@ -12,12 +13,17 @@ var _ vfs.VFS = (*GCFS)(nil)
 // GCFS is a virtual file system that calls a GC handler when the disk is full. The GC handler is responsible for freeing up space on the disk. The GCFS is a wrapper around another VFS.
 type GCFS struct {
 	vfs.VFS
-	multiplier   int
+	multiplier int
-	gcHanderFunc GCHandlerFunc
+
+	// protected by mu
+	gcHanderFunc                 GCHandlerFunc
+	lifetimeBytes, lifetimeFiles uint
+	reclaimedBytes, deletedFiles uint
+	mu                           sync.Mutex
 }
 
 // GCHandlerFunc is a function that is called when the disk is full and the GCFS needs to free up space. It is passed the VFS and the size of the file that needs to be written. Its up to the implementation to free up space. How much space is freed is also up to the implementation.
-type GCHandlerFunc func(vfs vfs.VFS, size int)
+type GCHandlerFunc func(vfs vfs.VFS, size uint) (reclaimedBytes uint, deletedFiles uint)
 
 func New(vfs vfs.VFS, multiplier int, gcHandlerFunc GCHandlerFunc) *GCFS {
 	if multiplier <= 0 {
@@ -30,13 +36,39 @@ func New(vfs vfs.VFS, multiplier int, gcHandlerFunc GCHandlerFunc) *GCFS {
 	}
 }
 
+// Stats returns the lifetime bytes, lifetime files, reclaimed bytes and deleted files.
+// The lifetime bytes and lifetime files are the total bytes and files that have been freed up by the GC handler.
+// The reclaimed bytes and deleted files are the bytes and files that have been freed up by the GC handler since the last execution.
+// The reclaimed bytes and deleted files are reset to 0 after the call to Stats.
+func (g *GCFS) Stats() (lifetimeBytes, lifetimeFiles, reclaimedBytes, deletedFiles uint) {
+	g.mu.Lock()
+	defer g.mu.Unlock()
+
+	g.lifetimeBytes += g.reclaimedBytes
+	g.lifetimeFiles += g.deletedFiles
+
+	lifetimeBytes = g.lifetimeBytes
+	lifetimeFiles = g.lifetimeFiles
+	reclaimedBytes = g.reclaimedBytes
+	deletedFiles = g.deletedFiles
+
+	g.reclaimedBytes = 0
+	g.deletedFiles = 0
+
+	return
+}
+
 // Set overrides the Set method of the VFS interface. It tries to set the key and src, if it fails due to disk full error, it calls the GC handler and tries again. If it still fails it returns the error.
 func (g *GCFS) Set(key string, src []byte) error {
+	g.mu.Lock()
+	defer g.mu.Unlock()
 	err := g.VFS.Set(key, src) // try to set the key and src
 
 	if err == vfserror.ErrDiskFull && g.gcHanderFunc != nil { // if the error is disk full and there is a GC handler
-		g.gcHanderFunc(g.VFS, len(src)*g.multiplier) // call the GC handler
+		reclaimedBytes, deletedFiles := g.gcHanderFunc(g.VFS, uint(len(src)*g.multiplier)) // call the GC handler
-		err = g.VFS.Set(key, src)                    // try again after GC if it still fails return the error
+		g.reclaimedBytes += reclaimedBytes
+		g.deletedFiles += deletedFiles
+		err = g.VFS.Set(key, src) // try again after GC if it still fails return the error
 	}
 
 	return err

vfs/gc/gc_test.go

@@ -6,7 +6,6 @@ import (
 	"s1d3sw1ped/SteamCache2/vfs/memory"
 	"sort"
 	"testing"
-	"time"
 
 	"golang.org/x/exp/rand"
 )
@@ -15,13 +14,11 @@ func TestGCSmallRandom(t *testing.T) {
 	t.Parallel()
 
 	m := memory.New(1024 * 1024 * 16)
-	gc := New(m, 10, func(vfs vfs.VFS, size int) {
+	gc := New(m, 10, func(vfs vfs.VFS, size uint) (uint, uint) {
-		tstart := time.Now()
 		deletions := 0
-		targetreclaim := int64(size)
+		var reclaimed uint
-		var reclaimed int64
 
-		t.Logf("GC starting to reclaim %d bytes", targetreclaim)
+		t.Logf("GC starting to reclaim %d bytes", size)
 
 		stats := vfs.StatAll()
 		sort.Slice(stats, func(i, j int) bool {
@@ -31,7 +28,7 @@ func TestGCSmallRandom(t *testing.T) {
 
 		// Delete the oldest files until we've reclaimed enough space.
 		for _, s := range stats {
-			sz := s.Size() // Get the size of the file
+			sz := uint(s.Size()) // Get the size of the file
 			err := vfs.Delete(s.Name())
 			if err != nil {
 				panic(err)
@@ -41,12 +38,11 @@ func TestGCSmallRandom(t *testing.T) {
 
 			// t.Logf("GC deleting %s, %v", s.Name(), s.AccessTime().Format(time.RFC3339Nano))
 
-			if reclaimed >= targetreclaim { // We've reclaimed enough space
+			if reclaimed >= size { // We've reclaimed enough space
 				break
 			}
 		}
-
+		return uint(reclaimed), uint(deletions)
-		t.Logf("GC took %v to reclaim %d bytes by deleting %d files", time.Since(tstart), reclaimed, deletions)
 	})
 
 	for i := 0; i < 10000; i++ {
@@ -70,13 +66,11 @@ func TestGCLargeRandom(t *testing.T) {
 	t.Parallel()
 
 	m := memory.New(1024 * 1024 * 16) // 16MB
-	gc := New(m, 10, func(vfs vfs.VFS, size int) {
+	gc := New(m, 10, func(vfs vfs.VFS, size uint) (uint, uint) {
-		tstart := time.Now()
 		deletions := 0
-		targetreclaim := int64(size)
+		var reclaimed uint
-		var reclaimed int64
 
-		t.Logf("GC starting to reclaim %d bytes", targetreclaim)
+		t.Logf("GC starting to reclaim %d bytes", size)
 
 		stats := vfs.StatAll()
 		sort.Slice(stats, func(i, j int) bool {
@@ -86,17 +80,17 @@ func TestGCLargeRandom(t *testing.T) {
 
 		// Delete the oldest files until we've reclaimed enough space.
 		for _, s := range stats {
-			sz := s.Size() // Get the size of the file
+			sz := uint(s.Size()) // Get the size of the file
 			vfs.Delete(s.Name())
 			reclaimed += sz // Track how much space we've reclaimed
 			deletions++     // Track how many files we've deleted
 
-			if reclaimed >= targetreclaim { // We've reclaimed enough space
+			if reclaimed >= size { // We've reclaimed enough space
 				break
 			}
 		}
 
-		t.Logf("GC took %v to reclaim %d bytes by deleting %d files", time.Since(tstart), reclaimed, deletions)
+		return uint(reclaimed), uint(deletions)
 	})
 
 	for i := 0; i < 10000; i++ {