13 Commits

Author SHA1 Message Date
931c43d7a8 fix: remove draft status from release configuration
All checks were successful
Release Tag / release (push) Successful in 9s
2025-01-22 21:01:35 -06:00
6fe80c82ad fix: reduce cache hits averaging size for improved performance
All checks were successful
Release Tag / release (push) Successful in 9s
2025-01-22 20:59:24 -06:00
4a69c4ba66 fix: add empty line in logging for improved readability
All checks were successful
Release Tag / release (push) Successful in 9s
2025-01-22 20:51:32 -06:00
2d0fe6571d feat: enhance garbage collection logging with total GC time and improved stat formatting
All checks were successful
Release Tag / release (push) Successful in 9s
2025-01-22 20:48:05 -06:00
550948951e feat: implement enhanced garbage collection statistics logging
All checks were successful
Release Tag / release (push) Successful in 12s
2025-01-22 20:27:12 -06:00
4a23eecae0 fix: reduce log interval to 1 second for more frequent statistics updates
All checks were successful
Release Tag / release (push) Successful in 28s
2025-01-22 19:37:30 -06:00
fed9bbe218 fix: go back to the old averaging
All checks were successful
Release Tag / release (push) Successful in 12s
2025-01-22 19:35:17 -06:00
7401c040dc feat: add configurations for memory only, disk only, and memory & disk modes
All checks were successful
Release Tag / release (push) Successful in 14s
2025-01-22 19:28:45 -06:00
ca069a20ee fix: track cache hits and misses in average cache state 2025-01-22 18:43:07 -06:00
3e8a92b865 fix: log memory statistics only if memory is enabled
All checks were successful
Release Tag / release (push) Successful in 9s
2025-01-22 18:29:19 -06:00
b7652ed7cc fix: initialize average cache state with cleared values
All checks were successful
Release Tag / release (push) Successful in 9s
2025-01-22 18:25:16 -06:00
08b8d0ce3d fix: enhance logging for memory and disk statistics in cache 2025-01-22 18:23:53 -06:00
53847db0e9 fix: improve cache state clearing and average calculation logic 2025-01-22 18:23:47 -06:00
12 changed files with 274 additions and 130 deletions

View File

@@ -36,7 +36,6 @@ changelog:
- "^test:" - "^test:"
release: release:
draft: true
name_template: '{{.ProjectName}}-{{.Version}}' name_template: '{{.ProjectName}}-{{.Version}}'
footer: >- footer: >-

26
.vscode/launch.json vendored
View File

@@ -5,7 +5,7 @@
"version": "0.2.0", "version": "0.2.0",
"configurations": [ "configurations": [
{ {
"name": "Launch Package", "name": "Launch Memory & Disk",
"type": "go", "type": "go",
"request": "launch", "request": "launch",
"mode": "auto", "mode": "auto",
@@ -18,6 +18,30 @@
"--disk-path", "--disk-path",
"tmp/disk", "tmp/disk",
], ],
},
{
"name": "Launch Disk Only",
"type": "go",
"request": "launch",
"mode": "auto",
"program": "${workspaceFolder}/main.go",
"args": [
"--disk",
"10G",
"--disk-path",
"tmp/disk",
],
},
{
"name": "Launch Memory Only",
"type": "go",
"request": "launch",
"mode": "auto",
"program": "${workspaceFolder}/main.go",
"args": [
"--memory",
"1G",
],
} }
] ]
} }

View File

@@ -46,9 +46,9 @@ func Execute() {
} }
func init() { func init() {
rootCmd.Flags().StringVarP(&memory, "memory", "m", "100MB", "The size of the memory cache") rootCmd.Flags().StringVarP(&memory, "memory", "m", "0", "The size of the memory cache")
rootCmd.Flags().IntVarP(&memorymultiplier, "memory-multiplier", "M", 10, "The multiplier for the memory cache") rootCmd.Flags().IntVarP(&memorymultiplier, "memory-gc", "M", 10, "The gc value for the memory cache")
rootCmd.Flags().StringVarP(&disk, "disk", "d", "10GB", "The size of the disk cache") rootCmd.Flags().StringVarP(&disk, "disk", "d", "0", "The size of the disk cache")
rootCmd.Flags().IntVarP(&diskmultiplier, "disk-multiplier", "D", 10, "The multiplier for the disk cache") rootCmd.Flags().IntVarP(&diskmultiplier, "disk-gc", "D", 100, "The gc value for the disk cache")
rootCmd.Flags().StringVarP(&diskpath, "disk-path", "p", "tmp/steamcache2-disk", "The path to the disk cache") rootCmd.Flags().StringVarP(&diskpath, "disk-path", "p", "", "The path to the disk cache")
} }

View File

@@ -14,11 +14,15 @@ type AvgCacheState struct {
// New creates a new average cache state with the given size. // New creates a new average cache state with the given size.
func New(size int) *AvgCacheState { func New(size int) *AvgCacheState {
return &AvgCacheState{ a := &AvgCacheState{
size: size, size: size,
avgs: make([]cachestate.CacheState, size), avgs: make([]cachestate.CacheState, size),
mu: sync.Mutex{}, mu: sync.Mutex{},
} }
a.Clear()
return a
} }
// Clear resets the average cache state to zero. // Clear resets the average cache state to zero.
@@ -26,7 +30,9 @@ func (a *AvgCacheState) Clear() {
a.mu.Lock() a.mu.Lock()
defer a.mu.Unlock() defer a.mu.Unlock()
a.avgs = make([]cachestate.CacheState, a.size) // zeroed for i := 0; i < len(a.avgs); i++ {
a.avgs[i] = cachestate.CacheStateMiss
}
} }
// Add adds a cache state to the average cache state. // Add adds a cache state to the average cache state.
@@ -45,16 +51,13 @@ func (a *AvgCacheState) Avg() float64 {
a.mu.Lock() a.mu.Lock()
defer a.mu.Unlock() defer a.mu.Unlock()
var hits, misses int var hits int
for _, cs := range a.avgs { for _, cs := range a.avgs {
switch cs { if cs == cachestate.CacheStateHit {
case cachestate.CacheStateHit:
hits++ hits++
case cachestate.CacheStateMiss:
misses++
} }
} }
total := hits + misses
return float64(hits) / float64(total) return float64(hits) / float64(len(a.avgs))
} }

View File

@@ -1,73 +1,42 @@
package steamcache package steamcache
import ( import (
"s1d3sw1ped/SteamCache2/steamcache/logger"
"s1d3sw1ped/SteamCache2/vfs" "s1d3sw1ped/SteamCache2/vfs"
"s1d3sw1ped/SteamCache2/vfs/cachestate" "s1d3sw1ped/SteamCache2/vfs/cachestate"
"time" "time"
"github.com/docker/go-units"
"golang.org/x/exp/rand" "golang.org/x/exp/rand"
) )
func randomgc(vfss vfs.VFS, stats []*vfs.FileInfo) int64 { // RandomGC randomly deletes files until we've reclaimed enough space.
// Pick a random file to delete func randomgc(vfss vfs.VFS, size uint) (uint, uint) {
randfile := stats[rand.Intn(len(stats))]
sz := randfile.Size() // Randomly delete files until we've reclaimed enough space.
err := vfss.Delete(randfile.Name()) random := func(vfss vfs.VFS, stats []*vfs.FileInfo) int64 {
if err != nil { randfile := stats[rand.Intn(len(stats))]
// If we failed to delete the file, log it and return 0 sz := randfile.Size()
// logger.Logger.Error().Err(err).Msgf("Failed to delete %s", randfile.Name()) err := vfss.Delete(randfile.Name())
return 0 if err != nil {
return 0
}
return sz
} }
return sz
}
func memorygc(vfss vfs.VFS, size int) {
tstart := time.Now()
deletions := 0 deletions := 0
targetreclaim := int64(size) targetreclaim := int64(size)
var reclaimed int64 var reclaimed int64
stats := vfss.StatAll() stats := vfss.StatAll()
for { for {
reclaimed += randomgc(vfss, stats) reclaimed += random(vfss, stats)
deletions++ deletions++
if reclaimed >= targetreclaim { if reclaimed >= targetreclaim {
break break
} }
} }
logger.Logger.Info(). return uint(reclaimed), uint(deletions)
Str("name", vfss.Name()).
Str("duration", time.Since(tstart).String()).
Str("reclaimed", units.HumanSize(float64(reclaimed))).
Int("deletions", deletions).
Msgf("GC")
}
func diskgc(vfss vfs.VFS, size int) {
tstart := time.Now()
deletions := 0
targetreclaim := int64(size)
var reclaimed int64
stats := vfss.StatAll()
for {
reclaimed += randomgc(vfss, stats)
deletions++
if reclaimed >= targetreclaim {
break
}
}
logger.Logger.Info().
Str("name", vfss.Name()).
Str("duration", time.Since(tstart).String()).
Str("reclaimed", units.HumanSize(float64(reclaimed))).
Int("deletions", deletions).
Msgf("GC")
} }
func cachehandler(fi *vfs.FileInfo, cs cachestate.CacheState) bool { func cachehandler(fi *vfs.FileInfo, cs cachestate.CacheState) bool {

View File

@@ -30,6 +30,9 @@ type SteamCache struct {
memory *memory.MemoryFS memory *memory.MemoryFS
disk *disk.DiskFS disk *disk.DiskFS
memorygc *gc.GCFS
diskgc *gc.GCFS
hits *avgcachestate.AvgCacheState hits *avgcachestate.AvgCacheState
dirty bool dirty bool
@@ -47,35 +50,64 @@ func New(address string, memorySize string, memoryMultiplier int, diskSize strin
panic(err) panic(err)
} }
m := memory.New(memorysize) c := cache.New(
d := disk.New(diskPath, disksize) cachehandler,
)
var m *memory.MemoryFS
var mgc *gc.GCFS
if memorysize > 0 {
m = memory.New(memorysize)
mgc = gc.New(m, memoryMultiplier, randomgc)
}
var d *disk.DiskFS
var dgc *gc.GCFS
if disksize > 0 {
d = disk.New(diskPath, disksize)
dgc = gc.New(d, diskMultiplier, randomgc)
}
// configure the cache to match the specified mode (memory only, disk only, or memory and disk) based on the provided sizes
if disksize == 0 && memorysize != 0 {
//memory only mode - no disk
c.SetSlow(mgc)
logger.Logger.Info().Bool("memory", true).Bool("disk", false).Msg("configuration")
} else if disksize != 0 && memorysize == 0 {
// disk only mode
c.SetSlow(dgc)
logger.Logger.Info().Bool("memory", false).Bool("disk", true).Msg("configuration")
} else if disksize != 0 && memorysize != 0 {
// memory and disk mode
c.SetFast(mgc)
c.SetSlow(dgc)
logger.Logger.Info().Bool("memory", true).Bool("disk", true).Msg("configuration")
} else {
// no memory or disk isn't a valid configuration
logger.Logger.Error().Bool("memory", false).Bool("disk", false).Msg("configuration invalid :( exiting")
os.Exit(1)
}
sc := &SteamCache{ sc := &SteamCache{
address: address, address: address,
vfs: syncfs.New( vfs: syncfs.New(c),
cache.New(
gc.New(
m,
memoryMultiplier,
memorygc,
),
gc.New(
d,
diskMultiplier,
diskgc,
),
cachehandler,
),
),
memory: m, memory: m,
disk: d, disk: d,
hits: avgcachestate.New(10000), memorygc: mgc,
diskgc: dgc,
hits: avgcachestate.New(100),
} }
if d.Size() > d.Capacity() { if d != nil {
diskgc(d, int(d.Size()-d.Capacity())) if d.Size() > d.Capacity() {
randomgc(d, uint(d.Size()-d.Capacity()))
}
} }
return sc return sc
@@ -89,7 +121,7 @@ func (sc *SteamCache) Run() {
sc.mu.Unlock() sc.mu.Unlock()
sc.LogStats() sc.LogStats()
t := time.NewTicker(10 * time.Second) t := time.NewTicker(1 * time.Second)
go func() { go func() {
for range t.C { for range t.C {
sc.LogStats() sc.LogStats()
@@ -111,11 +143,49 @@ func (sc *SteamCache) LogStats() {
sc.mu.Lock() sc.mu.Lock()
defer sc.mu.Unlock() defer sc.mu.Unlock()
if sc.dirty { if sc.dirty {
logger.Logger.Info().Msg("") // empty line to separate log entries for better readability
if sc.memory != nil { // only log memory if memory is enabled
lifetimeBytes, lifetimeFiles, reclaimedBytes, deletedFiles, gcTime := sc.memorygc.Stats()
logger.Logger.Info().
Str("size", units.HumanSize(float64(sc.memory.Size()))).
Str("capacity", units.HumanSize(float64(sc.memory.Capacity()))).
Str("files", fmt.Sprintf("%d", len(sc.memory.StatAll()))).
Msg("memory")
logger.Logger.Info().
Str("data_total", units.HumanSize(float64(lifetimeBytes))).
Uint("files_total", lifetimeFiles).
Str("data", units.HumanSize(float64(reclaimedBytes))).
Uint("files", deletedFiles).
Str("gc_time", gcTime.String()).
Msg("memory_gc")
}
if sc.disk != nil { // only log disk if disk is enabled
lifetimeBytes, lifetimeFiles, reclaimedBytes, deletedFiles, gcTime := sc.diskgc.Stats()
logger.Logger.Info().
Str("size", units.HumanSize(float64(sc.disk.Size()))).
Str("capacity", units.HumanSize(float64(sc.disk.Capacity()))).
Str("files", fmt.Sprintf("%d", len(sc.disk.StatAll()))).
Msg("disk")
logger.Logger.Info().
Str("data_total", units.HumanSize(float64(lifetimeBytes))).
Uint("files_total", lifetimeFiles).
Str("data", units.HumanSize(float64(reclaimedBytes))).
Uint("files", deletedFiles).
Str("gc_time", gcTime.String()).
Msg("disk_gc")
}
logger.Logger.Info(). logger.Logger.Info().
Str("memory", fmt.Sprintf("%s/%s", units.HumanSize(float64(sc.memory.Size())), units.HumanSize(float64(sc.memory.Capacity())))).Int("memory-files", len(sc.memory.StatAll())).
Str("disk", fmt.Sprintf("%s/%s", units.HumanSize(float64(sc.disk.Size())), units.HumanSize(float64(sc.disk.Capacity())))).Int("disk-files", len(sc.disk.StatAll())).
Str("hitrate", fmt.Sprintf("%.2f%%", sc.hits.Avg()*100)). Str("hitrate", fmt.Sprintf("%.2f%%", sc.hits.Avg()*100)).
Msg("stats") Msg("cache")
sc.dirty = false sc.dirty = false
} }
} }

25
vfs/cache/cache.go vendored
View File

@@ -21,23 +21,24 @@ type CacheFS struct {
type CacheHandler func(*vfs.FileInfo, cachestate.CacheState) bool type CacheHandler func(*vfs.FileInfo, cachestate.CacheState) bool
// New creates a new CacheFS. fast is used for caching, and slow is used for storage. fast should obviously be faster than slow. // New creates a new CacheFS. fast is used for caching, and slow is used for storage. fast should obviously be faster than slow.
func New(fast, slow vfs.VFS, cacheHandler CacheHandler) *CacheFS { func New(cacheHandler CacheHandler) *CacheFS {
if slow == nil {
panic("slow is nil")
}
if fast == slow {
panic("fast and slow are the same")
}
return &CacheFS{ return &CacheFS{
fast: fast,
slow: slow,
cacheHandler: cacheHandler, cacheHandler: cacheHandler,
} }
} }
func (c *CacheFS) SetSlow(vfs vfs.VFS) {
if vfs == nil {
panic("vfs is nil") // panic if the vfs is nil
}
c.slow = vfs
}
func (c *CacheFS) SetFast(vfs vfs.VFS) {
c.fast = vfs
}
// cacheState returns the state of the file at key. // cacheState returns the state of the file at key.
func (c *CacheFS) cacheState(key string) cachestate.CacheState { func (c *CacheFS) cacheState(key string) cachestate.CacheState {
if c.fast != nil { if c.fast != nil {

View File

@@ -20,7 +20,9 @@ func TestNew(t *testing.T) {
fast := testMemory() fast := testMemory()
slow := testMemory() slow := testMemory()
cache := New(fast, slow, nil) cache := New(nil)
cache.SetFast(fast)
cache.SetSlow(slow)
if cache == nil { if cache == nil {
t.Fatal("expected cache to be non-nil") t.Fatal("expected cache to be non-nil")
} }
@@ -35,7 +37,9 @@ func TestNewPanics(t *testing.T) {
} }
}() }()
New(nil, nil, nil) cache := New(nil)
cache.SetFast(nil)
cache.SetSlow(nil)
} }
func TestSetAndGet(t *testing.T) { func TestSetAndGet(t *testing.T) {
@@ -43,7 +47,9 @@ func TestSetAndGet(t *testing.T) {
fast := testMemory() fast := testMemory()
slow := testMemory() slow := testMemory()
cache := New(fast, slow, nil) cache := New(nil)
cache.SetFast(fast)
cache.SetSlow(slow)
key := "test" key := "test"
value := []byte("value") value := []byte("value")
@@ -66,7 +72,8 @@ func TestSetAndGetNoFast(t *testing.T) {
t.Parallel() t.Parallel()
slow := testMemory() slow := testMemory()
cache := New(nil, slow, nil) cache := New(nil)
cache.SetSlow(slow)
key := "test" key := "test"
value := []byte("value") value := []byte("value")
@@ -89,9 +96,11 @@ func TestCaching(t *testing.T) {
fast := testMemory() fast := testMemory()
slow := testMemory() slow := testMemory()
cache := New(fast, slow, func(fi *vfs.FileInfo, cs cachestate.CacheState) bool { cache := New(func(fi *vfs.FileInfo, cs cachestate.CacheState) bool {
return true return true
}) })
cache.SetFast(fast)
cache.SetSlow(slow)
key := "test" key := "test"
value := []byte("value") value := []byte("value")
@@ -148,7 +157,9 @@ func TestGetNotFound(t *testing.T) {
fast := testMemory() fast := testMemory()
slow := testMemory() slow := testMemory()
cache := New(fast, slow, nil) cache := New(nil)
cache.SetFast(fast)
cache.SetSlow(slow)
_, err := cache.Get("nonexistent") _, err := cache.Get("nonexistent")
if !errors.Is(err, vfserror.ErrNotFound) { if !errors.Is(err, vfserror.ErrNotFound) {
@@ -161,7 +172,9 @@ func TestDelete(t *testing.T) {
fast := testMemory() fast := testMemory()
slow := testMemory() slow := testMemory()
cache := New(fast, slow, nil) cache := New(nil)
cache.SetFast(fast)
cache.SetSlow(slow)
key := "test" key := "test"
value := []byte("value") value := []byte("value")
@@ -185,7 +198,9 @@ func TestStat(t *testing.T) {
fast := testMemory() fast := testMemory()
slow := testMemory() slow := testMemory()
cache := New(fast, slow, nil) cache := New(nil)
cache.SetFast(fast)
cache.SetSlow(slow)
key := "test" key := "test"
value := []byte("value") value := []byte("value")

View File

@@ -27,6 +27,24 @@ type DiskFS struct {
// New creates a new DiskFS. // New creates a new DiskFS.
func new(root string, capacity int64, skipinit bool) *DiskFS { func new(root string, capacity int64, skipinit bool) *DiskFS {
if capacity <= 0 {
panic("disk capacity must be greater than 0") // panic if the capacity is less than or equal to 0
}
if root == "" {
panic("disk root must not be empty") // panic if the root is empty
}
fi, err := os.Stat(root)
if err != nil {
if !os.IsNotExist(err) {
panic(err) // panic if the error is something other than not found
}
}
if !fi.IsDir() {
panic("disk root must be a directory") // panic if the root is not a directory
}
dfs := &DiskFS{ dfs := &DiskFS{
root: root, root: root,
info: make(map[string]*vfs.FileInfo), info: make(map[string]*vfs.FileInfo),
@@ -60,7 +78,12 @@ func (d *DiskFS) init() {
d.walk(d.root) d.walk(d.root)
d.sg.Wait() d.sg.Wait()
logger.Logger.Info().Str("name", d.Name()).Str("root", d.root).Str("capacity", units.HumanSize(float64(d.capacity))).Str("duration", time.Since(tstart).String()).Msg("init") logger.Logger.Info().
Str("name", d.Name()).
Str("root", d.root).
Str("capacity", units.HumanSize(float64(d.capacity))).
Str("duration", time.Since(tstart).String()).
Msg("init")
} }
func (d *DiskFS) walk(path string) { func (d *DiskFS) walk(path string) {

View File

@@ -4,6 +4,8 @@ import (
"fmt" "fmt"
"s1d3sw1ped/SteamCache2/vfs" "s1d3sw1ped/SteamCache2/vfs"
"s1d3sw1ped/SteamCache2/vfs/vfserror" "s1d3sw1ped/SteamCache2/vfs/vfserror"
"sync"
"time"
) )
// Ensure GCFS implements VFS. // Ensure GCFS implements VFS.
@@ -12,14 +14,23 @@ var _ vfs.VFS = (*GCFS)(nil)
// GCFS is a virtual file system that calls a GC handler when the disk is full. The GC handler is responsible for freeing up space on the disk. The GCFS is a wrapper around another VFS. // GCFS is a virtual file system that calls a GC handler when the disk is full. The GC handler is responsible for freeing up space on the disk. The GCFS is a wrapper around another VFS.
type GCFS struct { type GCFS struct {
vfs.VFS vfs.VFS
multiplier int multiplier int
gcHanderFunc GCHandlerFunc
// protected by mu
gcHanderFunc GCHandlerFunc
lifetimeBytes, lifetimeFiles uint
reclaimedBytes, deletedFiles uint
gcTime time.Duration
mu sync.Mutex
} }
// GCHandlerFunc is a function that is called when the disk is full and the GCFS needs to free up space. It is passed the VFS and the size of the file that needs to be written. Its up to the implementation to free up space. How much space is freed is also up to the implementation. // GCHandlerFunc is a function that is called when the disk is full and the GCFS needs to free up space. It is passed the VFS and the size of the file that needs to be written. Its up to the implementation to free up space. How much space is freed is also up to the implementation.
type GCHandlerFunc func(vfs vfs.VFS, size int) type GCHandlerFunc func(vfs vfs.VFS, size uint) (reclaimedBytes uint, deletedFiles uint)
func New(vfs vfs.VFS, multiplier int, gcHandlerFunc GCHandlerFunc) *GCFS { func New(vfs vfs.VFS, multiplier int, gcHandlerFunc GCHandlerFunc) *GCFS {
if multiplier <= 0 {
multiplier = 1 // if the multiplier is less than or equal to 0 set it to 1 will be slow but the user can set it to a higher value if they want
}
return &GCFS{ return &GCFS{
VFS: vfs, VFS: vfs,
multiplier: multiplier, multiplier: multiplier,
@@ -27,13 +38,44 @@ func New(vfs vfs.VFS, multiplier int, gcHandlerFunc GCHandlerFunc) *GCFS {
} }
} }
// Stats returns the lifetime bytes, lifetime files, reclaimed bytes and deleted files.
// The lifetime bytes and lifetime files are the total bytes and files that have been freed up by the GC handler.
// The reclaimed bytes and deleted files are the bytes and files that have been freed up by the GC handler since last call to Stats.
// The gc time is the total time spent in the GC handler since last call to Stats.
// The reclaimed bytes and deleted files and gc time are reset to 0 after the call to Stats.
func (g *GCFS) Stats() (lifetimeBytes, lifetimeFiles, reclaimedBytes, deletedFiles uint, gcTime time.Duration) {
g.mu.Lock()
defer g.mu.Unlock()
g.lifetimeBytes += g.reclaimedBytes
g.lifetimeFiles += g.deletedFiles
lifetimeBytes = g.lifetimeBytes
lifetimeFiles = g.lifetimeFiles
reclaimedBytes = g.reclaimedBytes
deletedFiles = g.deletedFiles
gcTime = g.gcTime
g.reclaimedBytes = 0
g.deletedFiles = 0
g.gcTime = time.Duration(0)
return
}
// Set overrides the Set method of the VFS interface. It tries to set the key and src, if it fails due to disk full error, it calls the GC handler and tries again. If it still fails it returns the error. // Set overrides the Set method of the VFS interface. It tries to set the key and src, if it fails due to disk full error, it calls the GC handler and tries again. If it still fails it returns the error.
func (g *GCFS) Set(key string, src []byte) error { func (g *GCFS) Set(key string, src []byte) error {
g.mu.Lock()
defer g.mu.Unlock()
err := g.VFS.Set(key, src) // try to set the key and src err := g.VFS.Set(key, src) // try to set the key and src
if err == vfserror.ErrDiskFull && g.gcHanderFunc != nil { // if the error is disk full and there is a GC handler if err == vfserror.ErrDiskFull && g.gcHanderFunc != nil { // if the error is disk full and there is a GC handler
g.gcHanderFunc(g.VFS, len(src)*g.multiplier) // call the GC handler tstart := time.Now()
err = g.VFS.Set(key, src) // try again after GC if it still fails return the error reclaimedBytes, deletedFiles := g.gcHanderFunc(g.VFS, uint(len(src)*g.multiplier)) // call the GC handler
g.gcTime += time.Since(tstart)
g.reclaimedBytes += reclaimedBytes
g.deletedFiles += deletedFiles
err = g.VFS.Set(key, src) // try again after GC if it still fails return the error
} }
return err return err

View File

@@ -6,7 +6,6 @@ import (
"s1d3sw1ped/SteamCache2/vfs/memory" "s1d3sw1ped/SteamCache2/vfs/memory"
"sort" "sort"
"testing" "testing"
"time"
"golang.org/x/exp/rand" "golang.org/x/exp/rand"
) )
@@ -15,13 +14,11 @@ func TestGCSmallRandom(t *testing.T) {
t.Parallel() t.Parallel()
m := memory.New(1024 * 1024 * 16) m := memory.New(1024 * 1024 * 16)
gc := New(m, 10, func(vfs vfs.VFS, size int) { gc := New(m, 10, func(vfs vfs.VFS, size uint) (uint, uint) {
tstart := time.Now()
deletions := 0 deletions := 0
targetreclaim := int64(size) var reclaimed uint
var reclaimed int64
t.Logf("GC starting to reclaim %d bytes", targetreclaim) t.Logf("GC starting to reclaim %d bytes", size)
stats := vfs.StatAll() stats := vfs.StatAll()
sort.Slice(stats, func(i, j int) bool { sort.Slice(stats, func(i, j int) bool {
@@ -31,7 +28,7 @@ func TestGCSmallRandom(t *testing.T) {
// Delete the oldest files until we've reclaimed enough space. // Delete the oldest files until we've reclaimed enough space.
for _, s := range stats { for _, s := range stats {
sz := s.Size() // Get the size of the file sz := uint(s.Size()) // Get the size of the file
err := vfs.Delete(s.Name()) err := vfs.Delete(s.Name())
if err != nil { if err != nil {
panic(err) panic(err)
@@ -41,12 +38,11 @@ func TestGCSmallRandom(t *testing.T) {
// t.Logf("GC deleting %s, %v", s.Name(), s.AccessTime().Format(time.RFC3339Nano)) // t.Logf("GC deleting %s, %v", s.Name(), s.AccessTime().Format(time.RFC3339Nano))
if reclaimed >= targetreclaim { // We've reclaimed enough space if reclaimed >= size { // We've reclaimed enough space
break break
} }
} }
return uint(reclaimed), uint(deletions)
t.Logf("GC took %v to reclaim %d bytes by deleting %d files", time.Since(tstart), reclaimed, deletions)
}) })
for i := 0; i < 10000; i++ { for i := 0; i < 10000; i++ {
@@ -70,13 +66,11 @@ func TestGCLargeRandom(t *testing.T) {
t.Parallel() t.Parallel()
m := memory.New(1024 * 1024 * 16) // 16MB m := memory.New(1024 * 1024 * 16) // 16MB
gc := New(m, 10, func(vfs vfs.VFS, size int) { gc := New(m, 10, func(vfs vfs.VFS, size uint) (uint, uint) {
tstart := time.Now()
deletions := 0 deletions := 0
targetreclaim := int64(size) var reclaimed uint
var reclaimed int64
t.Logf("GC starting to reclaim %d bytes", targetreclaim) t.Logf("GC starting to reclaim %d bytes", size)
stats := vfs.StatAll() stats := vfs.StatAll()
sort.Slice(stats, func(i, j int) bool { sort.Slice(stats, func(i, j int) bool {
@@ -86,17 +80,17 @@ func TestGCLargeRandom(t *testing.T) {
// Delete the oldest files until we've reclaimed enough space. // Delete the oldest files until we've reclaimed enough space.
for _, s := range stats { for _, s := range stats {
sz := s.Size() // Get the size of the file sz := uint(s.Size()) // Get the size of the file
vfs.Delete(s.Name()) vfs.Delete(s.Name())
reclaimed += sz // Track how much space we've reclaimed reclaimed += sz // Track how much space we've reclaimed
deletions++ // Track how many files we've deleted deletions++ // Track how many files we've deleted
if reclaimed >= targetreclaim { // We've reclaimed enough space if reclaimed >= size { // We've reclaimed enough space
break break
} }
} }
t.Logf("GC took %v to reclaim %d bytes by deleting %d files", time.Since(tstart), reclaimed, deletions) return uint(reclaimed), uint(deletions)
}) })
for i := 0; i < 10000; i++ { for i := 0; i < 10000; i++ {

View File

@@ -25,6 +25,10 @@ type MemoryFS struct {
// New creates a new MemoryFS. // New creates a new MemoryFS.
func New(capacity int64) *MemoryFS { func New(capacity int64) *MemoryFS {
if capacity <= 0 {
panic("memory capacity must be greater than 0") // panic if the capacity is less than or equal to 0
}
return &MemoryFS{ return &MemoryFS{
files: make(map[string]*file), files: make(map[string]*file),
capacity: capacity, capacity: capacity,