feat: add configurations for memory only, disk only, and memory & disk modes
All checks were successful
Release Tag / release (push) Successful in 14s

This commit is contained in:
2025-01-22 19:28:45 -06:00
parent ca069a20ee
commit 7401c040dc
8 changed files with 156 additions and 50 deletions

26
.vscode/launch.json vendored
View File

@@ -5,7 +5,7 @@
"version": "0.2.0", "version": "0.2.0",
"configurations": [ "configurations": [
{ {
"name": "Launch Package", "name": "Launch Memory & Disk",
"type": "go", "type": "go",
"request": "launch", "request": "launch",
"mode": "auto", "mode": "auto",
@@ -18,6 +18,30 @@
"--disk-path", "--disk-path",
"tmp/disk", "tmp/disk",
], ],
},
{
"name": "Launch Disk Only",
"type": "go",
"request": "launch",
"mode": "auto",
"program": "${workspaceFolder}/main.go",
"args": [
"--disk",
"10G",
"--disk-path",
"tmp/disk",
],
},
{
"name": "Launch Memory Only",
"type": "go",
"request": "launch",
"mode": "auto",
"program": "${workspaceFolder}/main.go",
"args": [
"--memory",
"1G",
],
} }
] ]
} }

View File

@@ -46,9 +46,9 @@ func Execute() {
} }
func init() { func init() {
rootCmd.Flags().StringVarP(&memory, "memory", "m", "100MB", "The size of the memory cache") rootCmd.Flags().StringVarP(&memory, "memory", "m", "0", "The size of the memory cache")
rootCmd.Flags().IntVarP(&memorymultiplier, "memory-multiplier", "M", 10, "The multiplier for the memory cache") rootCmd.Flags().IntVarP(&memorymultiplier, "memory-gc", "M", 10, "The gc value for the memory cache")
rootCmd.Flags().StringVarP(&disk, "disk", "d", "10GB", "The size of the disk cache") rootCmd.Flags().StringVarP(&disk, "disk", "d", "0", "The size of the disk cache")
rootCmd.Flags().IntVarP(&diskmultiplier, "disk-multiplier", "D", 10, "The multiplier for the disk cache") rootCmd.Flags().IntVarP(&diskmultiplier, "disk-gc", "D", 100, "The gc value for the disk cache")
rootCmd.Flags().StringVarP(&diskpath, "disk-path", "p", "tmp/steamcache2-disk", "The path to the disk cache") rootCmd.Flags().StringVarP(&diskpath, "disk-path", "p", "", "The path to the disk cache")
} }

View File

@@ -47,26 +47,63 @@ func New(address string, memorySize string, memoryMultiplier int, diskSize strin
panic(err) panic(err)
} }
m := memory.New(memorysize) c := cache.New(
d := disk.New(diskPath, disksize) cachehandler,
)
sc := &SteamCache{ var m *memory.MemoryFS
address: address, if memorysize > 0 {
vfs: syncfs.New( m = memory.New(memorysize)
cache.New( }
gc.New(
var d *disk.DiskFS
if disksize > 0 {
d = disk.New(diskPath, disksize)
}
// configure the cache to match the specified mode (memory only, disk only, or memory and disk) based on the provided sizes
if disksize == 0 && memorysize != 0 {
//memory only mode - no disk
logger.Logger.Info().Bool("memory", true).Bool("disk", false).Msg("configuration")
c.SetSlow(gc.New(
m, m,
memoryMultiplier, memoryMultiplier,
memorygc, memorygc,
), ))
gc.New( } else if disksize != 0 && memorysize == 0 {
// disk only mode
logger.Logger.Info().Bool("memory", false).Bool("disk", true).Msg("configuration")
c.SetSlow(gc.New(
d, d,
diskMultiplier, diskMultiplier,
diskgc, diskgc,
), ))
cachehandler, } else if disksize != 0 && memorysize != 0 {
), // memory and disk mode
),
logger.Logger.Info().Bool("memory", true).Bool("disk", true).Msg("configuration")
c.SetFast(gc.New(
m,
memoryMultiplier,
memorygc,
))
c.SetSlow(gc.New(
d,
diskMultiplier,
diskgc,
))
} else {
// no memory or disk isn't a valid configuration
logger.Logger.Error().Bool("memory", false).Bool("disk", false).Msg("configuration invalid :( exiting")
os.Exit(1)
}
sc := &SteamCache{
address: address,
vfs: syncfs.New(c),
memory: m, memory: m,
disk: d, disk: d,
@@ -74,9 +111,11 @@ func New(address string, memorySize string, memoryMultiplier int, diskSize strin
hits: avgcachestate.New(10000), hits: avgcachestate.New(10000),
} }
if d != nil {
if d.Size() > d.Capacity() { if d.Size() > d.Capacity() {
diskgc(d, int(d.Size()-d.Capacity())) diskgc(d, int(d.Size()-d.Capacity()))
} }
}
return sc return sc
} }
@@ -120,11 +159,13 @@ func (sc *SteamCache) LogStats() {
Msg("memory") Msg("memory")
} }
if sc.disk != nil { // only log disk if disk is enabled
logger.Logger.Info(). logger.Logger.Info().
Str("size", units.HumanSize(float64(sc.disk.Size()))). Str("size", units.HumanSize(float64(sc.disk.Size()))).
Str("capacity", units.HumanSize(float64(sc.disk.Capacity()))). Str("capacity", units.HumanSize(float64(sc.disk.Capacity()))).
Str("files", fmt.Sprintf("%d", len(sc.disk.StatAll()))). Str("files", fmt.Sprintf("%d", len(sc.disk.StatAll()))).
Msg("disk") Msg("disk")
}
logger.Logger.Info(). logger.Logger.Info().
Str("hitrate", fmt.Sprintf("%.2f%%", sc.hits.Avg()*100)). Str("hitrate", fmt.Sprintf("%.2f%%", sc.hits.Avg()*100)).

25
vfs/cache/cache.go vendored
View File

@@ -21,23 +21,24 @@ type CacheFS struct {
type CacheHandler func(*vfs.FileInfo, cachestate.CacheState) bool type CacheHandler func(*vfs.FileInfo, cachestate.CacheState) bool
// New creates a new CacheFS. fast is used for caching, and slow is used for storage. fast should obviously be faster than slow. // New creates a new CacheFS. fast is used for caching, and slow is used for storage. fast should obviously be faster than slow.
func New(fast, slow vfs.VFS, cacheHandler CacheHandler) *CacheFS { func New(cacheHandler CacheHandler) *CacheFS {
if slow == nil {
panic("slow is nil")
}
if fast == slow {
panic("fast and slow are the same")
}
return &CacheFS{ return &CacheFS{
fast: fast,
slow: slow,
cacheHandler: cacheHandler, cacheHandler: cacheHandler,
} }
} }
func (c *CacheFS) SetSlow(vfs vfs.VFS) {
if vfs == nil {
panic("vfs is nil") // panic if the vfs is nil
}
c.slow = vfs
}
func (c *CacheFS) SetFast(vfs vfs.VFS) {
c.fast = vfs
}
// cacheState returns the state of the file at key. // cacheState returns the state of the file at key.
func (c *CacheFS) cacheState(key string) cachestate.CacheState { func (c *CacheFS) cacheState(key string) cachestate.CacheState {
if c.fast != nil { if c.fast != nil {

View File

@@ -20,7 +20,9 @@ func TestNew(t *testing.T) {
fast := testMemory() fast := testMemory()
slow := testMemory() slow := testMemory()
cache := New(fast, slow, nil) cache := New(nil)
cache.SetFast(fast)
cache.SetSlow(slow)
if cache == nil { if cache == nil {
t.Fatal("expected cache to be non-nil") t.Fatal("expected cache to be non-nil")
} }
@@ -35,7 +37,9 @@ func TestNewPanics(t *testing.T) {
} }
}() }()
New(nil, nil, nil) cache := New(nil)
cache.SetFast(nil)
cache.SetSlow(nil)
} }
func TestSetAndGet(t *testing.T) { func TestSetAndGet(t *testing.T) {
@@ -43,7 +47,9 @@ func TestSetAndGet(t *testing.T) {
fast := testMemory() fast := testMemory()
slow := testMemory() slow := testMemory()
cache := New(fast, slow, nil) cache := New(nil)
cache.SetFast(fast)
cache.SetSlow(slow)
key := "test" key := "test"
value := []byte("value") value := []byte("value")
@@ -66,7 +72,8 @@ func TestSetAndGetNoFast(t *testing.T) {
t.Parallel() t.Parallel()
slow := testMemory() slow := testMemory()
cache := New(nil, slow, nil) cache := New(nil)
cache.SetSlow(slow)
key := "test" key := "test"
value := []byte("value") value := []byte("value")
@@ -89,9 +96,11 @@ func TestCaching(t *testing.T) {
fast := testMemory() fast := testMemory()
slow := testMemory() slow := testMemory()
cache := New(fast, slow, func(fi *vfs.FileInfo, cs cachestate.CacheState) bool { cache := New(func(fi *vfs.FileInfo, cs cachestate.CacheState) bool {
return true return true
}) })
cache.SetFast(fast)
cache.SetSlow(slow)
key := "test" key := "test"
value := []byte("value") value := []byte("value")
@@ -148,7 +157,9 @@ func TestGetNotFound(t *testing.T) {
fast := testMemory() fast := testMemory()
slow := testMemory() slow := testMemory()
cache := New(fast, slow, nil) cache := New(nil)
cache.SetFast(fast)
cache.SetSlow(slow)
_, err := cache.Get("nonexistent") _, err := cache.Get("nonexistent")
if !errors.Is(err, vfserror.ErrNotFound) { if !errors.Is(err, vfserror.ErrNotFound) {
@@ -161,7 +172,9 @@ func TestDelete(t *testing.T) {
fast := testMemory() fast := testMemory()
slow := testMemory() slow := testMemory()
cache := New(fast, slow, nil) cache := New(nil)
cache.SetFast(fast)
cache.SetSlow(slow)
key := "test" key := "test"
value := []byte("value") value := []byte("value")
@@ -185,7 +198,9 @@ func TestStat(t *testing.T) {
fast := testMemory() fast := testMemory()
slow := testMemory() slow := testMemory()
cache := New(fast, slow, nil) cache := New(nil)
cache.SetFast(fast)
cache.SetSlow(slow)
key := "test" key := "test"
value := []byte("value") value := []byte("value")

View File

@@ -27,6 +27,24 @@ type DiskFS struct {
// New creates a new DiskFS. // New creates a new DiskFS.
func new(root string, capacity int64, skipinit bool) *DiskFS { func new(root string, capacity int64, skipinit bool) *DiskFS {
if capacity <= 0 {
panic("disk capacity must be greater than 0") // panic if the capacity is less than or equal to 0
}
if root == "" {
panic("disk root must not be empty") // panic if the root is empty
}
fi, err := os.Stat(root)
if err != nil {
if !os.IsNotExist(err) {
panic(err) // panic if the error is something other than not found
}
}
if !fi.IsDir() {
panic("disk root must be a directory") // panic if the root is not a directory
}
dfs := &DiskFS{ dfs := &DiskFS{
root: root, root: root,
info: make(map[string]*vfs.FileInfo), info: make(map[string]*vfs.FileInfo),

View File

@@ -20,6 +20,9 @@ type GCFS struct {
type GCHandlerFunc func(vfs vfs.VFS, size int) type GCHandlerFunc func(vfs vfs.VFS, size int)
func New(vfs vfs.VFS, multiplier int, gcHandlerFunc GCHandlerFunc) *GCFS { func New(vfs vfs.VFS, multiplier int, gcHandlerFunc GCHandlerFunc) *GCFS {
if multiplier <= 0 {
multiplier = 1 // if the multiplier is less than or equal to 0 set it to 1 will be slow but the user can set it to a higher value if they want
}
return &GCFS{ return &GCFS{
VFS: vfs, VFS: vfs,
multiplier: multiplier, multiplier: multiplier,

View File

@@ -25,6 +25,10 @@ type MemoryFS struct {
// New creates a new MemoryFS. // New creates a new MemoryFS.
func New(capacity int64) *MemoryFS { func New(capacity int64) *MemoryFS {
if capacity <= 0 {
panic("memory capacity must be greater than 0") // panic if the capacity is less than or equal to 0
}
return &MemoryFS{ return &MemoryFS{
files: make(map[string]*file), files: make(map[string]*file),
capacity: capacity, capacity: capacity,