feat: add configurations for memory only, disk only, and memory & disk modes
All checks were successful
Release Tag / release (push) Successful in 14s
All checks were successful
Release Tag / release (push) Successful in 14s
This commit is contained in:
26
.vscode/launch.json
vendored
26
.vscode/launch.json
vendored
@@ -5,7 +5,7 @@
|
|||||||
"version": "0.2.0",
|
"version": "0.2.0",
|
||||||
"configurations": [
|
"configurations": [
|
||||||
{
|
{
|
||||||
"name": "Launch Package",
|
"name": "Launch Memory & Disk",
|
||||||
"type": "go",
|
"type": "go",
|
||||||
"request": "launch",
|
"request": "launch",
|
||||||
"mode": "auto",
|
"mode": "auto",
|
||||||
@@ -18,6 +18,30 @@
|
|||||||
"--disk-path",
|
"--disk-path",
|
||||||
"tmp/disk",
|
"tmp/disk",
|
||||||
],
|
],
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "Launch Disk Only",
|
||||||
|
"type": "go",
|
||||||
|
"request": "launch",
|
||||||
|
"mode": "auto",
|
||||||
|
"program": "${workspaceFolder}/main.go",
|
||||||
|
"args": [
|
||||||
|
"--disk",
|
||||||
|
"10G",
|
||||||
|
"--disk-path",
|
||||||
|
"tmp/disk",
|
||||||
|
],
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "Launch Memory Only",
|
||||||
|
"type": "go",
|
||||||
|
"request": "launch",
|
||||||
|
"mode": "auto",
|
||||||
|
"program": "${workspaceFolder}/main.go",
|
||||||
|
"args": [
|
||||||
|
"--memory",
|
||||||
|
"1G",
|
||||||
|
],
|
||||||
}
|
}
|
||||||
]
|
]
|
||||||
}
|
}
|
||||||
10
cmd/root.go
10
cmd/root.go
@@ -46,9 +46,9 @@ func Execute() {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func init() {
|
func init() {
|
||||||
rootCmd.Flags().StringVarP(&memory, "memory", "m", "100MB", "The size of the memory cache")
|
rootCmd.Flags().StringVarP(&memory, "memory", "m", "0", "The size of the memory cache")
|
||||||
rootCmd.Flags().IntVarP(&memorymultiplier, "memory-multiplier", "M", 10, "The multiplier for the memory cache")
|
rootCmd.Flags().IntVarP(&memorymultiplier, "memory-gc", "M", 10, "The gc value for the memory cache")
|
||||||
rootCmd.Flags().StringVarP(&disk, "disk", "d", "10GB", "The size of the disk cache")
|
rootCmd.Flags().StringVarP(&disk, "disk", "d", "0", "The size of the disk cache")
|
||||||
rootCmd.Flags().IntVarP(&diskmultiplier, "disk-multiplier", "D", 10, "The multiplier for the disk cache")
|
rootCmd.Flags().IntVarP(&diskmultiplier, "disk-gc", "D", 100, "The gc value for the disk cache")
|
||||||
rootCmd.Flags().StringVarP(&diskpath, "disk-path", "p", "tmp/steamcache2-disk", "The path to the disk cache")
|
rootCmd.Flags().StringVarP(&diskpath, "disk-path", "p", "", "The path to the disk cache")
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -47,26 +47,63 @@ func New(address string, memorySize string, memoryMultiplier int, diskSize strin
|
|||||||
panic(err)
|
panic(err)
|
||||||
}
|
}
|
||||||
|
|
||||||
m := memory.New(memorysize)
|
c := cache.New(
|
||||||
d := disk.New(diskPath, disksize)
|
cachehandler,
|
||||||
|
)
|
||||||
|
|
||||||
|
var m *memory.MemoryFS
|
||||||
|
if memorysize > 0 {
|
||||||
|
m = memory.New(memorysize)
|
||||||
|
}
|
||||||
|
|
||||||
|
var d *disk.DiskFS
|
||||||
|
if disksize > 0 {
|
||||||
|
d = disk.New(diskPath, disksize)
|
||||||
|
}
|
||||||
|
|
||||||
|
// configure the cache to match the specified mode (memory only, disk only, or memory and disk) based on the provided sizes
|
||||||
|
if disksize == 0 && memorysize != 0 {
|
||||||
|
//memory only mode - no disk
|
||||||
|
|
||||||
|
logger.Logger.Info().Bool("memory", true).Bool("disk", false).Msg("configuration")
|
||||||
|
c.SetSlow(gc.New(
|
||||||
|
m,
|
||||||
|
memoryMultiplier,
|
||||||
|
memorygc,
|
||||||
|
))
|
||||||
|
} else if disksize != 0 && memorysize == 0 {
|
||||||
|
// disk only mode
|
||||||
|
|
||||||
|
logger.Logger.Info().Bool("memory", false).Bool("disk", true).Msg("configuration")
|
||||||
|
c.SetSlow(gc.New(
|
||||||
|
d,
|
||||||
|
diskMultiplier,
|
||||||
|
diskgc,
|
||||||
|
))
|
||||||
|
} else if disksize != 0 && memorysize != 0 {
|
||||||
|
// memory and disk mode
|
||||||
|
|
||||||
|
logger.Logger.Info().Bool("memory", true).Bool("disk", true).Msg("configuration")
|
||||||
|
c.SetFast(gc.New(
|
||||||
|
m,
|
||||||
|
memoryMultiplier,
|
||||||
|
memorygc,
|
||||||
|
))
|
||||||
|
|
||||||
|
c.SetSlow(gc.New(
|
||||||
|
d,
|
||||||
|
diskMultiplier,
|
||||||
|
diskgc,
|
||||||
|
))
|
||||||
|
} else {
|
||||||
|
// no memory or disk isn't a valid configuration
|
||||||
|
logger.Logger.Error().Bool("memory", false).Bool("disk", false).Msg("configuration invalid :( exiting")
|
||||||
|
os.Exit(1)
|
||||||
|
}
|
||||||
|
|
||||||
sc := &SteamCache{
|
sc := &SteamCache{
|
||||||
address: address,
|
address: address,
|
||||||
vfs: syncfs.New(
|
vfs: syncfs.New(c),
|
||||||
cache.New(
|
|
||||||
gc.New(
|
|
||||||
m,
|
|
||||||
memoryMultiplier,
|
|
||||||
memorygc,
|
|
||||||
),
|
|
||||||
gc.New(
|
|
||||||
d,
|
|
||||||
diskMultiplier,
|
|
||||||
diskgc,
|
|
||||||
),
|
|
||||||
cachehandler,
|
|
||||||
),
|
|
||||||
),
|
|
||||||
|
|
||||||
memory: m,
|
memory: m,
|
||||||
disk: d,
|
disk: d,
|
||||||
@@ -74,8 +111,10 @@ func New(address string, memorySize string, memoryMultiplier int, diskSize strin
|
|||||||
hits: avgcachestate.New(10000),
|
hits: avgcachestate.New(10000),
|
||||||
}
|
}
|
||||||
|
|
||||||
if d.Size() > d.Capacity() {
|
if d != nil {
|
||||||
diskgc(d, int(d.Size()-d.Capacity()))
|
if d.Size() > d.Capacity() {
|
||||||
|
diskgc(d, int(d.Size()-d.Capacity()))
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
return sc
|
return sc
|
||||||
@@ -120,11 +159,13 @@ func (sc *SteamCache) LogStats() {
|
|||||||
Msg("memory")
|
Msg("memory")
|
||||||
}
|
}
|
||||||
|
|
||||||
logger.Logger.Info().
|
if sc.disk != nil { // only log disk if disk is enabled
|
||||||
Str("size", units.HumanSize(float64(sc.disk.Size()))).
|
logger.Logger.Info().
|
||||||
Str("capacity", units.HumanSize(float64(sc.disk.Capacity()))).
|
Str("size", units.HumanSize(float64(sc.disk.Size()))).
|
||||||
Str("files", fmt.Sprintf("%d", len(sc.disk.StatAll()))).
|
Str("capacity", units.HumanSize(float64(sc.disk.Capacity()))).
|
||||||
Msg("disk")
|
Str("files", fmt.Sprintf("%d", len(sc.disk.StatAll()))).
|
||||||
|
Msg("disk")
|
||||||
|
}
|
||||||
|
|
||||||
logger.Logger.Info().
|
logger.Logger.Info().
|
||||||
Str("hitrate", fmt.Sprintf("%.2f%%", sc.hits.Avg()*100)).
|
Str("hitrate", fmt.Sprintf("%.2f%%", sc.hits.Avg()*100)).
|
||||||
|
|||||||
25
vfs/cache/cache.go
vendored
25
vfs/cache/cache.go
vendored
@@ -21,23 +21,24 @@ type CacheFS struct {
|
|||||||
type CacheHandler func(*vfs.FileInfo, cachestate.CacheState) bool
|
type CacheHandler func(*vfs.FileInfo, cachestate.CacheState) bool
|
||||||
|
|
||||||
// New creates a new CacheFS. fast is used for caching, and slow is used for storage. fast should obviously be faster than slow.
|
// New creates a new CacheFS. fast is used for caching, and slow is used for storage. fast should obviously be faster than slow.
|
||||||
func New(fast, slow vfs.VFS, cacheHandler CacheHandler) *CacheFS {
|
func New(cacheHandler CacheHandler) *CacheFS {
|
||||||
if slow == nil {
|
|
||||||
panic("slow is nil")
|
|
||||||
}
|
|
||||||
|
|
||||||
if fast == slow {
|
|
||||||
panic("fast and slow are the same")
|
|
||||||
}
|
|
||||||
|
|
||||||
return &CacheFS{
|
return &CacheFS{
|
||||||
fast: fast,
|
|
||||||
slow: slow,
|
|
||||||
|
|
||||||
cacheHandler: cacheHandler,
|
cacheHandler: cacheHandler,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func (c *CacheFS) SetSlow(vfs vfs.VFS) {
|
||||||
|
if vfs == nil {
|
||||||
|
panic("vfs is nil") // panic if the vfs is nil
|
||||||
|
}
|
||||||
|
|
||||||
|
c.slow = vfs
|
||||||
|
}
|
||||||
|
|
||||||
|
func (c *CacheFS) SetFast(vfs vfs.VFS) {
|
||||||
|
c.fast = vfs
|
||||||
|
}
|
||||||
|
|
||||||
// cacheState returns the state of the file at key.
|
// cacheState returns the state of the file at key.
|
||||||
func (c *CacheFS) cacheState(key string) cachestate.CacheState {
|
func (c *CacheFS) cacheState(key string) cachestate.CacheState {
|
||||||
if c.fast != nil {
|
if c.fast != nil {
|
||||||
|
|||||||
31
vfs/cache/cache_test.go
vendored
31
vfs/cache/cache_test.go
vendored
@@ -20,7 +20,9 @@ func TestNew(t *testing.T) {
|
|||||||
fast := testMemory()
|
fast := testMemory()
|
||||||
slow := testMemory()
|
slow := testMemory()
|
||||||
|
|
||||||
cache := New(fast, slow, nil)
|
cache := New(nil)
|
||||||
|
cache.SetFast(fast)
|
||||||
|
cache.SetSlow(slow)
|
||||||
if cache == nil {
|
if cache == nil {
|
||||||
t.Fatal("expected cache to be non-nil")
|
t.Fatal("expected cache to be non-nil")
|
||||||
}
|
}
|
||||||
@@ -35,7 +37,9 @@ func TestNewPanics(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}()
|
}()
|
||||||
|
|
||||||
New(nil, nil, nil)
|
cache := New(nil)
|
||||||
|
cache.SetFast(nil)
|
||||||
|
cache.SetSlow(nil)
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestSetAndGet(t *testing.T) {
|
func TestSetAndGet(t *testing.T) {
|
||||||
@@ -43,7 +47,9 @@ func TestSetAndGet(t *testing.T) {
|
|||||||
|
|
||||||
fast := testMemory()
|
fast := testMemory()
|
||||||
slow := testMemory()
|
slow := testMemory()
|
||||||
cache := New(fast, slow, nil)
|
cache := New(nil)
|
||||||
|
cache.SetFast(fast)
|
||||||
|
cache.SetSlow(slow)
|
||||||
|
|
||||||
key := "test"
|
key := "test"
|
||||||
value := []byte("value")
|
value := []byte("value")
|
||||||
@@ -66,7 +72,8 @@ func TestSetAndGetNoFast(t *testing.T) {
|
|||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
slow := testMemory()
|
slow := testMemory()
|
||||||
cache := New(nil, slow, nil)
|
cache := New(nil)
|
||||||
|
cache.SetSlow(slow)
|
||||||
|
|
||||||
key := "test"
|
key := "test"
|
||||||
value := []byte("value")
|
value := []byte("value")
|
||||||
@@ -89,9 +96,11 @@ func TestCaching(t *testing.T) {
|
|||||||
|
|
||||||
fast := testMemory()
|
fast := testMemory()
|
||||||
slow := testMemory()
|
slow := testMemory()
|
||||||
cache := New(fast, slow, func(fi *vfs.FileInfo, cs cachestate.CacheState) bool {
|
cache := New(func(fi *vfs.FileInfo, cs cachestate.CacheState) bool {
|
||||||
return true
|
return true
|
||||||
})
|
})
|
||||||
|
cache.SetFast(fast)
|
||||||
|
cache.SetSlow(slow)
|
||||||
|
|
||||||
key := "test"
|
key := "test"
|
||||||
value := []byte("value")
|
value := []byte("value")
|
||||||
@@ -148,7 +157,9 @@ func TestGetNotFound(t *testing.T) {
|
|||||||
|
|
||||||
fast := testMemory()
|
fast := testMemory()
|
||||||
slow := testMemory()
|
slow := testMemory()
|
||||||
cache := New(fast, slow, nil)
|
cache := New(nil)
|
||||||
|
cache.SetFast(fast)
|
||||||
|
cache.SetSlow(slow)
|
||||||
|
|
||||||
_, err := cache.Get("nonexistent")
|
_, err := cache.Get("nonexistent")
|
||||||
if !errors.Is(err, vfserror.ErrNotFound) {
|
if !errors.Is(err, vfserror.ErrNotFound) {
|
||||||
@@ -161,7 +172,9 @@ func TestDelete(t *testing.T) {
|
|||||||
|
|
||||||
fast := testMemory()
|
fast := testMemory()
|
||||||
slow := testMemory()
|
slow := testMemory()
|
||||||
cache := New(fast, slow, nil)
|
cache := New(nil)
|
||||||
|
cache.SetFast(fast)
|
||||||
|
cache.SetSlow(slow)
|
||||||
|
|
||||||
key := "test"
|
key := "test"
|
||||||
value := []byte("value")
|
value := []byte("value")
|
||||||
@@ -185,7 +198,9 @@ func TestStat(t *testing.T) {
|
|||||||
|
|
||||||
fast := testMemory()
|
fast := testMemory()
|
||||||
slow := testMemory()
|
slow := testMemory()
|
||||||
cache := New(fast, slow, nil)
|
cache := New(nil)
|
||||||
|
cache.SetFast(fast)
|
||||||
|
cache.SetSlow(slow)
|
||||||
|
|
||||||
key := "test"
|
key := "test"
|
||||||
value := []byte("value")
|
value := []byte("value")
|
||||||
|
|||||||
@@ -27,6 +27,24 @@ type DiskFS struct {
|
|||||||
|
|
||||||
// New creates a new DiskFS.
|
// New creates a new DiskFS.
|
||||||
func new(root string, capacity int64, skipinit bool) *DiskFS {
|
func new(root string, capacity int64, skipinit bool) *DiskFS {
|
||||||
|
if capacity <= 0 {
|
||||||
|
panic("disk capacity must be greater than 0") // panic if the capacity is less than or equal to 0
|
||||||
|
}
|
||||||
|
|
||||||
|
if root == "" {
|
||||||
|
panic("disk root must not be empty") // panic if the root is empty
|
||||||
|
}
|
||||||
|
|
||||||
|
fi, err := os.Stat(root)
|
||||||
|
if err != nil {
|
||||||
|
if !os.IsNotExist(err) {
|
||||||
|
panic(err) // panic if the error is something other than not found
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if !fi.IsDir() {
|
||||||
|
panic("disk root must be a directory") // panic if the root is not a directory
|
||||||
|
}
|
||||||
|
|
||||||
dfs := &DiskFS{
|
dfs := &DiskFS{
|
||||||
root: root,
|
root: root,
|
||||||
info: make(map[string]*vfs.FileInfo),
|
info: make(map[string]*vfs.FileInfo),
|
||||||
|
|||||||
@@ -20,6 +20,9 @@ type GCFS struct {
|
|||||||
type GCHandlerFunc func(vfs vfs.VFS, size int)
|
type GCHandlerFunc func(vfs vfs.VFS, size int)
|
||||||
|
|
||||||
func New(vfs vfs.VFS, multiplier int, gcHandlerFunc GCHandlerFunc) *GCFS {
|
func New(vfs vfs.VFS, multiplier int, gcHandlerFunc GCHandlerFunc) *GCFS {
|
||||||
|
if multiplier <= 0 {
|
||||||
|
multiplier = 1 // if the multiplier is less than or equal to 0 set it to 1 will be slow but the user can set it to a higher value if they want
|
||||||
|
}
|
||||||
return &GCFS{
|
return &GCFS{
|
||||||
VFS: vfs,
|
VFS: vfs,
|
||||||
multiplier: multiplier,
|
multiplier: multiplier,
|
||||||
|
|||||||
@@ -25,6 +25,10 @@ type MemoryFS struct {
|
|||||||
|
|
||||||
// New creates a new MemoryFS.
|
// New creates a new MemoryFS.
|
||||||
func New(capacity int64) *MemoryFS {
|
func New(capacity int64) *MemoryFS {
|
||||||
|
if capacity <= 0 {
|
||||||
|
panic("memory capacity must be greater than 0") // panic if the capacity is less than or equal to 0
|
||||||
|
}
|
||||||
|
|
||||||
return &MemoryFS{
|
return &MemoryFS{
|
||||||
files: make(map[string]*file),
|
files: make(map[string]*file),
|
||||||
capacity: capacity,
|
capacity: capacity,
|
||||||
|
|||||||
Reference in New Issue
Block a user