Compare commits
40 Commits
1.0.0
..
6fc83063da
| Author | SHA1 | Date | |
|---|---|---|---|
| 6fc83063da | |||
| cd4b5e0598 | |||
| e583780b9c | |||
| c0ed8a590e | |||
| 09d3450738 | |||
| f01b15c8c6 | |||
| 27ac30d28a | |||
| 0424de1fca | |||
| 32c0d85bad | |||
| 6b51e61a87 | |||
| 1094505f20 | |||
| 93d04410bf | |||
| 62f7a38295 | |||
| c370e80888 | |||
| 9d0f4894e1 | |||
| 745ee37184 | |||
| 5a9b07f609 | |||
| d88db3d9c1 | |||
| 91075232cd | |||
| 1f588fa20a | |||
| d39a2ad4fa | |||
| 80cc75f372 | |||
| 53f5d1577c | |||
| 1186e14c56 | |||
| 3bb57b7741 | |||
| 8f2d1eff9b | |||
| 27242fdf05 | |||
| 55330ebd3c | |||
| e37dc930dd | |||
| 79f0814745 | |||
| 1feee2758c | |||
| 22f9196960 | |||
| 9d0a09d8be | |||
| f6f007bf47 | |||
| 232b290a44 | |||
| b55b95f5d8 | |||
| ca6c9a5861 | |||
| 16dce1f0c2 | |||
| 2be7b117ea | |||
| 88ecb1bc24 |
@@ -1,4 +1,4 @@
|
|||||||
name: Release Tag
|
name: Release
|
||||||
on:
|
on:
|
||||||
push:
|
push:
|
||||||
tags:
|
tags:
|
||||||
@@ -6,6 +6,7 @@ on:
|
|||||||
|
|
||||||
jobs:
|
jobs:
|
||||||
release:
|
release:
|
||||||
|
name: Build versioned release
|
||||||
runs-on: ubuntu-latest
|
runs-on: ubuntu-latest
|
||||||
steps:
|
steps:
|
||||||
- uses: actions/checkout@v4
|
- uses: actions/checkout@v4
|
||||||
|
|||||||
@@ -1,4 +1,4 @@
|
|||||||
name: PR Check
|
name: CI
|
||||||
on:
|
on:
|
||||||
- pull_request
|
- pull_request
|
||||||
|
|
||||||
|
|||||||
@@ -36,7 +36,6 @@ changelog:
|
|||||||
- "^test:"
|
- "^test:"
|
||||||
|
|
||||||
release:
|
release:
|
||||||
name_template: '{{.ProjectName}}-{{.Version}}'
|
|
||||||
footer: >-
|
footer: >-
|
||||||
|
|
||||||
---
|
---
|
||||||
|
|||||||
Vendored
+1
-25
@@ -5,7 +5,7 @@
|
|||||||
"version": "0.2.0",
|
"version": "0.2.0",
|
||||||
"configurations": [
|
"configurations": [
|
||||||
{
|
{
|
||||||
"name": "Launch Memory & Disk",
|
"name": "Launch Package",
|
||||||
"type": "go",
|
"type": "go",
|
||||||
"request": "launch",
|
"request": "launch",
|
||||||
"mode": "auto",
|
"mode": "auto",
|
||||||
@@ -18,30 +18,6 @@
|
|||||||
"--disk-path",
|
"--disk-path",
|
||||||
"tmp/disk",
|
"tmp/disk",
|
||||||
],
|
],
|
||||||
},
|
|
||||||
{
|
|
||||||
"name": "Launch Disk Only",
|
|
||||||
"type": "go",
|
|
||||||
"request": "launch",
|
|
||||||
"mode": "auto",
|
|
||||||
"program": "${workspaceFolder}/main.go",
|
|
||||||
"args": [
|
|
||||||
"--disk",
|
|
||||||
"10G",
|
|
||||||
"--disk-path",
|
|
||||||
"tmp/disk",
|
|
||||||
],
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"name": "Launch Memory Only",
|
|
||||||
"type": "go",
|
|
||||||
"request": "launch",
|
|
||||||
"mode": "auto",
|
|
||||||
"program": "${workspaceFolder}/main.go",
|
|
||||||
"args": [
|
|
||||||
"--memory",
|
|
||||||
"1G",
|
|
||||||
],
|
|
||||||
}
|
}
|
||||||
]
|
]
|
||||||
}
|
}
|
||||||
+5
-5
@@ -46,9 +46,9 @@ func Execute() {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func init() {
|
func init() {
|
||||||
rootCmd.Flags().StringVarP(&memory, "memory", "m", "0", "The size of the memory cache")
|
rootCmd.Flags().StringVarP(&memory, "memory", "m", "100MB", "The size of the memory cache")
|
||||||
rootCmd.Flags().IntVarP(&memorymultiplier, "memory-gc", "M", 10, "The gc value for the memory cache")
|
rootCmd.Flags().IntVarP(&memorymultiplier, "memory-multiplier", "M", 10, "The multiplier for the memory cache")
|
||||||
rootCmd.Flags().StringVarP(&disk, "disk", "d", "0", "The size of the disk cache")
|
rootCmd.Flags().StringVarP(&disk, "disk", "d", "10GB", "The size of the disk cache")
|
||||||
rootCmd.Flags().IntVarP(&diskmultiplier, "disk-gc", "D", 100, "The gc value for the disk cache")
|
rootCmd.Flags().IntVarP(&diskmultiplier, "disk-multiplier", "D", 10, "The multiplier for the disk cache")
|
||||||
rootCmd.Flags().StringVarP(&diskpath, "disk-path", "p", "", "The path to the disk cache")
|
rootCmd.Flags().StringVarP(&diskpath, "disk-path", "p", "tmp/steamcache2-disk", "The path to the disk cache")
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -14,15 +14,11 @@ type AvgCacheState struct {
|
|||||||
|
|
||||||
// New creates a new average cache state with the given size.
|
// New creates a new average cache state with the given size.
|
||||||
func New(size int) *AvgCacheState {
|
func New(size int) *AvgCacheState {
|
||||||
a := &AvgCacheState{
|
return &AvgCacheState{
|
||||||
size: size,
|
size: size,
|
||||||
avgs: make([]cachestate.CacheState, size),
|
avgs: make([]cachestate.CacheState, size),
|
||||||
mu: sync.Mutex{},
|
mu: sync.Mutex{},
|
||||||
}
|
}
|
||||||
|
|
||||||
a.Clear()
|
|
||||||
|
|
||||||
return a
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// Clear resets the average cache state to zero.
|
// Clear resets the average cache state to zero.
|
||||||
@@ -30,9 +26,7 @@ func (a *AvgCacheState) Clear() {
|
|||||||
a.mu.Lock()
|
a.mu.Lock()
|
||||||
defer a.mu.Unlock()
|
defer a.mu.Unlock()
|
||||||
|
|
||||||
for i := 0; i < len(a.avgs); i++ {
|
a.avgs = make([]cachestate.CacheState, a.size) // zeroed
|
||||||
a.avgs[i] = cachestate.CacheStateMiss
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// Add adds a cache state to the average cache state.
|
// Add adds a cache state to the average cache state.
|
||||||
@@ -51,13 +45,16 @@ func (a *AvgCacheState) Avg() float64 {
|
|||||||
a.mu.Lock()
|
a.mu.Lock()
|
||||||
defer a.mu.Unlock()
|
defer a.mu.Unlock()
|
||||||
|
|
||||||
var hits int
|
var hits, misses int
|
||||||
|
|
||||||
for _, cs := range a.avgs {
|
for _, cs := range a.avgs {
|
||||||
if cs == cachestate.CacheStateHit {
|
switch cs {
|
||||||
|
case cachestate.CacheStateHit:
|
||||||
hits++
|
hits++
|
||||||
|
case cachestate.CacheStateMiss:
|
||||||
|
misses++
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
total := hits + misses
|
||||||
|
|
||||||
return float64(hits) / float64(len(a.avgs))
|
return float64(hits) / float64(total)
|
||||||
}
|
}
|
||||||
|
|||||||
+38
-7
@@ -1,42 +1,73 @@
|
|||||||
package steamcache
|
package steamcache
|
||||||
|
|
||||||
import (
|
import (
|
||||||
|
"s1d3sw1ped/SteamCache2/steamcache/logger"
|
||||||
"s1d3sw1ped/SteamCache2/vfs"
|
"s1d3sw1ped/SteamCache2/vfs"
|
||||||
"s1d3sw1ped/SteamCache2/vfs/cachestate"
|
"s1d3sw1ped/SteamCache2/vfs/cachestate"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
|
"github.com/docker/go-units"
|
||||||
"golang.org/x/exp/rand"
|
"golang.org/x/exp/rand"
|
||||||
)
|
)
|
||||||
|
|
||||||
// RandomGC randomly deletes files until we've reclaimed enough space.
|
func randomgc(vfss vfs.VFS, stats []*vfs.FileInfo) int64 {
|
||||||
func randomgc(vfss vfs.VFS, size uint) (uint, uint) {
|
// Pick a random file to delete
|
||||||
|
|
||||||
// Randomly delete files until we've reclaimed enough space.
|
|
||||||
random := func(vfss vfs.VFS, stats []*vfs.FileInfo) int64 {
|
|
||||||
randfile := stats[rand.Intn(len(stats))]
|
randfile := stats[rand.Intn(len(stats))]
|
||||||
sz := randfile.Size()
|
sz := randfile.Size()
|
||||||
err := vfss.Delete(randfile.Name())
|
err := vfss.Delete(randfile.Name())
|
||||||
if err != nil {
|
if err != nil {
|
||||||
|
// If we failed to delete the file, log it and return 0
|
||||||
|
// logger.Logger.Error().Err(err).Msgf("Failed to delete %s", randfile.Name())
|
||||||
return 0
|
return 0
|
||||||
}
|
}
|
||||||
|
|
||||||
return sz
|
return sz
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func memorygc(vfss vfs.VFS, size int) {
|
||||||
|
tstart := time.Now()
|
||||||
deletions := 0
|
deletions := 0
|
||||||
targetreclaim := int64(size)
|
targetreclaim := int64(size)
|
||||||
var reclaimed int64
|
var reclaimed int64
|
||||||
|
|
||||||
stats := vfss.StatAll()
|
stats := vfss.StatAll()
|
||||||
for {
|
for {
|
||||||
reclaimed += random(vfss, stats)
|
reclaimed += randomgc(vfss, stats)
|
||||||
deletions++
|
deletions++
|
||||||
if reclaimed >= targetreclaim {
|
if reclaimed >= targetreclaim {
|
||||||
break
|
break
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
return uint(reclaimed), uint(deletions)
|
logger.Logger.Info().
|
||||||
|
Str("name", vfss.Name()).
|
||||||
|
Str("duration", time.Since(tstart).String()).
|
||||||
|
Str("reclaimed", units.HumanSize(float64(reclaimed))).
|
||||||
|
Int("deletions", deletions).
|
||||||
|
Msgf("GC")
|
||||||
|
}
|
||||||
|
|
||||||
|
func diskgc(vfss vfs.VFS, size int) {
|
||||||
|
tstart := time.Now()
|
||||||
|
deletions := 0
|
||||||
|
targetreclaim := int64(size)
|
||||||
|
var reclaimed int64
|
||||||
|
|
||||||
|
stats := vfss.StatAll()
|
||||||
|
for {
|
||||||
|
reclaimed += randomgc(vfss, stats)
|
||||||
|
deletions++
|
||||||
|
if reclaimed >= targetreclaim {
|
||||||
|
break
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
logger.Logger.Info().
|
||||||
|
Str("name", vfss.Name()).
|
||||||
|
Str("duration", time.Since(tstart).String()).
|
||||||
|
Str("reclaimed", units.HumanSize(float64(reclaimed))).
|
||||||
|
Int("deletions", deletions).
|
||||||
|
Msgf("GC")
|
||||||
}
|
}
|
||||||
|
|
||||||
func cachehandler(fi *vfs.FileInfo, cs cachestate.CacheState) bool {
|
func cachehandler(fi *vfs.FileInfo, cs cachestate.CacheState) bool {
|
||||||
|
|||||||
+23
-93
@@ -30,9 +30,6 @@ type SteamCache struct {
|
|||||||
memory *memory.MemoryFS
|
memory *memory.MemoryFS
|
||||||
disk *disk.DiskFS
|
disk *disk.DiskFS
|
||||||
|
|
||||||
memorygc *gc.GCFS
|
|
||||||
diskgc *gc.GCFS
|
|
||||||
|
|
||||||
hits *avgcachestate.AvgCacheState
|
hits *avgcachestate.AvgCacheState
|
||||||
|
|
||||||
dirty bool
|
dirty bool
|
||||||
@@ -50,64 +47,35 @@ func New(address string, memorySize string, memoryMultiplier int, diskSize strin
|
|||||||
panic(err)
|
panic(err)
|
||||||
}
|
}
|
||||||
|
|
||||||
c := cache.New(
|
m := memory.New(memorysize)
|
||||||
cachehandler,
|
d := disk.New(diskPath, disksize)
|
||||||
)
|
|
||||||
|
|
||||||
var m *memory.MemoryFS
|
|
||||||
var mgc *gc.GCFS
|
|
||||||
if memorysize > 0 {
|
|
||||||
m = memory.New(memorysize)
|
|
||||||
mgc = gc.New(m, memoryMultiplier, randomgc)
|
|
||||||
}
|
|
||||||
|
|
||||||
var d *disk.DiskFS
|
|
||||||
var dgc *gc.GCFS
|
|
||||||
if disksize > 0 {
|
|
||||||
d = disk.New(diskPath, disksize)
|
|
||||||
dgc = gc.New(d, diskMultiplier, randomgc)
|
|
||||||
}
|
|
||||||
|
|
||||||
// configure the cache to match the specified mode (memory only, disk only, or memory and disk) based on the provided sizes
|
|
||||||
if disksize == 0 && memorysize != 0 {
|
|
||||||
//memory only mode - no disk
|
|
||||||
|
|
||||||
c.SetSlow(mgc)
|
|
||||||
logger.Logger.Info().Bool("memory", true).Bool("disk", false).Msg("configuration")
|
|
||||||
} else if disksize != 0 && memorysize == 0 {
|
|
||||||
// disk only mode
|
|
||||||
|
|
||||||
c.SetSlow(dgc)
|
|
||||||
logger.Logger.Info().Bool("memory", false).Bool("disk", true).Msg("configuration")
|
|
||||||
} else if disksize != 0 && memorysize != 0 {
|
|
||||||
// memory and disk mode
|
|
||||||
|
|
||||||
c.SetFast(mgc)
|
|
||||||
c.SetSlow(dgc)
|
|
||||||
logger.Logger.Info().Bool("memory", true).Bool("disk", true).Msg("configuration")
|
|
||||||
} else {
|
|
||||||
// no memory or disk isn't a valid configuration
|
|
||||||
logger.Logger.Error().Bool("memory", false).Bool("disk", false).Msg("configuration invalid :( exiting")
|
|
||||||
os.Exit(1)
|
|
||||||
}
|
|
||||||
|
|
||||||
sc := &SteamCache{
|
sc := &SteamCache{
|
||||||
address: address,
|
address: address,
|
||||||
vfs: syncfs.New(c),
|
vfs: syncfs.New(
|
||||||
|
cache.New(
|
||||||
|
gc.New(
|
||||||
|
m,
|
||||||
|
memoryMultiplier,
|
||||||
|
memorygc,
|
||||||
|
),
|
||||||
|
gc.New(
|
||||||
|
d,
|
||||||
|
diskMultiplier,
|
||||||
|
diskgc,
|
||||||
|
),
|
||||||
|
cachehandler,
|
||||||
|
),
|
||||||
|
),
|
||||||
|
|
||||||
memory: m,
|
memory: m,
|
||||||
disk: d,
|
disk: d,
|
||||||
|
|
||||||
memorygc: mgc,
|
hits: avgcachestate.New(10000),
|
||||||
diskgc: dgc,
|
|
||||||
|
|
||||||
hits: avgcachestate.New(100),
|
|
||||||
}
|
}
|
||||||
|
|
||||||
if d != nil {
|
|
||||||
if d.Size() > d.Capacity() {
|
if d.Size() > d.Capacity() {
|
||||||
randomgc(d, uint(d.Size()-d.Capacity()))
|
diskgc(d, int(d.Size()-d.Capacity()))
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
return sc
|
return sc
|
||||||
@@ -121,7 +89,7 @@ func (sc *SteamCache) Run() {
|
|||||||
sc.mu.Unlock()
|
sc.mu.Unlock()
|
||||||
|
|
||||||
sc.LogStats()
|
sc.LogStats()
|
||||||
t := time.NewTicker(1 * time.Second)
|
t := time.NewTicker(10 * time.Second)
|
||||||
go func() {
|
go func() {
|
||||||
for range t.C {
|
for range t.C {
|
||||||
sc.LogStats()
|
sc.LogStats()
|
||||||
@@ -143,49 +111,11 @@ func (sc *SteamCache) LogStats() {
|
|||||||
sc.mu.Lock()
|
sc.mu.Lock()
|
||||||
defer sc.mu.Unlock()
|
defer sc.mu.Unlock()
|
||||||
if sc.dirty {
|
if sc.dirty {
|
||||||
|
|
||||||
logger.Logger.Info().Msg("") // empty line to separate log entries for better readability
|
|
||||||
|
|
||||||
if sc.memory != nil { // only log memory if memory is enabled
|
|
||||||
lifetimeBytes, lifetimeFiles, reclaimedBytes, deletedFiles, gcTime := sc.memorygc.Stats()
|
|
||||||
|
|
||||||
logger.Logger.Info().
|
|
||||||
Str("size", units.HumanSize(float64(sc.memory.Size()))).
|
|
||||||
Str("capacity", units.HumanSize(float64(sc.memory.Capacity()))).
|
|
||||||
Str("files", fmt.Sprintf("%d", len(sc.memory.StatAll()))).
|
|
||||||
Msg("memory")
|
|
||||||
|
|
||||||
logger.Logger.Info().
|
|
||||||
Str("data_total", units.HumanSize(float64(lifetimeBytes))).
|
|
||||||
Uint("files_total", lifetimeFiles).
|
|
||||||
Str("data", units.HumanSize(float64(reclaimedBytes))).
|
|
||||||
Uint("files", deletedFiles).
|
|
||||||
Str("gc_time", gcTime.String()).
|
|
||||||
Msg("memory_gc")
|
|
||||||
}
|
|
||||||
|
|
||||||
if sc.disk != nil { // only log disk if disk is enabled
|
|
||||||
lifetimeBytes, lifetimeFiles, reclaimedBytes, deletedFiles, gcTime := sc.diskgc.Stats()
|
|
||||||
|
|
||||||
logger.Logger.Info().
|
|
||||||
Str("size", units.HumanSize(float64(sc.disk.Size()))).
|
|
||||||
Str("capacity", units.HumanSize(float64(sc.disk.Capacity()))).
|
|
||||||
Str("files", fmt.Sprintf("%d", len(sc.disk.StatAll()))).
|
|
||||||
Msg("disk")
|
|
||||||
|
|
||||||
logger.Logger.Info().
|
|
||||||
Str("data_total", units.HumanSize(float64(lifetimeBytes))).
|
|
||||||
Uint("files_total", lifetimeFiles).
|
|
||||||
Str("data", units.HumanSize(float64(reclaimedBytes))).
|
|
||||||
Uint("files", deletedFiles).
|
|
||||||
Str("gc_time", gcTime.String()).
|
|
||||||
Msg("disk_gc")
|
|
||||||
}
|
|
||||||
|
|
||||||
logger.Logger.Info().
|
logger.Logger.Info().
|
||||||
|
Str("memory", fmt.Sprintf("%s/%s", units.HumanSize(float64(sc.memory.Size())), units.HumanSize(float64(sc.memory.Capacity())))).Int("memory-files", len(sc.memory.StatAll())).
|
||||||
|
Str("disk", fmt.Sprintf("%s/%s", units.HumanSize(float64(sc.disk.Size())), units.HumanSize(float64(sc.disk.Capacity())))).Int("disk-files", len(sc.disk.StatAll())).
|
||||||
Str("hitrate", fmt.Sprintf("%.2f%%", sc.hits.Avg()*100)).
|
Str("hitrate", fmt.Sprintf("%.2f%%", sc.hits.Avg()*100)).
|
||||||
Msg("cache")
|
Msg("stats")
|
||||||
|
|
||||||
sc.dirty = false
|
sc.dirty = false
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
Vendored
+12
-13
@@ -21,24 +21,23 @@ type CacheFS struct {
|
|||||||
type CacheHandler func(*vfs.FileInfo, cachestate.CacheState) bool
|
type CacheHandler func(*vfs.FileInfo, cachestate.CacheState) bool
|
||||||
|
|
||||||
// New creates a new CacheFS. fast is used for caching, and slow is used for storage. fast should obviously be faster than slow.
|
// New creates a new CacheFS. fast is used for caching, and slow is used for storage. fast should obviously be faster than slow.
|
||||||
func New(cacheHandler CacheHandler) *CacheFS {
|
func New(fast, slow vfs.VFS, cacheHandler CacheHandler) *CacheFS {
|
||||||
|
if slow == nil {
|
||||||
|
panic("slow is nil")
|
||||||
|
}
|
||||||
|
|
||||||
|
if fast == slow {
|
||||||
|
panic("fast and slow are the same")
|
||||||
|
}
|
||||||
|
|
||||||
return &CacheFS{
|
return &CacheFS{
|
||||||
|
fast: fast,
|
||||||
|
slow: slow,
|
||||||
|
|
||||||
cacheHandler: cacheHandler,
|
cacheHandler: cacheHandler,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func (c *CacheFS) SetSlow(vfs vfs.VFS) {
|
|
||||||
if vfs == nil {
|
|
||||||
panic("vfs is nil") // panic if the vfs is nil
|
|
||||||
}
|
|
||||||
|
|
||||||
c.slow = vfs
|
|
||||||
}
|
|
||||||
|
|
||||||
func (c *CacheFS) SetFast(vfs vfs.VFS) {
|
|
||||||
c.fast = vfs
|
|
||||||
}
|
|
||||||
|
|
||||||
// cacheState returns the state of the file at key.
|
// cacheState returns the state of the file at key.
|
||||||
func (c *CacheFS) cacheState(key string) cachestate.CacheState {
|
func (c *CacheFS) cacheState(key string) cachestate.CacheState {
|
||||||
if c.fast != nil {
|
if c.fast != nil {
|
||||||
|
|||||||
Vendored
+8
-23
@@ -20,9 +20,7 @@ func TestNew(t *testing.T) {
|
|||||||
fast := testMemory()
|
fast := testMemory()
|
||||||
slow := testMemory()
|
slow := testMemory()
|
||||||
|
|
||||||
cache := New(nil)
|
cache := New(fast, slow, nil)
|
||||||
cache.SetFast(fast)
|
|
||||||
cache.SetSlow(slow)
|
|
||||||
if cache == nil {
|
if cache == nil {
|
||||||
t.Fatal("expected cache to be non-nil")
|
t.Fatal("expected cache to be non-nil")
|
||||||
}
|
}
|
||||||
@@ -37,9 +35,7 @@ func TestNewPanics(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}()
|
}()
|
||||||
|
|
||||||
cache := New(nil)
|
New(nil, nil, nil)
|
||||||
cache.SetFast(nil)
|
|
||||||
cache.SetSlow(nil)
|
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestSetAndGet(t *testing.T) {
|
func TestSetAndGet(t *testing.T) {
|
||||||
@@ -47,9 +43,7 @@ func TestSetAndGet(t *testing.T) {
|
|||||||
|
|
||||||
fast := testMemory()
|
fast := testMemory()
|
||||||
slow := testMemory()
|
slow := testMemory()
|
||||||
cache := New(nil)
|
cache := New(fast, slow, nil)
|
||||||
cache.SetFast(fast)
|
|
||||||
cache.SetSlow(slow)
|
|
||||||
|
|
||||||
key := "test"
|
key := "test"
|
||||||
value := []byte("value")
|
value := []byte("value")
|
||||||
@@ -72,8 +66,7 @@ func TestSetAndGetNoFast(t *testing.T) {
|
|||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
slow := testMemory()
|
slow := testMemory()
|
||||||
cache := New(nil)
|
cache := New(nil, slow, nil)
|
||||||
cache.SetSlow(slow)
|
|
||||||
|
|
||||||
key := "test"
|
key := "test"
|
||||||
value := []byte("value")
|
value := []byte("value")
|
||||||
@@ -96,11 +89,9 @@ func TestCaching(t *testing.T) {
|
|||||||
|
|
||||||
fast := testMemory()
|
fast := testMemory()
|
||||||
slow := testMemory()
|
slow := testMemory()
|
||||||
cache := New(func(fi *vfs.FileInfo, cs cachestate.CacheState) bool {
|
cache := New(fast, slow, func(fi *vfs.FileInfo, cs cachestate.CacheState) bool {
|
||||||
return true
|
return true
|
||||||
})
|
})
|
||||||
cache.SetFast(fast)
|
|
||||||
cache.SetSlow(slow)
|
|
||||||
|
|
||||||
key := "test"
|
key := "test"
|
||||||
value := []byte("value")
|
value := []byte("value")
|
||||||
@@ -157,9 +148,7 @@ func TestGetNotFound(t *testing.T) {
|
|||||||
|
|
||||||
fast := testMemory()
|
fast := testMemory()
|
||||||
slow := testMemory()
|
slow := testMemory()
|
||||||
cache := New(nil)
|
cache := New(fast, slow, nil)
|
||||||
cache.SetFast(fast)
|
|
||||||
cache.SetSlow(slow)
|
|
||||||
|
|
||||||
_, err := cache.Get("nonexistent")
|
_, err := cache.Get("nonexistent")
|
||||||
if !errors.Is(err, vfserror.ErrNotFound) {
|
if !errors.Is(err, vfserror.ErrNotFound) {
|
||||||
@@ -172,9 +161,7 @@ func TestDelete(t *testing.T) {
|
|||||||
|
|
||||||
fast := testMemory()
|
fast := testMemory()
|
||||||
slow := testMemory()
|
slow := testMemory()
|
||||||
cache := New(nil)
|
cache := New(fast, slow, nil)
|
||||||
cache.SetFast(fast)
|
|
||||||
cache.SetSlow(slow)
|
|
||||||
|
|
||||||
key := "test"
|
key := "test"
|
||||||
value := []byte("value")
|
value := []byte("value")
|
||||||
@@ -198,9 +185,7 @@ func TestStat(t *testing.T) {
|
|||||||
|
|
||||||
fast := testMemory()
|
fast := testMemory()
|
||||||
slow := testMemory()
|
slow := testMemory()
|
||||||
cache := New(nil)
|
cache := New(fast, slow, nil)
|
||||||
cache.SetFast(fast)
|
|
||||||
cache.SetSlow(slow)
|
|
||||||
|
|
||||||
key := "test"
|
key := "test"
|
||||||
value := []byte("value")
|
value := []byte("value")
|
||||||
|
|||||||
+1
-24
@@ -27,24 +27,6 @@ type DiskFS struct {
|
|||||||
|
|
||||||
// New creates a new DiskFS.
|
// New creates a new DiskFS.
|
||||||
func new(root string, capacity int64, skipinit bool) *DiskFS {
|
func new(root string, capacity int64, skipinit bool) *DiskFS {
|
||||||
if capacity <= 0 {
|
|
||||||
panic("disk capacity must be greater than 0") // panic if the capacity is less than or equal to 0
|
|
||||||
}
|
|
||||||
|
|
||||||
if root == "" {
|
|
||||||
panic("disk root must not be empty") // panic if the root is empty
|
|
||||||
}
|
|
||||||
|
|
||||||
fi, err := os.Stat(root)
|
|
||||||
if err != nil {
|
|
||||||
if !os.IsNotExist(err) {
|
|
||||||
panic(err) // panic if the error is something other than not found
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if !fi.IsDir() {
|
|
||||||
panic("disk root must be a directory") // panic if the root is not a directory
|
|
||||||
}
|
|
||||||
|
|
||||||
dfs := &DiskFS{
|
dfs := &DiskFS{
|
||||||
root: root,
|
root: root,
|
||||||
info: make(map[string]*vfs.FileInfo),
|
info: make(map[string]*vfs.FileInfo),
|
||||||
@@ -78,12 +60,7 @@ func (d *DiskFS) init() {
|
|||||||
d.walk(d.root)
|
d.walk(d.root)
|
||||||
d.sg.Wait()
|
d.sg.Wait()
|
||||||
|
|
||||||
logger.Logger.Info().
|
logger.Logger.Info().Str("name", d.Name()).Str("root", d.root).Str("capacity", units.HumanSize(float64(d.capacity))).Str("duration", time.Since(tstart).String()).Msg("init")
|
||||||
Str("name", d.Name()).
|
|
||||||
Str("root", d.root).
|
|
||||||
Str("capacity", units.HumanSize(float64(d.capacity))).
|
|
||||||
Str("duration", time.Since(tstart).String()).
|
|
||||||
Msg("init")
|
|
||||||
}
|
}
|
||||||
|
|
||||||
func (d *DiskFS) walk(path string) {
|
func (d *DiskFS) walk(path string) {
|
||||||
|
|||||||
+2
-44
@@ -4,8 +4,6 @@ import (
|
|||||||
"fmt"
|
"fmt"
|
||||||
"s1d3sw1ped/SteamCache2/vfs"
|
"s1d3sw1ped/SteamCache2/vfs"
|
||||||
"s1d3sw1ped/SteamCache2/vfs/vfserror"
|
"s1d3sw1ped/SteamCache2/vfs/vfserror"
|
||||||
"sync"
|
|
||||||
"time"
|
|
||||||
)
|
)
|
||||||
|
|
||||||
// Ensure GCFS implements VFS.
|
// Ensure GCFS implements VFS.
|
||||||
@@ -15,22 +13,13 @@ var _ vfs.VFS = (*GCFS)(nil)
|
|||||||
type GCFS struct {
|
type GCFS struct {
|
||||||
vfs.VFS
|
vfs.VFS
|
||||||
multiplier int
|
multiplier int
|
||||||
|
|
||||||
// protected by mu
|
|
||||||
gcHanderFunc GCHandlerFunc
|
gcHanderFunc GCHandlerFunc
|
||||||
lifetimeBytes, lifetimeFiles uint
|
|
||||||
reclaimedBytes, deletedFiles uint
|
|
||||||
gcTime time.Duration
|
|
||||||
mu sync.Mutex
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// GCHandlerFunc is a function that is called when the disk is full and the GCFS needs to free up space. It is passed the VFS and the size of the file that needs to be written. Its up to the implementation to free up space. How much space is freed is also up to the implementation.
|
// GCHandlerFunc is a function that is called when the disk is full and the GCFS needs to free up space. It is passed the VFS and the size of the file that needs to be written. Its up to the implementation to free up space. How much space is freed is also up to the implementation.
|
||||||
type GCHandlerFunc func(vfs vfs.VFS, size uint) (reclaimedBytes uint, deletedFiles uint)
|
type GCHandlerFunc func(vfs vfs.VFS, size int)
|
||||||
|
|
||||||
func New(vfs vfs.VFS, multiplier int, gcHandlerFunc GCHandlerFunc) *GCFS {
|
func New(vfs vfs.VFS, multiplier int, gcHandlerFunc GCHandlerFunc) *GCFS {
|
||||||
if multiplier <= 0 {
|
|
||||||
multiplier = 1 // if the multiplier is less than or equal to 0 set it to 1 will be slow but the user can set it to a higher value if they want
|
|
||||||
}
|
|
||||||
return &GCFS{
|
return &GCFS{
|
||||||
VFS: vfs,
|
VFS: vfs,
|
||||||
multiplier: multiplier,
|
multiplier: multiplier,
|
||||||
@@ -38,43 +27,12 @@ func New(vfs vfs.VFS, multiplier int, gcHandlerFunc GCHandlerFunc) *GCFS {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// Stats returns the lifetime bytes, lifetime files, reclaimed bytes and deleted files.
|
|
||||||
// The lifetime bytes and lifetime files are the total bytes and files that have been freed up by the GC handler.
|
|
||||||
// The reclaimed bytes and deleted files are the bytes and files that have been freed up by the GC handler since last call to Stats.
|
|
||||||
// The gc time is the total time spent in the GC handler since last call to Stats.
|
|
||||||
// The reclaimed bytes and deleted files and gc time are reset to 0 after the call to Stats.
|
|
||||||
func (g *GCFS) Stats() (lifetimeBytes, lifetimeFiles, reclaimedBytes, deletedFiles uint, gcTime time.Duration) {
|
|
||||||
g.mu.Lock()
|
|
||||||
defer g.mu.Unlock()
|
|
||||||
|
|
||||||
g.lifetimeBytes += g.reclaimedBytes
|
|
||||||
g.lifetimeFiles += g.deletedFiles
|
|
||||||
|
|
||||||
lifetimeBytes = g.lifetimeBytes
|
|
||||||
lifetimeFiles = g.lifetimeFiles
|
|
||||||
reclaimedBytes = g.reclaimedBytes
|
|
||||||
deletedFiles = g.deletedFiles
|
|
||||||
gcTime = g.gcTime
|
|
||||||
|
|
||||||
g.reclaimedBytes = 0
|
|
||||||
g.deletedFiles = 0
|
|
||||||
g.gcTime = time.Duration(0)
|
|
||||||
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
// Set overrides the Set method of the VFS interface. It tries to set the key and src, if it fails due to disk full error, it calls the GC handler and tries again. If it still fails it returns the error.
|
// Set overrides the Set method of the VFS interface. It tries to set the key and src, if it fails due to disk full error, it calls the GC handler and tries again. If it still fails it returns the error.
|
||||||
func (g *GCFS) Set(key string, src []byte) error {
|
func (g *GCFS) Set(key string, src []byte) error {
|
||||||
g.mu.Lock()
|
|
||||||
defer g.mu.Unlock()
|
|
||||||
err := g.VFS.Set(key, src) // try to set the key and src
|
err := g.VFS.Set(key, src) // try to set the key and src
|
||||||
|
|
||||||
if err == vfserror.ErrDiskFull && g.gcHanderFunc != nil { // if the error is disk full and there is a GC handler
|
if err == vfserror.ErrDiskFull && g.gcHanderFunc != nil { // if the error is disk full and there is a GC handler
|
||||||
tstart := time.Now()
|
g.gcHanderFunc(g.VFS, len(src)*g.multiplier) // call the GC handler
|
||||||
reclaimedBytes, deletedFiles := g.gcHanderFunc(g.VFS, uint(len(src)*g.multiplier)) // call the GC handler
|
|
||||||
g.gcTime += time.Since(tstart)
|
|
||||||
g.reclaimedBytes += reclaimedBytes
|
|
||||||
g.deletedFiles += deletedFiles
|
|
||||||
err = g.VFS.Set(key, src) // try again after GC if it still fails return the error
|
err = g.VFS.Set(key, src) // try again after GC if it still fails return the error
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
+18
-12
@@ -6,6 +6,7 @@ import (
|
|||||||
"s1d3sw1ped/SteamCache2/vfs/memory"
|
"s1d3sw1ped/SteamCache2/vfs/memory"
|
||||||
"sort"
|
"sort"
|
||||||
"testing"
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
"golang.org/x/exp/rand"
|
"golang.org/x/exp/rand"
|
||||||
)
|
)
|
||||||
@@ -14,11 +15,13 @@ func TestGCSmallRandom(t *testing.T) {
|
|||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
m := memory.New(1024 * 1024 * 16)
|
m := memory.New(1024 * 1024 * 16)
|
||||||
gc := New(m, 10, func(vfs vfs.VFS, size uint) (uint, uint) {
|
gc := New(m, 10, func(vfs vfs.VFS, size int) {
|
||||||
|
tstart := time.Now()
|
||||||
deletions := 0
|
deletions := 0
|
||||||
var reclaimed uint
|
targetreclaim := int64(size)
|
||||||
|
var reclaimed int64
|
||||||
|
|
||||||
t.Logf("GC starting to reclaim %d bytes", size)
|
t.Logf("GC starting to reclaim %d bytes", targetreclaim)
|
||||||
|
|
||||||
stats := vfs.StatAll()
|
stats := vfs.StatAll()
|
||||||
sort.Slice(stats, func(i, j int) bool {
|
sort.Slice(stats, func(i, j int) bool {
|
||||||
@@ -28,7 +31,7 @@ func TestGCSmallRandom(t *testing.T) {
|
|||||||
|
|
||||||
// Delete the oldest files until we've reclaimed enough space.
|
// Delete the oldest files until we've reclaimed enough space.
|
||||||
for _, s := range stats {
|
for _, s := range stats {
|
||||||
sz := uint(s.Size()) // Get the size of the file
|
sz := s.Size() // Get the size of the file
|
||||||
err := vfs.Delete(s.Name())
|
err := vfs.Delete(s.Name())
|
||||||
if err != nil {
|
if err != nil {
|
||||||
panic(err)
|
panic(err)
|
||||||
@@ -38,11 +41,12 @@ func TestGCSmallRandom(t *testing.T) {
|
|||||||
|
|
||||||
// t.Logf("GC deleting %s, %v", s.Name(), s.AccessTime().Format(time.RFC3339Nano))
|
// t.Logf("GC deleting %s, %v", s.Name(), s.AccessTime().Format(time.RFC3339Nano))
|
||||||
|
|
||||||
if reclaimed >= size { // We've reclaimed enough space
|
if reclaimed >= targetreclaim { // We've reclaimed enough space
|
||||||
break
|
break
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
return uint(reclaimed), uint(deletions)
|
|
||||||
|
t.Logf("GC took %v to reclaim %d bytes by deleting %d files", time.Since(tstart), reclaimed, deletions)
|
||||||
})
|
})
|
||||||
|
|
||||||
for i := 0; i < 10000; i++ {
|
for i := 0; i < 10000; i++ {
|
||||||
@@ -66,11 +70,13 @@ func TestGCLargeRandom(t *testing.T) {
|
|||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
m := memory.New(1024 * 1024 * 16) // 16MB
|
m := memory.New(1024 * 1024 * 16) // 16MB
|
||||||
gc := New(m, 10, func(vfs vfs.VFS, size uint) (uint, uint) {
|
gc := New(m, 10, func(vfs vfs.VFS, size int) {
|
||||||
|
tstart := time.Now()
|
||||||
deletions := 0
|
deletions := 0
|
||||||
var reclaimed uint
|
targetreclaim := int64(size)
|
||||||
|
var reclaimed int64
|
||||||
|
|
||||||
t.Logf("GC starting to reclaim %d bytes", size)
|
t.Logf("GC starting to reclaim %d bytes", targetreclaim)
|
||||||
|
|
||||||
stats := vfs.StatAll()
|
stats := vfs.StatAll()
|
||||||
sort.Slice(stats, func(i, j int) bool {
|
sort.Slice(stats, func(i, j int) bool {
|
||||||
@@ -80,17 +86,17 @@ func TestGCLargeRandom(t *testing.T) {
|
|||||||
|
|
||||||
// Delete the oldest files until we've reclaimed enough space.
|
// Delete the oldest files until we've reclaimed enough space.
|
||||||
for _, s := range stats {
|
for _, s := range stats {
|
||||||
sz := uint(s.Size()) // Get the size of the file
|
sz := s.Size() // Get the size of the file
|
||||||
vfs.Delete(s.Name())
|
vfs.Delete(s.Name())
|
||||||
reclaimed += sz // Track how much space we've reclaimed
|
reclaimed += sz // Track how much space we've reclaimed
|
||||||
deletions++ // Track how many files we've deleted
|
deletions++ // Track how many files we've deleted
|
||||||
|
|
||||||
if reclaimed >= size { // We've reclaimed enough space
|
if reclaimed >= targetreclaim { // We've reclaimed enough space
|
||||||
break
|
break
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
return uint(reclaimed), uint(deletions)
|
t.Logf("GC took %v to reclaim %d bytes by deleting %d files", time.Since(tstart), reclaimed, deletions)
|
||||||
})
|
})
|
||||||
|
|
||||||
for i := 0; i < 10000; i++ {
|
for i := 0; i < 10000; i++ {
|
||||||
|
|||||||
@@ -25,10 +25,6 @@ type MemoryFS struct {
|
|||||||
|
|
||||||
// New creates a new MemoryFS.
|
// New creates a new MemoryFS.
|
||||||
func New(capacity int64) *MemoryFS {
|
func New(capacity int64) *MemoryFS {
|
||||||
if capacity <= 0 {
|
|
||||||
panic("memory capacity must be greater than 0") // panic if the capacity is less than or equal to 0
|
|
||||||
}
|
|
||||||
|
|
||||||
return &MemoryFS{
|
return &MemoryFS{
|
||||||
files: make(map[string]*file),
|
files: make(map[string]*file),
|
||||||
capacity: capacity,
|
capacity: capacity,
|
||||||
|
|||||||
Reference in New Issue
Block a user