32 Commits

Author SHA1 Message Date
s1d3sw1ped 32c0d85bad chore: remove unused extra files from goreleaser configuration 2025-01-22 10:03:14 -06:00
s1d3sw1ped 6b51e61a87 chore: remove unused goreleaser checksum script 2025-01-22 10:02:19 -06:00
s1d3sw1ped 1094505f20 fix: increase hits buffer size in AvgCacheState initialization
Release / Build versioned release (push) Successful in 8s
2025-01-22 00:02:11 -06:00
s1d3sw1ped 93d04410bf feat: integrate zerolog for structured logging and replace fmt with logger
Release / Build versioned release (push) Successful in 20s
2025-01-21 23:33:14 -06:00
s1d3sw1ped 62f7a38295 refactor: replace log statements with fmt for console output
Release / Build versioned release (push) Successful in 16s
2025-01-21 20:56:58 -06:00
s1d3sw1ped c370e80888 fix: initialize the avgcachestatus
Release / Build versioned release (push) Successful in 14s
2025-01-21 20:20:48 -06:00
s1d3sw1ped 9d0f4894e1 refactor: comment out logging for request duration in ServeHTTP method
Release / Build versioned release (push) Successful in 16s
2025-01-21 20:03:11 -06:00
s1d3sw1ped 745ee37184 feat: add AvgCacheState to track average cache hits and misses 2025-01-21 20:02:34 -06:00
s1d3sw1ped 5a9b07f609 refactor: streamline release and test workflows by removing unnecessary steps 2025-01-21 15:49:50 -06:00
s1d3sw1ped d88db3d9c1 refactor: remove old CI workflows and consolidate into a new test workflow
Release / Build versioned release (push) Successful in 14s
2025-01-21 15:37:31 -06:00
s1d3sw1ped 91075232cd refactor: remove unnecessary go generate step from goreleaser configuration
CI / Lint (push) Successful in 6s
CI / Build (push) Successful in 13s
CI / Test (push) Successful in 57s
2025-01-21 15:26:30 -06:00
s1d3sw1ped 1f588fa20a fix: handle error from http.ListenAndServe in SteamCache
CI / Lint (push) Successful in 7s
CI / Build (push) Successful in 13s
CI / Test (push) Successful in 59s
2025-01-21 15:24:08 -06:00
s1d3sw1ped d39a2ad4fa refactor: remove address flag and set default value in command configuration 2025-01-21 15:24:01 -06:00
s1d3sw1ped 80cc75f372 docs: update README with garbage collection details and Windows hosts file override instructions 2025-01-21 15:23:24 -06:00
s1d3sw1ped 53f5d1577c refactor: simplify tag release workflow by removing unnecessary steps
CI / Lint (push) Successful in 6s
CI / Build (push) Successful in 17s
CI / Test (push) Successful in 59s
2025-01-21 13:38:23 -06:00
s1d3sw1ped 1186e14c56 fix: update Gitea URLs in goreleaser configuration forehead
CI / Lint (push) Successful in 6s
CI / Build (push) Successful in 13s
CI / Test (push) Successful in 1m1s
Release versioned tag / Build versioned release (push) Successful in 1m11s
2025-01-21 13:37:12 -06:00
s1d3sw1ped 3bb57b7741 refactor: update goreleaser configuration for improved build and release process
CI / Lint (push) Successful in 6s
CI / Build (push) Successful in 13s
CI / Test (push) Successful in 1m0s
Release versioned tag / Build versioned release (push) Failing after 1m7s
2025-01-21 13:34:51 -06:00
s1d3sw1ped 8f2d1eff9b refactor: remove commented build steps and add checksum script for binaries
CI / Lint (push) Successful in 6s
CI / Build (push) Successful in 14s
CI / Test (push) Successful in 59s
Release versioned tag / Build versioned release (push) Failing after 1m5s
2025-01-21 13:18:21 -06:00
s1d3sw1ped 27242fdf05 test
CI / Lint (push) Successful in 6s
CI / Build (push) Successful in 14s
CI / Test (push) Successful in 1m0s
Release versioned tag / Build versioned release (push) Failing after 1m25s
2025-01-21 13:07:42 -06:00
s1d3sw1ped 55330ebd3c get the entire repo depth for changelogs
CI / Lint (push) Successful in 6s
CI / Build (push) Successful in 13s
CI / Test (push) Successful in 58s
Release versioned tag / Build versioned release (push) Failing after 1m6s
2025-01-21 13:03:23 -06:00
s1d3sw1ped e37dc930dd test goreleaser
CI / Lint (push) Successful in 6s
CI / Build (push) Successful in 13s
CI / Test (push) Successful in 59s
Release versioned tag / Build versioned release (push) Failing after 1m3s
2025-01-21 12:51:17 -06:00
s1d3sw1ped 79f0814745 refactor CI workflows to streamline linting and release processes
CI / Lint (push) Successful in 6s
CI / Build (push) Successful in 14s
CI / Test (push) Successful in 57s
Release versioned tag / Build versioned release (push) Failing after 1m5s
2025-01-21 12:36:31 -06:00
s1d3sw1ped 1feee2758c fixed bad character
CI / Lint (push) Successful in 7s
CI / Build (push) Successful in 14s
CI / Test (push) Successful in 59s
Release versioned tag / Lint (push) Failing after 6s
Release versioned tag / Build versioned release (push) Failing after 20s
Release versioned tag / Test (push) Successful in 58s
2025-01-21 12:32:20 -06:00
s1d3sw1ped 22f9196960 enable parallel execution for cache tests
CI / Lint (push) Successful in 7s
CI / Build (push) Successful in 15s
CI / Test (push) Successful in 58s
2025-01-21 12:27:49 -06:00
s1d3sw1ped 9d0a09d8be minor fixes to gc_test to lower the log spam
CI / Lint (push) Successful in 7s
CI / Build (push) Successful in 16s
CI / Test (push) Successful in 58s
2025-01-21 12:25:44 -06:00
s1d3sw1ped f6f007bf47 d for disable
CI / Lint (push) Successful in 7s
CI / Build (push) Successful in 15s
CI / Test (push) Successful in 59s
2025-01-21 12:22:20 -06:00
s1d3sw1ped 232b290a44 disable errcheck
CI / Lint (push) Failing after 7s
CI / Build (push) Successful in 20s
CI / Test (push) Successful in 59s
2025-01-21 12:21:05 -06:00
s1d3sw1ped b55b95f5d8 fixed a invisible character in yaml
CI / Lint (push) Failing after 27s
CI / Build (push) Successful in 27s
CI / Test (push) Successful in 1m11s
2025-01-21 12:07:31 -06:00
s1d3sw1ped ca6c9a5861 added CI
CI / Build (push) Has been cancelled
CI / Lint (push) Has been cancelled
CI / Test (push) Has been cancelled
2025-01-21 12:06:25 -06:00
s1d3sw1ped 16dce1f0c2 it works 2025-01-21 11:52:04 -06:00
s1d3sw1ped 2be7b117ea lottsa stuff 2025-01-19 20:17:57 -06:00
s1d3sw1ped 88ecb1bc24 initial commit 2025-01-18 20:03:40 -06:00
19 changed files with 188 additions and 328 deletions
+2 -1
View File
@@ -1,4 +1,4 @@
name: Release Tag name: Release
on: on:
push: push:
tags: tags:
@@ -6,6 +6,7 @@ on:
jobs: jobs:
release: release:
name: Build versioned release
runs-on: ubuntu-latest runs-on: ubuntu-latest
steps: steps:
- uses: actions/checkout@v4 - uses: actions/checkout@v4
+1 -1
View File
@@ -1,4 +1,4 @@
name: PR Check name: CI
on: on:
- pull_request - pull_request
-1
View File
@@ -1,3 +1,2 @@
dist/
tmp/ tmp/
__*.exe __*.exe
+7 -10
View File
@@ -5,15 +5,13 @@ before:
- go mod tidy - go mod tidy
builds: builds:
- ldflags: - env:
- -X s1d3sw1ped/SteamCache2/version.Version={{.Version}} - CGO_ENABLED=0
env: goos:
- CGO_ENABLED=0 - linux
goos: - windows
- linux goarch:
- windows - amd64
goarch:
- amd64
archives: archives:
- format: tar.gz - format: tar.gz
@@ -36,7 +34,6 @@ changelog:
- "^test:" - "^test:"
release: release:
name_template: '{{.ProjectName}}-{{.Version}}'
footer: >- footer: >-
--- ---
+4 -31
View File
@@ -5,42 +5,15 @@
"version": "0.2.0", "version": "0.2.0",
"configurations": [ "configurations": [
{ {
"name": "Launch Memory & Disk", "name": "Launch Package",
"type": "go", "type": "go",
"request": "launch", "request": "launch",
"mode": "auto", "mode": "auto",
"program": "${workspaceFolder}/main.go", "program": "${workspaceFolder}/main.go",
"args": [ "args": [
"--memory", "--memory", "1G",
"1G", "--disk", "10G",
"--disk", "--disk-path", "tmp/disk",
"10G",
"--disk-path",
"tmp/disk",
],
},
{
"name": "Launch Disk Only",
"type": "go",
"request": "launch",
"mode": "auto",
"program": "${workspaceFolder}/main.go",
"args": [
"--disk",
"10G",
"--disk-path",
"tmp/disk",
],
},
{
"name": "Launch Memory Only",
"type": "go",
"request": "launch",
"mode": "auto",
"program": "${workspaceFolder}/main.go",
"args": [
"--memory",
"1G",
], ],
} }
] ]
+1 -2
View File
@@ -55,8 +55,7 @@ This will direct any requests to `lancache.steamcontent.com` to your SteamCache2
## License ## License
See the [LICENSE](LICENSE) file for details. This project is licensed. See the [LICENSE](LICENSE) file for details.
But just for clarity this covers all files in this project unless stated in the individual file.
## Acknowledgements ## Acknowledgements
+26 -5
View File
@@ -1,3 +1,24 @@
/*
Copyright © 2025 s1d3sw1ped
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in
all copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
THE SOFTWARE.
*/
package cmd package cmd
import ( import (
@@ -46,9 +67,9 @@ func Execute() {
} }
func init() { func init() {
rootCmd.Flags().StringVarP(&memory, "memory", "m", "0", "The size of the memory cache") rootCmd.Flags().StringVarP(&memory, "memory", "m", "100MB", "The size of the memory cache")
rootCmd.Flags().IntVarP(&memorymultiplier, "memory-gc", "M", 10, "The gc value for the memory cache") rootCmd.Flags().IntVarP(&memorymultiplier, "memory-multiplier", "M", 10, "The multiplier for the memory cache")
rootCmd.Flags().StringVarP(&disk, "disk", "d", "0", "The size of the disk cache") rootCmd.Flags().StringVarP(&disk, "disk", "d", "10GB", "The size of the disk cache")
rootCmd.Flags().IntVarP(&diskmultiplier, "disk-gc", "D", 100, "The gc value for the disk cache") rootCmd.Flags().IntVarP(&diskmultiplier, "disk-multiplier", "D", 10, "The multiplier for the disk cache")
rootCmd.Flags().StringVarP(&diskpath, "disk-path", "p", "", "The path to the disk cache") rootCmd.Flags().StringVarP(&diskpath, "disk-path", "p", "tmp/steamcache2-disk", "The path to the disk cache")
} }
-23
View File
@@ -1,23 +0,0 @@
package cmd
import (
"fmt"
"os"
"s1d3sw1ped/SteamCache2/version"
"github.com/spf13/cobra"
)
// versionCmd represents the version command
var versionCmd = &cobra.Command{
Use: "version",
Short: "prints the version of SteamCache2",
Long: `Prints the version of SteamCache2. This command is useful for checking the version of the application.`,
Run: func(cmd *cobra.Command, args []string) {
fmt.Fprintln(os.Stderr, "SteamCache2", version.Version)
},
}
func init() {
rootCmd.AddCommand(versionCmd)
}
+22 -4
View File
@@ -1,9 +1,27 @@
/*
Copyright © 2025 s1d3sw1ped
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in
all copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
THE SOFTWARE.
*/
package main package main
import ( import "s1d3sw1ped/SteamCache2/cmd"
"s1d3sw1ped/SteamCache2/cmd"
_ "s1d3sw1ped/SteamCache2/version" // Import the version package for global version variable
)
func main() { func main() {
cmd.Execute() cmd.Execute()
+9 -12
View File
@@ -14,15 +14,11 @@ type AvgCacheState struct {
// New creates a new average cache state with the given size. // New creates a new average cache state with the given size.
func New(size int) *AvgCacheState { func New(size int) *AvgCacheState {
a := &AvgCacheState{ return &AvgCacheState{
size: size, size: size,
avgs: make([]cachestate.CacheState, size), avgs: make([]cachestate.CacheState, size),
mu: sync.Mutex{}, mu: sync.Mutex{},
} }
a.Clear()
return a
} }
// Clear resets the average cache state to zero. // Clear resets the average cache state to zero.
@@ -30,9 +26,7 @@ func (a *AvgCacheState) Clear() {
a.mu.Lock() a.mu.Lock()
defer a.mu.Unlock() defer a.mu.Unlock()
for i := 0; i < len(a.avgs); i++ { a.avgs = make([]cachestate.CacheState, a.size) // zeroed
a.avgs[i] = cachestate.CacheStateMiss
}
} }
// Add adds a cache state to the average cache state. // Add adds a cache state to the average cache state.
@@ -51,13 +45,16 @@ func (a *AvgCacheState) Avg() float64 {
a.mu.Lock() a.mu.Lock()
defer a.mu.Unlock() defer a.mu.Unlock()
var hits int var hits, misses int
for _, cs := range a.avgs { for _, cs := range a.avgs {
if cs == cachestate.CacheStateHit { switch cs {
case cachestate.CacheStateHit:
hits++ hits++
case cachestate.CacheStateMiss:
misses++
} }
} }
total := hits + misses
return float64(hits) / float64(len(a.avgs)) return float64(hits) / float64(total)
} }
+46 -15
View File
@@ -1,42 +1,73 @@
package steamcache package steamcache
import ( import (
"s1d3sw1ped/SteamCache2/steamcache/logger"
"s1d3sw1ped/SteamCache2/vfs" "s1d3sw1ped/SteamCache2/vfs"
"s1d3sw1ped/SteamCache2/vfs/cachestate" "s1d3sw1ped/SteamCache2/vfs/cachestate"
"time" "time"
"github.com/docker/go-units"
"golang.org/x/exp/rand" "golang.org/x/exp/rand"
) )
// RandomGC randomly deletes files until we've reclaimed enough space. func randomgc(vfss vfs.VFS, stats []*vfs.FileInfo) int64 {
func randomgc(vfss vfs.VFS, size uint) (uint, uint) { // Pick a random file to delete
randfile := stats[rand.Intn(len(stats))]
// Randomly delete files until we've reclaimed enough space. sz := randfile.Size()
random := func(vfss vfs.VFS, stats []*vfs.FileInfo) int64 { err := vfss.Delete(randfile.Name())
randfile := stats[rand.Intn(len(stats))] if err != nil {
sz := randfile.Size() // If we failed to delete the file, log it and return 0
err := vfss.Delete(randfile.Name()) // logger.Logger.Error().Err(err).Msgf("Failed to delete %s", randfile.Name())
if err != nil { return 0
return 0
}
return sz
} }
return sz
}
func memorygc(vfss vfs.VFS, size int) {
tstart := time.Now()
deletions := 0 deletions := 0
targetreclaim := int64(size) targetreclaim := int64(size)
var reclaimed int64 var reclaimed int64
stats := vfss.StatAll() stats := vfss.StatAll()
for { for {
reclaimed += random(vfss, stats) reclaimed += randomgc(vfss, stats)
deletions++ deletions++
if reclaimed >= targetreclaim { if reclaimed >= targetreclaim {
break break
} }
} }
return uint(reclaimed), uint(deletions) logger.Logger.Info().
Str("name", vfss.Name()).
Str("duration", time.Since(tstart).String()).
Str("reclaimed", units.HumanSize(float64(reclaimed))).
Int("deletions", deletions).
Msgf("GC")
}
func diskgc(vfss vfs.VFS, size int) {
tstart := time.Now()
deletions := 0
targetreclaim := int64(size)
var reclaimed int64
stats := vfss.StatAll()
for {
reclaimed += randomgc(vfss, stats)
deletions++
if reclaimed >= targetreclaim {
break
}
}
logger.Logger.Info().
Str("name", vfss.Name()).
Str("duration", time.Since(tstart).String()).
Str("reclaimed", units.HumanSize(float64(reclaimed))).
Int("deletions", deletions).
Msgf("GC")
} }
func cachehandler(fi *vfs.FileInfo, cs cachestate.CacheState) bool { func cachehandler(fi *vfs.FileInfo, cs cachestate.CacheState) bool {
+25 -96
View File
@@ -8,7 +8,6 @@ import (
"os" "os"
"s1d3sw1ped/SteamCache2/steamcache/avgcachestate" "s1d3sw1ped/SteamCache2/steamcache/avgcachestate"
"s1d3sw1ped/SteamCache2/steamcache/logger" "s1d3sw1ped/SteamCache2/steamcache/logger"
"s1d3sw1ped/SteamCache2/version"
"s1d3sw1ped/SteamCache2/vfs" "s1d3sw1ped/SteamCache2/vfs"
"s1d3sw1ped/SteamCache2/vfs/cache" "s1d3sw1ped/SteamCache2/vfs/cache"
"s1d3sw1ped/SteamCache2/vfs/cachestate" "s1d3sw1ped/SteamCache2/vfs/cachestate"
@@ -30,9 +29,6 @@ type SteamCache struct {
memory *memory.MemoryFS memory *memory.MemoryFS
disk *disk.DiskFS disk *disk.DiskFS
memorygc *gc.GCFS
diskgc *gc.GCFS
hits *avgcachestate.AvgCacheState hits *avgcachestate.AvgCacheState
dirty bool dirty bool
@@ -50,78 +46,49 @@ func New(address string, memorySize string, memoryMultiplier int, diskSize strin
panic(err) panic(err)
} }
c := cache.New( m := memory.New(memorysize)
cachehandler, d := disk.New(diskPath, disksize)
)
var m *memory.MemoryFS
var mgc *gc.GCFS
if memorysize > 0 {
m = memory.New(memorysize)
mgc = gc.New(m, memoryMultiplier, randomgc)
}
var d *disk.DiskFS
var dgc *gc.GCFS
if disksize > 0 {
d = disk.New(diskPath, disksize)
dgc = gc.New(d, diskMultiplier, randomgc)
}
// configure the cache to match the specified mode (memory only, disk only, or memory and disk) based on the provided sizes
if disksize == 0 && memorysize != 0 {
//memory only mode - no disk
c.SetSlow(mgc)
logger.Logger.Info().Bool("memory", true).Bool("disk", false).Msg("configuration")
} else if disksize != 0 && memorysize == 0 {
// disk only mode
c.SetSlow(dgc)
logger.Logger.Info().Bool("memory", false).Bool("disk", true).Msg("configuration")
} else if disksize != 0 && memorysize != 0 {
// memory and disk mode
c.SetFast(mgc)
c.SetSlow(dgc)
logger.Logger.Info().Bool("memory", true).Bool("disk", true).Msg("configuration")
} else {
// no memory or disk isn't a valid configuration
logger.Logger.Error().Bool("memory", false).Bool("disk", false).Msg("configuration invalid :( exiting")
os.Exit(1)
}
sc := &SteamCache{ sc := &SteamCache{
address: address, address: address,
vfs: syncfs.New(c), vfs: syncfs.New(
cache.New(
gc.New(
m,
memoryMultiplier,
memorygc,
),
gc.New(
d,
diskMultiplier,
diskgc,
),
cachehandler,
),
),
memory: m, memory: m,
disk: d, disk: d,
memorygc: mgc, hits: avgcachestate.New(10000),
diskgc: dgc,
hits: avgcachestate.New(100),
} }
if d != nil { if d.Size() > d.Capacity() {
if d.Size() > d.Capacity() { diskgc(d, int(d.Size()-d.Capacity()))
randomgc(d, uint(d.Size()-d.Capacity()))
}
} }
return sc return sc
} }
func (sc *SteamCache) Run() { func (sc *SteamCache) Run() {
logger.Logger.Info().Str("address", sc.address).Str("version", version.Version).Msg("listening") logger.Logger.Info().Str("address", sc.address).Msg("listening")
sc.mu.Lock() sc.mu.Lock()
sc.dirty = true sc.dirty = true
sc.mu.Unlock() sc.mu.Unlock()
sc.LogStats() sc.LogStats()
t := time.NewTicker(1 * time.Second) t := time.NewTicker(10 * time.Second)
go func() { go func() {
for range t.C { for range t.C {
sc.LogStats() sc.LogStats()
@@ -143,49 +110,11 @@ func (sc *SteamCache) LogStats() {
sc.mu.Lock() sc.mu.Lock()
defer sc.mu.Unlock() defer sc.mu.Unlock()
if sc.dirty { if sc.dirty {
logger.Logger.Info().Msg("") // empty line to separate log entries for better readability
if sc.memory != nil { // only log memory if memory is enabled
lifetimeBytes, lifetimeFiles, reclaimedBytes, deletedFiles, gcTime := sc.memorygc.Stats()
logger.Logger.Info().
Str("size", units.HumanSize(float64(sc.memory.Size()))).
Str("capacity", units.HumanSize(float64(sc.memory.Capacity()))).
Str("files", fmt.Sprintf("%d", len(sc.memory.StatAll()))).
Msg("memory")
logger.Logger.Info().
Str("data_total", units.HumanSize(float64(lifetimeBytes))).
Uint("files_total", lifetimeFiles).
Str("data", units.HumanSize(float64(reclaimedBytes))).
Uint("files", deletedFiles).
Str("gc_time", gcTime.String()).
Msg("memory_gc")
}
if sc.disk != nil { // only log disk if disk is enabled
lifetimeBytes, lifetimeFiles, reclaimedBytes, deletedFiles, gcTime := sc.diskgc.Stats()
logger.Logger.Info().
Str("size", units.HumanSize(float64(sc.disk.Size()))).
Str("capacity", units.HumanSize(float64(sc.disk.Capacity()))).
Str("files", fmt.Sprintf("%d", len(sc.disk.StatAll()))).
Msg("disk")
logger.Logger.Info().
Str("data_total", units.HumanSize(float64(lifetimeBytes))).
Uint("files_total", lifetimeFiles).
Str("data", units.HumanSize(float64(reclaimedBytes))).
Uint("files", deletedFiles).
Str("gc_time", gcTime.String()).
Msg("disk_gc")
}
logger.Logger.Info(). logger.Logger.Info().
Str("memory", fmt.Sprintf("%s/%s", units.HumanSize(float64(sc.memory.Size())), units.HumanSize(float64(sc.memory.Capacity())))).Int("memory-files", len(sc.memory.StatAll())).
Str("disk", fmt.Sprintf("%s/%s", units.HumanSize(float64(sc.disk.Size())), units.HumanSize(float64(sc.disk.Capacity())))).Int("disk-files", len(sc.disk.StatAll())).
Str("hitrate", fmt.Sprintf("%.2f%%", sc.hits.Avg()*100)). Str("hitrate", fmt.Sprintf("%.2f%%", sc.hits.Avg()*100)).
Msg("cache") Msg("stats")
sc.dirty = false sc.dirty = false
} }
} }
-3
View File
@@ -1,3 +0,0 @@
package version
var Version string
+12 -13
View File
@@ -21,24 +21,23 @@ type CacheFS struct {
type CacheHandler func(*vfs.FileInfo, cachestate.CacheState) bool type CacheHandler func(*vfs.FileInfo, cachestate.CacheState) bool
// New creates a new CacheFS. fast is used for caching, and slow is used for storage. fast should obviously be faster than slow. // New creates a new CacheFS. fast is used for caching, and slow is used for storage. fast should obviously be faster than slow.
func New(cacheHandler CacheHandler) *CacheFS { func New(fast, slow vfs.VFS, cacheHandler CacheHandler) *CacheFS {
if slow == nil {
panic("slow is nil")
}
if fast == slow {
panic("fast and slow are the same")
}
return &CacheFS{ return &CacheFS{
fast: fast,
slow: slow,
cacheHandler: cacheHandler, cacheHandler: cacheHandler,
} }
} }
func (c *CacheFS) SetSlow(vfs vfs.VFS) {
if vfs == nil {
panic("vfs is nil") // panic if the vfs is nil
}
c.slow = vfs
}
func (c *CacheFS) SetFast(vfs vfs.VFS) {
c.fast = vfs
}
// cacheState returns the state of the file at key. // cacheState returns the state of the file at key.
func (c *CacheFS) cacheState(key string) cachestate.CacheState { func (c *CacheFS) cacheState(key string) cachestate.CacheState {
if c.fast != nil { if c.fast != nil {
+8 -23
View File
@@ -20,9 +20,7 @@ func TestNew(t *testing.T) {
fast := testMemory() fast := testMemory()
slow := testMemory() slow := testMemory()
cache := New(nil) cache := New(fast, slow, nil)
cache.SetFast(fast)
cache.SetSlow(slow)
if cache == nil { if cache == nil {
t.Fatal("expected cache to be non-nil") t.Fatal("expected cache to be non-nil")
} }
@@ -37,9 +35,7 @@ func TestNewPanics(t *testing.T) {
} }
}() }()
cache := New(nil) New(nil, nil, nil)
cache.SetFast(nil)
cache.SetSlow(nil)
} }
func TestSetAndGet(t *testing.T) { func TestSetAndGet(t *testing.T) {
@@ -47,9 +43,7 @@ func TestSetAndGet(t *testing.T) {
fast := testMemory() fast := testMemory()
slow := testMemory() slow := testMemory()
cache := New(nil) cache := New(fast, slow, nil)
cache.SetFast(fast)
cache.SetSlow(slow)
key := "test" key := "test"
value := []byte("value") value := []byte("value")
@@ -72,8 +66,7 @@ func TestSetAndGetNoFast(t *testing.T) {
t.Parallel() t.Parallel()
slow := testMemory() slow := testMemory()
cache := New(nil) cache := New(nil, slow, nil)
cache.SetSlow(slow)
key := "test" key := "test"
value := []byte("value") value := []byte("value")
@@ -96,11 +89,9 @@ func TestCaching(t *testing.T) {
fast := testMemory() fast := testMemory()
slow := testMemory() slow := testMemory()
cache := New(func(fi *vfs.FileInfo, cs cachestate.CacheState) bool { cache := New(fast, slow, func(fi *vfs.FileInfo, cs cachestate.CacheState) bool {
return true return true
}) })
cache.SetFast(fast)
cache.SetSlow(slow)
key := "test" key := "test"
value := []byte("value") value := []byte("value")
@@ -157,9 +148,7 @@ func TestGetNotFound(t *testing.T) {
fast := testMemory() fast := testMemory()
slow := testMemory() slow := testMemory()
cache := New(nil) cache := New(fast, slow, nil)
cache.SetFast(fast)
cache.SetSlow(slow)
_, err := cache.Get("nonexistent") _, err := cache.Get("nonexistent")
if !errors.Is(err, vfserror.ErrNotFound) { if !errors.Is(err, vfserror.ErrNotFound) {
@@ -172,9 +161,7 @@ func TestDelete(t *testing.T) {
fast := testMemory() fast := testMemory()
slow := testMemory() slow := testMemory()
cache := New(nil) cache := New(fast, slow, nil)
cache.SetFast(fast)
cache.SetSlow(slow)
key := "test" key := "test"
value := []byte("value") value := []byte("value")
@@ -198,9 +185,7 @@ func TestStat(t *testing.T) {
fast := testMemory() fast := testMemory()
slow := testMemory() slow := testMemory()
cache := New(nil) cache := New(fast, slow, nil)
cache.SetFast(fast)
cache.SetSlow(slow)
key := "test" key := "test"
value := []byte("value") value := []byte("value")
+1 -24
View File
@@ -27,24 +27,6 @@ type DiskFS struct {
// New creates a new DiskFS. // New creates a new DiskFS.
func new(root string, capacity int64, skipinit bool) *DiskFS { func new(root string, capacity int64, skipinit bool) *DiskFS {
if capacity <= 0 {
panic("disk capacity must be greater than 0") // panic if the capacity is less than or equal to 0
}
if root == "" {
panic("disk root must not be empty") // panic if the root is empty
}
fi, err := os.Stat(root)
if err != nil {
if !os.IsNotExist(err) {
panic(err) // panic if the error is something other than not found
}
}
if !fi.IsDir() {
panic("disk root must be a directory") // panic if the root is not a directory
}
dfs := &DiskFS{ dfs := &DiskFS{
root: root, root: root,
info: make(map[string]*vfs.FileInfo), info: make(map[string]*vfs.FileInfo),
@@ -78,12 +60,7 @@ func (d *DiskFS) init() {
d.walk(d.root) d.walk(d.root)
d.sg.Wait() d.sg.Wait()
logger.Logger.Info(). logger.Logger.Info().Str("name", d.Name()).Str("root", d.root).Str("capacity", units.HumanSize(float64(d.capacity))).Str("duration", time.Since(tstart).String()).Msg("init")
Str("name", d.Name()).
Str("root", d.root).
Str("capacity", units.HumanSize(float64(d.capacity))).
Str("duration", time.Since(tstart).String()).
Msg("init")
} }
func (d *DiskFS) walk(path string) { func (d *DiskFS) walk(path string) {
+5 -47
View File
@@ -4,8 +4,6 @@ import (
"fmt" "fmt"
"s1d3sw1ped/SteamCache2/vfs" "s1d3sw1ped/SteamCache2/vfs"
"s1d3sw1ped/SteamCache2/vfs/vfserror" "s1d3sw1ped/SteamCache2/vfs/vfserror"
"sync"
"time"
) )
// Ensure GCFS implements VFS. // Ensure GCFS implements VFS.
@@ -14,23 +12,14 @@ var _ vfs.VFS = (*GCFS)(nil)
// GCFS is a virtual file system that calls a GC handler when the disk is full. The GC handler is responsible for freeing up space on the disk. The GCFS is a wrapper around another VFS. // GCFS is a virtual file system that calls a GC handler when the disk is full. The GC handler is responsible for freeing up space on the disk. The GCFS is a wrapper around another VFS.
type GCFS struct { type GCFS struct {
vfs.VFS vfs.VFS
multiplier int multiplier int
gcHanderFunc GCHandlerFunc
// protected by mu
gcHanderFunc GCHandlerFunc
lifetimeBytes, lifetimeFiles uint
reclaimedBytes, deletedFiles uint
gcTime time.Duration
mu sync.Mutex
} }
// GCHandlerFunc is a function that is called when the disk is full and the GCFS needs to free up space. It is passed the VFS and the size of the file that needs to be written. Its up to the implementation to free up space. How much space is freed is also up to the implementation. // GCHandlerFunc is a function that is called when the disk is full and the GCFS needs to free up space. It is passed the VFS and the size of the file that needs to be written. Its up to the implementation to free up space. How much space is freed is also up to the implementation.
type GCHandlerFunc func(vfs vfs.VFS, size uint) (reclaimedBytes uint, deletedFiles uint) type GCHandlerFunc func(vfs vfs.VFS, size int)
func New(vfs vfs.VFS, multiplier int, gcHandlerFunc GCHandlerFunc) *GCFS { func New(vfs vfs.VFS, multiplier int, gcHandlerFunc GCHandlerFunc) *GCFS {
if multiplier <= 0 {
multiplier = 1 // if the multiplier is less than or equal to 0 set it to 1 will be slow but the user can set it to a higher value if they want
}
return &GCFS{ return &GCFS{
VFS: vfs, VFS: vfs,
multiplier: multiplier, multiplier: multiplier,
@@ -38,44 +27,13 @@ func New(vfs vfs.VFS, multiplier int, gcHandlerFunc GCHandlerFunc) *GCFS {
} }
} }
// Stats returns the lifetime bytes, lifetime files, reclaimed bytes and deleted files.
// The lifetime bytes and lifetime files are the total bytes and files that have been freed up by the GC handler.
// The reclaimed bytes and deleted files are the bytes and files that have been freed up by the GC handler since last call to Stats.
// The gc time is the total time spent in the GC handler since last call to Stats.
// The reclaimed bytes and deleted files and gc time are reset to 0 after the call to Stats.
func (g *GCFS) Stats() (lifetimeBytes, lifetimeFiles, reclaimedBytes, deletedFiles uint, gcTime time.Duration) {
g.mu.Lock()
defer g.mu.Unlock()
g.lifetimeBytes += g.reclaimedBytes
g.lifetimeFiles += g.deletedFiles
lifetimeBytes = g.lifetimeBytes
lifetimeFiles = g.lifetimeFiles
reclaimedBytes = g.reclaimedBytes
deletedFiles = g.deletedFiles
gcTime = g.gcTime
g.reclaimedBytes = 0
g.deletedFiles = 0
g.gcTime = time.Duration(0)
return
}
// Set overrides the Set method of the VFS interface. It tries to set the key and src, if it fails due to disk full error, it calls the GC handler and tries again. If it still fails it returns the error. // Set overrides the Set method of the VFS interface. It tries to set the key and src, if it fails due to disk full error, it calls the GC handler and tries again. If it still fails it returns the error.
func (g *GCFS) Set(key string, src []byte) error { func (g *GCFS) Set(key string, src []byte) error {
g.mu.Lock()
defer g.mu.Unlock()
err := g.VFS.Set(key, src) // try to set the key and src err := g.VFS.Set(key, src) // try to set the key and src
if err == vfserror.ErrDiskFull && g.gcHanderFunc != nil { // if the error is disk full and there is a GC handler if err == vfserror.ErrDiskFull && g.gcHanderFunc != nil { // if the error is disk full and there is a GC handler
tstart := time.Now() g.gcHanderFunc(g.VFS, len(src)*g.multiplier) // call the GC handler
reclaimedBytes, deletedFiles := g.gcHanderFunc(g.VFS, uint(len(src)*g.multiplier)) // call the GC handler err = g.VFS.Set(key, src) // try again after GC if it still fails return the error
g.gcTime += time.Since(tstart)
g.reclaimedBytes += reclaimedBytes
g.deletedFiles += deletedFiles
err = g.VFS.Set(key, src) // try again after GC if it still fails return the error
} }
return err return err
+18 -12
View File
@@ -6,6 +6,7 @@ import (
"s1d3sw1ped/SteamCache2/vfs/memory" "s1d3sw1ped/SteamCache2/vfs/memory"
"sort" "sort"
"testing" "testing"
"time"
"golang.org/x/exp/rand" "golang.org/x/exp/rand"
) )
@@ -14,11 +15,13 @@ func TestGCSmallRandom(t *testing.T) {
t.Parallel() t.Parallel()
m := memory.New(1024 * 1024 * 16) m := memory.New(1024 * 1024 * 16)
gc := New(m, 10, func(vfs vfs.VFS, size uint) (uint, uint) { gc := New(m, 10, func(vfs vfs.VFS, size int) {
tstart := time.Now()
deletions := 0 deletions := 0
var reclaimed uint targetreclaim := int64(size)
var reclaimed int64
t.Logf("GC starting to reclaim %d bytes", size) t.Logf("GC starting to reclaim %d bytes", targetreclaim)
stats := vfs.StatAll() stats := vfs.StatAll()
sort.Slice(stats, func(i, j int) bool { sort.Slice(stats, func(i, j int) bool {
@@ -28,7 +31,7 @@ func TestGCSmallRandom(t *testing.T) {
// Delete the oldest files until we've reclaimed enough space. // Delete the oldest files until we've reclaimed enough space.
for _, s := range stats { for _, s := range stats {
sz := uint(s.Size()) // Get the size of the file sz := s.Size() // Get the size of the file
err := vfs.Delete(s.Name()) err := vfs.Delete(s.Name())
if err != nil { if err != nil {
panic(err) panic(err)
@@ -38,11 +41,12 @@ func TestGCSmallRandom(t *testing.T) {
// t.Logf("GC deleting %s, %v", s.Name(), s.AccessTime().Format(time.RFC3339Nano)) // t.Logf("GC deleting %s, %v", s.Name(), s.AccessTime().Format(time.RFC3339Nano))
if reclaimed >= size { // We've reclaimed enough space if reclaimed >= targetreclaim { // We've reclaimed enough space
break break
} }
} }
return uint(reclaimed), uint(deletions)
t.Logf("GC took %v to reclaim %d bytes by deleting %d files", time.Since(tstart), reclaimed, deletions)
}) })
for i := 0; i < 10000; i++ { for i := 0; i < 10000; i++ {
@@ -66,11 +70,13 @@ func TestGCLargeRandom(t *testing.T) {
t.Parallel() t.Parallel()
m := memory.New(1024 * 1024 * 16) // 16MB m := memory.New(1024 * 1024 * 16) // 16MB
gc := New(m, 10, func(vfs vfs.VFS, size uint) (uint, uint) { gc := New(m, 10, func(vfs vfs.VFS, size int) {
tstart := time.Now()
deletions := 0 deletions := 0
var reclaimed uint targetreclaim := int64(size)
var reclaimed int64
t.Logf("GC starting to reclaim %d bytes", size) t.Logf("GC starting to reclaim %d bytes", targetreclaim)
stats := vfs.StatAll() stats := vfs.StatAll()
sort.Slice(stats, func(i, j int) bool { sort.Slice(stats, func(i, j int) bool {
@@ -80,17 +86,17 @@ func TestGCLargeRandom(t *testing.T) {
// Delete the oldest files until we've reclaimed enough space. // Delete the oldest files until we've reclaimed enough space.
for _, s := range stats { for _, s := range stats {
sz := uint(s.Size()) // Get the size of the file sz := s.Size() // Get the size of the file
vfs.Delete(s.Name()) vfs.Delete(s.Name())
reclaimed += sz // Track how much space we've reclaimed reclaimed += sz // Track how much space we've reclaimed
deletions++ // Track how many files we've deleted deletions++ // Track how many files we've deleted
if reclaimed >= size { // We've reclaimed enough space if reclaimed >= targetreclaim { // We've reclaimed enough space
break break
} }
} }
return uint(reclaimed), uint(deletions) t.Logf("GC took %v to reclaim %d bytes by deleting %d files", time.Since(tstart), reclaimed, deletions)
}) })
for i := 0; i < 10000; i++ { for i := 0; i < 10000; i++ {
-4
View File
@@ -25,10 +25,6 @@ type MemoryFS struct {
// New creates a new MemoryFS. // New creates a new MemoryFS.
func New(capacity int64) *MemoryFS { func New(capacity int64) *MemoryFS {
if capacity <= 0 {
panic("memory capacity must be greater than 0") // panic if the capacity is less than or equal to 0
}
return &MemoryFS{ return &MemoryFS{
files: make(map[string]*file), files: make(map[string]*file),
capacity: capacity, capacity: capacity,