25 Commits

Author SHA1 Message Date
0d263be2ca Merge pull request 'feat: update dependencies and improve caching mechanism' (#2) from feature/blazing-sun-speed into main
All checks were successful
Release Tag / release (push) Successful in 1m17s
Reviewed-on: s1d3sw1ped/SteamCache2#2
2025-07-12 13:18:53 +00:00
63a1c21861 fix: update action versions to use main branch for consistency
All checks were successful
PR Check / check-and-test (pull_request) Successful in 1m13s
2025-07-12 07:32:35 -05:00
0a73e46f90 fix: remove golangci-lint-action from PR workflow
All checks were successful
PR Check / check-and-test (pull_request) Successful in 1m42s
2025-07-12 07:16:48 -05:00
6f1158edeb fix: update action versions to use main branch for consistency
Some checks failed
PR Check / check-and-test (pull_request) Failing after 1m4s
2025-07-12 07:10:14 -05:00
93b682cfa5 chore: update action versions to latest in CI workflow
Some checks failed
PR Check / check-and-test (pull_request) Failing after 15s
2025-07-12 07:08:25 -05:00
f378d0e81f feat: update dependencies and improve caching mechanism
Some checks failed
PR Check / check-and-test (pull_request) Failing after 2m11s
- Added Prometheus client library for metrics collection.
- Refactored garbage collection strategy from random deletion to LRU (Least Recently Used) deletion.
- Introduced per-key locking in cache to prevent race conditions.
- Enhanced logging with structured log messages for cache hits and misses.
- Implemented a retry mechanism for upstream requests with exponential backoff.
- Updated Go modules and indirect dependencies for better compatibility and performance.
- Removed unused sync filesystem implementation.
- Added version initialization to ensure a default version string.
2025-07-12 06:43:00 -05:00
8c1bb695b8 fix: enhance logging to handle empty upstream values
All checks were successful
Release Tag / release (push) Successful in 10s
2025-01-23 11:31:28 -06:00
f58951fd92 fix: removed specific to my dev setup config 2025-01-23 11:31:13 -06:00
70786da8c6 fix: improve logging readability and remove configuration messages
All checks were successful
Release Tag / release (push) Successful in 10s
2025-01-23 11:25:18 -06:00
e24af47697 feat: add upstream and verbose flags to command line interface
All checks were successful
Release Tag / release (push) Successful in 13s
feat: add upstream support allowing to chain cache servers if needed
fix: tweaked garbage collection to be better
2025-01-23 11:14:39 -06:00
931c43d7a8 fix: remove draft status from release configuration
All checks were successful
Release Tag / release (push) Successful in 9s
2025-01-22 21:01:35 -06:00
6fe80c82ad fix: reduce cache hits averaging size for improved performance
All checks were successful
Release Tag / release (push) Successful in 9s
2025-01-22 20:59:24 -06:00
4a69c4ba66 fix: add empty line in logging for improved readability
All checks were successful
Release Tag / release (push) Successful in 9s
2025-01-22 20:51:32 -06:00
2d0fe6571d feat: enhance garbage collection logging with total GC time and improved stat formatting
All checks were successful
Release Tag / release (push) Successful in 9s
2025-01-22 20:48:05 -06:00
550948951e feat: implement enhanced garbage collection statistics logging
All checks were successful
Release Tag / release (push) Successful in 12s
2025-01-22 20:27:12 -06:00
4a23eecae0 fix: reduce log interval to 1 second for more frequent statistics updates
All checks were successful
Release Tag / release (push) Successful in 28s
2025-01-22 19:37:30 -06:00
fed9bbe218 fix: go back to the old averaging
All checks were successful
Release Tag / release (push) Successful in 12s
2025-01-22 19:35:17 -06:00
7401c040dc feat: add configurations for memory only, disk only, and memory & disk modes
All checks were successful
Release Tag / release (push) Successful in 14s
2025-01-22 19:28:45 -06:00
ca069a20ee fix: track cache hits and misses in average cache state 2025-01-22 18:43:07 -06:00
3e8a92b865 fix: log memory statistics only if memory is enabled
All checks were successful
Release Tag / release (push) Successful in 9s
2025-01-22 18:29:19 -06:00
b7652ed7cc fix: initialize average cache state with cleared values
All checks were successful
Release Tag / release (push) Successful in 9s
2025-01-22 18:25:16 -06:00
08b8d0ce3d fix: enhance logging for memory and disk statistics in cache 2025-01-22 18:23:53 -06:00
53847db0e9 fix: improve cache state clearing and average calculation logic 2025-01-22 18:23:47 -06:00
d11c294010 chore: update goreleaser configuration for draft releases and name template
All checks were successful
Release Tag / release (push) Successful in 8s
2025-01-22 18:11:33 -06:00
f951f5fd8e chore: rename workflows for clarity and consistency
All checks were successful
Release Tag / release (push) Successful in 9s
2025-01-22 18:00:55 -06:00
24 changed files with 796 additions and 348 deletions

View File

@@ -1,4 +1,4 @@
name: Release
name: Release Tag
on:
push:
tags:
@@ -6,17 +6,16 @@ on:
jobs:
release:
name: Build versioned release
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
- uses: actions/checkout@main
with:
fetch-depth: 0
- run: git fetch --force --tags
- uses: actions/setup-go@v5
- uses: actions/setup-go@main
with:
go-version-file: 'go.mod'
- uses: goreleaser/goreleaser-action@v6
- uses: goreleaser/goreleaser-action@master
with:
distribution: goreleaser
version: 'latest'

View File

@@ -1,4 +1,4 @@
name: CI
name: PR Check
on:
- pull_request
@@ -6,14 +6,10 @@ jobs:
check-and-test:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
- uses: actions/setup-go@v5
- uses: actions/checkout@main
- uses: actions/setup-go@main
with:
go-version-file: 'go.mod'
- run: go mod tidy
- uses: golangci/golangci-lint-action@v3
with:
args: -D errcheck
version: latest
- run: go build ./...
- run: go test -race -v -shuffle=on ./...

4
.gitignore vendored
View File

@@ -1,3 +1,5 @@
dist/
tmp/
__*.exe
__*.exe
.smashed.txt
.smashignore

View File

@@ -36,8 +36,9 @@ changelog:
- "^test:"
release:
name_template: '{{.ProjectName}}-{{.Version}}'
footer: >-
---
Released by [GoReleaser](https://github.com/goreleaser/goreleaser).

29
.vscode/launch.json vendored
View File

@@ -5,7 +5,7 @@
"version": "0.2.0",
"configurations": [
{
"name": "Launch Package",
"name": "Launch Memory & Disk",
"type": "go",
"request": "launch",
"mode": "auto",
@@ -17,6 +17,33 @@
"10G",
"--disk-path",
"tmp/disk",
"--verbose",
],
},
{
"name": "Launch Disk Only",
"type": "go",
"request": "launch",
"mode": "auto",
"program": "${workspaceFolder}/main.go",
"args": [
"--disk",
"10G",
"--disk-path",
"tmp/disk",
"--verbose",
],
},
{
"name": "Launch Memory Only",
"type": "go",
"request": "launch",
"mode": "auto",
"program": "${workspaceFolder}/main.go",
"args": [
"--memory",
"1G",
"--verbose",
],
}
]

View File

@@ -3,7 +3,10 @@ package cmd
import (
"os"
"s1d3sw1ped/SteamCache2/steamcache"
"s1d3sw1ped/SteamCache2/steamcache/logger"
"s1d3sw1ped/SteamCache2/version"
"github.com/rs/zerolog"
"github.com/spf13/cobra"
)
@@ -13,6 +16,11 @@ var (
disk string
diskmultiplier int
diskpath string
upstream string
pprof bool
logLevel string
logFormat string
)
var rootCmd = &cobra.Command{
@@ -24,6 +32,28 @@ var rootCmd = &cobra.Command{
By caching game files, SteamCache2 ensures that subsequent downloads of the same files are served from the local cache,
significantly improving download times and reducing the load on the internet connection.`,
Run: func(cmd *cobra.Command, args []string) {
// Configure logging
switch logLevel {
case "debug":
zerolog.SetGlobalLevel(zerolog.DebugLevel)
case "error":
zerolog.SetGlobalLevel(zerolog.ErrorLevel)
case "info":
zerolog.SetGlobalLevel(zerolog.InfoLevel)
default:
zerolog.SetGlobalLevel(zerolog.InfoLevel) // Default to info level if not specified
}
var writer zerolog.ConsoleWriter
if logFormat == "json" {
writer = zerolog.ConsoleWriter{Out: os.Stderr, NoColor: true}
} else {
writer = zerolog.ConsoleWriter{Out: os.Stderr}
}
logger.Logger = zerolog.New(writer).With().Timestamp().Logger()
logger.Logger.Info().
Msg("starting SteamCache2 " + version.Version)
sc := steamcache.New(
":80",
memory,
@@ -31,8 +61,17 @@ var rootCmd = &cobra.Command{
disk,
diskmultiplier,
diskpath,
upstream,
pprof,
)
logger.Logger.Info().
Msg("starting SteamCache2 on port 80")
sc.Run()
logger.Logger.Info().Msg("SteamCache2 stopped")
os.Exit(0)
},
}
@@ -46,9 +85,17 @@ func Execute() {
}
func init() {
rootCmd.Flags().StringVarP(&memory, "memory", "m", "100MB", "The size of the memory cache")
rootCmd.Flags().IntVarP(&memorymultiplier, "memory-multiplier", "M", 10, "The multiplier for the memory cache")
rootCmd.Flags().StringVarP(&disk, "disk", "d", "10GB", "The size of the disk cache")
rootCmd.Flags().IntVarP(&diskmultiplier, "disk-multiplier", "D", 10, "The multiplier for the disk cache")
rootCmd.Flags().StringVarP(&diskpath, "disk-path", "p", "tmp/steamcache2-disk", "The path to the disk cache")
rootCmd.Flags().StringVarP(&memory, "memory", "m", "0", "The size of the memory cache")
rootCmd.Flags().IntVarP(&memorymultiplier, "memory-gc", "M", 10, "The gc value for the memory cache")
rootCmd.Flags().StringVarP(&disk, "disk", "d", "0", "The size of the disk cache")
rootCmd.Flags().IntVarP(&diskmultiplier, "disk-gc", "D", 100, "The gc value for the disk cache")
rootCmd.Flags().StringVarP(&diskpath, "disk-path", "p", "", "The path to the disk cache")
rootCmd.Flags().StringVarP(&upstream, "upstream", "u", "", "The upstream server to proxy requests overrides the host header from the client but forwards the original host header to the upstream server")
rootCmd.Flags().BoolVarP(&pprof, "pprof", "P", false, "Enable pprof")
rootCmd.Flags().MarkHidden("pprof")
rootCmd.Flags().StringVarP(&logLevel, "log-level", "l", "info", "Logging level: debug, info, error")
rootCmd.Flags().StringVarP(&logFormat, "log-format", "f", "console", "Logging format: json, console")
}

10
go.mod
View File

@@ -4,15 +4,23 @@ go 1.23.0
require (
github.com/docker/go-units v0.5.0
github.com/prometheus/client_golang v1.22.0
github.com/rs/zerolog v1.33.0
github.com/spf13/cobra v1.8.1
golang.org/x/exp v0.0.0-20250106191152-7588d65b2ba8
)
require (
github.com/beorn7/perks v1.0.1 // indirect
github.com/cespare/xxhash/v2 v2.3.0 // indirect
github.com/inconshreveable/mousetrap v1.1.0 // indirect
github.com/mattn/go-colorable v0.1.13 // indirect
github.com/mattn/go-isatty v0.0.19 // indirect
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
github.com/prometheus/client_model v0.6.1 // indirect
github.com/prometheus/common v0.62.0 // indirect
github.com/prometheus/procfs v0.15.1 // indirect
github.com/spf13/pflag v1.0.5 // indirect
golang.org/x/sys v0.12.0 // indirect
golang.org/x/sys v0.30.0 // indirect
google.golang.org/protobuf v1.36.5 // indirect
)

32
go.sum
View File

@@ -1,16 +1,40 @@
github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw=
github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs=
github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
github.com/coreos/go-systemd/v22 v22.5.0/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc=
github.com/cpuguy83/go-md2man/v2 v2.0.4/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o=
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/docker/go-units v0.5.0 h1:69rxXcBk27SvSaaxTtLh/8llcHD8vYHT7WSdRZ/jvr4=
github.com/docker/go-units v0.5.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk=
github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA=
github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8=
github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU=
github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8=
github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw=
github.com/klauspost/compress v1.18.0 h1:c/Cqfb0r+Yi+JtIEq73FWXVkRonBlf0CRNYc8Zttxdo=
github.com/klauspost/compress v1.18.0/go.mod h1:2Pp+KzxcywXVXMr50+X0Q/Lsb43OQHYWRCY2AiWywWQ=
github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc=
github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw=
github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA=
github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg=
github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM=
github.com/mattn/go-isatty v0.0.19 h1:JITubQf0MOLdlGRuRq+jtsDlekdYPia9ZFsB8h/APPA=
github.com/mattn/go-isatty v0.0.19/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y=
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA=
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ=
github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
github.com/prometheus/client_golang v1.22.0 h1:rb93p9lokFEsctTys46VnV1kLCDpVZ0a/Y92Vm0Zc6Q=
github.com/prometheus/client_golang v1.22.0/go.mod h1:R7ljNsLXhuQXYZYtw6GAE9AZg8Y7vEW5scdCXrWRXC0=
github.com/prometheus/client_model v0.6.1 h1:ZKSh/rekM+n3CeS952MLRAdFwIKqeY8b62p8ais2e9E=
github.com/prometheus/client_model v0.6.1/go.mod h1:OrxVMOVHjw3lKMa8+x6HeMGkHMQyHDk9E3jmP2AmGiY=
github.com/prometheus/common v0.62.0 h1:xasJaQlnWAeyHdUBeGjXmutelfJHWMRr+Fg4QszZ2Io=
github.com/prometheus/common v0.62.0/go.mod h1:vyBcEuLSvWos9B1+CyL7JZ2up+uFzXhkqml0W5zIY1I=
github.com/prometheus/procfs v0.15.1 h1:YagwOFzUgYfKKHX6Dr+sHT7km/hxC76UB0learggepc=
github.com/prometheus/procfs v0.15.1/go.mod h1:fB45yRUv8NstnjriLhBQLuOUt+WW4BsoGhij/e3PBqk=
github.com/rs/xid v1.5.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg=
github.com/rs/zerolog v1.33.0 h1:1cU2KZkvPxNyfgEmhHAz/1A9Bz+llsdYzklWFzgp0r8=
github.com/rs/zerolog v1.33.0/go.mod h1:/7mN4D5sKwJLZQ2b/znpjC3/GQWY/xaDXUM0kKWRHss=
@@ -19,11 +43,17 @@ github.com/spf13/cobra v1.8.1 h1:e5/vxKd/rZsfSJMUX1agtjeTDf+qv1/JdBF8gg5k9ZM=
github.com/spf13/cobra v1.8.1/go.mod h1:wHxEcudfqmLYa8iTfL+OuZPbBZkmvliBWKIezN3kD9Y=
github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA=
github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA=
github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
golang.org/x/exp v0.0.0-20250106191152-7588d65b2ba8 h1:yqrTHse8TCMW1M1ZCP+VAR/l0kKxwaAIqN/il7x4voA=
golang.org/x/exp v0.0.0-20250106191152-7588d65b2ba8/go.mod h1:tujkw807nyEEAamNbDrEGzRav+ilXA7PCRAd6xsmwiU=
golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.12.0 h1:CM0HF96J0hcLAwsHPJZjfdNzs0gftsLfgKt57wWHJ0o=
golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.30.0 h1:QjkSwP/36a20jFYWkSue1YwXzLmsV5Gfq7Eiy72C1uc=
golang.org/x/sys v0.30.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
google.golang.org/protobuf v1.36.5 h1:tPhr+woSbjfYvY6/GPufUoYizxw1cF/yFoxJ2fmpwlM=
google.golang.org/protobuf v1.36.5/go.mod h1:9fA7Ob0pmnwhb644+1+CVWFRbNajQ6iRojtC/QF5bRE=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=

View File

@@ -14,11 +14,15 @@ type AvgCacheState struct {
// New creates a new average cache state with the given size.
func New(size int) *AvgCacheState {
return &AvgCacheState{
a := &AvgCacheState{
size: size,
avgs: make([]cachestate.CacheState, size),
mu: sync.Mutex{},
}
a.Clear()
return a
}
// Clear resets the average cache state to zero.
@@ -26,7 +30,9 @@ func (a *AvgCacheState) Clear() {
a.mu.Lock()
defer a.mu.Unlock()
a.avgs = make([]cachestate.CacheState, a.size) // zeroed
for i := 0; i < len(a.avgs); i++ {
a.avgs[i] = cachestate.CacheStateMiss
}
}
// Add adds a cache state to the average cache state.
@@ -45,16 +51,13 @@ func (a *AvgCacheState) Avg() float64 {
a.mu.Lock()
defer a.mu.Unlock()
var hits, misses int
var hits int
for _, cs := range a.avgs {
switch cs {
case cachestate.CacheStateHit:
if cs == cachestate.CacheStateHit {
hits++
case cachestate.CacheStateMiss:
misses++
}
}
total := hits + misses
return float64(hits) / float64(total)
return float64(hits) / float64(len(a.avgs))
}

View File

@@ -1,75 +1,44 @@
package steamcache
import (
"s1d3sw1ped/SteamCache2/steamcache/logger"
"runtime/debug"
"s1d3sw1ped/SteamCache2/vfs"
"s1d3sw1ped/SteamCache2/vfs/cachestate"
"sort"
"time"
"github.com/docker/go-units"
"golang.org/x/exp/rand"
)
func randomgc(vfss vfs.VFS, stats []*vfs.FileInfo) int64 {
// Pick a random file to delete
randfile := stats[rand.Intn(len(stats))]
sz := randfile.Size()
err := vfss.Delete(randfile.Name())
if err != nil {
// If we failed to delete the file, log it and return 0
// logger.Logger.Error().Err(err).Msgf("Failed to delete %s", randfile.Name())
return 0
}
return sz
func init() {
// Set the GC percentage to 50%. This is a good balance between performance and memory usage.
debug.SetGCPercent(50)
}
func memorygc(vfss vfs.VFS, size int) {
tstart := time.Now()
// lruGC deletes files in LRU order until enough space is reclaimed.
func lruGC(vfss vfs.VFS, size uint) (uint, uint) {
deletions := 0
targetreclaim := int64(size)
var reclaimed int64
var reclaimed uint
stats := vfss.StatAll()
for {
reclaimed += randomgc(vfss, stats)
sort.Slice(stats, func(i, j int) bool {
return stats[i].AccessTime().Before(stats[j].AccessTime())
})
for _, s := range stats {
sz := uint(s.Size())
err := vfss.Delete(s.Name())
if err != nil {
continue
}
reclaimed += sz
deletions++
if reclaimed >= targetreclaim {
if reclaimed >= size {
break
}
}
logger.Logger.Info().
Str("name", vfss.Name()).
Str("duration", time.Since(tstart).String()).
Str("reclaimed", units.HumanSize(float64(reclaimed))).
Int("deletions", deletions).
Msgf("GC")
}
func diskgc(vfss vfs.VFS, size int) {
tstart := time.Now()
deletions := 0
targetreclaim := int64(size)
var reclaimed int64
stats := vfss.StatAll()
for {
reclaimed += randomgc(vfss, stats)
deletions++
if reclaimed >= targetreclaim {
break
}
}
logger.Logger.Info().
Str("name", vfss.Name()).
Str("duration", time.Since(tstart).String()).
Str("reclaimed", units.HumanSize(float64(reclaimed))).
Int("deletions", deletions).
Msgf("GC")
return reclaimed, uint(deletions)
}
func cachehandler(fi *vfs.FileInfo, cs cachestate.CacheState) bool {
return time.Since(fi.AccessTime()) < time.Minute*10 // Put files in the cache if they've been accessed twice in the last 10 minutes
return time.Since(fi.AccessTime()) < time.Second*10 // Put hot files in the fast vfs if equipped
}

View File

@@ -1,9 +1,7 @@
package logger
import (
"os"
"github.com/rs/zerolog"
)
var Logger = zerolog.New(zerolog.ConsoleWriter{Out: os.Stderr}).With().Timestamp().Logger()
var Logger zerolog.Logger

View File

@@ -1,42 +1,64 @@
package steamcache
import (
"fmt"
"io"
"net/http"
"net/url"
"os"
"s1d3sw1ped/SteamCache2/steamcache/avgcachestate"
"s1d3sw1ped/SteamCache2/steamcache/logger"
"s1d3sw1ped/SteamCache2/version"
"s1d3sw1ped/SteamCache2/vfs"
"s1d3sw1ped/SteamCache2/vfs/cache"
"s1d3sw1ped/SteamCache2/vfs/cachestate"
"s1d3sw1ped/SteamCache2/vfs/disk"
"s1d3sw1ped/SteamCache2/vfs/gc"
"s1d3sw1ped/SteamCache2/vfs/memory"
syncfs "s1d3sw1ped/SteamCache2/vfs/sync"
// syncfs "s1d3sw1ped/SteamCache2/vfs/sync"
"strings"
"sync"
"time"
pprof "net/http/pprof"
"github.com/docker/go-units"
"github.com/prometheus/client_golang/prometheus"
"github.com/prometheus/client_golang/prometheus/promauto"
"github.com/prometheus/client_golang/prometheus/promhttp"
)
var (
requestsTotal = promauto.NewCounterVec(
prometheus.CounterOpts{
Name: "http_requests_total",
Help: "Total number of HTTP requests",
},
[]string{"method", "status"},
)
cacheHitRate = promauto.NewGauge(
prometheus.GaugeOpts{
Name: "cache_hit_rate",
Help: "Cache hit rate",
},
)
)
type SteamCache struct {
address string
vfs vfs.VFS
pprof bool
address string
upstream string
vfs vfs.VFS
memory *memory.MemoryFS
disk *disk.DiskFS
hits *avgcachestate.AvgCacheState
memorygc *gc.GCFS
diskgc *gc.GCFS
dirty bool
mu sync.Mutex
hits *avgcachestate.AvgCacheState
}
func New(address string, memorySize string, memoryMultiplier int, diskSize string, diskMultiplier int, diskPath string) *SteamCache {
func New(address string, memorySize string, memoryMultiplier int, diskSize string, diskMultiplier int, diskPath, upstream string, pprof bool) *SteamCache {
memorysize, err := units.FromHumanSize(memorySize)
if err != nil {
panic(err)
@@ -47,54 +69,77 @@ func New(address string, memorySize string, memoryMultiplier int, diskSize strin
panic(err)
}
m := memory.New(memorysize)
d := disk.New(diskPath, disksize)
c := cache.New(
cachehandler,
)
var m *memory.MemoryFS
var mgc *gc.GCFS
if memorysize > 0 {
m = memory.New(memorysize)
mgc = gc.New(m, memoryMultiplier, lruGC)
}
var d *disk.DiskFS
var dgc *gc.GCFS
if disksize > 0 {
d = disk.New(diskPath, disksize)
dgc = gc.New(d, diskMultiplier, lruGC)
}
// configure the cache to match the specified mode (memory only, disk only, or memory and disk) based on the provided sizes
if disksize == 0 && memorysize != 0 {
//memory only mode - no disk
c.SetSlow(mgc)
} else if disksize != 0 && memorysize == 0 {
// disk only mode
c.SetSlow(dgc)
} else if disksize != 0 && memorysize != 0 {
// memory and disk mode
c.SetFast(mgc)
c.SetSlow(dgc)
} else {
// no memory or disk isn't a valid configuration
logger.Logger.Error().Bool("memory", false).Bool("disk", false).Msg("configuration invalid :( exiting")
os.Exit(1)
}
sc := &SteamCache{
address: address,
vfs: syncfs.New(
cache.New(
gc.New(
m,
memoryMultiplier,
memorygc,
),
gc.New(
d,
diskMultiplier,
diskgc,
),
cachehandler,
),
),
pprof: pprof,
upstream: upstream,
address: address,
// vfs: syncfs.New(c),
vfs: c,
memory: m,
disk: d,
hits: avgcachestate.New(10000),
memorygc: mgc,
diskgc: dgc,
hits: avgcachestate.New(100),
}
if d.Size() > d.Capacity() {
diskgc(d, int(d.Size()-d.Capacity()))
if d != nil {
if d.Size() > d.Capacity() {
lruGC(d, uint(d.Size()-d.Capacity()))
}
}
return sc
}
func (sc *SteamCache) Run() {
logger.Logger.Info().Str("address", sc.address).Str("version", version.Version).Msg("listening")
sc.mu.Lock()
sc.dirty = true
sc.mu.Unlock()
sc.LogStats()
t := time.NewTicker(10 * time.Second)
go func() {
for range t.C {
sc.LogStats()
if sc.upstream != "" {
_, err := http.Get(sc.upstream)
if err != nil {
logger.Logger.Error().Err(err).Str("upstream", sc.upstream).Msg("Failed to connect to upstream server")
os.Exit(1)
}
}()
}
err := http.ListenAndServe(sc.address, sc)
if err != nil {
@@ -107,21 +152,21 @@ func (sc *SteamCache) Run() {
}
}
func (sc *SteamCache) LogStats() {
sc.mu.Lock()
defer sc.mu.Unlock()
if sc.dirty {
logger.Logger.Info().
Str("memory", fmt.Sprintf("%s/%s", units.HumanSize(float64(sc.memory.Size())), units.HumanSize(float64(sc.memory.Capacity())))).Int("memory-files", len(sc.memory.StatAll())).
Str("disk", fmt.Sprintf("%s/%s", units.HumanSize(float64(sc.disk.Size())), units.HumanSize(float64(sc.disk.Capacity())))).Int("disk-files", len(sc.disk.StatAll())).
Str("hitrate", fmt.Sprintf("%.2f%%", sc.hits.Avg()*100)).
Msg("stats")
sc.dirty = false
}
}
func (sc *SteamCache) ServeHTTP(w http.ResponseWriter, r *http.Request) {
if sc.pprof && r.URL.Path == "/debug/pprof/" {
pprof.Index(w, r)
return
} else if sc.pprof && strings.HasPrefix(r.URL.Path, "/debug/pprof/") {
pprof.Handler(strings.TrimPrefix(r.URL.Path, "/debug/pprof/")).ServeHTTP(w, r)
return
}
if r.URL.Path == "/metrics" {
promhttp.Handler().ServeHTTP(w, r)
return
}
if r.Method != http.MethodGet {
requestsTotal.WithLabelValues(r.Method, "405").Inc()
http.Error(w, "Only GET method is supported", http.StatusMethodNotAllowed)
return
}
@@ -133,67 +178,107 @@ func (sc *SteamCache) ServeHTTP(w http.ResponseWriter, r *http.Request) {
return
}
if r.Header.Get("User-Agent") != "Valve/Steam HTTP Client 1.0" {
http.Error(w, "Only Valve/Steam HTTP Client 1.0 is supported", http.StatusForbidden)
tstart := time.Now()
cacheKey := strings.ReplaceAll(r.URL.String()[1:], "\\", "/") // replace all backslashes with forward slashes shouldn't be necessary but just in case
if cacheKey == "" {
requestsTotal.WithLabelValues(r.Method, "400").Inc()
http.Error(w, "Invalid URL", http.StatusBadRequest)
return
}
if strings.Contains(r.URL.String(), "manifest") {
w.Header().Add("X-LanCache-Processed-By", "SteamCache2")
forward(w, r)
return
}
// tstart := time.Now()
// defer func() {
// logger.Logger.Info().Str("method", r.Method).Str("url", r.URL.String()).Str("status", w.Header().Get("X-LanCache-Status")).Dur("duration", time.Since(tstart)).Msg("Request")
// }()
sc.mu.Lock()
sc.dirty = true
sc.mu.Unlock()
w.Header().Add("X-LanCache-Processed-By", "SteamCache2") // SteamPrefill uses this header to determine if the request was processed by the cache maybe steam uses it too
cacheKey := r.URL.String()
// if vfs is also a vfs.GetSer, we can use it to get the cache state
data, err := sc.vfs.Get(cacheKey)
if err == nil {
sc.hits.Add(cachestate.CacheStateHit)
w.Header().Add("X-LanCache-Status", "HIT")
requestsTotal.WithLabelValues(r.Method, "200").Inc()
cacheHitRate.Set(sc.hits.Avg())
w.Write(data)
logger.Logger.Info().
Str("key", cacheKey).
Str("host", r.Host).
Str("status", "HIT").
Int64("size", int64(len(data))).
Dur("duration", time.Since(tstart)).
Msg("request")
return
}
htt := "http://"
if r.Header.Get("X-Sls-Https") == "enable" {
htt = "https://"
var req *http.Request
if sc.upstream != "" { // if an upstream server is configured, proxy the request to the upstream server
ur, err := url.JoinPath(sc.upstream, r.URL.String())
if err != nil {
requestsTotal.WithLabelValues(r.Method, "500").Inc()
http.Error(w, "Failed to join URL path", http.StatusInternalServerError)
return
}
req, err = http.NewRequest(http.MethodGet, ur, nil)
if err != nil {
requestsTotal.WithLabelValues(r.Method, "500").Inc()
http.Error(w, "Failed to create request", http.StatusInternalServerError)
return
}
req.Host = r.Host
} else { // if no upstream server is configured, proxy the request to the host specified in the request
host := r.Host
if r.Header.Get("X-Sls-Https") == "enable" {
host = "https://" + host
} else {
host = "http://" + host
}
ur, err := url.JoinPath(host, r.URL.String())
if err != nil {
requestsTotal.WithLabelValues(r.Method, "500").Inc()
http.Error(w, "Failed to join URL path", http.StatusInternalServerError)
return
}
req, err = http.NewRequest(http.MethodGet, ur, nil)
if err != nil {
requestsTotal.WithLabelValues(r.Method, "500").Inc()
http.Error(w, "Failed to create request", http.StatusInternalServerError)
return
}
}
base := htt + r.Host
hosturl, err := url.JoinPath(base, cacheKey)
if err != nil {
http.Error(w, "Failed to join URL path", http.StatusInternalServerError)
return
// Copy headers from the original request to the new request
for key, values := range r.Header {
for _, value := range values {
req.Header.Add(key, value)
}
}
resp, err := http.Get(hosturl)
if err != nil {
// req.Header.Add("X-Sls-Https", r.Header.Get("X-Sls-Https"))
// req.Header.Add("User-Agent", r.Header.Get("User-Agent"))
// Retry logic
backoffSchedule := []time.Duration{1 * time.Second, 3 * time.Second, 10 * time.Second}
var resp *http.Response
for i, backoff := range backoffSchedule {
resp, err = http.DefaultClient.Do(req)
if err == nil && resp.StatusCode == http.StatusOK {
break
}
if i < len(backoffSchedule)-1 {
time.Sleep(backoff)
}
}
if err != nil || resp.StatusCode != http.StatusOK {
requestsTotal.WithLabelValues(r.Method, "500").Inc()
http.Error(w, "Failed to fetch the requested URL", http.StatusInternalServerError)
return
}
defer resp.Body.Close()
if resp.StatusCode != http.StatusOK {
http.Error(w, "Failed to fetch the requested URL", resp.StatusCode)
return
}
body, err := io.ReadAll(resp.Body)
if err != nil {
requestsTotal.WithLabelValues(r.Method, "500").Inc()
http.Error(w, "Failed to read response body", http.StatusInternalServerError)
return
}
@@ -201,42 +286,16 @@ func (sc *SteamCache) ServeHTTP(w http.ResponseWriter, r *http.Request) {
sc.vfs.Set(cacheKey, body)
sc.hits.Add(cachestate.CacheStateMiss)
w.Header().Add("X-LanCache-Status", "MISS")
w.Write(body)
}
func forward(w http.ResponseWriter, r *http.Request) {
htt := "http://"
if r.Header.Get("X-Sls-Https") == "enable" {
htt = "https://"
}
base := htt + r.Host
cacheKey := r.URL.String()
hosturl, err := url.JoinPath(base, cacheKey)
if err != nil {
http.Error(w, "Failed to join URL path", http.StatusInternalServerError)
return
}
resp, err := http.Get(hosturl)
if err != nil {
http.Error(w, "Failed to fetch the requested URL", http.StatusInternalServerError)
return
}
defer resp.Body.Close()
if resp.StatusCode != http.StatusOK {
http.Error(w, "Failed to fetch the requested URL", resp.StatusCode)
return
}
body, err := io.ReadAll(resp.Body)
if err != nil {
http.Error(w, "Failed to read response body", http.StatusInternalServerError)
return
}
requestsTotal.WithLabelValues(r.Method, "200").Inc()
cacheHitRate.Set(sc.hits.Avg())
w.Write(body)
logger.Logger.Info().
Str("key", cacheKey).
Str("host", r.Host).
Str("status", "MISS").
Int64("size", int64(len(body))).
Dur("duration", time.Since(tstart)).
Msg("request")
}

View File

@@ -0,0 +1,65 @@
package steamcache
import (
"os"
"path/filepath"
"testing"
)
func TestCaching(t *testing.T) {
t.Parallel()
td := t.TempDir()
os.WriteFile(filepath.Join(td, "key2"), []byte("value2"), 0644)
sc := New("localhost:8080", "1GB", 10, "1GB", 100, td, "", false)
if err := sc.vfs.Set("key", []byte("value")); err != nil {
t.Errorf("Set failed: %v", err)
}
if err := sc.vfs.Set("key1", []byte("value1")); err != nil {
t.Errorf("Set failed: %v", err)
}
if sc.diskgc.Size() != 17 {
t.Errorf("Size failed: got %d, want %d", sc.diskgc.Size(), 17)
}
if sc.vfs.Size() != 17 {
t.Errorf("Size failed: got %d, want %d", sc.vfs.Size(), 17)
}
if d, err := sc.vfs.Get("key"); err != nil {
t.Errorf("Get failed: %v", err)
} else if string(d) != "value" {
t.Errorf("Get failed: got %s, want %s", d, "value")
}
if d, err := sc.vfs.Get("key1"); err != nil {
t.Errorf("Get failed: %v", err)
} else if string(d) != "value1" {
t.Errorf("Get failed: got %s, want %s", d, "value1")
}
if d, err := sc.vfs.Get("key2"); err != nil {
t.Errorf("Get failed: %v", err)
} else if string(d) != "value2" {
t.Errorf("Get failed: got %s, want %s", d, "value2")
}
if sc.diskgc.Size() != 17 {
t.Errorf("Size failed: got %d, want %d", sc.diskgc.Size(), 17)
}
if sc.vfs.Size() != 17 {
t.Errorf("Size failed: got %d, want %d", sc.vfs.Size(), 17)
}
sc.memory.Delete("key2")
os.Remove(filepath.Join(td, "key2"))
if _, err := sc.vfs.Get("key2"); err == nil {
t.Errorf("Get failed: got nil, want error")
}
}

View File

@@ -1,3 +1,9 @@
package version
var Version string
func init() {
if Version == "" {
Version = "0.0.0-dev"
}
}

51
vfs/cache/cache.go vendored
View File

@@ -5,6 +5,7 @@ import (
"s1d3sw1ped/SteamCache2/vfs"
"s1d3sw1ped/SteamCache2/vfs/cachestate"
"s1d3sw1ped/SteamCache2/vfs/vfserror"
"sync"
)
// Ensure CacheFS implements VFS.
@@ -16,28 +17,38 @@ type CacheFS struct {
slow vfs.VFS
cacheHandler CacheHandler
keyLocks sync.Map // map[string]*sync.RWMutex for per-key locks
}
type CacheHandler func(*vfs.FileInfo, cachestate.CacheState) bool
// New creates a new CacheFS. fast is used for caching, and slow is used for storage. fast should obviously be faster than slow.
func New(fast, slow vfs.VFS, cacheHandler CacheHandler) *CacheFS {
if slow == nil {
panic("slow is nil")
}
if fast == slow {
panic("fast and slow are the same")
}
func New(cacheHandler CacheHandler) *CacheFS {
return &CacheFS{
fast: fast,
slow: slow,
cacheHandler: cacheHandler,
keyLocks: sync.Map{},
}
}
func (c *CacheFS) SetSlow(vfs vfs.VFS) {
if vfs == nil {
panic("vfs is nil") // panic if the vfs is nil
}
c.slow = vfs
}
func (c *CacheFS) SetFast(vfs vfs.VFS) {
c.fast = vfs
}
// getKeyLock returns a RWMutex for the given key, creating it if necessary.
func (c *CacheFS) getKeyLock(key string) *sync.RWMutex {
mu, _ := c.keyLocks.LoadOrStore(key, &sync.RWMutex{})
return mu.(*sync.RWMutex)
}
// cacheState returns the state of the file at key.
func (c *CacheFS) cacheState(key string) cachestate.CacheState {
if c.fast != nil {
@@ -64,6 +75,10 @@ func (c *CacheFS) Size() int64 {
// Set sets the file at key to src. If the file is already in the cache, it is replaced.
func (c *CacheFS) Set(key string, src []byte) error {
mu := c.getKeyLock(key)
mu.Lock()
defer mu.Unlock()
state := c.cacheState(key)
switch state {
@@ -81,6 +96,10 @@ func (c *CacheFS) Set(key string, src []byte) error {
// Delete deletes the file at key from the cache.
func (c *CacheFS) Delete(key string) error {
mu := c.getKeyLock(key)
mu.Lock()
defer mu.Unlock()
if c.fast != nil {
c.fast.Delete(key)
}
@@ -95,6 +114,10 @@ func (c *CacheFS) Get(key string) ([]byte, error) {
// GetS returns the file at key. If the file is not in the cache, it is fetched from the storage. It also returns the cache state.
func (c *CacheFS) GetS(key string) ([]byte, cachestate.CacheState, error) {
mu := c.getKeyLock(key)
mu.RLock()
defer mu.RUnlock()
state := c.cacheState(key)
switch state {
@@ -129,6 +152,10 @@ func (c *CacheFS) GetS(key string) ([]byte, cachestate.CacheState, error) {
// Stat returns information about the file at key.
// Warning: This will return information about the file in the fastest storage its in.
func (c *CacheFS) Stat(key string) (*vfs.FileInfo, error) {
mu := c.getKeyLock(key)
mu.RLock()
defer mu.RUnlock()
state := c.cacheState(key)
switch state {

View File

@@ -20,7 +20,9 @@ func TestNew(t *testing.T) {
fast := testMemory()
slow := testMemory()
cache := New(fast, slow, nil)
cache := New(nil)
cache.SetFast(fast)
cache.SetSlow(slow)
if cache == nil {
t.Fatal("expected cache to be non-nil")
}
@@ -35,7 +37,9 @@ func TestNewPanics(t *testing.T) {
}
}()
New(nil, nil, nil)
cache := New(nil)
cache.SetFast(nil)
cache.SetSlow(nil)
}
func TestSetAndGet(t *testing.T) {
@@ -43,7 +47,9 @@ func TestSetAndGet(t *testing.T) {
fast := testMemory()
slow := testMemory()
cache := New(fast, slow, nil)
cache := New(nil)
cache.SetFast(fast)
cache.SetSlow(slow)
key := "test"
value := []byte("value")
@@ -66,7 +72,8 @@ func TestSetAndGetNoFast(t *testing.T) {
t.Parallel()
slow := testMemory()
cache := New(nil, slow, nil)
cache := New(nil)
cache.SetSlow(slow)
key := "test"
value := []byte("value")
@@ -89,9 +96,11 @@ func TestCaching(t *testing.T) {
fast := testMemory()
slow := testMemory()
cache := New(fast, slow, func(fi *vfs.FileInfo, cs cachestate.CacheState) bool {
cache := New(func(fi *vfs.FileInfo, cs cachestate.CacheState) bool {
return true
})
cache.SetFast(fast)
cache.SetSlow(slow)
key := "test"
value := []byte("value")
@@ -148,7 +157,9 @@ func TestGetNotFound(t *testing.T) {
fast := testMemory()
slow := testMemory()
cache := New(fast, slow, nil)
cache := New(nil)
cache.SetFast(fast)
cache.SetSlow(slow)
_, err := cache.Get("nonexistent")
if !errors.Is(err, vfserror.ErrNotFound) {
@@ -161,7 +172,9 @@ func TestDelete(t *testing.T) {
fast := testMemory()
slow := testMemory()
cache := New(fast, slow, nil)
cache := New(nil)
cache.SetFast(fast)
cache.SetSlow(slow)
key := "test"
value := []byte("value")
@@ -185,7 +198,9 @@ func TestStat(t *testing.T) {
fast := testMemory()
slow := testMemory()
cache := New(fast, slow, nil)
cache := New(nil)
cache.SetFast(fast)
cache.SetSlow(slow)
key := "test"
value := []byte("value")

View File

@@ -1,11 +1,13 @@
package disk
import (
"fmt"
"os"
"path/filepath"
"s1d3sw1ped/SteamCache2/steamcache/logger"
"s1d3sw1ped/SteamCache2/vfs"
"s1d3sw1ped/SteamCache2/vfs/vfserror"
"strings"
"sync"
"time"
@@ -23,16 +25,44 @@ type DiskFS struct {
capacity int64
mu sync.Mutex
sg sync.WaitGroup
bytePool sync.Pool // Pool for []byte slices
}
// New creates a new DiskFS.
func new(root string, capacity int64, skipinit bool) *DiskFS {
if capacity <= 0 {
panic("disk capacity must be greater than 0") // panic if the capacity is less than or equal to 0
}
if root == "" {
panic("disk root must not be empty") // panic if the root is empty
}
fi, err := os.Stat(root)
if err != nil {
if !os.IsNotExist(err) {
panic(err) // panic if the error is something other than not found
}
os.Mkdir(root, 0755) // create the root directory if it does not exist
fi, err = os.Stat(root) // re-stat to get the file info
if err != nil {
panic(err) // panic if the re-stat fails
}
}
if !fi.IsDir() {
panic("disk root must be a directory") // panic if the root is not a directory
}
dfs := &DiskFS{
root: root,
info: make(map[string]*vfs.FileInfo),
capacity: capacity,
mu: sync.Mutex{},
sg: sync.WaitGroup{},
bytePool: sync.Pool{
New: func() interface{} { return make([]byte, 0) }, // Initial capacity for pooled slices is 0, will grow as needed
},
}
os.MkdirAll(dfs.root, 0755)
@@ -53,14 +83,19 @@ func NewSkipInit(root string, capacity int64) *DiskFS {
}
func (d *DiskFS) init() {
// logger.Logger.Info().Str("name", d.Name()).Str("root", d.root).Str("capacity", units.HumanSize(float64(d.capacity))).Msg("init")
tstart := time.Now()
d.walk(d.root)
d.sg.Wait()
logger.Logger.Info().Str("name", d.Name()).Str("root", d.root).Str("capacity", units.HumanSize(float64(d.capacity))).Str("duration", time.Since(tstart).String()).Msg("init")
logger.Logger.Info().
Str("name", d.Name()).
Str("root", d.root).
Str("capacity", units.HumanSize(float64(d.capacity))).
Str("size", units.HumanSize(float64(d.Size()))).
Str("files", fmt.Sprint(len(d.info))).
Str("duration", time.Since(tstart).String()).
Msg("init")
}
func (d *DiskFS) walk(path string) {
@@ -82,11 +117,10 @@ func (d *DiskFS) walk(path string) {
}
d.mu.Lock()
k := npath[len(d.root)+1:]
k := strings.ReplaceAll(npath[len(d.root)+1:], "\\", "/")
d.info[k] = vfs.NewFileInfoFromOS(info, k)
d.mu.Unlock()
// logger.Logger.Debug().Str("name", d.Name()).Str("root", d.root).Str("capacity", units.HumanSize(float64(d.capacity))).Str("path", npath).Msg("init")
return nil
})
}()
@@ -101,9 +135,10 @@ func (d *DiskFS) Name() string {
}
func (d *DiskFS) Size() int64 {
var size int64
d.mu.Lock()
defer d.mu.Unlock()
var size int64
for _, v := range d.info {
size += v.Size()
}
@@ -111,6 +146,13 @@ func (d *DiskFS) Size() int64 {
}
func (d *DiskFS) Set(key string, src []byte) error {
if key == "" {
return vfserror.ErrInvalidKey
}
if key[0] == '/' {
return vfserror.ErrInvalidKey
}
if d.capacity > 0 {
if size := d.Size() + int64(len(src)); size > d.capacity {
return vfserror.ErrDiskFull
@@ -123,12 +165,12 @@ func (d *DiskFS) Set(key string, src []byte) error {
d.mu.Lock()
defer d.mu.Unlock()
os.MkdirAll(filepath.Join(d.root, filepath.Dir(key)), 0755)
if err := os.WriteFile(filepath.Join(d.root, key), src, 0644); err != nil {
os.MkdirAll(d.root+"/"+filepath.Dir(key), 0755)
if err := os.WriteFile(d.root+"/"+key, src, 0644); err != nil {
return err
}
fi, err := os.Stat(filepath.Join(d.root, key))
fi, err := os.Stat(d.root + "/" + key)
if err != nil {
panic(err)
}
@@ -140,6 +182,13 @@ func (d *DiskFS) Set(key string, src []byte) error {
// Delete deletes the value of key.
func (d *DiskFS) Delete(key string) error {
if key == "" {
return vfserror.ErrInvalidKey
}
if key[0] == '/' {
return vfserror.ErrInvalidKey
}
_, err := d.Stat(key)
if err != nil {
return err
@@ -147,6 +196,7 @@ func (d *DiskFS) Delete(key string) error {
d.mu.Lock()
defer d.mu.Unlock()
delete(d.info, key)
if err := os.Remove(filepath.Join(d.root, key)); err != nil {
return err
@@ -157,6 +207,13 @@ func (d *DiskFS) Delete(key string) error {
// Get gets the value of key and returns it.
func (d *DiskFS) Get(key string) ([]byte, error) {
if key == "" {
return nil, vfserror.ErrInvalidKey
}
if key[0] == '/' {
return nil, vfserror.ErrInvalidKey
}
_, err := d.Stat(key)
if err != nil {
return nil, err
@@ -170,30 +227,35 @@ func (d *DiskFS) Get(key string) ([]byte, error) {
return nil, err
}
return data, nil
// Use pooled slice for return if possible, but since ReadFile allocates new, copy to pool if beneficial
dst := d.bytePool.Get().([]byte)
if cap(dst) < len(data) {
dst = make([]byte, len(data)) // create a new slice if the pool slice is too small
} else {
dst = dst[:len(data)] // reuse the pool slice, but resize it to fit
}
dst = dst[:len(data)]
copy(dst, data)
return dst, nil
}
// Stat returns the FileInfo of key. If key is not found in the cache, it will stat the file on disk. If the file is not found on disk, it will return vfs.ErrNotFound.
func (d *DiskFS) Stat(key string) (*vfs.FileInfo, error) {
d.mu.Lock()
fi, ok := d.info[key]
d.mu.Unlock() // unlock before statting the file
if !ok {
fii, err := os.Stat(filepath.Join(d.root, key))
if err != nil {
return nil, vfserror.ErrNotFound
}
d.mu.Lock() // relock to update the info map
defer d.mu.Unlock() // nothing else needs to unlock before returning
d.info[key] = vfs.NewFileInfoFromOS(fii, key)
fi = d.info[key]
// fallthrough to return fi with shiny new info
if key == "" {
return nil, vfserror.ErrInvalidKey
}
if key[0] == '/' {
return nil, vfserror.ErrInvalidKey
}
return fi, nil
d.mu.Lock()
defer d.mu.Unlock()
if fi, ok := d.info[key]; !ok {
return nil, vfserror.ErrNotFound
} else {
return fi, nil
}
}
func (m *DiskFS) StatAll() []*vfs.FileInfo {

View File

@@ -85,3 +85,62 @@ func TestInit(t *testing.T) {
t.Errorf("Stat failed: got %s, want %s", s.Name(), "key")
}
}
func TestDiskSizeDiscrepancy(t *testing.T) {
t.Parallel()
td := t.TempDir()
assumedSize := int64(6 + 5 + 6) // 6 + 5 + 6 bytes for key, key1, key2
os.WriteFile(filepath.Join(td, "key2"), []byte("value2"), 0644)
m := New(td, 1024)
if 6 != m.Size() {
t.Errorf("Size failed: got %d, want %d", m.Size(), 6)
}
if err := m.Set("key", []byte("value")); err != nil {
t.Errorf("Set failed: %v", err)
}
if err := m.Set("key1", []byte("value1")); err != nil {
t.Errorf("Set failed: %v", err)
}
if assumedSize != m.Size() {
t.Errorf("Size failed: got %d, want %d", m.Size(), assumedSize)
}
if d, err := m.Get("key"); err != nil {
t.Errorf("Get failed: %v", err)
} else if string(d) != "value" {
t.Errorf("Get failed: got %s, want %s", d, "value")
}
if d, err := m.Get("key1"); err != nil {
t.Errorf("Get failed: %v", err)
} else if string(d) != "value1" {
t.Errorf("Get failed: got %s, want %s", d, "value1")
}
m = New(td, 1024)
if assumedSize != m.Size() {
t.Errorf("Size failed: got %d, want %d", m.Size(), assumedSize)
}
if d, err := m.Get("key"); err != nil {
t.Errorf("Get failed: %v", err)
} else if string(d) != "value" {
t.Errorf("Get failed: got %s, want %s", d, "value")
}
if d, err := m.Get("key1"); err != nil {
t.Errorf("Get failed: %v", err)
} else if string(d) != "value1" {
t.Errorf("Get failed: got %s, want %s", d, "value1")
}
if assumedSize != m.Size() {
t.Errorf("Size failed: got %d, want %d", m.Size(), assumedSize)
}
}

View File

@@ -12,9 +12,9 @@ type FileInfo struct {
ATime time.Time
}
func NewFileInfo(name string, size int64, modTime time.Time) *FileInfo {
func NewFileInfo(key string, size int64, modTime time.Time) *FileInfo {
return &FileInfo{
name: name,
name: key,
size: size,
MTime: modTime,
ATime: time.Now(),

View File

@@ -4,6 +4,8 @@ import (
"fmt"
"s1d3sw1ped/SteamCache2/vfs"
"s1d3sw1ped/SteamCache2/vfs/vfserror"
"sync"
"time"
)
// Ensure GCFS implements VFS.
@@ -12,14 +14,23 @@ var _ vfs.VFS = (*GCFS)(nil)
// GCFS is a virtual file system that calls a GC handler when the disk is full. The GC handler is responsible for freeing up space on the disk. The GCFS is a wrapper around another VFS.
type GCFS struct {
vfs.VFS
multiplier int
gcHanderFunc GCHandlerFunc
multiplier int
// protected by mu
gcHanderFunc GCHandlerFunc
lifetimeBytes, lifetimeFiles uint
reclaimedBytes, deletedFiles uint
gcTime time.Duration
mu sync.Mutex
}
// GCHandlerFunc is a function that is called when the disk is full and the GCFS needs to free up space. It is passed the VFS and the size of the file that needs to be written. Its up to the implementation to free up space. How much space is freed is also up to the implementation.
type GCHandlerFunc func(vfs vfs.VFS, size int)
type GCHandlerFunc func(vfs vfs.VFS, size uint) (reclaimedBytes uint, deletedFiles uint)
func New(vfs vfs.VFS, multiplier int, gcHandlerFunc GCHandlerFunc) *GCFS {
if multiplier <= 0 {
multiplier = 1 // if the multiplier is less than or equal to 0 set it to 1 will be slow but the user can set it to a higher value if they want
}
return &GCFS{
VFS: vfs,
multiplier: multiplier,
@@ -27,13 +38,44 @@ func New(vfs vfs.VFS, multiplier int, gcHandlerFunc GCHandlerFunc) *GCFS {
}
}
// Stats returns the lifetime bytes, lifetime files, reclaimed bytes and deleted files.
// The lifetime bytes and lifetime files are the total bytes and files that have been freed up by the GC handler.
// The reclaimed bytes and deleted files are the bytes and files that have been freed up by the GC handler since last call to Stats.
// The gc time is the total time spent in the GC handler since last call to Stats.
// The reclaimed bytes and deleted files and gc time are reset to 0 after the call to Stats.
func (g *GCFS) Stats() (lifetimeBytes, lifetimeFiles, reclaimedBytes, deletedFiles uint, gcTime time.Duration) {
g.mu.Lock()
defer g.mu.Unlock()
g.lifetimeBytes += g.reclaimedBytes
g.lifetimeFiles += g.deletedFiles
lifetimeBytes = g.lifetimeBytes
lifetimeFiles = g.lifetimeFiles
reclaimedBytes = g.reclaimedBytes
deletedFiles = g.deletedFiles
gcTime = g.gcTime
g.reclaimedBytes = 0
g.deletedFiles = 0
g.gcTime = time.Duration(0)
return
}
// Set overrides the Set method of the VFS interface. It tries to set the key and src, if it fails due to disk full error, it calls the GC handler and tries again. If it still fails it returns the error.
func (g *GCFS) Set(key string, src []byte) error {
g.mu.Lock()
defer g.mu.Unlock()
err := g.VFS.Set(key, src) // try to set the key and src
if err == vfserror.ErrDiskFull && g.gcHanderFunc != nil { // if the error is disk full and there is a GC handler
g.gcHanderFunc(g.VFS, len(src)*g.multiplier) // call the GC handler
err = g.VFS.Set(key, src) // try again after GC if it still fails return the error
tstart := time.Now()
reclaimedBytes, deletedFiles := g.gcHanderFunc(g.VFS, uint(len(src)*g.multiplier)) // call the GC handler
g.gcTime += time.Since(tstart)
g.reclaimedBytes += reclaimedBytes
g.deletedFiles += deletedFiles
err = g.VFS.Set(key, src) // try again after GC if it still fails return the error
}
return err

View File

@@ -6,7 +6,6 @@ import (
"s1d3sw1ped/SteamCache2/vfs/memory"
"sort"
"testing"
"time"
"golang.org/x/exp/rand"
)
@@ -15,13 +14,11 @@ func TestGCSmallRandom(t *testing.T) {
t.Parallel()
m := memory.New(1024 * 1024 * 16)
gc := New(m, 10, func(vfs vfs.VFS, size int) {
tstart := time.Now()
gc := New(m, 10, func(vfs vfs.VFS, size uint) (uint, uint) {
deletions := 0
targetreclaim := int64(size)
var reclaimed int64
var reclaimed uint
t.Logf("GC starting to reclaim %d bytes", targetreclaim)
t.Logf("GC starting to reclaim %d bytes", size)
stats := vfs.StatAll()
sort.Slice(stats, func(i, j int) bool {
@@ -31,7 +28,7 @@ func TestGCSmallRandom(t *testing.T) {
// Delete the oldest files until we've reclaimed enough space.
for _, s := range stats {
sz := s.Size() // Get the size of the file
sz := uint(s.Size()) // Get the size of the file
err := vfs.Delete(s.Name())
if err != nil {
panic(err)
@@ -41,12 +38,11 @@ func TestGCSmallRandom(t *testing.T) {
// t.Logf("GC deleting %s, %v", s.Name(), s.AccessTime().Format(time.RFC3339Nano))
if reclaimed >= targetreclaim { // We've reclaimed enough space
if reclaimed >= size { // We've reclaimed enough space
break
}
}
t.Logf("GC took %v to reclaim %d bytes by deleting %d files", time.Since(tstart), reclaimed, deletions)
return uint(reclaimed), uint(deletions)
})
for i := 0; i < 10000; i++ {
@@ -70,13 +66,11 @@ func TestGCLargeRandom(t *testing.T) {
t.Parallel()
m := memory.New(1024 * 1024 * 16) // 16MB
gc := New(m, 10, func(vfs vfs.VFS, size int) {
tstart := time.Now()
gc := New(m, 10, func(vfs vfs.VFS, size uint) (uint, uint) {
deletions := 0
targetreclaim := int64(size)
var reclaimed int64
var reclaimed uint
t.Logf("GC starting to reclaim %d bytes", targetreclaim)
t.Logf("GC starting to reclaim %d bytes", size)
stats := vfs.StatAll()
sort.Slice(stats, func(i, j int) bool {
@@ -86,17 +80,17 @@ func TestGCLargeRandom(t *testing.T) {
// Delete the oldest files until we've reclaimed enough space.
for _, s := range stats {
sz := s.Size() // Get the size of the file
sz := uint(s.Size()) // Get the size of the file
vfs.Delete(s.Name())
reclaimed += sz // Track how much space we've reclaimed
deletions++ // Track how many files we've deleted
if reclaimed >= targetreclaim { // We've reclaimed enough space
if reclaimed >= size { // We've reclaimed enough space
break
}
}
t.Logf("GC took %v to reclaim %d bytes by deleting %d files", time.Since(tstart), reclaimed, deletions)
return uint(reclaimed), uint(deletions)
})
for i := 0; i < 10000; i++ {

View File

@@ -1,10 +1,13 @@
package memory
import (
"s1d3sw1ped/SteamCache2/steamcache/logger"
"s1d3sw1ped/SteamCache2/vfs"
"s1d3sw1ped/SteamCache2/vfs/vfserror"
"sync"
"time"
"github.com/docker/go-units"
)
// Ensure MemoryFS implements VFS.
@@ -21,14 +24,28 @@ type MemoryFS struct {
files map[string]*file
capacity int64
mu sync.Mutex
bytePool sync.Pool // Pool for []byte slices
}
// New creates a new MemoryFS.
func New(capacity int64) *MemoryFS {
if capacity <= 0 {
panic("memory capacity must be greater than 0") // panic if the capacity is less than or equal to 0
}
logger.Logger.Info().
Str("name", "MemoryFS").
Str("capacity", units.HumanSize(float64(capacity))).
Msg("init")
return &MemoryFS{
files: make(map[string]*file),
capacity: capacity,
mu: sync.Mutex{},
bytePool: sync.Pool{
New: func() interface{} { return make([]byte, 0) }, // Initial capacity for pooled slices
},
}
}
@@ -63,15 +80,23 @@ func (m *MemoryFS) Set(key string, src []byte) error {
m.mu.Lock()
defer m.mu.Unlock()
// Use pooled slice
data := m.bytePool.Get().([]byte)
if cap(data) < len(src) {
data = make([]byte, len(src)) // expand the slice if the pool slice is too small
} else {
data = data[:len(src)] // reuse the pool slice, but resize it to fit
}
copy(data, src)
m.files[key] = &file{
fileinfo: vfs.NewFileInfo(
key,
int64(len(src)),
time.Now(),
),
data: make([]byte, len(src)),
data: data,
}
copy(m.files[key].data, src)
return nil
}
@@ -85,6 +110,11 @@ func (m *MemoryFS) Delete(key string) error {
m.mu.Lock()
defer m.mu.Unlock()
// Return data to pool
if f, ok := m.files[key]; ok {
m.bytePool.Put(f.data)
}
delete(m.files, key)
return nil
@@ -103,6 +133,12 @@ func (m *MemoryFS) Get(key string) ([]byte, error) {
dst := make([]byte, len(m.files[key].data))
copy(dst, m.files[key].data)
logger.Logger.Debug().
Str("name", key).
Str("status", "GET").
Int64("size", int64(len(dst))).
Msg("get file from memory")
return dst, nil
}

View File

@@ -1,76 +1,76 @@
package sync
import (
"fmt"
"s1d3sw1ped/SteamCache2/vfs"
"sync"
)
// import (
// "fmt"
// "s1d3sw1ped/SteamCache2/vfs"
// "sync"
// )
// Ensure SyncFS implements VFS.
var _ vfs.VFS = (*SyncFS)(nil)
// // Ensure SyncFS implements VFS.
// var _ vfs.VFS = (*SyncFS)(nil)
type SyncFS struct {
vfs vfs.VFS
mu sync.RWMutex
}
// type SyncFS struct {
// vfs vfs.VFS
// mu sync.RWMutex
// }
func New(vfs vfs.VFS) *SyncFS {
return &SyncFS{
vfs: vfs,
mu: sync.RWMutex{},
}
}
// func New(vfs vfs.VFS) *SyncFS {
// return &SyncFS{
// vfs: vfs,
// mu: sync.RWMutex{},
// }
// }
// Name returns the name of the file system.
func (sfs *SyncFS) Name() string {
return fmt.Sprintf("SyncFS(%s)", sfs.vfs.Name())
}
// // Name returns the name of the file system.
// func (sfs *SyncFS) Name() string {
// return fmt.Sprintf("SyncFS(%s)", sfs.vfs.Name())
// }
// Size returns the total size of all files in the file system.
func (sfs *SyncFS) Size() int64 {
sfs.mu.RLock()
defer sfs.mu.RUnlock()
// // Size returns the total size of all files in the file system.
// func (sfs *SyncFS) Size() int64 {
// sfs.mu.RLock()
// defer sfs.mu.RUnlock()
return sfs.vfs.Size()
}
// return sfs.vfs.Size()
// }
// Set sets the value of key as src.
// Setting the same key multiple times, the last set call takes effect.
func (sfs *SyncFS) Set(key string, src []byte) error {
sfs.mu.Lock()
defer sfs.mu.Unlock()
// // Set sets the value of key as src.
// // Setting the same key multiple times, the last set call takes effect.
// func (sfs *SyncFS) Set(key string, src []byte) error {
// sfs.mu.Lock()
// defer sfs.mu.Unlock()
return sfs.vfs.Set(key, src)
}
// return sfs.vfs.Set(key, src)
// }
// Delete deletes the value of key.
func (sfs *SyncFS) Delete(key string) error {
sfs.mu.Lock()
defer sfs.mu.Unlock()
// // Delete deletes the value of key.
// func (sfs *SyncFS) Delete(key string) error {
// sfs.mu.Lock()
// defer sfs.mu.Unlock()
return sfs.vfs.Delete(key)
}
// return sfs.vfs.Delete(key)
// }
// Get gets the value of key to dst, and returns dst no matter whether or not there is an error.
func (sfs *SyncFS) Get(key string) ([]byte, error) {
sfs.mu.RLock()
defer sfs.mu.RUnlock()
// // Get gets the value of key to dst, and returns dst no matter whether or not there is an error.
// func (sfs *SyncFS) Get(key string) ([]byte, error) {
// sfs.mu.RLock()
// defer sfs.mu.RUnlock()
return sfs.vfs.Get(key)
}
// return sfs.vfs.Get(key)
// }
// Stat returns the FileInfo of key.
func (sfs *SyncFS) Stat(key string) (*vfs.FileInfo, error) {
sfs.mu.RLock()
defer sfs.mu.RUnlock()
// // Stat returns the FileInfo of key.
// func (sfs *SyncFS) Stat(key string) (*vfs.FileInfo, error) {
// sfs.mu.RLock()
// defer sfs.mu.RUnlock()
return sfs.vfs.Stat(key)
}
// return sfs.vfs.Stat(key)
// }
// StatAll returns the FileInfo of all keys.
func (sfs *SyncFS) StatAll() []*vfs.FileInfo {
sfs.mu.RLock()
defer sfs.mu.RUnlock()
// // StatAll returns the FileInfo of all keys.
// func (sfs *SyncFS) StatAll() []*vfs.FileInfo {
// sfs.mu.RLock()
// defer sfs.mu.RUnlock()
return sfs.vfs.StatAll()
}
// return sfs.vfs.StatAll()
// }

View File

@@ -3,6 +3,9 @@ package vfserror
import "errors"
var (
// ErrInvalidKey is returned when a key is invalid.
ErrInvalidKey = errors.New("vfs: invalid key")
// ErrUnreachable is returned when a code path is unreachable.
ErrUnreachable = errors.New("unreachable")