Compare commits
40 Commits
1.0.6
..
6fc83063da
| Author | SHA1 | Date | |
|---|---|---|---|
 s1d3sw1ped
|
6fc83063da | ||
|
s1d3sw1ped
|
cd4b5e0598 | ||
|
s1d3sw1ped
|
e583780b9c | ||
|
s1d3sw1ped
|
c0ed8a590e | ||
|
s1d3sw1ped
|
09d3450738 | ||
|
s1d3sw1ped
|
f01b15c8c6 | ||
|
s1d3sw1ped
|
27ac30d28a | ||
|
s1d3sw1ped
|
0424de1fca | ||
|
s1d3sw1ped
|
32c0d85bad | ||
|
s1d3sw1ped
|
6b51e61a87 | ||
|
s1d3sw1ped
|
1094505f20 | ||
|
s1d3sw1ped
|
93d04410bf | ||
|
s1d3sw1ped
|
62f7a38295 | ||
|
s1d3sw1ped
|
c370e80888 | ||
|
s1d3sw1ped
|
9d0f4894e1 | ||
|
s1d3sw1ped
|
745ee37184 | ||
|
s1d3sw1ped
|
5a9b07f609 | ||
|
s1d3sw1ped
|
d88db3d9c1 | ||
|
s1d3sw1ped
|
91075232cd | ||
|
s1d3sw1ped
|
1f588fa20a | ||
|
s1d3sw1ped
|
d39a2ad4fa | ||
|
s1d3sw1ped
|
80cc75f372 | ||
|
s1d3sw1ped
|
53f5d1577c | ||
|
s1d3sw1ped
|
1186e14c56 | ||
|
s1d3sw1ped
|
3bb57b7741 | ||
|
s1d3sw1ped
|
8f2d1eff9b | ||
|
s1d3sw1ped
|
27242fdf05 | ||
|
s1d3sw1ped
|
55330ebd3c | ||
|
s1d3sw1ped
|
e37dc930dd | ||
|
s1d3sw1ped
|
79f0814745 | ||
|
s1d3sw1ped
|
1feee2758c | ||
|
s1d3sw1ped
|
22f9196960 | ||
|
s1d3sw1ped
|
9d0a09d8be | ||
|
s1d3sw1ped
|
f6f007bf47 | ||
|
s1d3sw1ped
|
232b290a44 | ||
|
s1d3sw1ped
|
b55b95f5d8 | ||
|
s1d3sw1ped
|
ca6c9a5861 | ||
|
s1d3sw1ped
|
16dce1f0c2 | ||
|
s1d3sw1ped
|
2be7b117ea | ||
|
s1d3sw1ped
|
88ecb1bc24 |
@@ -1,4 +1,4 @@
|
||||
name: Release Tag
|
||||
name: Release
|
||||
on:
|
||||
push:
|
||||
tags:
|
||||
@@ -6,16 +6,17 @@ on:
|
||||
|
||||
jobs:
|
||||
release:
|
||||
name: Build versioned release
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@main
|
||||
- uses: actions/checkout@v4
|
||||
with:
|
||||
fetch-depth: 0
|
||||
- run: git fetch --force --tags
|
||||
- uses: actions/setup-go@main
|
||||
- uses: actions/setup-go@v5
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
- uses: goreleaser/goreleaser-action@master
|
||||
- uses: goreleaser/goreleaser-action@v6
|
||||
with:
|
||||
distribution: goreleaser
|
||||
version: 'latest'
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
name: PR Check
|
||||
name: CI
|
||||
on:
|
||||
- pull_request
|
||||
|
||||
@@ -6,10 +6,14 @@ jobs:
|
||||
check-and-test:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@main
|
||||
- uses: actions/setup-go@main
|
||||
- uses: actions/checkout@v4
|
||||
- uses: actions/setup-go@v5
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
- run: go mod tidy
|
||||
- uses: golangci/golangci-lint-action@v3
|
||||
with:
|
||||
args: -D errcheck
|
||||
version: latest
|
||||
- run: go build ./...
|
||||
- run: go test -race -v -shuffle=on ./...
|
||||
+1
-3
@@ -1,5 +1,3 @@
|
||||
dist/
|
||||
tmp/
|
||||
__*.exe
|
||||
.smashed.txt
|
||||
.smashignore
|
||||
__*.exe
|
||||
+3
-4
@@ -16,7 +16,7 @@ builds:
|
||||
- amd64
|
||||
|
||||
archives:
|
||||
- formats: tar.gz
|
||||
- format: tar.gz
|
||||
name_template: >-
|
||||
{{ .ProjectName }}_
|
||||
{{- title .Os }}_
|
||||
@@ -26,7 +26,7 @@ archives:
|
||||
{{- if .Arm }}v{{ .Arm }}{{ end }}
|
||||
format_overrides:
|
||||
- goos: windows
|
||||
formats: zip
|
||||
format: zip
|
||||
|
||||
changelog:
|
||||
sort: asc
|
||||
@@ -36,9 +36,8 @@ changelog:
|
||||
- "^test:"
|
||||
|
||||
release:
|
||||
name_template: '{{.ProjectName}}-{{.Version}}'
|
||||
footer: >-
|
||||
|
||||
|
||||
---
|
||||
|
||||
Released by [GoReleaser](https://github.com/goreleaser/goreleaser).
|
||||
|
||||
Vendored
+1
-28
@@ -5,7 +5,7 @@
|
||||
"version": "0.2.0",
|
||||
"configurations": [
|
||||
{
|
||||
"name": "Launch Memory & Disk",
|
||||
"name": "Launch Package",
|
||||
"type": "go",
|
||||
"request": "launch",
|
||||
"mode": "auto",
|
||||
@@ -17,33 +17,6 @@
|
||||
"10G",
|
||||
"--disk-path",
|
||||
"tmp/disk",
|
||||
"--verbose",
|
||||
],
|
||||
},
|
||||
{
|
||||
"name": "Launch Disk Only",
|
||||
"type": "go",
|
||||
"request": "launch",
|
||||
"mode": "auto",
|
||||
"program": "${workspaceFolder}/main.go",
|
||||
"args": [
|
||||
"--disk",
|
||||
"10G",
|
||||
"--disk-path",
|
||||
"tmp/disk",
|
||||
"--verbose",
|
||||
],
|
||||
},
|
||||
{
|
||||
"name": "Launch Memory Only",
|
||||
"type": "go",
|
||||
"request": "launch",
|
||||
"mode": "auto",
|
||||
"program": "${workspaceFolder}/main.go",
|
||||
"args": [
|
||||
"--memory",
|
||||
"1G",
|
||||
"--verbose",
|
||||
],
|
||||
}
|
||||
]
|
||||
|
||||
+6
-63
@@ -1,29 +1,18 @@
|
||||
// cmd/root.go
|
||||
package cmd
|
||||
|
||||
import (
|
||||
"os"
|
||||
"runtime"
|
||||
"s1d3sw1ped/SteamCache2/steamcache"
|
||||
"s1d3sw1ped/SteamCache2/steamcache/logger"
|
||||
"s1d3sw1ped/SteamCache2/version"
|
||||
|
||||
"github.com/rs/zerolog"
|
||||
"github.com/spf13/cobra"
|
||||
)
|
||||
|
||||
var (
|
||||
threads int
|
||||
|
||||
memory string
|
||||
memorymultiplier int
|
||||
disk string
|
||||
diskmultiplier int
|
||||
diskpath string
|
||||
upstream string
|
||||
|
||||
logLevel string
|
||||
logFormat string
|
||||
)
|
||||
|
||||
var rootCmd = &cobra.Command{
|
||||
@@ -35,54 +24,15 @@ var rootCmd = &cobra.Command{
|
||||
By caching game files, SteamCache2 ensures that subsequent downloads of the same files are served from the local cache,
|
||||
significantly improving download times and reducing the load on the internet connection.`,
|
||||
Run: func(cmd *cobra.Command, args []string) {
|
||||
// Configure logging
|
||||
switch logLevel {
|
||||
case "debug":
|
||||
zerolog.SetGlobalLevel(zerolog.DebugLevel)
|
||||
case "error":
|
||||
zerolog.SetGlobalLevel(zerolog.ErrorLevel)
|
||||
case "info":
|
||||
zerolog.SetGlobalLevel(zerolog.InfoLevel)
|
||||
default:
|
||||
zerolog.SetGlobalLevel(zerolog.InfoLevel) // Default to info level if not specified
|
||||
}
|
||||
var writer zerolog.ConsoleWriter
|
||||
if logFormat == "json" {
|
||||
writer = zerolog.ConsoleWriter{Out: os.Stderr, NoColor: true}
|
||||
} else {
|
||||
writer = zerolog.ConsoleWriter{Out: os.Stderr}
|
||||
}
|
||||
logger.Logger = zerolog.New(writer).With().Timestamp().Logger()
|
||||
|
||||
logger.Logger.Info().
|
||||
Msg("SteamCache2 " + version.Version + " starting...")
|
||||
|
||||
address := ":80"
|
||||
|
||||
if runtime.GOMAXPROCS(-1) != threads {
|
||||
runtime.GOMAXPROCS(threads)
|
||||
logger.Logger.Info().
|
||||
Int("threads", threads).
|
||||
Msg("Maximum number of threads set")
|
||||
}
|
||||
|
||||
sc := steamcache.New(
|
||||
address,
|
||||
":80",
|
||||
memory,
|
||||
memorymultiplier,
|
||||
disk,
|
||||
diskmultiplier,
|
||||
diskpath,
|
||||
upstream,
|
||||
)
|
||||
|
||||
logger.Logger.Info().
|
||||
Msg("SteamCache2 " + version.Version + " started on " + address)
|
||||
|
||||
sc.Run()
|
||||
|
||||
logger.Logger.Info().Msg("SteamCache2 stopped")
|
||||
os.Exit(0)
|
||||
},
|
||||
}
|
||||
|
||||
@@ -96,16 +46,9 @@ func Execute() {
|
||||
}
|
||||
|
||||
func init() {
|
||||
rootCmd.Flags().IntVarP(&threads, "threads", "t", runtime.GOMAXPROCS(-1), "Number of worker threads to use for processing requests")
|
||||
|
||||
rootCmd.Flags().StringVarP(&memory, "memory", "m", "0", "The size of the memory cache")
|
||||
rootCmd.Flags().IntVarP(&memorymultiplier, "memory-gc", "M", 10, "The gc value for the memory cache")
|
||||
rootCmd.Flags().StringVarP(&disk, "disk", "d", "0", "The size of the disk cache")
|
||||
rootCmd.Flags().IntVarP(&diskmultiplier, "disk-gc", "D", 100, "The gc value for the disk cache")
|
||||
rootCmd.Flags().StringVarP(&diskpath, "disk-path", "p", "", "The path to the disk cache")
|
||||
|
||||
rootCmd.Flags().StringVarP(&upstream, "upstream", "u", "", "The upstream server to proxy requests overrides the host header from the client but forwards the original host header to the upstream server")
|
||||
|
||||
rootCmd.Flags().StringVarP(&logLevel, "log-level", "l", "info", "Logging level: debug, info, error")
|
||||
rootCmd.Flags().StringVarP(&logFormat, "log-format", "f", "console", "Logging format: json, console")
|
||||
rootCmd.Flags().StringVarP(&memory, "memory", "m", "100MB", "The size of the memory cache")
|
||||
rootCmd.Flags().IntVarP(&memorymultiplier, "memory-multiplier", "M", 10, "The multiplier for the memory cache")
|
||||
rootCmd.Flags().StringVarP(&disk, "disk", "d", "10GB", "The size of the disk cache")
|
||||
rootCmd.Flags().IntVarP(&diskmultiplier, "disk-multiplier", "D", 10, "The multiplier for the disk cache")
|
||||
rootCmd.Flags().StringVarP(&diskpath, "disk-path", "p", "tmp/steamcache2-disk", "The path to the disk cache")
|
||||
}
|
||||
|
||||
@@ -1,4 +1,3 @@
|
||||
// cmd/version.go
|
||||
package cmd
|
||||
|
||||
import (
|
||||
|
||||
@@ -4,22 +4,15 @@ go 1.23.0
|
||||
|
||||
require (
|
||||
github.com/docker/go-units v0.5.0
|
||||
github.com/prometheus/client_golang v1.22.0
|
||||
github.com/rs/zerolog v1.33.0
|
||||
github.com/spf13/cobra v1.8.1
|
||||
golang.org/x/exp v0.0.0-20250106191152-7588d65b2ba8
|
||||
)
|
||||
|
||||
require (
|
||||
github.com/beorn7/perks v1.0.1 // indirect
|
||||
github.com/cespare/xxhash/v2 v2.3.0 // indirect
|
||||
github.com/inconshreveable/mousetrap v1.1.0 // indirect
|
||||
github.com/mattn/go-colorable v0.1.13 // indirect
|
||||
github.com/mattn/go-isatty v0.0.19 // indirect
|
||||
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
|
||||
github.com/prometheus/client_model v0.6.1 // indirect
|
||||
github.com/prometheus/common v0.62.0 // indirect
|
||||
github.com/prometheus/procfs v0.15.1 // indirect
|
||||
github.com/spf13/pflag v1.0.5 // indirect
|
||||
golang.org/x/sys v0.30.0 // indirect
|
||||
google.golang.org/protobuf v1.36.5 // indirect
|
||||
golang.org/x/sys v0.12.0 // indirect
|
||||
)
|
||||
|
||||
@@ -1,40 +1,16 @@
|
||||
github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
|
||||
github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw=
|
||||
github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs=
|
||||
github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
|
||||
github.com/coreos/go-systemd/v22 v22.5.0/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc=
|
||||
github.com/cpuguy83/go-md2man/v2 v2.0.4/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o=
|
||||
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
|
||||
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||
github.com/docker/go-units v0.5.0 h1:69rxXcBk27SvSaaxTtLh/8llcHD8vYHT7WSdRZ/jvr4=
|
||||
github.com/docker/go-units v0.5.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk=
|
||||
github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA=
|
||||
github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8=
|
||||
github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU=
|
||||
github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8=
|
||||
github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw=
|
||||
github.com/klauspost/compress v1.18.0 h1:c/Cqfb0r+Yi+JtIEq73FWXVkRonBlf0CRNYc8Zttxdo=
|
||||
github.com/klauspost/compress v1.18.0/go.mod h1:2Pp+KzxcywXVXMr50+X0Q/Lsb43OQHYWRCY2AiWywWQ=
|
||||
github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc=
|
||||
github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw=
|
||||
github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA=
|
||||
github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg=
|
||||
github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM=
|
||||
github.com/mattn/go-isatty v0.0.19 h1:JITubQf0MOLdlGRuRq+jtsDlekdYPia9ZFsB8h/APPA=
|
||||
github.com/mattn/go-isatty v0.0.19/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y=
|
||||
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA=
|
||||
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ=
|
||||
github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
|
||||
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
|
||||
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
|
||||
github.com/prometheus/client_golang v1.22.0 h1:rb93p9lokFEsctTys46VnV1kLCDpVZ0a/Y92Vm0Zc6Q=
|
||||
github.com/prometheus/client_golang v1.22.0/go.mod h1:R7ljNsLXhuQXYZYtw6GAE9AZg8Y7vEW5scdCXrWRXC0=
|
||||
github.com/prometheus/client_model v0.6.1 h1:ZKSh/rekM+n3CeS952MLRAdFwIKqeY8b62p8ais2e9E=
|
||||
github.com/prometheus/client_model v0.6.1/go.mod h1:OrxVMOVHjw3lKMa8+x6HeMGkHMQyHDk9E3jmP2AmGiY=
|
||||
github.com/prometheus/common v0.62.0 h1:xasJaQlnWAeyHdUBeGjXmutelfJHWMRr+Fg4QszZ2Io=
|
||||
github.com/prometheus/common v0.62.0/go.mod h1:vyBcEuLSvWos9B1+CyL7JZ2up+uFzXhkqml0W5zIY1I=
|
||||
github.com/prometheus/procfs v0.15.1 h1:YagwOFzUgYfKKHX6Dr+sHT7km/hxC76UB0learggepc=
|
||||
github.com/prometheus/procfs v0.15.1/go.mod h1:fB45yRUv8NstnjriLhBQLuOUt+WW4BsoGhij/e3PBqk=
|
||||
github.com/rs/xid v1.5.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg=
|
||||
github.com/rs/zerolog v1.33.0 h1:1cU2KZkvPxNyfgEmhHAz/1A9Bz+llsdYzklWFzgp0r8=
|
||||
github.com/rs/zerolog v1.33.0/go.mod h1:/7mN4D5sKwJLZQ2b/znpjC3/GQWY/xaDXUM0kKWRHss=
|
||||
@@ -43,15 +19,11 @@ github.com/spf13/cobra v1.8.1 h1:e5/vxKd/rZsfSJMUX1agtjeTDf+qv1/JdBF8gg5k9ZM=
|
||||
github.com/spf13/cobra v1.8.1/go.mod h1:wHxEcudfqmLYa8iTfL+OuZPbBZkmvliBWKIezN3kD9Y=
|
||||
github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA=
|
||||
github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
|
||||
github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA=
|
||||
github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
|
||||
golang.org/x/exp v0.0.0-20250106191152-7588d65b2ba8 h1:yqrTHse8TCMW1M1ZCP+VAR/l0kKxwaAIqN/il7x4voA=
|
||||
golang.org/x/exp v0.0.0-20250106191152-7588d65b2ba8/go.mod h1:tujkw807nyEEAamNbDrEGzRav+ilXA7PCRAd6xsmwiU=
|
||||
golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.12.0 h1:CM0HF96J0hcLAwsHPJZjfdNzs0gftsLfgKt57wWHJ0o=
|
||||
golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.30.0 h1:QjkSwP/36a20jFYWkSue1YwXzLmsV5Gfq7Eiy72C1uc=
|
||||
golang.org/x/sys v0.30.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
|
||||
google.golang.org/protobuf v1.36.5 h1:tPhr+woSbjfYvY6/GPufUoYizxw1cF/yFoxJ2fmpwlM=
|
||||
google.golang.org/protobuf v1.36.5/go.mod h1:9fA7Ob0pmnwhb644+1+CVWFRbNajQ6iRojtC/QF5bRE=
|
||||
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
|
||||
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
|
||||
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
|
||||
|
||||
@@ -0,0 +1,60 @@
|
||||
package avgcachestate
|
||||
|
||||
import (
|
||||
"s1d3sw1ped/SteamCache2/vfs/cachestate"
|
||||
"sync"
|
||||
)
|
||||
|
||||
// AvgCacheState is a cache state that averages the last N cache states.
|
||||
type AvgCacheState struct {
|
||||
size int
|
||||
avgs []cachestate.CacheState
|
||||
mu sync.Mutex
|
||||
}
|
||||
|
||||
// New creates a new average cache state with the given size.
|
||||
func New(size int) *AvgCacheState {
|
||||
return &AvgCacheState{
|
||||
size: size,
|
||||
avgs: make([]cachestate.CacheState, size),
|
||||
mu: sync.Mutex{},
|
||||
}
|
||||
}
|
||||
|
||||
// Clear resets the average cache state to zero.
|
||||
func (a *AvgCacheState) Clear() {
|
||||
a.mu.Lock()
|
||||
defer a.mu.Unlock()
|
||||
|
||||
a.avgs = make([]cachestate.CacheState, a.size) // zeroed
|
||||
}
|
||||
|
||||
// Add adds a cache state to the average cache state.
|
||||
func (a *AvgCacheState) Add(cs cachestate.CacheState) {
|
||||
a.mu.Lock()
|
||||
defer a.mu.Unlock()
|
||||
|
||||
a.avgs = append(a.avgs, cs)
|
||||
if len(a.avgs) > a.size {
|
||||
a.avgs = a.avgs[1:]
|
||||
}
|
||||
}
|
||||
|
||||
// Avg returns the average cache state.
|
||||
func (a *AvgCacheState) Avg() float64 {
|
||||
a.mu.Lock()
|
||||
defer a.mu.Unlock()
|
||||
|
||||
var hits, misses int
|
||||
for _, cs := range a.avgs {
|
||||
switch cs {
|
||||
case cachestate.CacheStateHit:
|
||||
hits++
|
||||
case cachestate.CacheStateMiss:
|
||||
misses++
|
||||
}
|
||||
}
|
||||
total := hits + misses
|
||||
|
||||
return float64(hits) / float64(total)
|
||||
}
|
||||
@@ -0,0 +1,75 @@
|
||||
package steamcache
|
||||
|
||||
import (
|
||||
"s1d3sw1ped/SteamCache2/steamcache/logger"
|
||||
"s1d3sw1ped/SteamCache2/vfs"
|
||||
"s1d3sw1ped/SteamCache2/vfs/cachestate"
|
||||
"time"
|
||||
|
||||
"github.com/docker/go-units"
|
||||
"golang.org/x/exp/rand"
|
||||
)
|
||||
|
||||
func randomgc(vfss vfs.VFS, stats []*vfs.FileInfo) int64 {
|
||||
// Pick a random file to delete
|
||||
randfile := stats[rand.Intn(len(stats))]
|
||||
sz := randfile.Size()
|
||||
err := vfss.Delete(randfile.Name())
|
||||
if err != nil {
|
||||
// If we failed to delete the file, log it and return 0
|
||||
// logger.Logger.Error().Err(err).Msgf("Failed to delete %s", randfile.Name())
|
||||
return 0
|
||||
}
|
||||
|
||||
return sz
|
||||
}
|
||||
|
||||
func memorygc(vfss vfs.VFS, size int) {
|
||||
tstart := time.Now()
|
||||
deletions := 0
|
||||
targetreclaim := int64(size)
|
||||
var reclaimed int64
|
||||
|
||||
stats := vfss.StatAll()
|
||||
for {
|
||||
reclaimed += randomgc(vfss, stats)
|
||||
deletions++
|
||||
if reclaimed >= targetreclaim {
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
logger.Logger.Info().
|
||||
Str("name", vfss.Name()).
|
||||
Str("duration", time.Since(tstart).String()).
|
||||
Str("reclaimed", units.HumanSize(float64(reclaimed))).
|
||||
Int("deletions", deletions).
|
||||
Msgf("GC")
|
||||
}
|
||||
|
||||
func diskgc(vfss vfs.VFS, size int) {
|
||||
tstart := time.Now()
|
||||
deletions := 0
|
||||
targetreclaim := int64(size)
|
||||
var reclaimed int64
|
||||
|
||||
stats := vfss.StatAll()
|
||||
for {
|
||||
reclaimed += randomgc(vfss, stats)
|
||||
deletions++
|
||||
if reclaimed >= targetreclaim {
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
logger.Logger.Info().
|
||||
Str("name", vfss.Name()).
|
||||
Str("duration", time.Since(tstart).String()).
|
||||
Str("reclaimed", units.HumanSize(float64(reclaimed))).
|
||||
Int("deletions", deletions).
|
||||
Msgf("GC")
|
||||
}
|
||||
|
||||
func cachehandler(fi *vfs.FileInfo, cs cachestate.CacheState) bool {
|
||||
return time.Since(fi.AccessTime()) < time.Minute*10 // Put files in the cache if they've been accessed twice in the last 10 minutes
|
||||
}
|
||||
@@ -1,8 +1,9 @@
|
||||
// steamcache/logger/logger.go
|
||||
package logger
|
||||
|
||||
import (
|
||||
"os"
|
||||
|
||||
"github.com/rs/zerolog"
|
||||
)
|
||||
|
||||
var Logger zerolog.Logger
|
||||
var Logger = zerolog.New(zerolog.ConsoleWriter{Out: os.Stderr}).With().Timestamp().Logger()
|
||||
|
||||
+160
-288
@@ -1,74 +1,42 @@
|
||||
// steamcache/steamcache.go
|
||||
package steamcache
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"io"
|
||||
"net"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"os"
|
||||
"s1d3sw1ped/SteamCache2/steamcache/avgcachestate"
|
||||
"s1d3sw1ped/SteamCache2/steamcache/logger"
|
||||
"s1d3sw1ped/SteamCache2/version"
|
||||
"s1d3sw1ped/SteamCache2/vfs"
|
||||
"s1d3sw1ped/SteamCache2/vfs/cache"
|
||||
"s1d3sw1ped/SteamCache2/vfs/cachestate"
|
||||
"s1d3sw1ped/SteamCache2/vfs/disk"
|
||||
"s1d3sw1ped/SteamCache2/vfs/gc"
|
||||
"s1d3sw1ped/SteamCache2/vfs/memory"
|
||||
syncfs "s1d3sw1ped/SteamCache2/vfs/sync"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/docker/go-units"
|
||||
"github.com/prometheus/client_golang/prometheus"
|
||||
"github.com/prometheus/client_golang/prometheus/promauto"
|
||||
"github.com/prometheus/client_golang/prometheus/promhttp"
|
||||
)
|
||||
|
||||
var (
|
||||
requestsTotal = promauto.NewCounterVec(
|
||||
prometheus.CounterOpts{
|
||||
Name: "http_requests_total",
|
||||
Help: "Total number of HTTP requests",
|
||||
},
|
||||
[]string{"method", "status"},
|
||||
)
|
||||
|
||||
cacheStatusTotal = promauto.NewCounterVec(
|
||||
prometheus.CounterOpts{
|
||||
Name: "cache_status_total",
|
||||
Help: "Total cache status counts",
|
||||
},
|
||||
[]string{"status"},
|
||||
)
|
||||
|
||||
responseTime = promauto.NewHistogram(
|
||||
prometheus.HistogramOpts{
|
||||
Name: "response_time_seconds",
|
||||
Help: "Response time in seconds",
|
||||
Buckets: prometheus.DefBuckets,
|
||||
},
|
||||
)
|
||||
)
|
||||
|
||||
type SteamCache struct {
|
||||
address string
|
||||
upstream string
|
||||
|
||||
vfs vfs.VFS
|
||||
address string
|
||||
vfs vfs.VFS
|
||||
|
||||
memory *memory.MemoryFS
|
||||
disk *disk.DiskFS
|
||||
|
||||
memorygc *gc.GCFS
|
||||
diskgc *gc.GCFS
|
||||
hits *avgcachestate.AvgCacheState
|
||||
|
||||
server *http.Server
|
||||
client *http.Client
|
||||
cancel context.CancelFunc
|
||||
wg sync.WaitGroup
|
||||
dirty bool
|
||||
mu sync.Mutex
|
||||
}
|
||||
|
||||
func New(address string, memorySize string, memoryMultiplier int, diskSize string, diskMultiplier int, diskPath, upstream string) *SteamCache {
|
||||
func New(address string, memorySize string, memoryMultiplier int, diskSize string, diskMultiplier int, diskPath string) *SteamCache {
|
||||
memorysize, err := units.FromHumanSize(memorySize)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
@@ -79,133 +47,81 @@ func New(address string, memorySize string, memoryMultiplier int, diskSize strin
|
||||
panic(err)
|
||||
}
|
||||
|
||||
c := cache.New(
|
||||
gc.PromotionDecider,
|
||||
)
|
||||
|
||||
var m *memory.MemoryFS
|
||||
var mgc *gc.GCFS
|
||||
if memorysize > 0 {
|
||||
m = memory.New(memorysize)
|
||||
mgc = gc.New(m, memoryMultiplier, gc.LRUGC)
|
||||
}
|
||||
|
||||
var d *disk.DiskFS
|
||||
var dgc *gc.GCFS
|
||||
if disksize > 0 {
|
||||
d = disk.New(diskPath, disksize)
|
||||
dgc = gc.New(d, diskMultiplier, gc.LRUGC)
|
||||
}
|
||||
|
||||
// configure the cache to match the specified mode (memory only, disk only, or memory and disk) based on the provided sizes
|
||||
if disksize == 0 && memorysize != 0 {
|
||||
//memory only mode - no disk
|
||||
|
||||
c.SetSlow(mgc)
|
||||
} else if disksize != 0 && memorysize == 0 {
|
||||
// disk only mode
|
||||
|
||||
c.SetSlow(dgc)
|
||||
} else if disksize != 0 && memorysize != 0 {
|
||||
// memory and disk mode
|
||||
|
||||
c.SetFast(mgc)
|
||||
c.SetSlow(dgc)
|
||||
} else {
|
||||
// no memory or disk isn't a valid configuration
|
||||
logger.Logger.Error().Bool("memory", false).Bool("disk", false).Msg("configuration invalid :( exiting")
|
||||
os.Exit(1)
|
||||
}
|
||||
|
||||
transport := &http.Transport{
|
||||
MaxIdleConns: 100,
|
||||
MaxIdleConnsPerHost: 10,
|
||||
IdleConnTimeout: 90 * time.Second,
|
||||
DialContext: (&net.Dialer{
|
||||
Timeout: 30 * time.Second,
|
||||
KeepAlive: 30 * time.Second,
|
||||
}).DialContext,
|
||||
TLSHandshakeTimeout: 10 * time.Second,
|
||||
ResponseHeaderTimeout: 10 * time.Second,
|
||||
ExpectContinueTimeout: 1 * time.Second,
|
||||
}
|
||||
|
||||
client := &http.Client{
|
||||
Transport: transport,
|
||||
Timeout: 60 * time.Second,
|
||||
}
|
||||
m := memory.New(memorysize)
|
||||
d := disk.New(diskPath, disksize)
|
||||
|
||||
sc := &SteamCache{
|
||||
upstream: upstream,
|
||||
address: address,
|
||||
vfs: c,
|
||||
memory: m,
|
||||
disk: d,
|
||||
memorygc: mgc,
|
||||
diskgc: dgc,
|
||||
client: client,
|
||||
server: &http.Server{
|
||||
Addr: address,
|
||||
ReadTimeout: 5 * time.Second,
|
||||
WriteTimeout: 10 * time.Second,
|
||||
IdleTimeout: 120 * time.Second,
|
||||
},
|
||||
address: address,
|
||||
vfs: syncfs.New(
|
||||
cache.New(
|
||||
gc.New(
|
||||
m,
|
||||
memoryMultiplier,
|
||||
memorygc,
|
||||
),
|
||||
gc.New(
|
||||
d,
|
||||
diskMultiplier,
|
||||
diskgc,
|
||||
),
|
||||
cachehandler,
|
||||
),
|
||||
),
|
||||
|
||||
memory: m,
|
||||
disk: d,
|
||||
|
||||
hits: avgcachestate.New(10000),
|
||||
}
|
||||
|
||||
if d != nil {
|
||||
if d.Size() > d.Capacity() {
|
||||
gc.LRUGC(d, uint(d.Size()-d.Capacity()))
|
||||
}
|
||||
if d.Size() > d.Capacity() {
|
||||
diskgc(d, int(d.Size()-d.Capacity()))
|
||||
}
|
||||
|
||||
return sc
|
||||
}
|
||||
|
||||
func (sc *SteamCache) Run() {
|
||||
if sc.upstream != "" {
|
||||
resp, err := sc.client.Get(sc.upstream)
|
||||
if err != nil || resp.StatusCode != http.StatusOK {
|
||||
logger.Logger.Error().Err(err).Str("upstream", sc.upstream).Msg("Failed to connect to upstream server")
|
||||
os.Exit(1)
|
||||
}
|
||||
resp.Body.Close()
|
||||
}
|
||||
logger.Logger.Info().Str("address", sc.address).Str("version", version.Version).Msg("listening")
|
||||
|
||||
sc.server.Handler = sc
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
sc.cancel = cancel
|
||||
sc.mu.Lock()
|
||||
sc.dirty = true
|
||||
sc.mu.Unlock()
|
||||
|
||||
sc.wg.Add(1)
|
||||
sc.LogStats()
|
||||
t := time.NewTicker(10 * time.Second)
|
||||
go func() {
|
||||
defer sc.wg.Done()
|
||||
err := sc.server.ListenAndServe()
|
||||
if err != nil && err != http.ErrServerClosed {
|
||||
logger.Logger.Error().Err(err).Msg("Failed to start SteamCache2")
|
||||
os.Exit(1)
|
||||
for range t.C {
|
||||
sc.LogStats()
|
||||
}
|
||||
}()
|
||||
|
||||
<-ctx.Done()
|
||||
sc.server.Shutdown(ctx)
|
||||
sc.wg.Wait()
|
||||
err := http.ListenAndServe(sc.address, sc)
|
||||
if err != nil {
|
||||
if err == http.ErrServerClosed {
|
||||
logger.Logger.Info().Msg("shutdown")
|
||||
return
|
||||
}
|
||||
logger.Logger.Error().Err(err).Msg("Failed to start SteamCache2")
|
||||
os.Exit(1)
|
||||
}
|
||||
}
|
||||
|
||||
func (sc *SteamCache) Shutdown() {
|
||||
if sc.cancel != nil {
|
||||
sc.cancel()
|
||||
func (sc *SteamCache) LogStats() {
|
||||
sc.mu.Lock()
|
||||
defer sc.mu.Unlock()
|
||||
if sc.dirty {
|
||||
logger.Logger.Info().
|
||||
Str("memory", fmt.Sprintf("%s/%s", units.HumanSize(float64(sc.memory.Size())), units.HumanSize(float64(sc.memory.Capacity())))).Int("memory-files", len(sc.memory.StatAll())).
|
||||
Str("disk", fmt.Sprintf("%s/%s", units.HumanSize(float64(sc.disk.Size())), units.HumanSize(float64(sc.disk.Capacity())))).Int("disk-files", len(sc.disk.StatAll())).
|
||||
Str("hitrate", fmt.Sprintf("%.2f%%", sc.hits.Avg()*100)).
|
||||
Msg("stats")
|
||||
sc.dirty = false
|
||||
}
|
||||
sc.wg.Wait()
|
||||
}
|
||||
|
||||
func (sc *SteamCache) ServeHTTP(w http.ResponseWriter, r *http.Request) {
|
||||
if r.URL.Path == "/metrics" {
|
||||
promhttp.Handler().ServeHTTP(w, r)
|
||||
return
|
||||
}
|
||||
|
||||
if r.Method != http.MethodGet {
|
||||
requestsTotal.WithLabelValues(r.Method, "405").Inc()
|
||||
logger.Logger.Warn().Str("method", r.Method).Msg("Only GET method is supported")
|
||||
http.Error(w, "Only GET method is supported", http.StatusMethodNotAllowed)
|
||||
return
|
||||
}
|
||||
@@ -217,154 +133,110 @@ func (sc *SteamCache) ServeHTTP(w http.ResponseWriter, r *http.Request) {
|
||||
return
|
||||
}
|
||||
|
||||
if strings.HasPrefix(r.URL.String(), "/depot/") {
|
||||
// trim the query parameters from the URL path
|
||||
// this is necessary because the cache key should not include query parameters
|
||||
r.URL.Path = strings.Split(r.URL.Path, "?")[0]
|
||||
|
||||
tstart := time.Now()
|
||||
defer func() { responseTime.Observe(time.Since(tstart).Seconds()) }()
|
||||
|
||||
cacheKey := strings.ReplaceAll(r.URL.String()[1:], "\\", "/") // replace all backslashes with forward slashes shouldn't be necessary but just in case
|
||||
|
||||
if cacheKey == "" {
|
||||
requestsTotal.WithLabelValues(r.Method, "400").Inc()
|
||||
logger.Logger.Warn().Str("url", r.URL.String()).Msg("Invalid URL")
|
||||
http.Error(w, "Invalid URL", http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
|
||||
w.Header().Add("X-LanCache-Processed-By", "SteamCache2") // SteamPrefill uses this header to determine if the request was processed by the cache maybe steam uses it too
|
||||
|
||||
reader, err := sc.vfs.Open(cacheKey)
|
||||
if err == nil {
|
||||
defer reader.Close()
|
||||
w.Header().Add("X-LanCache-Status", "HIT")
|
||||
|
||||
io.Copy(w, reader)
|
||||
|
||||
logger.Logger.Info().
|
||||
Str("key", cacheKey).
|
||||
Str("host", r.Host).
|
||||
Str("status", "HIT").
|
||||
Dur("duration", time.Since(tstart)).
|
||||
Msg("request")
|
||||
|
||||
requestsTotal.WithLabelValues(r.Method, "200").Inc()
|
||||
cacheStatusTotal.WithLabelValues("HIT").Inc()
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
var req *http.Request
|
||||
if sc.upstream != "" { // if an upstream server is configured, proxy the request to the upstream server
|
||||
ur, err := url.JoinPath(sc.upstream, r.URL.String())
|
||||
if err != nil {
|
||||
requestsTotal.WithLabelValues(r.Method, "500").Inc()
|
||||
logger.Logger.Error().Err(err).Str("upstream", sc.upstream).Msg("Failed to join URL path")
|
||||
http.Error(w, "Failed to join URL path", http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
|
||||
req, err = http.NewRequest(http.MethodGet, ur, nil)
|
||||
if err != nil {
|
||||
requestsTotal.WithLabelValues(r.Method, "500").Inc()
|
||||
logger.Logger.Error().Err(err).Str("upstream", sc.upstream).Msg("Failed to create request")
|
||||
http.Error(w, "Failed to create request", http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
req.Host = r.Host
|
||||
} else { // if no upstream server is configured, proxy the request to the host specified in the request
|
||||
host := r.Host
|
||||
if r.Header.Get("X-Sls-Https") == "enable" {
|
||||
host = "https://" + host
|
||||
} else {
|
||||
host = "http://" + host
|
||||
}
|
||||
|
||||
ur, err := url.JoinPath(host, r.URL.String())
|
||||
if err != nil {
|
||||
requestsTotal.WithLabelValues(r.Method, "500").Inc()
|
||||
logger.Logger.Error().Err(err).Str("host", host).Msg("Failed to join URL path")
|
||||
http.Error(w, "Failed to join URL path", http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
|
||||
req, err = http.NewRequest(http.MethodGet, ur, nil)
|
||||
if err != nil {
|
||||
requestsTotal.WithLabelValues(r.Method, "500").Inc()
|
||||
logger.Logger.Error().Err(err).Str("host", host).Msg("Failed to create request")
|
||||
http.Error(w, "Failed to create request", http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
// Copy headers from the original request to the new request
|
||||
for key, values := range r.Header {
|
||||
for _, value := range values {
|
||||
req.Header.Add(key, value)
|
||||
}
|
||||
}
|
||||
|
||||
// Retry logic
|
||||
backoffSchedule := []time.Duration{1 * time.Second, 3 * time.Second, 10 * time.Second}
|
||||
var resp *http.Response
|
||||
for i, backoff := range backoffSchedule {
|
||||
resp, err = sc.client.Do(req)
|
||||
if err == nil && resp.StatusCode == http.StatusOK {
|
||||
break
|
||||
}
|
||||
if i < len(backoffSchedule)-1 {
|
||||
time.Sleep(backoff)
|
||||
}
|
||||
}
|
||||
if err != nil || resp.StatusCode != http.StatusOK {
|
||||
requestsTotal.WithLabelValues(r.Method, "500 upstream host "+r.Host).Inc()
|
||||
logger.Logger.Error().Err(err).Str("url", req.URL.String()).Msg("Failed to fetch the requested URL")
|
||||
http.Error(w, "Failed to fetch the requested URL", http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
size := resp.ContentLength
|
||||
|
||||
writer, err := sc.vfs.Create(cacheKey, size)
|
||||
if err != nil {
|
||||
http.Error(w, err.Error(), http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
defer writer.Close()
|
||||
|
||||
w.Header().Add("X-LanCache-Status", "MISS")
|
||||
|
||||
io.Copy(io.MultiWriter(w, writer), resp.Body)
|
||||
|
||||
logger.Logger.Info().
|
||||
Str("key", cacheKey).
|
||||
Str("host", r.Host).
|
||||
Str("status", "MISS").
|
||||
Dur("duration", time.Since(tstart)).
|
||||
Msg("request")
|
||||
|
||||
requestsTotal.WithLabelValues(r.Method, "200").Inc()
|
||||
cacheStatusTotal.WithLabelValues("MISS").Inc()
|
||||
|
||||
if r.Header.Get("User-Agent") != "Valve/Steam HTTP Client 1.0" {
|
||||
http.Error(w, "Only Valve/Steam HTTP Client 1.0 is supported", http.StatusForbidden)
|
||||
return
|
||||
}
|
||||
|
||||
if r.URL.Path == "/favicon.ico" {
|
||||
w.WriteHeader(http.StatusNoContent)
|
||||
if strings.Contains(r.URL.String(), "manifest") {
|
||||
w.Header().Add("X-LanCache-Processed-By", "SteamCache2")
|
||||
forward(w, r)
|
||||
return
|
||||
}
|
||||
|
||||
if r.URL.Path == "/robots.txt" {
|
||||
w.Header().Set("Content-Type", "text/plain")
|
||||
w.WriteHeader(http.StatusOK)
|
||||
w.Write([]byte("User-agent: *\nDisallow: /\n"))
|
||||
// tstart := time.Now()
|
||||
// defer func() {
|
||||
// logger.Logger.Info().Str("method", r.Method).Str("url", r.URL.String()).Str("status", w.Header().Get("X-LanCache-Status")).Dur("duration", time.Since(tstart)).Msg("Request")
|
||||
// }()
|
||||
|
||||
sc.mu.Lock()
|
||||
sc.dirty = true
|
||||
sc.mu.Unlock()
|
||||
|
||||
w.Header().Add("X-LanCache-Processed-By", "SteamCache2") // SteamPrefill uses this header to determine if the request was processed by the cache maybe steam uses it too
|
||||
|
||||
cacheKey := r.URL.String()
|
||||
|
||||
// if vfs is also a vfs.GetSer, we can use it to get the cache state
|
||||
|
||||
data, err := sc.vfs.Get(cacheKey)
|
||||
if err == nil {
|
||||
sc.hits.Add(cachestate.CacheStateHit)
|
||||
w.Header().Add("X-LanCache-Status", "HIT")
|
||||
w.Write(data)
|
||||
return
|
||||
}
|
||||
|
||||
requestsTotal.WithLabelValues(r.Method, "404").Inc()
|
||||
logger.Logger.Warn().Str("url", r.URL.String()).Msg("Not found")
|
||||
http.Error(w, "Not found", http.StatusNotFound)
|
||||
htt := "http://"
|
||||
if r.Header.Get("X-Sls-Https") == "enable" {
|
||||
htt = "https://"
|
||||
}
|
||||
|
||||
base := htt + r.Host
|
||||
|
||||
hosturl, err := url.JoinPath(base, cacheKey)
|
||||
if err != nil {
|
||||
http.Error(w, "Failed to join URL path", http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
|
||||
resp, err := http.Get(hosturl)
|
||||
if err != nil {
|
||||
http.Error(w, "Failed to fetch the requested URL", http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
http.Error(w, "Failed to fetch the requested URL", resp.StatusCode)
|
||||
return
|
||||
}
|
||||
|
||||
body, err := io.ReadAll(resp.Body)
|
||||
if err != nil {
|
||||
http.Error(w, "Failed to read response body", http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
|
||||
sc.vfs.Set(cacheKey, body)
|
||||
sc.hits.Add(cachestate.CacheStateMiss)
|
||||
w.Header().Add("X-LanCache-Status", "MISS")
|
||||
w.Write(body)
|
||||
}
|
||||
|
||||
func forward(w http.ResponseWriter, r *http.Request) {
|
||||
htt := "http://"
|
||||
if r.Header.Get("X-Sls-Https") == "enable" {
|
||||
htt = "https://"
|
||||
}
|
||||
|
||||
base := htt + r.Host
|
||||
|
||||
cacheKey := r.URL.String()
|
||||
|
||||
hosturl, err := url.JoinPath(base, cacheKey)
|
||||
if err != nil {
|
||||
http.Error(w, "Failed to join URL path", http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
|
||||
resp, err := http.Get(hosturl)
|
||||
if err != nil {
|
||||
http.Error(w, "Failed to fetch the requested URL", http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
http.Error(w, "Failed to fetch the requested URL", resp.StatusCode)
|
||||
return
|
||||
}
|
||||
|
||||
body, err := io.ReadAll(resp.Body)
|
||||
if err != nil {
|
||||
http.Error(w, "Failed to read response body", http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
|
||||
w.Write(body)
|
||||
}
|
||||
|
||||
@@ -1,110 +0,0 @@
|
||||
// steamcache/steamcache_test.go
|
||||
package steamcache
|
||||
|
||||
import (
|
||||
"io"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestCaching(t *testing.T) {
|
||||
td := t.TempDir()
|
||||
|
||||
os.WriteFile(filepath.Join(td, "key2"), []byte("value2"), 0644)
|
||||
|
||||
sc := New("localhost:8080", "1G", 10, "1G", 100, td, "")
|
||||
|
||||
w, err := sc.vfs.Create("key", 5)
|
||||
if err != nil {
|
||||
t.Errorf("Create failed: %v", err)
|
||||
}
|
||||
w.Write([]byte("value"))
|
||||
w.Close()
|
||||
|
||||
w, err = sc.vfs.Create("key1", 6)
|
||||
if err != nil {
|
||||
t.Errorf("Create failed: %v", err)
|
||||
}
|
||||
w.Write([]byte("value1"))
|
||||
w.Close()
|
||||
|
||||
if sc.diskgc.Size() != 17 {
|
||||
t.Errorf("Size failed: got %d, want %d", sc.diskgc.Size(), 17)
|
||||
}
|
||||
|
||||
if sc.vfs.Size() != 17 {
|
||||
t.Errorf("Size failed: got %d, want %d", sc.vfs.Size(), 17)
|
||||
}
|
||||
|
||||
rc, err := sc.vfs.Open("key")
|
||||
if err != nil {
|
||||
t.Errorf("Open failed: %v", err)
|
||||
}
|
||||
d, _ := io.ReadAll(rc)
|
||||
rc.Close()
|
||||
if string(d) != "value" {
|
||||
t.Errorf("Get failed: got %s, want %s", d, "value")
|
||||
}
|
||||
|
||||
rc, err = sc.vfs.Open("key1")
|
||||
if err != nil {
|
||||
t.Errorf("Open failed: %v", err)
|
||||
}
|
||||
d, _ = io.ReadAll(rc)
|
||||
rc.Close()
|
||||
if string(d) != "value1" {
|
||||
t.Errorf("Get failed: got %s, want %s", d, "value1")
|
||||
}
|
||||
|
||||
rc, err = sc.vfs.Open("key2")
|
||||
if err != nil {
|
||||
t.Errorf("Open failed: %v", err)
|
||||
}
|
||||
d, _ = io.ReadAll(rc)
|
||||
rc.Close()
|
||||
if string(d) != "value2" {
|
||||
t.Errorf("Get failed: got %s, want %s", d, "value2")
|
||||
}
|
||||
|
||||
if sc.diskgc.Size() != 17 {
|
||||
t.Errorf("Size failed: got %d, want %d", sc.diskgc.Size(), 17)
|
||||
}
|
||||
|
||||
if sc.vfs.Size() != 17 {
|
||||
t.Errorf("Size failed: got %d, want %d", sc.vfs.Size(), 17)
|
||||
}
|
||||
|
||||
sc.memory.Delete("key2")
|
||||
os.Remove(filepath.Join(td, "key2"))
|
||||
|
||||
if _, err := sc.vfs.Open("key2"); err == nil {
|
||||
t.Errorf("Open failed: got nil, want error")
|
||||
}
|
||||
}
|
||||
|
||||
func TestCacheMissAndHit(t *testing.T) {
|
||||
sc := New("localhost:8080", "0", 0, "1G", 100, t.TempDir(), "")
|
||||
|
||||
key := "testkey"
|
||||
value := []byte("testvalue")
|
||||
|
||||
// Simulate miss: but since no upstream, skip full ServeHTTP, test VFS
|
||||
w, err := sc.vfs.Create(key, int64(len(value)))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
w.Write(value)
|
||||
w.Close()
|
||||
|
||||
rc, err := sc.vfs.Open(key)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
got, _ := io.ReadAll(rc)
|
||||
rc.Close()
|
||||
|
||||
if string(got) != string(value) {
|
||||
t.Errorf("expected %s, got %s", value, got)
|
||||
}
|
||||
}
|
||||
@@ -1,10 +1,3 @@
|
||||
// version/version.go
|
||||
package version
|
||||
|
||||
var Version string
|
||||
|
||||
func init() {
|
||||
if Version == "" {
|
||||
Version = "0.0.0-dev"
|
||||
}
|
||||
}
|
||||
|
||||
Vendored
+63
-104
@@ -1,13 +1,10 @@
|
||||
// vfs/cache/cache.go
|
||||
package cache
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"io"
|
||||
"s1d3sw1ped/SteamCache2/vfs"
|
||||
"s1d3sw1ped/SteamCache2/vfs/cachestate"
|
||||
"s1d3sw1ped/SteamCache2/vfs/vfserror"
|
||||
"sync"
|
||||
)
|
||||
|
||||
// Ensure CacheFS implements VFS.
|
||||
@@ -19,38 +16,28 @@ type CacheFS struct {
|
||||
slow vfs.VFS
|
||||
|
||||
cacheHandler CacheHandler
|
||||
|
||||
keyLocks sync.Map // map[string]*sync.RWMutex for per-key locks
|
||||
}
|
||||
|
||||
type CacheHandler func(*vfs.FileInfo, cachestate.CacheState) bool
|
||||
|
||||
// New creates a new CacheFS. fast is used for caching, and slow is used for storage. fast should obviously be faster than slow.
|
||||
func New(cacheHandler CacheHandler) *CacheFS {
|
||||
func New(fast, slow vfs.VFS, cacheHandler CacheHandler) *CacheFS {
|
||||
if slow == nil {
|
||||
panic("slow is nil")
|
||||
}
|
||||
|
||||
if fast == slow {
|
||||
panic("fast and slow are the same")
|
||||
}
|
||||
|
||||
return &CacheFS{
|
||||
fast: fast,
|
||||
slow: slow,
|
||||
|
||||
cacheHandler: cacheHandler,
|
||||
keyLocks: sync.Map{},
|
||||
}
|
||||
}
|
||||
|
||||
func (c *CacheFS) SetSlow(vfs vfs.VFS) {
|
||||
if vfs == nil {
|
||||
panic("vfs is nil") // panic if the vfs is nil
|
||||
}
|
||||
|
||||
c.slow = vfs
|
||||
}
|
||||
|
||||
func (c *CacheFS) SetFast(vfs vfs.VFS) {
|
||||
c.fast = vfs
|
||||
}
|
||||
|
||||
// getKeyLock returns a RWMutex for the given key, creating it if necessary.
|
||||
func (c *CacheFS) getKeyLock(key string) *sync.RWMutex {
|
||||
mu, _ := c.keyLocks.LoadOrStore(key, &sync.RWMutex{})
|
||||
return mu.(*sync.RWMutex)
|
||||
}
|
||||
|
||||
// cacheState returns the state of the file at key.
|
||||
func (c *CacheFS) cacheState(key string) cachestate.CacheState {
|
||||
if c.fast != nil {
|
||||
@@ -75,64 +62,8 @@ func (c *CacheFS) Size() int64 {
|
||||
return c.slow.Size()
|
||||
}
|
||||
|
||||
// Delete deletes the file at key from the cache.
|
||||
func (c *CacheFS) Delete(key string) error {
|
||||
mu := c.getKeyLock(key)
|
||||
mu.Lock()
|
||||
defer mu.Unlock()
|
||||
|
||||
if c.fast != nil {
|
||||
c.fast.Delete(key)
|
||||
}
|
||||
return c.slow.Delete(key)
|
||||
}
|
||||
|
||||
// Open returns the file at key. If the file is not in the cache, it is fetched from the storage.
|
||||
func (c *CacheFS) Open(key string) (io.ReadCloser, error) {
|
||||
mu := c.getKeyLock(key)
|
||||
mu.RLock()
|
||||
defer mu.RUnlock()
|
||||
|
||||
state := c.cacheState(key)
|
||||
|
||||
switch state {
|
||||
case cachestate.CacheStateHit:
|
||||
// if c.fast == nil then cacheState cannot be CacheStateHit so we can safely ignore the check
|
||||
return c.fast.Open(key)
|
||||
case cachestate.CacheStateMiss:
|
||||
slowReader, err := c.slow.Open(key)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
sstat, _ := c.slow.Stat(key)
|
||||
if sstat != nil && c.fast != nil { // file found in slow storage and fast storage is available
|
||||
// We are accessing the file from the slow storage, and the file has been accessed less then a minute ago so it popular, so we should update the fast storage with the latest file.
|
||||
if c.cacheHandler != nil && c.cacheHandler(sstat, state) {
|
||||
fastWriter, err := c.fast.Create(key, sstat.Size())
|
||||
if err == nil {
|
||||
return &teeReadCloser{
|
||||
Reader: io.TeeReader(slowReader, fastWriter),
|
||||
closers: []io.Closer{slowReader, fastWriter},
|
||||
}, nil
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return slowReader, nil
|
||||
case cachestate.CacheStateNotFound:
|
||||
return nil, vfserror.ErrNotFound
|
||||
}
|
||||
|
||||
panic(vfserror.ErrUnreachable)
|
||||
}
|
||||
|
||||
// Create creates a new file at key. If the file is already in the cache, it is replaced.
|
||||
func (c *CacheFS) Create(key string, size int64) (io.WriteCloser, error) {
|
||||
mu := c.getKeyLock(key)
|
||||
mu.Lock()
|
||||
defer mu.Unlock()
|
||||
|
||||
// Set sets the file at key to src. If the file is already in the cache, it is replaced.
|
||||
func (c *CacheFS) Set(key string, src []byte) error {
|
||||
state := c.cacheState(key)
|
||||
|
||||
switch state {
|
||||
@@ -140,9 +71,56 @@ func (c *CacheFS) Create(key string, size int64) (io.WriteCloser, error) {
|
||||
if c.fast != nil {
|
||||
c.fast.Delete(key)
|
||||
}
|
||||
return c.slow.Create(key, size)
|
||||
return c.slow.Set(key, src)
|
||||
case cachestate.CacheStateMiss, cachestate.CacheStateNotFound:
|
||||
return c.slow.Create(key, size)
|
||||
return c.slow.Set(key, src)
|
||||
}
|
||||
|
||||
panic(vfserror.ErrUnreachable)
|
||||
}
|
||||
|
||||
// Delete deletes the file at key from the cache.
|
||||
func (c *CacheFS) Delete(key string) error {
|
||||
if c.fast != nil {
|
||||
c.fast.Delete(key)
|
||||
}
|
||||
return c.slow.Delete(key)
|
||||
}
|
||||
|
||||
// Get returns the file at key. If the file is not in the cache, it is fetched from the storage.
|
||||
func (c *CacheFS) Get(key string) ([]byte, error) {
|
||||
src, _, err := c.GetS(key)
|
||||
return src, err
|
||||
}
|
||||
|
||||
// GetS returns the file at key. If the file is not in the cache, it is fetched from the storage. It also returns the cache state.
|
||||
func (c *CacheFS) GetS(key string) ([]byte, cachestate.CacheState, error) {
|
||||
state := c.cacheState(key)
|
||||
|
||||
switch state {
|
||||
case cachestate.CacheStateHit:
|
||||
// if c.fast == nil then cacheState cannot be CacheStateHit so we can safely ignore the check
|
||||
src, err := c.fast.Get(key)
|
||||
return src, state, err
|
||||
case cachestate.CacheStateMiss:
|
||||
src, err := c.slow.Get(key)
|
||||
if err != nil {
|
||||
return nil, state, err
|
||||
}
|
||||
|
||||
sstat, _ := c.slow.Stat(key)
|
||||
if sstat != nil && c.fast != nil { // file found in slow storage and fast storage is available
|
||||
// We are accessing the file from the slow storage, and the file has been accessed less then a minute ago so it popular, so we should update the fast storage with the latest file.
|
||||
if c.cacheHandler != nil && c.cacheHandler(sstat, state) {
|
||||
if err := c.fast.Set(key, src); err != nil {
|
||||
return nil, state, err
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return src, state, nil
|
||||
case cachestate.CacheStateNotFound:
|
||||
return nil, state, vfserror.ErrNotFound
|
||||
}
|
||||
|
||||
panic(vfserror.ErrUnreachable)
|
||||
@@ -151,10 +129,6 @@ func (c *CacheFS) Create(key string, size int64) (io.WriteCloser, error) {
|
||||
// Stat returns information about the file at key.
|
||||
// Warning: This will return information about the file in the fastest storage its in.
|
||||
func (c *CacheFS) Stat(key string) (*vfs.FileInfo, error) {
|
||||
mu := c.getKeyLock(key)
|
||||
mu.RLock()
|
||||
defer mu.RUnlock()
|
||||
|
||||
state := c.cacheState(key)
|
||||
|
||||
switch state {
|
||||
@@ -175,18 +149,3 @@ func (c *CacheFS) Stat(key string) (*vfs.FileInfo, error) {
|
||||
func (c *CacheFS) StatAll() []*vfs.FileInfo {
|
||||
return c.slow.StatAll()
|
||||
}
|
||||
|
||||
type teeReadCloser struct {
|
||||
io.Reader
|
||||
closers []io.Closer
|
||||
}
|
||||
|
||||
func (t *teeReadCloser) Close() error {
|
||||
var err error
|
||||
for _, c := range t.closers {
|
||||
if e := c.Close(); e != nil {
|
||||
err = e
|
||||
}
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
Vendored
+69
-65
@@ -1,9 +1,7 @@
|
||||
// vfs/cache/cache_test.go
|
||||
package cache
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"io"
|
||||
"testing"
|
||||
|
||||
"s1d3sw1ped/SteamCache2/vfs"
|
||||
@@ -17,175 +15,184 @@ func testMemory() vfs.VFS {
|
||||
}
|
||||
|
||||
func TestNew(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
fast := testMemory()
|
||||
slow := testMemory()
|
||||
|
||||
cache := New(nil)
|
||||
cache.SetFast(fast)
|
||||
cache.SetSlow(slow)
|
||||
cache := New(fast, slow, nil)
|
||||
if cache == nil {
|
||||
t.Fatal("expected cache to be non-nil")
|
||||
}
|
||||
}
|
||||
|
||||
func TestNewPanics(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
defer func() {
|
||||
if r := recover(); r == nil {
|
||||
t.Fatal("expected panic but did not get one")
|
||||
}
|
||||
}()
|
||||
|
||||
cache := New(nil)
|
||||
cache.SetFast(nil)
|
||||
cache.SetSlow(nil)
|
||||
New(nil, nil, nil)
|
||||
}
|
||||
|
||||
func TestCreateAndOpen(t *testing.T) {
|
||||
func TestSetAndGet(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
fast := testMemory()
|
||||
slow := testMemory()
|
||||
cache := New(nil)
|
||||
cache.SetFast(fast)
|
||||
cache.SetSlow(slow)
|
||||
cache := New(fast, slow, nil)
|
||||
|
||||
key := "test"
|
||||
value := []byte("value")
|
||||
|
||||
w, err := cache.Create(key, int64(len(value)))
|
||||
if err != nil {
|
||||
if err := cache.Set(key, value); err != nil {
|
||||
t.Fatalf("unexpected error: %v", err)
|
||||
}
|
||||
w.Write(value)
|
||||
w.Close()
|
||||
|
||||
rc, err := cache.Open(key)
|
||||
got, err := cache.Get(key)
|
||||
if err != nil {
|
||||
t.Fatalf("unexpected error: %v", err)
|
||||
}
|
||||
got, _ := io.ReadAll(rc)
|
||||
rc.Close()
|
||||
|
||||
if string(got) != string(value) {
|
||||
t.Fatalf("expected %s, got %s", value, got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCreateAndOpenNoFast(t *testing.T) {
|
||||
func TestSetAndGetNoFast(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
slow := testMemory()
|
||||
cache := New(nil)
|
||||
cache.SetSlow(slow)
|
||||
cache := New(nil, slow, nil)
|
||||
|
||||
key := "test"
|
||||
value := []byte("value")
|
||||
|
||||
w, err := cache.Create(key, int64(len(value)))
|
||||
if err != nil {
|
||||
if err := cache.Set(key, value); err != nil {
|
||||
t.Fatalf("unexpected error: %v", err)
|
||||
}
|
||||
w.Write(value)
|
||||
w.Close()
|
||||
|
||||
rc, err := cache.Open(key)
|
||||
got, err := cache.Get(key)
|
||||
if err != nil {
|
||||
t.Fatalf("unexpected error: %v", err)
|
||||
}
|
||||
got, _ := io.ReadAll(rc)
|
||||
rc.Close()
|
||||
|
||||
if string(got) != string(value) {
|
||||
t.Fatalf("expected %s, got %s", value, got)
|
||||
}
|
||||
}
|
||||
func TestCaching(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
func TestCachingPromotion(t *testing.T) {
|
||||
fast := testMemory()
|
||||
slow := testMemory()
|
||||
cache := New(func(fi *vfs.FileInfo, cs cachestate.CacheState) bool {
|
||||
cache := New(fast, slow, func(fi *vfs.FileInfo, cs cachestate.CacheState) bool {
|
||||
return true
|
||||
})
|
||||
cache.SetFast(fast)
|
||||
cache.SetSlow(slow)
|
||||
|
||||
key := "test"
|
||||
value := []byte("value")
|
||||
|
||||
ws, _ := slow.Create(key, int64(len(value)))
|
||||
ws.Write(value)
|
||||
ws.Close()
|
||||
if err := fast.Set(key, value); err != nil {
|
||||
t.Fatalf("unexpected error: %v", err)
|
||||
}
|
||||
|
||||
rc, err := cache.Open(key)
|
||||
if err := slow.Set(key, value); err != nil {
|
||||
t.Fatalf("unexpected error: %v", err)
|
||||
}
|
||||
|
||||
_, state, err := cache.GetS(key)
|
||||
if err != nil {
|
||||
t.Fatalf("unexpected error: %v", err)
|
||||
}
|
||||
got, _ := io.ReadAll(rc)
|
||||
rc.Close()
|
||||
if state != cachestate.CacheStateHit {
|
||||
t.Fatalf("expected %v, got %v", cachestate.CacheStateHit, state)
|
||||
}
|
||||
|
||||
err = fast.Delete(key)
|
||||
if err != nil {
|
||||
t.Fatalf("unexpected error: %v", err)
|
||||
}
|
||||
|
||||
got, state, err := cache.GetS(key)
|
||||
if err != nil {
|
||||
t.Fatalf("unexpected error: %v", err)
|
||||
}
|
||||
if state != cachestate.CacheStateMiss {
|
||||
t.Fatalf("expected %v, got %v", cachestate.CacheStateMiss, state)
|
||||
}
|
||||
|
||||
if string(got) != string(value) {
|
||||
t.Fatalf("expected %s, got %s", value, got)
|
||||
}
|
||||
|
||||
// Check if promoted to fast
|
||||
_, err = fast.Open(key)
|
||||
err = cache.Delete(key)
|
||||
if err != nil {
|
||||
t.Error("Expected promotion to fast cache")
|
||||
t.Fatalf("unexpected error: %v", err)
|
||||
}
|
||||
|
||||
_, state, err = cache.GetS(key)
|
||||
if !errors.Is(err, vfserror.ErrNotFound) {
|
||||
t.Fatalf("expected %v, got %v", vfserror.ErrNotFound, err)
|
||||
}
|
||||
if state != cachestate.CacheStateNotFound {
|
||||
t.Fatalf("expected %v, got %v", cachestate.CacheStateNotFound, state)
|
||||
}
|
||||
}
|
||||
|
||||
func TestOpenNotFound(t *testing.T) {
|
||||
func TestGetNotFound(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
fast := testMemory()
|
||||
slow := testMemory()
|
||||
cache := New(nil)
|
||||
cache.SetFast(fast)
|
||||
cache.SetSlow(slow)
|
||||
cache := New(fast, slow, nil)
|
||||
|
||||
_, err := cache.Open("nonexistent")
|
||||
_, err := cache.Get("nonexistent")
|
||||
if !errors.Is(err, vfserror.ErrNotFound) {
|
||||
t.Fatalf("expected %v, got %v", vfserror.ErrNotFound, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDelete(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
fast := testMemory()
|
||||
slow := testMemory()
|
||||
cache := New(nil)
|
||||
cache.SetFast(fast)
|
||||
cache.SetSlow(slow)
|
||||
cache := New(fast, slow, nil)
|
||||
|
||||
key := "test"
|
||||
value := []byte("value")
|
||||
|
||||
w, err := cache.Create(key, int64(len(value)))
|
||||
if err != nil {
|
||||
if err := cache.Set(key, value); err != nil {
|
||||
t.Fatalf("unexpected error: %v", err)
|
||||
}
|
||||
w.Write(value)
|
||||
w.Close()
|
||||
|
||||
if err := cache.Delete(key); err != nil {
|
||||
t.Fatalf("unexpected error: %v", err)
|
||||
}
|
||||
|
||||
_, err = cache.Open(key)
|
||||
_, err := cache.Get(key)
|
||||
if !errors.Is(err, vfserror.ErrNotFound) {
|
||||
t.Fatalf("expected %v, got %v", vfserror.ErrNotFound, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestStat(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
fast := testMemory()
|
||||
slow := testMemory()
|
||||
cache := New(nil)
|
||||
cache.SetFast(fast)
|
||||
cache.SetSlow(slow)
|
||||
cache := New(fast, slow, nil)
|
||||
|
||||
key := "test"
|
||||
value := []byte("value")
|
||||
|
||||
w, err := cache.Create(key, int64(len(value)))
|
||||
if err != nil {
|
||||
if err := cache.Set(key, value); err != nil {
|
||||
t.Fatalf("unexpected error: %v", err)
|
||||
}
|
||||
w.Write(value)
|
||||
w.Close()
|
||||
|
||||
info, err := cache.Stat(key)
|
||||
if err != nil {
|
||||
@@ -195,7 +202,4 @@ func TestStat(t *testing.T) {
|
||||
if info == nil {
|
||||
t.Fatal("expected file info to be non-nil")
|
||||
}
|
||||
if info.Size() != int64(len(value)) {
|
||||
t.Errorf("expected size %d, got %d", len(value), info.Size())
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,4 +1,3 @@
|
||||
// vfs/cachestate/cachestate.go
|
||||
package cachestate
|
||||
|
||||
import "s1d3sw1ped/SteamCache2/vfs/vfserror"
|
||||
|
||||
+102
-354
@@ -1,52 +1,15 @@
|
||||
// vfs/disk/disk.go
|
||||
package disk
|
||||
|
||||
import (
|
||||
"container/list"
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"s1d3sw1ped/SteamCache2/steamcache/logger"
|
||||
"s1d3sw1ped/SteamCache2/vfs"
|
||||
"s1d3sw1ped/SteamCache2/vfs/vfserror"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/docker/go-units"
|
||||
"github.com/prometheus/client_golang/prometheus"
|
||||
"github.com/prometheus/client_golang/prometheus/promauto"
|
||||
)
|
||||
|
||||
var (
|
||||
diskCapacityBytes = promauto.NewGauge(
|
||||
prometheus.GaugeOpts{
|
||||
Name: "disk_cache_capacity_bytes",
|
||||
Help: "Total capacity of the disk cache in bytes",
|
||||
},
|
||||
)
|
||||
|
||||
diskSizeBytes = promauto.NewGauge(
|
||||
prometheus.GaugeOpts{
|
||||
Name: "disk_cache_size_bytes",
|
||||
Help: "Total size of the disk cache in bytes",
|
||||
},
|
||||
)
|
||||
|
||||
diskReadBytes = promauto.NewCounter(
|
||||
prometheus.CounterOpts{
|
||||
Name: "disk_cache_read_bytes_total",
|
||||
Help: "Total number of bytes read from the disk cache",
|
||||
},
|
||||
)
|
||||
|
||||
diskWriteBytes = promauto.NewCounter(
|
||||
prometheus.CounterOpts{
|
||||
Name: "disk_cache_write_bytes_total",
|
||||
Help: "Total number of bytes written to the disk cache",
|
||||
},
|
||||
)
|
||||
)
|
||||
|
||||
// Ensure DiskFS implements VFS.
|
||||
@@ -58,92 +21,24 @@ type DiskFS struct {
|
||||
|
||||
info map[string]*vfs.FileInfo
|
||||
capacity int64
|
||||
size int64
|
||||
mu sync.RWMutex
|
||||
keyLocks sync.Map // map[string]*sync.RWMutex
|
||||
LRU *lruList
|
||||
}
|
||||
|
||||
// lruList for LRU eviction
|
||||
type lruList struct {
|
||||
list *list.List
|
||||
elem map[string]*list.Element
|
||||
}
|
||||
|
||||
func newLruList() *lruList {
|
||||
return &lruList{
|
||||
list: list.New(),
|
||||
elem: make(map[string]*list.Element),
|
||||
}
|
||||
}
|
||||
|
||||
func (l *lruList) MoveToFront(key string) {
|
||||
if e, ok := l.elem[key]; ok {
|
||||
l.list.MoveToFront(e)
|
||||
}
|
||||
}
|
||||
|
||||
func (l *lruList) Add(key string, fi *vfs.FileInfo) *list.Element {
|
||||
e := l.list.PushFront(fi)
|
||||
l.elem[key] = e
|
||||
return e
|
||||
}
|
||||
|
||||
func (l *lruList) Remove(key string) {
|
||||
if e, ok := l.elem[key]; ok {
|
||||
l.list.Remove(e)
|
||||
delete(l.elem, key)
|
||||
}
|
||||
}
|
||||
|
||||
func (l *lruList) Back() *vfs.FileInfo {
|
||||
if e := l.list.Back(); e != nil {
|
||||
return e.Value.(*vfs.FileInfo)
|
||||
}
|
||||
return nil
|
||||
mu sync.Mutex
|
||||
sg sync.WaitGroup
|
||||
}
|
||||
|
||||
// New creates a new DiskFS.
|
||||
func new(root string, capacity int64, skipinit bool) *DiskFS {
|
||||
if capacity <= 0 {
|
||||
panic("disk capacity must be greater than 0") // panic if the capacity is less than or equal to 0
|
||||
}
|
||||
|
||||
if root == "" {
|
||||
panic("disk root must not be empty") // panic if the root is empty
|
||||
}
|
||||
|
||||
fi, err := os.Stat(root)
|
||||
if err != nil {
|
||||
if !os.IsNotExist(err) {
|
||||
panic(err) // panic if the error is something other than not found
|
||||
}
|
||||
os.Mkdir(root, 0755) // create the root directory if it does not exist
|
||||
fi, err = os.Stat(root) // re-stat to get the file info
|
||||
if err != nil {
|
||||
panic(err) // panic if the re-stat fails
|
||||
}
|
||||
}
|
||||
if !fi.IsDir() {
|
||||
panic("disk root must be a directory") // panic if the root is not a directory
|
||||
}
|
||||
|
||||
dfs := &DiskFS{
|
||||
root: root,
|
||||
info: make(map[string]*vfs.FileInfo),
|
||||
capacity: capacity,
|
||||
mu: sync.RWMutex{},
|
||||
keyLocks: sync.Map{},
|
||||
LRU: newLruList(),
|
||||
mu: sync.Mutex{},
|
||||
sg: sync.WaitGroup{},
|
||||
}
|
||||
|
||||
os.MkdirAll(dfs.root, 0755)
|
||||
|
||||
diskCapacityBytes.Set(float64(dfs.capacity))
|
||||
|
||||
if !skipinit {
|
||||
dfs.init()
|
||||
diskSizeBytes.Set(float64(dfs.Size()))
|
||||
}
|
||||
|
||||
return dfs
|
||||
@@ -158,39 +53,43 @@ func NewSkipInit(root string, capacity int64) *DiskFS {
|
||||
}
|
||||
|
||||
func (d *DiskFS) init() {
|
||||
// logger.Logger.Info().Str("name", d.Name()).Str("root", d.root).Str("capacity", units.HumanSize(float64(d.capacity))).Msg("init")
|
||||
|
||||
tstart := time.Now()
|
||||
|
||||
err := filepath.Walk(d.root, func(npath string, info os.FileInfo, err error) error {
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
d.walk(d.root)
|
||||
d.sg.Wait()
|
||||
|
||||
if info.IsDir() {
|
||||
logger.Logger.Info().Str("name", d.Name()).Str("root", d.root).Str("capacity", units.HumanSize(float64(d.capacity))).Str("duration", time.Since(tstart).String()).Msg("init")
|
||||
}
|
||||
|
||||
func (d *DiskFS) walk(path string) {
|
||||
d.sg.Add(1)
|
||||
go func() {
|
||||
defer d.sg.Done()
|
||||
filepath.Walk(path, func(npath string, info os.FileInfo, err error) error {
|
||||
if path == npath {
|
||||
return nil
|
||||
}
|
||||
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if info.IsDir() {
|
||||
d.walk(npath)
|
||||
return filepath.SkipDir
|
||||
}
|
||||
|
||||
d.mu.Lock()
|
||||
k := npath[len(d.root)+1:]
|
||||
d.info[k] = vfs.NewFileInfoFromOS(info, k)
|
||||
d.mu.Unlock()
|
||||
|
||||
// logger.Logger.Debug().Str("name", d.Name()).Str("root", d.root).Str("capacity", units.HumanSize(float64(d.capacity))).Str("path", npath).Msg("init")
|
||||
return nil
|
||||
}
|
||||
|
||||
d.mu.Lock()
|
||||
k := strings.ReplaceAll(npath[len(d.root)+1:], "\\", "/")
|
||||
fi := vfs.NewFileInfoFromOS(info, k)
|
||||
d.info[k] = fi
|
||||
d.LRU.Add(k, fi)
|
||||
d.size += info.Size()
|
||||
d.mu.Unlock()
|
||||
|
||||
return nil
|
||||
})
|
||||
if err != nil {
|
||||
logger.Logger.Error().Err(err).Msg("Walk failed")
|
||||
}
|
||||
|
||||
logger.Logger.Info().
|
||||
Str("name", d.Name()).
|
||||
Str("root", d.root).
|
||||
Str("capacity", units.HumanSize(float64(d.capacity))).
|
||||
Str("size", units.HumanSize(float64(d.Size()))).
|
||||
Str("files", fmt.Sprint(len(d.info))).
|
||||
Str("duration", time.Since(tstart).String()).
|
||||
Msg("init")
|
||||
})
|
||||
}()
|
||||
}
|
||||
|
||||
func (d *DiskFS) Capacity() int64 {
|
||||
@@ -202,259 +101,108 @@ func (d *DiskFS) Name() string {
|
||||
}
|
||||
|
||||
func (d *DiskFS) Size() int64 {
|
||||
d.mu.RLock()
|
||||
defer d.mu.RUnlock()
|
||||
return d.size
|
||||
var size int64
|
||||
d.mu.Lock()
|
||||
defer d.mu.Unlock()
|
||||
for _, v := range d.info {
|
||||
size += v.Size()
|
||||
}
|
||||
return size
|
||||
}
|
||||
|
||||
func (d *DiskFS) getKeyLock(key string) *sync.RWMutex {
|
||||
mu, _ := d.keyLocks.LoadOrStore(key, &sync.RWMutex{})
|
||||
return mu.(*sync.RWMutex)
|
||||
}
|
||||
|
||||
func (d *DiskFS) Create(key string, size int64) (io.WriteCloser, error) {
|
||||
if key == "" {
|
||||
return nil, vfserror.ErrInvalidKey
|
||||
}
|
||||
if key[0] == '/' {
|
||||
return nil, vfserror.ErrInvalidKey
|
||||
}
|
||||
|
||||
// Sanitize key to prevent path traversal
|
||||
key = filepath.Clean(key)
|
||||
key = strings.ReplaceAll(key, "\\", "/") // Ensure forward slashes for consistency
|
||||
if strings.Contains(key, "..") {
|
||||
return nil, vfserror.ErrInvalidKey
|
||||
}
|
||||
|
||||
d.mu.RLock()
|
||||
func (d *DiskFS) Set(key string, src []byte) error {
|
||||
if d.capacity > 0 {
|
||||
if d.size+size > d.capacity {
|
||||
d.mu.RUnlock()
|
||||
return nil, vfserror.ErrDiskFull
|
||||
if size := d.Size() + int64(len(src)); size > d.capacity {
|
||||
return vfserror.ErrDiskFull
|
||||
}
|
||||
}
|
||||
d.mu.RUnlock()
|
||||
|
||||
keyMu := d.getKeyLock(key)
|
||||
keyMu.Lock()
|
||||
defer keyMu.Unlock()
|
||||
|
||||
// Check again after lock
|
||||
d.mu.Lock()
|
||||
if fi, exists := d.info[key]; exists {
|
||||
d.size -= fi.Size()
|
||||
d.LRU.Remove(key)
|
||||
delete(d.info, key)
|
||||
path := filepath.Join(d.root, key)
|
||||
os.Remove(path) // Ignore error, as file might not exist or other issues
|
||||
if _, err := d.Stat(key); err == nil {
|
||||
d.Delete(key)
|
||||
}
|
||||
d.mu.Unlock()
|
||||
|
||||
path := filepath.Join(d.root, key)
|
||||
path = strings.ReplaceAll(path, "\\", "/") // Ensure forward slashes for consistency
|
||||
dir := filepath.Dir(path)
|
||||
if err := os.MkdirAll(dir, 0755); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
file, err := os.Create(path)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return &diskWriteCloser{
|
||||
Writer: file,
|
||||
onClose: func(n int64) error {
|
||||
fi, err := os.Stat(path)
|
||||
if err != nil {
|
||||
os.Remove(path)
|
||||
return err
|
||||
}
|
||||
|
||||
d.mu.Lock()
|
||||
finfo := vfs.NewFileInfoFromOS(fi, key)
|
||||
d.info[key] = finfo
|
||||
d.LRU.Add(key, finfo)
|
||||
d.size += n
|
||||
d.mu.Unlock()
|
||||
|
||||
diskWriteBytes.Add(float64(n))
|
||||
diskSizeBytes.Set(float64(d.Size()))
|
||||
|
||||
return nil
|
||||
},
|
||||
key: key,
|
||||
file: file,
|
||||
}, nil
|
||||
}
|
||||
|
||||
type diskWriteCloser struct {
|
||||
io.Writer
|
||||
onClose func(int64) error
|
||||
n int64
|
||||
key string
|
||||
file *os.File
|
||||
}
|
||||
|
||||
func (wc *diskWriteCloser) Write(p []byte) (int, error) {
|
||||
n, err := wc.Writer.Write(p)
|
||||
wc.n += int64(n)
|
||||
return n, err
|
||||
}
|
||||
|
||||
func (wc *diskWriteCloser) Close() error {
|
||||
err := wc.file.Close()
|
||||
if e := wc.onClose(wc.n); e != nil {
|
||||
os.Remove(wc.file.Name())
|
||||
return e
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
// Delete deletes the value of key.
|
||||
func (d *DiskFS) Delete(key string) error {
|
||||
if key == "" {
|
||||
return vfserror.ErrInvalidKey
|
||||
}
|
||||
if key[0] == '/' {
|
||||
return vfserror.ErrInvalidKey
|
||||
}
|
||||
|
||||
// Sanitize key to prevent path traversal
|
||||
key = filepath.Clean(key)
|
||||
key = strings.ReplaceAll(key, "\\", "/") // Ensure forward slashes for consistency
|
||||
if strings.Contains(key, "..") {
|
||||
return vfserror.ErrInvalidKey
|
||||
}
|
||||
|
||||
keyMu := d.getKeyLock(key)
|
||||
keyMu.Lock()
|
||||
defer keyMu.Unlock()
|
||||
|
||||
d.mu.Lock()
|
||||
fi, exists := d.info[key]
|
||||
if !exists {
|
||||
d.mu.Unlock()
|
||||
return vfserror.ErrNotFound
|
||||
}
|
||||
d.size -= fi.Size()
|
||||
d.LRU.Remove(key)
|
||||
delete(d.info, key)
|
||||
d.mu.Unlock()
|
||||
|
||||
path := filepath.Join(d.root, key)
|
||||
path = strings.ReplaceAll(path, "\\", "/") // Ensure forward slashes for consistency
|
||||
if err := os.Remove(path); err != nil {
|
||||
defer d.mu.Unlock()
|
||||
os.MkdirAll(filepath.Join(d.root, filepath.Dir(key)), 0755)
|
||||
if err := os.WriteFile(filepath.Join(d.root, key), src, 0644); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
diskSizeBytes.Set(float64(d.Size()))
|
||||
fi, err := os.Stat(filepath.Join(d.root, key))
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
|
||||
d.info[key] = vfs.NewFileInfoFromOS(fi, key)
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// Open opens the file at key and returns it.
|
||||
func (d *DiskFS) Open(key string) (io.ReadCloser, error) {
|
||||
if key == "" {
|
||||
return nil, vfserror.ErrInvalidKey
|
||||
// Delete deletes the value of key.
|
||||
func (d *DiskFS) Delete(key string) error {
|
||||
_, err := d.Stat(key)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if key[0] == '/' {
|
||||
return nil, vfserror.ErrInvalidKey
|
||||
}
|
||||
|
||||
// Sanitize key to prevent path traversal
|
||||
key = filepath.Clean(key)
|
||||
key = strings.ReplaceAll(key, "\\", "/") // Ensure forward slashes for consistency
|
||||
if strings.Contains(key, "..") {
|
||||
return nil, vfserror.ErrInvalidKey
|
||||
}
|
||||
|
||||
keyMu := d.getKeyLock(key)
|
||||
keyMu.RLock()
|
||||
defer keyMu.RUnlock()
|
||||
|
||||
d.mu.Lock()
|
||||
fi, exists := d.info[key]
|
||||
if !exists {
|
||||
d.mu.Unlock()
|
||||
return nil, vfserror.ErrNotFound
|
||||
defer d.mu.Unlock()
|
||||
delete(d.info, key)
|
||||
if err := os.Remove(filepath.Join(d.root, key)); err != nil {
|
||||
return err
|
||||
}
|
||||
fi.ATime = time.Now()
|
||||
d.LRU.MoveToFront(key)
|
||||
d.mu.Unlock()
|
||||
|
||||
path := filepath.Join(d.root, key)
|
||||
path = strings.ReplaceAll(path, "\\", "/") // Ensure forward slashes for consistency
|
||||
file, err := os.Open(path)
|
||||
return nil
|
||||
}
|
||||
|
||||
// Get gets the value of key and returns it.
|
||||
func (d *DiskFS) Get(key string) ([]byte, error) {
|
||||
_, err := d.Stat(key)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// Update metrics on close
|
||||
return &readCloser{
|
||||
ReadCloser: file,
|
||||
onClose: func(n int64) {
|
||||
diskReadBytes.Add(float64(n))
|
||||
},
|
||||
}, nil
|
||||
}
|
||||
d.mu.Lock()
|
||||
defer d.mu.Unlock()
|
||||
|
||||
type readCloser struct {
|
||||
io.ReadCloser
|
||||
onClose func(int64)
|
||||
n int64
|
||||
}
|
||||
data, err := os.ReadFile(filepath.Join(d.root, key))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
func (rc *readCloser) Read(p []byte) (int, error) {
|
||||
n, err := rc.ReadCloser.Read(p)
|
||||
rc.n += int64(n)
|
||||
return n, err
|
||||
}
|
||||
|
||||
func (rc *readCloser) Close() error {
|
||||
err := rc.ReadCloser.Close()
|
||||
rc.onClose(rc.n)
|
||||
return err
|
||||
return data, nil
|
||||
}
|
||||
|
||||
// Stat returns the FileInfo of key. If key is not found in the cache, it will stat the file on disk. If the file is not found on disk, it will return vfs.ErrNotFound.
|
||||
func (d *DiskFS) Stat(key string) (*vfs.FileInfo, error) {
|
||||
if key == "" {
|
||||
return nil, vfserror.ErrInvalidKey
|
||||
}
|
||||
if key[0] == '/' {
|
||||
return nil, vfserror.ErrInvalidKey
|
||||
d.mu.Lock()
|
||||
fi, ok := d.info[key]
|
||||
d.mu.Unlock() // unlock before statting the file
|
||||
|
||||
if !ok {
|
||||
fii, err := os.Stat(filepath.Join(d.root, key))
|
||||
if err != nil {
|
||||
return nil, vfserror.ErrNotFound
|
||||
}
|
||||
|
||||
d.mu.Lock() // relock to update the info map
|
||||
defer d.mu.Unlock() // nothing else needs to unlock before returning
|
||||
|
||||
d.info[key] = vfs.NewFileInfoFromOS(fii, key)
|
||||
fi = d.info[key]
|
||||
// fallthrough to return fi with shiny new info
|
||||
}
|
||||
|
||||
// Sanitize key to prevent path traversal
|
||||
key = filepath.Clean(key)
|
||||
key = strings.ReplaceAll(key, "\\", "/") // Ensure forward slashes for consistency
|
||||
if strings.Contains(key, "..") {
|
||||
return nil, vfserror.ErrInvalidKey
|
||||
}
|
||||
|
||||
keyMu := d.getKeyLock(key)
|
||||
keyMu.RLock()
|
||||
defer keyMu.RUnlock()
|
||||
|
||||
d.mu.RLock()
|
||||
defer d.mu.RUnlock()
|
||||
|
||||
if fi, ok := d.info[key]; !ok {
|
||||
return nil, vfserror.ErrNotFound
|
||||
} else {
|
||||
return fi, nil
|
||||
}
|
||||
return fi, nil
|
||||
}
|
||||
|
||||
func (d *DiskFS) StatAll() []*vfs.FileInfo {
|
||||
d.mu.RLock()
|
||||
defer d.mu.RUnlock()
|
||||
func (m *DiskFS) StatAll() []*vfs.FileInfo {
|
||||
m.mu.Lock()
|
||||
defer m.mu.Unlock()
|
||||
|
||||
// hard copy the file info to prevent modification of the original file info or the other way around
|
||||
files := make([]*vfs.FileInfo, 0, len(d.info))
|
||||
for _, v := range d.info {
|
||||
files := make([]*vfs.FileInfo, 0, len(m.info))
|
||||
for _, v := range m.info {
|
||||
fi := *v
|
||||
files = append(files, &fi)
|
||||
}
|
||||
|
||||
+50
-144
@@ -1,181 +1,87 @@
|
||||
// vfs/disk/disk_test.go
|
||||
package disk
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"s1d3sw1ped/SteamCache2/vfs/vfserror"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestCreateAndOpen(t *testing.T) {
|
||||
func TestAllDisk(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
m := NewSkipInit(t.TempDir(), 1024)
|
||||
key := "key"
|
||||
value := []byte("value")
|
||||
|
||||
w, err := m.Create(key, int64(len(value)))
|
||||
if err != nil {
|
||||
t.Fatalf("Create failed: %v", err)
|
||||
if err := m.Set("key", []byte("value")); err != nil {
|
||||
t.Errorf("Set failed: %v", err)
|
||||
}
|
||||
w.Write(value)
|
||||
w.Close()
|
||||
|
||||
rc, err := m.Open(key)
|
||||
if err != nil {
|
||||
t.Fatalf("Open failed: %v", err)
|
||||
if err := m.Set("key", []byte("value1")); err != nil {
|
||||
t.Errorf("Set failed: %v", err)
|
||||
}
|
||||
got, _ := io.ReadAll(rc)
|
||||
rc.Close()
|
||||
|
||||
if string(got) != string(value) {
|
||||
t.Fatalf("expected %s, got %s", value, got)
|
||||
if d, err := m.Get("key"); err != nil {
|
||||
t.Errorf("Get failed: %v", err)
|
||||
} else if string(d) != "value1" {
|
||||
t.Errorf("Get failed: got %s, want %s", d, "value1")
|
||||
}
|
||||
|
||||
if err := m.Delete("key"); err != nil {
|
||||
t.Errorf("Delete failed: %v", err)
|
||||
}
|
||||
|
||||
if _, err := m.Get("key"); err == nil {
|
||||
t.Errorf("Get failed: got nil, want %v", vfserror.ErrNotFound)
|
||||
}
|
||||
|
||||
if err := m.Delete("key"); err == nil {
|
||||
t.Errorf("Delete failed: got nil, want %v", vfserror.ErrNotFound)
|
||||
}
|
||||
|
||||
if _, err := m.Stat("key"); err == nil {
|
||||
t.Errorf("Stat failed: got nil, want %v", vfserror.ErrNotFound)
|
||||
}
|
||||
|
||||
if err := m.Set("key", []byte("value")); err != nil {
|
||||
t.Errorf("Set failed: %v", err)
|
||||
}
|
||||
|
||||
if _, err := m.Stat("key"); err != nil {
|
||||
t.Errorf("Stat failed: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestOverwrite(t *testing.T) {
|
||||
m := NewSkipInit(t.TempDir(), 1024)
|
||||
key := "key"
|
||||
value1 := []byte("value1")
|
||||
value2 := []byte("value2")
|
||||
func TestLimited(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
w, err := m.Create(key, int64(len(value1)))
|
||||
if err != nil {
|
||||
t.Fatalf("Create failed: %v", err)
|
||||
}
|
||||
w.Write(value1)
|
||||
w.Close()
|
||||
|
||||
w, err = m.Create(key, int64(len(value2)))
|
||||
if err != nil {
|
||||
t.Fatalf("Create failed: %v", err)
|
||||
}
|
||||
w.Write(value2)
|
||||
w.Close()
|
||||
|
||||
rc, err := m.Open(key)
|
||||
if err != nil {
|
||||
t.Fatalf("Open failed: %v", err)
|
||||
}
|
||||
got, _ := io.ReadAll(rc)
|
||||
rc.Close()
|
||||
|
||||
if string(got) != string(value2) {
|
||||
t.Fatalf("expected %s, got %s", value2, got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDelete(t *testing.T) {
|
||||
m := NewSkipInit(t.TempDir(), 1024)
|
||||
key := "key"
|
||||
value := []byte("value")
|
||||
|
||||
w, err := m.Create(key, int64(len(value)))
|
||||
if err != nil {
|
||||
t.Fatalf("Create failed: %v", err)
|
||||
}
|
||||
w.Write(value)
|
||||
w.Close()
|
||||
|
||||
if err := m.Delete(key); err != nil {
|
||||
t.Fatalf("Delete failed: %v", err)
|
||||
}
|
||||
|
||||
_, err = m.Open(key)
|
||||
if !errors.Is(err, vfserror.ErrNotFound) {
|
||||
t.Fatalf("expected %v, got %v", vfserror.ErrNotFound, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCapacityLimit(t *testing.T) {
|
||||
m := NewSkipInit(t.TempDir(), 10)
|
||||
for i := 0; i < 11; i++ {
|
||||
w, err := m.Create(fmt.Sprintf("key%d", i), 1)
|
||||
if err != nil && i < 10 {
|
||||
t.Errorf("Create failed: %v", err)
|
||||
if err := m.Set(fmt.Sprintf("key%d", i), []byte("1")); err != nil && i < 10 {
|
||||
t.Errorf("Set failed: %v", err)
|
||||
} else if i == 10 && err == nil {
|
||||
t.Errorf("Create succeeded: got nil, want %v", vfserror.ErrDiskFull)
|
||||
}
|
||||
if i < 10 {
|
||||
w.Write([]byte("1"))
|
||||
w.Close()
|
||||
t.Errorf("Set succeeded: got nil, want %v", vfserror.ErrDiskFull)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestInitExistingFiles(t *testing.T) {
|
||||
func TestInit(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
td := t.TempDir()
|
||||
|
||||
path := filepath.Join(td, "test", "key")
|
||||
|
||||
os.MkdirAll(filepath.Dir(path), 0755)
|
||||
|
||||
os.WriteFile(path, []byte("value"), 0644)
|
||||
|
||||
m := New(td, 10)
|
||||
rc, err := m.Open("test/key")
|
||||
if err != nil {
|
||||
t.Fatalf("Open failed: %v", err)
|
||||
}
|
||||
got, _ := io.ReadAll(rc)
|
||||
rc.Close()
|
||||
|
||||
if string(got) != "value" {
|
||||
t.Errorf("expected value, got %s", got)
|
||||
if _, err := m.Get("test/key"); err != nil {
|
||||
t.Errorf("Get failed: %v", err)
|
||||
}
|
||||
|
||||
s, err := m.Stat("test/key")
|
||||
if err != nil {
|
||||
t.Fatalf("Stat failed: %v", err)
|
||||
}
|
||||
if s == nil {
|
||||
t.Error("Stat returned nil")
|
||||
}
|
||||
if s != nil && s.Name() != "test/key" {
|
||||
t.Errorf("Stat failed: got %s, want %s", s.Name(), "test/key")
|
||||
}
|
||||
}
|
||||
|
||||
func TestSizeConsistency(t *testing.T) {
|
||||
td := t.TempDir()
|
||||
os.WriteFile(filepath.Join(td, "key2"), []byte("value2"), 0644)
|
||||
|
||||
m := New(td, 1024)
|
||||
if m.Size() != 6 {
|
||||
t.Errorf("Size failed: got %d, want 6", m.Size())
|
||||
}
|
||||
|
||||
w, err := m.Create("key", 5)
|
||||
if err != nil {
|
||||
t.Errorf("Create failed: %v", err)
|
||||
}
|
||||
w.Write([]byte("value"))
|
||||
w.Close()
|
||||
|
||||
w, err = m.Create("key1", 6)
|
||||
if err != nil {
|
||||
t.Errorf("Create failed: %v", err)
|
||||
}
|
||||
w.Write([]byte("value1"))
|
||||
w.Close()
|
||||
|
||||
assumedSize := int64(6 + 5 + 6)
|
||||
if assumedSize != m.Size() {
|
||||
t.Errorf("Size failed: got %d, want %d", m.Size(), assumedSize)
|
||||
}
|
||||
|
||||
rc, err := m.Open("key")
|
||||
if err != nil {
|
||||
t.Errorf("Open failed: %v", err)
|
||||
}
|
||||
d, _ := io.ReadAll(rc)
|
||||
rc.Close()
|
||||
if string(d) != "value" {
|
||||
t.Errorf("Get failed: got %s, want value", d)
|
||||
}
|
||||
|
||||
m = New(td, 1024)
|
||||
if assumedSize != m.Size() {
|
||||
t.Errorf("Size failed: got %d, want %d", m.Size(), assumedSize)
|
||||
s, _ := m.Stat("test/key")
|
||||
if s.Name() != "test/key" {
|
||||
t.Errorf("Stat failed: got %s, want %s", s.Name(), "key")
|
||||
}
|
||||
}
|
||||
|
||||
+2
-3
@@ -1,4 +1,3 @@
|
||||
// vfs/fileinfo.go
|
||||
package vfs
|
||||
|
||||
import (
|
||||
@@ -13,9 +12,9 @@ type FileInfo struct {
|
||||
ATime time.Time
|
||||
}
|
||||
|
||||
func NewFileInfo(key string, size int64, modTime time.Time) *FileInfo {
|
||||
func NewFileInfo(name string, size int64, modTime time.Time) *FileInfo {
|
||||
return &FileInfo{
|
||||
name: key,
|
||||
name: name,
|
||||
size: size,
|
||||
MTime: modTime,
|
||||
ATime: time.Now(),
|
||||
|
||||
+8
-82
@@ -1,98 +1,25 @@
|
||||
// vfs/gc/gc.go
|
||||
package gc
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"io"
|
||||
"s1d3sw1ped/SteamCache2/steamcache/logger"
|
||||
"s1d3sw1ped/SteamCache2/vfs"
|
||||
"s1d3sw1ped/SteamCache2/vfs/cachestate"
|
||||
"s1d3sw1ped/SteamCache2/vfs/disk"
|
||||
"s1d3sw1ped/SteamCache2/vfs/memory"
|
||||
"s1d3sw1ped/SteamCache2/vfs/vfserror"
|
||||
"time"
|
||||
)
|
||||
|
||||
// LRUGC deletes files in LRU order until enough space is reclaimed.
|
||||
func LRUGC(vfss vfs.VFS, size uint) {
|
||||
attempts := 0
|
||||
deletions := 0
|
||||
var reclaimed uint
|
||||
|
||||
for reclaimed < size {
|
||||
if attempts > 10 {
|
||||
logger.Logger.Debug().
|
||||
Int("attempts", attempts).
|
||||
Msg("GC: Too many attempts to reclaim space, giving up")
|
||||
return
|
||||
}
|
||||
attempts++
|
||||
switch fs := vfss.(type) {
|
||||
case *disk.DiskFS:
|
||||
fi := fs.LRU.Back()
|
||||
if fi == nil {
|
||||
break
|
||||
}
|
||||
sz := uint(fi.Size())
|
||||
err := fs.Delete(fi.Name())
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
reclaimed += sz
|
||||
deletions++
|
||||
case *memory.MemoryFS:
|
||||
fi := fs.LRU.Back()
|
||||
if fi == nil {
|
||||
break
|
||||
}
|
||||
sz := uint(fi.Size())
|
||||
err := fs.Delete(fi.Name())
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
reclaimed += sz
|
||||
deletions++
|
||||
default:
|
||||
// Fallback to old method if not supported
|
||||
stats := vfss.StatAll()
|
||||
if len(stats) == 0 {
|
||||
break
|
||||
}
|
||||
fi := stats[0] // Assume sorted or pick first
|
||||
sz := uint(fi.Size())
|
||||
err := vfss.Delete(fi.Name())
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
reclaimed += sz
|
||||
deletions++
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func PromotionDecider(fi *vfs.FileInfo, cs cachestate.CacheState) bool {
|
||||
return time.Since(fi.AccessTime()) < time.Second*60 // Put hot files in the fast vfs if equipped
|
||||
}
|
||||
|
||||
// Ensure GCFS implements VFS.
|
||||
var _ vfs.VFS = (*GCFS)(nil)
|
||||
|
||||
// GCFS is a virtual file system that calls a GC handler when the disk is full. The GC handler is responsible for freeing up space on the disk. The GCFS is a wrapper around another VFS.
|
||||
type GCFS struct {
|
||||
vfs.VFS
|
||||
multiplier int
|
||||
|
||||
// protected by mu
|
||||
multiplier int
|
||||
gcHanderFunc GCHandlerFunc
|
||||
}
|
||||
|
||||
// GCHandlerFunc is a function that is called when the disk is full and the GCFS needs to free up space. It is passed the VFS and the size of the file that needs to be written. Its up to the implementation to free up space. How much space is freed is also up to the implementation.
|
||||
type GCHandlerFunc func(vfs vfs.VFS, size uint)
|
||||
type GCHandlerFunc func(vfs vfs.VFS, size int)
|
||||
|
||||
func New(vfs vfs.VFS, multiplier int, gcHandlerFunc GCHandlerFunc) *GCFS {
|
||||
if multiplier <= 0 {
|
||||
multiplier = 1 // if the multiplier is less than or equal to 0 set it to 1 will be slow but the user can set it to a higher value if they want
|
||||
}
|
||||
return &GCFS{
|
||||
VFS: vfs,
|
||||
multiplier: multiplier,
|
||||
@@ -100,17 +27,16 @@ func New(vfs vfs.VFS, multiplier int, gcHandlerFunc GCHandlerFunc) *GCFS {
|
||||
}
|
||||
}
|
||||
|
||||
// Create overrides the Create method of the VFS interface. It tries to create the key, if it fails due to disk full error, it calls the GC handler and tries again. If it still fails it returns the error.
|
||||
func (g *GCFS) Create(key string, size int64) (io.WriteCloser, error) {
|
||||
w, err := g.VFS.Create(key, size) // try to create the key
|
||||
// Set overrides the Set method of the VFS interface. It tries to set the key and src, if it fails due to disk full error, it calls the GC handler and tries again. If it still fails it returns the error.
|
||||
func (g *GCFS) Set(key string, src []byte) error {
|
||||
err := g.VFS.Set(key, src) // try to set the key and src
|
||||
|
||||
// if it fails due to disk full error, call the GC handler and try again in a loop that will continue until it succeeds or the error is not disk full
|
||||
if err == vfserror.ErrDiskFull && g.gcHanderFunc != nil { // if the error is disk full and there is a GC handler
|
||||
g.gcHanderFunc(g.VFS, uint(size*int64(g.multiplier))) // call the GC handler
|
||||
w, err = g.VFS.Create(key, size)
|
||||
g.gcHanderFunc(g.VFS, len(src)*g.multiplier) // call the GC handler
|
||||
err = g.VFS.Set(key, src) // try again after GC if it still fails return the error
|
||||
}
|
||||
|
||||
return w, err
|
||||
return err
|
||||
}
|
||||
|
||||
func (g *GCFS) Name() string {
|
||||
|
||||
+90
-52
@@ -1,73 +1,111 @@
|
||||
// vfs/gc/gc_test.go
|
||||
package gc
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"s1d3sw1ped/SteamCache2/vfs"
|
||||
"s1d3sw1ped/SteamCache2/vfs/memory"
|
||||
"s1d3sw1ped/SteamCache2/vfs/vfserror"
|
||||
"sort"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"golang.org/x/exp/rand"
|
||||
)
|
||||
|
||||
func TestGCOnFull(t *testing.T) {
|
||||
m := memory.New(10)
|
||||
gc := New(m, 2, LRUGC)
|
||||
func TestGCSmallRandom(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
for i := 0; i < 5; i++ {
|
||||
w, err := gc.Create(fmt.Sprintf("key%d", i), 2)
|
||||
if err != nil {
|
||||
t.Fatalf("Create failed: %v", err)
|
||||
m := memory.New(1024 * 1024 * 16)
|
||||
gc := New(m, 10, func(vfs vfs.VFS, size int) {
|
||||
tstart := time.Now()
|
||||
deletions := 0
|
||||
targetreclaim := int64(size)
|
||||
var reclaimed int64
|
||||
|
||||
t.Logf("GC starting to reclaim %d bytes", targetreclaim)
|
||||
|
||||
stats := vfs.StatAll()
|
||||
sort.Slice(stats, func(i, j int) bool {
|
||||
// Sort by access time so we can remove the oldest files first.
|
||||
return stats[i].AccessTime().Before(stats[j].AccessTime())
|
||||
})
|
||||
|
||||
// Delete the oldest files until we've reclaimed enough space.
|
||||
for _, s := range stats {
|
||||
sz := s.Size() // Get the size of the file
|
||||
err := vfs.Delete(s.Name())
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
reclaimed += sz // Track how much space we've reclaimed
|
||||
deletions++ // Track how many files we've deleted
|
||||
|
||||
// t.Logf("GC deleting %s, %v", s.Name(), s.AccessTime().Format(time.RFC3339Nano))
|
||||
|
||||
if reclaimed >= targetreclaim { // We've reclaimed enough space
|
||||
break
|
||||
}
|
||||
}
|
||||
w.Write([]byte("ab"))
|
||||
w.Close()
|
||||
}
|
||||
|
||||
// Cache full at 10 bytes
|
||||
w, err := gc.Create("key5", 2)
|
||||
if err != nil {
|
||||
t.Fatalf("Create failed: %v", err)
|
||||
}
|
||||
w.Write([]byte("cd"))
|
||||
w.Close()
|
||||
t.Logf("GC took %v to reclaim %d bytes by deleting %d files", time.Since(tstart), reclaimed, deletions)
|
||||
})
|
||||
|
||||
if gc.Size() > 10 {
|
||||
t.Errorf("Size exceeded: %d > 10", gc.Size())
|
||||
}
|
||||
|
||||
// Check if older keys were evicted
|
||||
_, err = m.Open("key0")
|
||||
if err == nil {
|
||||
t.Error("Expected key0 to be evicted")
|
||||
}
|
||||
}
|
||||
|
||||
func TestNoGCNeeded(t *testing.T) {
|
||||
m := memory.New(20)
|
||||
gc := New(m, 2, LRUGC)
|
||||
|
||||
for i := 0; i < 5; i++ {
|
||||
w, err := gc.Create(fmt.Sprintf("key%d", i), 2)
|
||||
if err != nil {
|
||||
t.Fatalf("Create failed: %v", err)
|
||||
for i := 0; i < 10000; i++ {
|
||||
if err := gc.Set(fmt.Sprintf("key:%d", i), genRandomData(1024*1, 1024*4)); err != nil {
|
||||
t.Errorf("Set failed: %v", err)
|
||||
}
|
||||
w.Write([]byte("ab"))
|
||||
w.Close()
|
||||
}
|
||||
|
||||
if gc.Size() != 10 {
|
||||
t.Errorf("Size: got %d, want 10", gc.Size())
|
||||
if gc.Size() > 1024*1024*16 {
|
||||
t.Errorf("MemoryFS size is %d, want <= 1024", m.Size())
|
||||
}
|
||||
}
|
||||
|
||||
func TestGCInsufficientSpace(t *testing.T) {
|
||||
m := memory.New(5)
|
||||
gc := New(m, 1, LRUGC)
|
||||
func genRandomData(min int, max int) []byte {
|
||||
data := make([]byte, rand.Intn(max-min)+min)
|
||||
rand.Read(data)
|
||||
return data
|
||||
}
|
||||
|
||||
w, err := gc.Create("key0", 10)
|
||||
if err == nil {
|
||||
w.Close()
|
||||
t.Error("Expected ErrDiskFull")
|
||||
} else if !errors.Is(err, vfserror.ErrDiskFull) {
|
||||
t.Errorf("Unexpected error: %v", err)
|
||||
func TestGCLargeRandom(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
m := memory.New(1024 * 1024 * 16) // 16MB
|
||||
gc := New(m, 10, func(vfs vfs.VFS, size int) {
|
||||
tstart := time.Now()
|
||||
deletions := 0
|
||||
targetreclaim := int64(size)
|
||||
var reclaimed int64
|
||||
|
||||
t.Logf("GC starting to reclaim %d bytes", targetreclaim)
|
||||
|
||||
stats := vfs.StatAll()
|
||||
sort.Slice(stats, func(i, j int) bool {
|
||||
// Sort by access time so we can remove the oldest files first.
|
||||
return stats[i].AccessTime().Before(stats[j].AccessTime())
|
||||
})
|
||||
|
||||
// Delete the oldest files until we've reclaimed enough space.
|
||||
for _, s := range stats {
|
||||
sz := s.Size() // Get the size of the file
|
||||
vfs.Delete(s.Name())
|
||||
reclaimed += sz // Track how much space we've reclaimed
|
||||
deletions++ // Track how many files we've deleted
|
||||
|
||||
if reclaimed >= targetreclaim { // We've reclaimed enough space
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
t.Logf("GC took %v to reclaim %d bytes by deleting %d files", time.Since(tstart), reclaimed, deletions)
|
||||
})
|
||||
|
||||
for i := 0; i < 10000; i++ {
|
||||
if err := gc.Set(fmt.Sprintf("key:%d", i), genRandomData(1024, 1024*1024)); err != nil {
|
||||
t.Errorf("Set failed: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
if gc.Size() > 1024*1024*16 {
|
||||
t.Errorf("MemoryFS size is %d, want <= 1024", m.Size())
|
||||
}
|
||||
}
|
||||
|
||||
+58
-199
@@ -1,49 +1,10 @@
|
||||
// vfs/memory/memory.go
|
||||
package memory
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"container/list"
|
||||
"io"
|
||||
"s1d3sw1ped/SteamCache2/steamcache/logger"
|
||||
"s1d3sw1ped/SteamCache2/vfs"
|
||||
"s1d3sw1ped/SteamCache2/vfs/vfserror"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/docker/go-units"
|
||||
"github.com/prometheus/client_golang/prometheus"
|
||||
"github.com/prometheus/client_golang/prometheus/promauto"
|
||||
)
|
||||
|
||||
var (
|
||||
memoryCapacityBytes = promauto.NewGauge(
|
||||
prometheus.GaugeOpts{
|
||||
Name: "memory_cache_capacity_bytes",
|
||||
Help: "Total capacity of the memory cache in bytes",
|
||||
},
|
||||
)
|
||||
|
||||
memorySizeBytes = promauto.NewGauge(
|
||||
prometheus.GaugeOpts{
|
||||
Name: "memory_cache_size_bytes",
|
||||
Help: "Total size of the memory cache in bytes",
|
||||
},
|
||||
)
|
||||
|
||||
memoryReadBytes = promauto.NewCounter(
|
||||
prometheus.CounterOpts{
|
||||
Name: "memory_cache_read_bytes_total",
|
||||
Help: "Total number of bytes read from the memory cache",
|
||||
},
|
||||
)
|
||||
|
||||
memoryWriteBytes = promauto.NewCounter(
|
||||
prometheus.CounterOpts{
|
||||
Name: "memory_cache_write_bytes_total",
|
||||
Help: "Total number of bytes written to the memory cache",
|
||||
},
|
||||
)
|
||||
)
|
||||
|
||||
// Ensure MemoryFS implements VFS.
|
||||
@@ -59,74 +20,16 @@ type file struct {
|
||||
type MemoryFS struct {
|
||||
files map[string]*file
|
||||
capacity int64
|
||||
size int64
|
||||
mu sync.RWMutex
|
||||
keyLocks sync.Map // map[string]*sync.RWMutex
|
||||
LRU *lruList
|
||||
}
|
||||
|
||||
// lruList for LRU eviction
|
||||
type lruList struct {
|
||||
list *list.List
|
||||
elem map[string]*list.Element
|
||||
}
|
||||
|
||||
func newLruList() *lruList {
|
||||
return &lruList{
|
||||
list: list.New(),
|
||||
elem: make(map[string]*list.Element),
|
||||
}
|
||||
}
|
||||
|
||||
func (l *lruList) MoveToFront(key string) {
|
||||
if e, ok := l.elem[key]; ok {
|
||||
l.list.MoveToFront(e)
|
||||
}
|
||||
}
|
||||
|
||||
func (l *lruList) Add(key string, fi *vfs.FileInfo) *list.Element {
|
||||
e := l.list.PushFront(fi)
|
||||
l.elem[key] = e
|
||||
return e
|
||||
}
|
||||
|
||||
func (l *lruList) Remove(key string) {
|
||||
if e, ok := l.elem[key]; ok {
|
||||
l.list.Remove(e)
|
||||
delete(l.elem, key)
|
||||
}
|
||||
}
|
||||
|
||||
func (l *lruList) Back() *vfs.FileInfo {
|
||||
if e := l.list.Back(); e != nil {
|
||||
return e.Value.(*vfs.FileInfo)
|
||||
}
|
||||
return nil
|
||||
mu sync.Mutex
|
||||
}
|
||||
|
||||
// New creates a new MemoryFS.
|
||||
func New(capacity int64) *MemoryFS {
|
||||
if capacity <= 0 {
|
||||
panic("memory capacity must be greater than 0") // panic if the capacity is less than or equal to 0
|
||||
}
|
||||
|
||||
logger.Logger.Info().
|
||||
Str("name", "MemoryFS").
|
||||
Str("capacity", units.HumanSize(float64(capacity))).
|
||||
Msg("init")
|
||||
|
||||
mfs := &MemoryFS{
|
||||
return &MemoryFS{
|
||||
files: make(map[string]*file),
|
||||
capacity: capacity,
|
||||
mu: sync.RWMutex{},
|
||||
keyLocks: sync.Map{},
|
||||
LRU: newLruList(),
|
||||
mu: sync.Mutex{},
|
||||
}
|
||||
|
||||
memoryCapacityBytes.Set(float64(capacity))
|
||||
memorySizeBytes.Set(float64(mfs.Size()))
|
||||
|
||||
return mfs
|
||||
}
|
||||
|
||||
func (m *MemoryFS) Capacity() int64 {
|
||||
@@ -138,118 +41,74 @@ func (m *MemoryFS) Name() string {
|
||||
}
|
||||
|
||||
func (m *MemoryFS) Size() int64 {
|
||||
m.mu.RLock()
|
||||
defer m.mu.RUnlock()
|
||||
return m.size
|
||||
}
|
||||
|
||||
func (m *MemoryFS) getKeyLock(key string) *sync.RWMutex {
|
||||
mu, _ := m.keyLocks.LoadOrStore(key, &sync.RWMutex{})
|
||||
return mu.(*sync.RWMutex)
|
||||
}
|
||||
|
||||
func (m *MemoryFS) Create(key string, size int64) (io.WriteCloser, error) {
|
||||
m.mu.RLock()
|
||||
if m.capacity > 0 {
|
||||
if m.size+size > m.capacity {
|
||||
m.mu.RUnlock()
|
||||
return nil, vfserror.ErrDiskFull
|
||||
}
|
||||
}
|
||||
m.mu.RUnlock()
|
||||
|
||||
keyMu := m.getKeyLock(key)
|
||||
keyMu.Lock()
|
||||
defer keyMu.Unlock()
|
||||
|
||||
buf := &bytes.Buffer{}
|
||||
|
||||
return &memWriteCloser{
|
||||
Writer: buf,
|
||||
onClose: func() error {
|
||||
data := buf.Bytes()
|
||||
m.mu.Lock()
|
||||
if f, exists := m.files[key]; exists {
|
||||
m.size -= int64(len(f.data))
|
||||
m.LRU.Remove(key)
|
||||
}
|
||||
fi := vfs.NewFileInfo(key, int64(len(data)), time.Now())
|
||||
m.files[key] = &file{
|
||||
fileinfo: fi,
|
||||
data: data,
|
||||
}
|
||||
m.LRU.Add(key, fi)
|
||||
m.size += int64(len(data))
|
||||
m.mu.Unlock()
|
||||
|
||||
memoryWriteBytes.Add(float64(len(data)))
|
||||
memorySizeBytes.Set(float64(m.Size()))
|
||||
|
||||
return nil
|
||||
},
|
||||
}, nil
|
||||
}
|
||||
|
||||
type memWriteCloser struct {
|
||||
io.Writer
|
||||
onClose func() error
|
||||
}
|
||||
|
||||
func (wc *memWriteCloser) Close() error {
|
||||
return wc.onClose()
|
||||
}
|
||||
|
||||
func (m *MemoryFS) Delete(key string) error {
|
||||
keyMu := m.getKeyLock(key)
|
||||
keyMu.Lock()
|
||||
defer keyMu.Unlock()
|
||||
var size int64
|
||||
|
||||
m.mu.Lock()
|
||||
f, exists := m.files[key]
|
||||
if !exists {
|
||||
m.mu.Unlock()
|
||||
return vfserror.ErrNotFound
|
||||
}
|
||||
m.size -= int64(len(f.data))
|
||||
m.LRU.Remove(key)
|
||||
delete(m.files, key)
|
||||
m.mu.Unlock()
|
||||
defer m.mu.Unlock()
|
||||
|
||||
memorySizeBytes.Set(float64(m.Size()))
|
||||
for _, v := range m.files {
|
||||
size += int64(len(v.data))
|
||||
}
|
||||
|
||||
return size
|
||||
}
|
||||
|
||||
func (m *MemoryFS) Set(key string, src []byte) error {
|
||||
if m.capacity > 0 {
|
||||
if size := m.Size() + int64(len(src)); size > m.capacity {
|
||||
return vfserror.ErrDiskFull
|
||||
}
|
||||
}
|
||||
|
||||
m.mu.Lock()
|
||||
defer m.mu.Unlock()
|
||||
|
||||
m.files[key] = &file{
|
||||
fileinfo: vfs.NewFileInfo(
|
||||
key,
|
||||
int64(len(src)),
|
||||
time.Now(),
|
||||
),
|
||||
data: make([]byte, len(src)),
|
||||
}
|
||||
copy(m.files[key].data, src)
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (m *MemoryFS) Open(key string) (io.ReadCloser, error) {
|
||||
keyMu := m.getKeyLock(key)
|
||||
keyMu.RLock()
|
||||
defer keyMu.RUnlock()
|
||||
func (m *MemoryFS) Delete(key string) error {
|
||||
_, err := m.Stat(key)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
m.mu.Lock()
|
||||
f, exists := m.files[key]
|
||||
if !exists {
|
||||
m.mu.Unlock()
|
||||
return nil, vfserror.ErrNotFound
|
||||
defer m.mu.Unlock()
|
||||
|
||||
delete(m.files, key)
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (m *MemoryFS) Get(key string) ([]byte, error) {
|
||||
_, err := m.Stat(key)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
f.fileinfo.ATime = time.Now()
|
||||
m.LRU.MoveToFront(key)
|
||||
dataCopy := make([]byte, len(f.data))
|
||||
copy(dataCopy, f.data)
|
||||
m.mu.Unlock()
|
||||
|
||||
memoryReadBytes.Add(float64(len(dataCopy)))
|
||||
memorySizeBytes.Set(float64(m.Size()))
|
||||
m.mu.Lock()
|
||||
defer m.mu.Unlock()
|
||||
|
||||
return io.NopCloser(bytes.NewReader(dataCopy)), nil
|
||||
m.files[key].fileinfo.ATime = time.Now()
|
||||
dst := make([]byte, len(m.files[key].data))
|
||||
copy(dst, m.files[key].data)
|
||||
|
||||
return dst, nil
|
||||
}
|
||||
|
||||
func (m *MemoryFS) Stat(key string) (*vfs.FileInfo, error) {
|
||||
keyMu := m.getKeyLock(key)
|
||||
keyMu.RLock()
|
||||
defer keyMu.RUnlock()
|
||||
|
||||
m.mu.RLock()
|
||||
defer m.mu.RUnlock()
|
||||
m.mu.Lock()
|
||||
defer m.mu.Unlock()
|
||||
|
||||
f, ok := m.files[key]
|
||||
if !ok {
|
||||
@@ -260,8 +119,8 @@ func (m *MemoryFS) Stat(key string) (*vfs.FileInfo, error) {
|
||||
}
|
||||
|
||||
func (m *MemoryFS) StatAll() []*vfs.FileInfo {
|
||||
m.mu.RLock()
|
||||
defer m.mu.RUnlock()
|
||||
m.mu.Lock()
|
||||
defer m.mu.Unlock()
|
||||
|
||||
// hard copy the file info to prevent modification of the original file info or the other way around
|
||||
files := make([]*vfs.FileInfo, 0, len(m.files))
|
||||
|
||||
+40
-106
@@ -1,129 +1,63 @@
|
||||
// vfs/memory/memory_test.go
|
||||
package memory
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"s1d3sw1ped/SteamCache2/vfs/vfserror"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestCreateAndOpen(t *testing.T) {
|
||||
func TestAllMemory(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
m := New(1024)
|
||||
key := "key"
|
||||
value := []byte("value")
|
||||
|
||||
w, err := m.Create(key, int64(len(value)))
|
||||
if err != nil {
|
||||
t.Fatalf("Create failed: %v", err)
|
||||
if err := m.Set("key", []byte("value")); err != nil {
|
||||
t.Errorf("Set failed: %v", err)
|
||||
}
|
||||
w.Write(value)
|
||||
w.Close()
|
||||
|
||||
rc, err := m.Open(key)
|
||||
if err != nil {
|
||||
t.Fatalf("Open failed: %v", err)
|
||||
if err := m.Set("key", []byte("value1")); err != nil {
|
||||
t.Errorf("Set failed: %v", err)
|
||||
}
|
||||
got, _ := io.ReadAll(rc)
|
||||
rc.Close()
|
||||
|
||||
if string(got) != string(value) {
|
||||
t.Fatalf("expected %s, got %s", value, got)
|
||||
if d, err := m.Get("key"); err != nil {
|
||||
t.Errorf("Get failed: %v", err)
|
||||
} else if string(d) != "value1" {
|
||||
t.Errorf("Get failed: got %s, want %s", d, "value1")
|
||||
}
|
||||
|
||||
if err := m.Delete("key"); err != nil {
|
||||
t.Errorf("Delete failed: %v", err)
|
||||
}
|
||||
|
||||
if _, err := m.Get("key"); err == nil {
|
||||
t.Errorf("Get failed: got nil, want %v", vfserror.ErrNotFound)
|
||||
}
|
||||
|
||||
if err := m.Delete("key"); err == nil {
|
||||
t.Errorf("Delete failed: got nil, want %v", vfserror.ErrNotFound)
|
||||
}
|
||||
|
||||
if _, err := m.Stat("key"); err == nil {
|
||||
t.Errorf("Stat failed: got nil, want %v", vfserror.ErrNotFound)
|
||||
}
|
||||
|
||||
if err := m.Set("key", []byte("value")); err != nil {
|
||||
t.Errorf("Set failed: %v", err)
|
||||
}
|
||||
|
||||
if _, err := m.Stat("key"); err != nil {
|
||||
t.Errorf("Stat failed: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestOverwrite(t *testing.T) {
|
||||
m := New(1024)
|
||||
key := "key"
|
||||
value1 := []byte("value1")
|
||||
value2 := []byte("value2")
|
||||
func TestLimited(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
w, err := m.Create(key, int64(len(value1)))
|
||||
if err != nil {
|
||||
t.Fatalf("Create failed: %v", err)
|
||||
}
|
||||
w.Write(value1)
|
||||
w.Close()
|
||||
|
||||
w, err = m.Create(key, int64(len(value2)))
|
||||
if err != nil {
|
||||
t.Fatalf("Create failed: %v", err)
|
||||
}
|
||||
w.Write(value2)
|
||||
w.Close()
|
||||
|
||||
rc, err := m.Open(key)
|
||||
if err != nil {
|
||||
t.Fatalf("Open failed: %v", err)
|
||||
}
|
||||
got, _ := io.ReadAll(rc)
|
||||
rc.Close()
|
||||
|
||||
if string(got) != string(value2) {
|
||||
t.Fatalf("expected %s, got %s", value2, got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDelete(t *testing.T) {
|
||||
m := New(1024)
|
||||
key := "key"
|
||||
value := []byte("value")
|
||||
|
||||
w, err := m.Create(key, int64(len(value)))
|
||||
if err != nil {
|
||||
t.Fatalf("Create failed: %v", err)
|
||||
}
|
||||
w.Write(value)
|
||||
w.Close()
|
||||
|
||||
if err := m.Delete(key); err != nil {
|
||||
t.Fatalf("Delete failed: %v", err)
|
||||
}
|
||||
|
||||
_, err = m.Open(key)
|
||||
if !errors.Is(err, vfserror.ErrNotFound) {
|
||||
t.Fatalf("expected %v, got %v", vfserror.ErrNotFound, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCapacityLimit(t *testing.T) {
|
||||
m := New(10)
|
||||
for i := 0; i < 11; i++ {
|
||||
w, err := m.Create(fmt.Sprintf("key%d", i), 1)
|
||||
if err != nil && i < 10 {
|
||||
t.Errorf("Create failed: %v", err)
|
||||
if err := m.Set(fmt.Sprintf("key%d", i), []byte("1")); err != nil && i < 10 {
|
||||
t.Errorf("Set failed: %v", err)
|
||||
} else if i == 10 && err == nil {
|
||||
t.Errorf("Create succeeded: got nil, want %v", vfserror.ErrDiskFull)
|
||||
}
|
||||
if i < 10 {
|
||||
w.Write([]byte("1"))
|
||||
w.Close()
|
||||
t.Errorf("Set succeeded: got nil, want %v", vfserror.ErrDiskFull)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestStat(t *testing.T) {
|
||||
m := New(1024)
|
||||
key := "key"
|
||||
value := []byte("value")
|
||||
|
||||
w, err := m.Create(key, int64(len(value)))
|
||||
if err != nil {
|
||||
t.Fatalf("Create failed: %v", err)
|
||||
}
|
||||
w.Write(value)
|
||||
w.Close()
|
||||
|
||||
info, err := m.Stat(key)
|
||||
if err != nil {
|
||||
t.Fatalf("unexpected error: %v", err)
|
||||
}
|
||||
|
||||
if info == nil {
|
||||
t.Fatal("expected file info to be non-nil")
|
||||
}
|
||||
if info.Size() != int64(len(value)) {
|
||||
t.Errorf("expected size %d, got %d", len(value), info.Size())
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,76 @@
|
||||
package sync
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"s1d3sw1ped/SteamCache2/vfs"
|
||||
"sync"
|
||||
)
|
||||
|
||||
// Ensure SyncFS implements VFS.
|
||||
var _ vfs.VFS = (*SyncFS)(nil)
|
||||
|
||||
type SyncFS struct {
|
||||
vfs vfs.VFS
|
||||
mu sync.RWMutex
|
||||
}
|
||||
|
||||
func New(vfs vfs.VFS) *SyncFS {
|
||||
return &SyncFS{
|
||||
vfs: vfs,
|
||||
mu: sync.RWMutex{},
|
||||
}
|
||||
}
|
||||
|
||||
// Name returns the name of the file system.
|
||||
func (sfs *SyncFS) Name() string {
|
||||
return fmt.Sprintf("SyncFS(%s)", sfs.vfs.Name())
|
||||
}
|
||||
|
||||
// Size returns the total size of all files in the file system.
|
||||
func (sfs *SyncFS) Size() int64 {
|
||||
sfs.mu.RLock()
|
||||
defer sfs.mu.RUnlock()
|
||||
|
||||
return sfs.vfs.Size()
|
||||
}
|
||||
|
||||
// Set sets the value of key as src.
|
||||
// Setting the same key multiple times, the last set call takes effect.
|
||||
func (sfs *SyncFS) Set(key string, src []byte) error {
|
||||
sfs.mu.Lock()
|
||||
defer sfs.mu.Unlock()
|
||||
|
||||
return sfs.vfs.Set(key, src)
|
||||
}
|
||||
|
||||
// Delete deletes the value of key.
|
||||
func (sfs *SyncFS) Delete(key string) error {
|
||||
sfs.mu.Lock()
|
||||
defer sfs.mu.Unlock()
|
||||
|
||||
return sfs.vfs.Delete(key)
|
||||
}
|
||||
|
||||
// Get gets the value of key to dst, and returns dst no matter whether or not there is an error.
|
||||
func (sfs *SyncFS) Get(key string) ([]byte, error) {
|
||||
sfs.mu.RLock()
|
||||
defer sfs.mu.RUnlock()
|
||||
|
||||
return sfs.vfs.Get(key)
|
||||
}
|
||||
|
||||
// Stat returns the FileInfo of key.
|
||||
func (sfs *SyncFS) Stat(key string) (*vfs.FileInfo, error) {
|
||||
sfs.mu.RLock()
|
||||
defer sfs.mu.RUnlock()
|
||||
|
||||
return sfs.vfs.Stat(key)
|
||||
}
|
||||
|
||||
// StatAll returns the FileInfo of all keys.
|
||||
func (sfs *SyncFS) StatAll() []*vfs.FileInfo {
|
||||
sfs.mu.RLock()
|
||||
defer sfs.mu.RUnlock()
|
||||
|
||||
return sfs.vfs.StatAll()
|
||||
}
|
||||
+5
-7
@@ -1,8 +1,5 @@
|
||||
// vfs/vfs.go
|
||||
package vfs
|
||||
|
||||
import "io"
|
||||
|
||||
// VFS is the interface that wraps the basic methods of a virtual file system.
|
||||
type VFS interface {
|
||||
// Name returns the name of the file system.
|
||||
@@ -11,14 +8,15 @@ type VFS interface {
|
||||
// Size returns the total size of all files in the file system.
|
||||
Size() int64
|
||||
|
||||
// Create creates a new file at key with expected size.
|
||||
Create(key string, size int64) (io.WriteCloser, error)
|
||||
// Set sets the value of key as src.
|
||||
// Setting the same key multiple times, the last set call takes effect.
|
||||
Set(key string, src []byte) error
|
||||
|
||||
// Delete deletes the value of key.
|
||||
Delete(key string) error
|
||||
|
||||
// Open opens the file at key.
|
||||
Open(key string) (io.ReadCloser, error)
|
||||
// Get gets the value of key to dst, and returns dst no matter whether or not there is an error.
|
||||
Get(key string) ([]byte, error)
|
||||
|
||||
// Stat returns the FileInfo of key.
|
||||
Stat(key string) (*FileInfo, error)
|
||||
|
||||
@@ -1,12 +1,8 @@
|
||||
// vfs/vfserror/vfserror.go
|
||||
package vfserror
|
||||
|
||||
import "errors"
|
||||
|
||||
var (
|
||||
// ErrInvalidKey is returned when a key is invalid.
|
||||
ErrInvalidKey = errors.New("vfs: invalid key")
|
||||
|
||||
// ErrUnreachable is returned when a code path is unreachable.
|
||||
ErrUnreachable = errors.New("unreachable")
|
||||
|
||||
|
||||
Reference in New Issue
Block a user