feat: add upstream and verbose flags to command line interface
All checks were successful
Release Tag / release (push) Successful in 13s
All checks were successful
Release Tag / release (push) Successful in 13s
feat: add upstream support allowing to chain cache servers if needed fix: tweaked garbage collection to be better
This commit is contained in:
9
.vscode/launch.json
vendored
9
.vscode/launch.json
vendored
@@ -17,6 +17,9 @@
|
|||||||
"10G",
|
"10G",
|
||||||
"--disk-path",
|
"--disk-path",
|
||||||
"tmp/disk",
|
"tmp/disk",
|
||||||
|
"--upstream",
|
||||||
|
"http://192.168.2.88:80",
|
||||||
|
"--verbose",
|
||||||
],
|
],
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
@@ -30,6 +33,9 @@
|
|||||||
"10G",
|
"10G",
|
||||||
"--disk-path",
|
"--disk-path",
|
||||||
"tmp/disk",
|
"tmp/disk",
|
||||||
|
"--upstream",
|
||||||
|
"http://192.168.2.88:80",
|
||||||
|
"--verbose",
|
||||||
],
|
],
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
@@ -41,6 +47,9 @@
|
|||||||
"args": [
|
"args": [
|
||||||
"--memory",
|
"--memory",
|
||||||
"1G",
|
"1G",
|
||||||
|
"--upstream",
|
||||||
|
"http://192.168.2.88:80",
|
||||||
|
"--verbose",
|
||||||
],
|
],
|
||||||
}
|
}
|
||||||
]
|
]
|
||||||
|
|||||||
17
cmd/root.go
17
cmd/root.go
@@ -4,6 +4,7 @@ import (
|
|||||||
"os"
|
"os"
|
||||||
"s1d3sw1ped/SteamCache2/steamcache"
|
"s1d3sw1ped/SteamCache2/steamcache"
|
||||||
|
|
||||||
|
"github.com/rs/zerolog"
|
||||||
"github.com/spf13/cobra"
|
"github.com/spf13/cobra"
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -13,6 +14,10 @@ var (
|
|||||||
disk string
|
disk string
|
||||||
diskmultiplier int
|
diskmultiplier int
|
||||||
diskpath string
|
diskpath string
|
||||||
|
upstream string
|
||||||
|
|
||||||
|
pprof bool
|
||||||
|
verbose bool
|
||||||
)
|
)
|
||||||
|
|
||||||
var rootCmd = &cobra.Command{
|
var rootCmd = &cobra.Command{
|
||||||
@@ -24,6 +29,10 @@ var rootCmd = &cobra.Command{
|
|||||||
By caching game files, SteamCache2 ensures that subsequent downloads of the same files are served from the local cache,
|
By caching game files, SteamCache2 ensures that subsequent downloads of the same files are served from the local cache,
|
||||||
significantly improving download times and reducing the load on the internet connection.`,
|
significantly improving download times and reducing the load on the internet connection.`,
|
||||||
Run: func(cmd *cobra.Command, args []string) {
|
Run: func(cmd *cobra.Command, args []string) {
|
||||||
|
if verbose {
|
||||||
|
zerolog.SetGlobalLevel(zerolog.DebugLevel)
|
||||||
|
}
|
||||||
|
|
||||||
sc := steamcache.New(
|
sc := steamcache.New(
|
||||||
":80",
|
":80",
|
||||||
memory,
|
memory,
|
||||||
@@ -31,6 +40,8 @@ var rootCmd = &cobra.Command{
|
|||||||
disk,
|
disk,
|
||||||
diskmultiplier,
|
diskmultiplier,
|
||||||
diskpath,
|
diskpath,
|
||||||
|
upstream,
|
||||||
|
pprof,
|
||||||
)
|
)
|
||||||
sc.Run()
|
sc.Run()
|
||||||
},
|
},
|
||||||
@@ -51,4 +62,10 @@ func init() {
|
|||||||
rootCmd.Flags().StringVarP(&disk, "disk", "d", "0", "The size of the disk cache")
|
rootCmd.Flags().StringVarP(&disk, "disk", "d", "0", "The size of the disk cache")
|
||||||
rootCmd.Flags().IntVarP(&diskmultiplier, "disk-gc", "D", 100, "The gc value for the disk cache")
|
rootCmd.Flags().IntVarP(&diskmultiplier, "disk-gc", "D", 100, "The gc value for the disk cache")
|
||||||
rootCmd.Flags().StringVarP(&diskpath, "disk-path", "p", "", "The path to the disk cache")
|
rootCmd.Flags().StringVarP(&diskpath, "disk-path", "p", "", "The path to the disk cache")
|
||||||
|
|
||||||
|
rootCmd.Flags().StringVarP(&upstream, "upstream", "u", "", "The upstream server to proxy requests overrides the host header from the client but forwards the original host header to the upstream server")
|
||||||
|
|
||||||
|
rootCmd.Flags().BoolVarP(&pprof, "pprof", "P", false, "Enable pprof")
|
||||||
|
rootCmd.Flags().MarkHidden("pprof")
|
||||||
|
rootCmd.Flags().BoolVarP(&verbose, "verbose", "v", false, "Enable verbose logging")
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -1,6 +1,7 @@
|
|||||||
package steamcache
|
package steamcache
|
||||||
|
|
||||||
import (
|
import (
|
||||||
|
"runtime/debug"
|
||||||
"s1d3sw1ped/SteamCache2/vfs"
|
"s1d3sw1ped/SteamCache2/vfs"
|
||||||
"s1d3sw1ped/SteamCache2/vfs/cachestate"
|
"s1d3sw1ped/SteamCache2/vfs/cachestate"
|
||||||
"time"
|
"time"
|
||||||
@@ -8,9 +9,13 @@ import (
|
|||||||
"golang.org/x/exp/rand"
|
"golang.org/x/exp/rand"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
func init() {
|
||||||
|
// Set the GC percentage to 50%. This is a good balance between performance and memory usage.
|
||||||
|
debug.SetGCPercent(50)
|
||||||
|
}
|
||||||
|
|
||||||
// RandomGC randomly deletes files until we've reclaimed enough space.
|
// RandomGC randomly deletes files until we've reclaimed enough space.
|
||||||
func randomgc(vfss vfs.VFS, size uint) (uint, uint) {
|
func randomgc(vfss vfs.VFS, size uint) (uint, uint) {
|
||||||
|
|
||||||
// Randomly delete files until we've reclaimed enough space.
|
// Randomly delete files until we've reclaimed enough space.
|
||||||
random := func(vfss vfs.VFS, stats []*vfs.FileInfo) int64 {
|
random := func(vfss vfs.VFS, stats []*vfs.FileInfo) int64 {
|
||||||
randfile := stats[rand.Intn(len(stats))]
|
randfile := stats[rand.Intn(len(stats))]
|
||||||
@@ -29,16 +34,16 @@ func randomgc(vfss vfs.VFS, size uint) (uint, uint) {
|
|||||||
|
|
||||||
stats := vfss.StatAll()
|
stats := vfss.StatAll()
|
||||||
for {
|
for {
|
||||||
reclaimed += random(vfss, stats)
|
|
||||||
deletions++
|
|
||||||
if reclaimed >= targetreclaim {
|
if reclaimed >= targetreclaim {
|
||||||
break
|
break
|
||||||
}
|
}
|
||||||
|
reclaimed += random(vfss, stats)
|
||||||
|
deletions++
|
||||||
}
|
}
|
||||||
|
|
||||||
return uint(reclaimed), uint(deletions)
|
return uint(reclaimed), uint(deletions)
|
||||||
}
|
}
|
||||||
|
|
||||||
func cachehandler(fi *vfs.FileInfo, cs cachestate.CacheState) bool {
|
func cachehandler(fi *vfs.FileInfo, cs cachestate.CacheState) bool {
|
||||||
return time.Since(fi.AccessTime()) < time.Minute*10 // Put files in the cache if they've been accessed twice in the last 10 minutes
|
return time.Since(fi.AccessTime()) < time.Second*10 // Put hot files in the fast vfs if equipped
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -6,4 +6,8 @@ import (
|
|||||||
"github.com/rs/zerolog"
|
"github.com/rs/zerolog"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
func init() {
|
||||||
|
zerolog.SetGlobalLevel(zerolog.InfoLevel)
|
||||||
|
}
|
||||||
|
|
||||||
var Logger = zerolog.New(zerolog.ConsoleWriter{Out: os.Stderr}).With().Timestamp().Logger()
|
var Logger = zerolog.New(zerolog.ConsoleWriter{Out: os.Stderr}).With().Timestamp().Logger()
|
||||||
|
|||||||
@@ -6,6 +6,7 @@ import (
|
|||||||
"net/http"
|
"net/http"
|
||||||
"net/url"
|
"net/url"
|
||||||
"os"
|
"os"
|
||||||
|
"runtime"
|
||||||
"s1d3sw1ped/SteamCache2/steamcache/avgcachestate"
|
"s1d3sw1ped/SteamCache2/steamcache/avgcachestate"
|
||||||
"s1d3sw1ped/SteamCache2/steamcache/logger"
|
"s1d3sw1ped/SteamCache2/steamcache/logger"
|
||||||
"s1d3sw1ped/SteamCache2/version"
|
"s1d3sw1ped/SteamCache2/version"
|
||||||
@@ -20,12 +21,17 @@ import (
|
|||||||
"sync"
|
"sync"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
|
pprof "net/http/pprof"
|
||||||
|
|
||||||
"github.com/docker/go-units"
|
"github.com/docker/go-units"
|
||||||
)
|
)
|
||||||
|
|
||||||
type SteamCache struct {
|
type SteamCache struct {
|
||||||
address string
|
pprof bool
|
||||||
vfs vfs.VFS
|
address string
|
||||||
|
upstream string
|
||||||
|
|
||||||
|
vfs vfs.VFS
|
||||||
|
|
||||||
memory *memory.MemoryFS
|
memory *memory.MemoryFS
|
||||||
disk *disk.DiskFS
|
disk *disk.DiskFS
|
||||||
@@ -39,7 +45,7 @@ type SteamCache struct {
|
|||||||
mu sync.Mutex
|
mu sync.Mutex
|
||||||
}
|
}
|
||||||
|
|
||||||
func New(address string, memorySize string, memoryMultiplier int, diskSize string, diskMultiplier int, diskPath string) *SteamCache {
|
func New(address string, memorySize string, memoryMultiplier int, diskSize string, diskMultiplier int, diskPath, upstream string, pprof bool) *SteamCache {
|
||||||
memorysize, err := units.FromHumanSize(memorySize)
|
memorysize, err := units.FromHumanSize(memorySize)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
panic(err)
|
panic(err)
|
||||||
@@ -92,8 +98,10 @@ func New(address string, memorySize string, memoryMultiplier int, diskSize strin
|
|||||||
}
|
}
|
||||||
|
|
||||||
sc := &SteamCache{
|
sc := &SteamCache{
|
||||||
address: address,
|
pprof: pprof,
|
||||||
vfs: syncfs.New(c),
|
upstream: upstream,
|
||||||
|
address: address,
|
||||||
|
vfs: syncfs.New(c),
|
||||||
|
|
||||||
memory: m,
|
memory: m,
|
||||||
disk: d,
|
disk: d,
|
||||||
@@ -116,6 +124,15 @@ func New(address string, memorySize string, memoryMultiplier int, diskSize strin
|
|||||||
func (sc *SteamCache) Run() {
|
func (sc *SteamCache) Run() {
|
||||||
logger.Logger.Info().Str("address", sc.address).Str("version", version.Version).Msg("listening")
|
logger.Logger.Info().Str("address", sc.address).Str("version", version.Version).Msg("listening")
|
||||||
|
|
||||||
|
if sc.upstream != "" {
|
||||||
|
_, err := http.Get(sc.upstream)
|
||||||
|
if err != nil {
|
||||||
|
logger.Logger.Error().Err(err).Str("upstream", sc.upstream).Msg("Failed to connect to upstream server")
|
||||||
|
os.Exit(1)
|
||||||
|
}
|
||||||
|
logger.Logger.Info().Str("upstream", sc.upstream).Msg("connected")
|
||||||
|
}
|
||||||
|
|
||||||
sc.mu.Lock()
|
sc.mu.Lock()
|
||||||
sc.dirty = true
|
sc.dirty = true
|
||||||
sc.mu.Unlock()
|
sc.mu.Unlock()
|
||||||
@@ -182,6 +199,15 @@ func (sc *SteamCache) LogStats() {
|
|||||||
Msg("disk_gc")
|
Msg("disk_gc")
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// log golang Garbage Collection stats
|
||||||
|
var m runtime.MemStats
|
||||||
|
runtime.ReadMemStats(&m)
|
||||||
|
|
||||||
|
logger.Logger.Info().
|
||||||
|
Str("alloc", units.HumanSize(float64(m.Alloc))).
|
||||||
|
Str("sys", units.HumanSize(float64(m.Sys))).
|
||||||
|
Msg("go_gc")
|
||||||
|
|
||||||
logger.Logger.Info().
|
logger.Logger.Info().
|
||||||
Str("hitrate", fmt.Sprintf("%.2f%%", sc.hits.Avg()*100)).
|
Str("hitrate", fmt.Sprintf("%.2f%%", sc.hits.Avg()*100)).
|
||||||
Msg("cache")
|
Msg("cache")
|
||||||
@@ -191,6 +217,14 @@ func (sc *SteamCache) LogStats() {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func (sc *SteamCache) ServeHTTP(w http.ResponseWriter, r *http.Request) {
|
func (sc *SteamCache) ServeHTTP(w http.ResponseWriter, r *http.Request) {
|
||||||
|
if sc.pprof && r.URL.Path == "/debug/pprof/" {
|
||||||
|
pprof.Index(w, r)
|
||||||
|
return
|
||||||
|
} else if sc.pprof && strings.HasPrefix(r.URL.Path, "/debug/pprof/") {
|
||||||
|
pprof.Handler(strings.TrimPrefix(r.URL.Path, "/debug/pprof/")).ServeHTTP(w, r)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
if r.Method != http.MethodGet {
|
if r.Method != http.MethodGet {
|
||||||
http.Error(w, "Only GET method is supported", http.StatusMethodNotAllowed)
|
http.Error(w, "Only GET method is supported", http.StatusMethodNotAllowed)
|
||||||
return
|
return
|
||||||
@@ -203,54 +237,68 @@ func (sc *SteamCache) ServeHTTP(w http.ResponseWriter, r *http.Request) {
|
|||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
if r.Header.Get("User-Agent") != "Valve/Steam HTTP Client 1.0" {
|
|
||||||
http.Error(w, "Only Valve/Steam HTTP Client 1.0 is supported", http.StatusForbidden)
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
if strings.Contains(r.URL.String(), "manifest") {
|
|
||||||
w.Header().Add("X-LanCache-Processed-By", "SteamCache2")
|
|
||||||
forward(w, r)
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
// tstart := time.Now()
|
|
||||||
// defer func() {
|
|
||||||
// logger.Logger.Info().Str("method", r.Method).Str("url", r.URL.String()).Str("status", w.Header().Get("X-LanCache-Status")).Dur("duration", time.Since(tstart)).Msg("Request")
|
|
||||||
// }()
|
|
||||||
|
|
||||||
sc.mu.Lock()
|
sc.mu.Lock()
|
||||||
sc.dirty = true
|
sc.dirty = true
|
||||||
sc.mu.Unlock()
|
sc.mu.Unlock()
|
||||||
|
|
||||||
w.Header().Add("X-LanCache-Processed-By", "SteamCache2") // SteamPrefill uses this header to determine if the request was processed by the cache maybe steam uses it too
|
w.Header().Add("X-LanCache-Processed-By", "SteamCache2") // SteamPrefill uses this header to determine if the request was processed by the cache maybe steam uses it too
|
||||||
|
|
||||||
cacheKey := r.URL.String()
|
cacheKey := strings.ReplaceAll(r.URL.String()[1:], "\\", "/") // replace all backslashes with forward slashes shouldn't be necessary but just in case
|
||||||
|
if cacheKey == "" {
|
||||||
// if vfs is also a vfs.GetSer, we can use it to get the cache state
|
http.Error(w, "Invalid URL", http.StatusBadRequest)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
data, err := sc.vfs.Get(cacheKey)
|
data, err := sc.vfs.Get(cacheKey)
|
||||||
if err == nil {
|
if err == nil {
|
||||||
sc.hits.Add(cachestate.CacheStateHit)
|
sc.hits.Add(cachestate.CacheStateHit)
|
||||||
w.Header().Add("X-LanCache-Status", "HIT")
|
w.Header().Add("X-LanCache-Status", "HIT")
|
||||||
w.Write(data)
|
w.Write(data)
|
||||||
|
logger.Logger.Debug().Str("key", r.URL.String()).Msg("cache")
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
htt := "http://"
|
var req *http.Request
|
||||||
if r.Header.Get("X-Sls-Https") == "enable" {
|
if sc.upstream != "" { // if an upstream server is configured, proxy the request to the upstream server
|
||||||
htt = "https://"
|
ur, err := url.JoinPath(sc.upstream, r.URL.String())
|
||||||
|
if err != nil {
|
||||||
|
http.Error(w, "Failed to join URL path", http.StatusInternalServerError)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
req, err = http.NewRequest(http.MethodGet, ur, nil)
|
||||||
|
if err != nil {
|
||||||
|
http.Error(w, "Failed to create request", http.StatusInternalServerError)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
req.Host = r.Host
|
||||||
|
logger.Logger.Debug().Str("key", cacheKey).Str("host", sc.upstream).Msg("upstream")
|
||||||
|
} else { // if no upstream server is configured, proxy the request to the host specified in the request
|
||||||
|
host := r.Host
|
||||||
|
if r.Header.Get("X-Sls-Https") == "enable" {
|
||||||
|
host = "https://" + host
|
||||||
|
} else {
|
||||||
|
host = "http://" + host
|
||||||
|
}
|
||||||
|
|
||||||
|
ur, err := url.JoinPath(host, r.URL.String())
|
||||||
|
if err != nil {
|
||||||
|
http.Error(w, "Failed to join URL path", http.StatusInternalServerError)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
req, err = http.NewRequest(http.MethodGet, ur, nil)
|
||||||
|
if err != nil {
|
||||||
|
http.Error(w, "Failed to create request", http.StatusInternalServerError)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
logger.Logger.Debug().Str("key", cacheKey).Str("host", host).Msg("forward")
|
||||||
}
|
}
|
||||||
|
|
||||||
base := htt + r.Host
|
req.Header.Add("X-Sls-Https", r.Header.Get("X-Sls-Https"))
|
||||||
|
req.Header.Add("User-Agent", r.Header.Get("User-Agent"))
|
||||||
hosturl, err := url.JoinPath(base, cacheKey)
|
resp, err := http.DefaultClient.Do(req)
|
||||||
if err != nil {
|
|
||||||
http.Error(w, "Failed to join URL path", http.StatusInternalServerError)
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
resp, err := http.Get(hosturl)
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
http.Error(w, "Failed to fetch the requested URL", http.StatusInternalServerError)
|
http.Error(w, "Failed to fetch the requested URL", http.StatusInternalServerError)
|
||||||
return
|
return
|
||||||
@@ -273,40 +321,3 @@ func (sc *SteamCache) ServeHTTP(w http.ResponseWriter, r *http.Request) {
|
|||||||
w.Header().Add("X-LanCache-Status", "MISS")
|
w.Header().Add("X-LanCache-Status", "MISS")
|
||||||
w.Write(body)
|
w.Write(body)
|
||||||
}
|
}
|
||||||
|
|
||||||
func forward(w http.ResponseWriter, r *http.Request) {
|
|
||||||
htt := "http://"
|
|
||||||
if r.Header.Get("X-Sls-Https") == "enable" {
|
|
||||||
htt = "https://"
|
|
||||||
}
|
|
||||||
|
|
||||||
base := htt + r.Host
|
|
||||||
|
|
||||||
cacheKey := r.URL.String()
|
|
||||||
|
|
||||||
hosturl, err := url.JoinPath(base, cacheKey)
|
|
||||||
if err != nil {
|
|
||||||
http.Error(w, "Failed to join URL path", http.StatusInternalServerError)
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
resp, err := http.Get(hosturl)
|
|
||||||
if err != nil {
|
|
||||||
http.Error(w, "Failed to fetch the requested URL", http.StatusInternalServerError)
|
|
||||||
return
|
|
||||||
}
|
|
||||||
defer resp.Body.Close()
|
|
||||||
|
|
||||||
if resp.StatusCode != http.StatusOK {
|
|
||||||
http.Error(w, "Failed to fetch the requested URL", resp.StatusCode)
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
body, err := io.ReadAll(resp.Body)
|
|
||||||
if err != nil {
|
|
||||||
http.Error(w, "Failed to read response body", http.StatusInternalServerError)
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
w.Write(body)
|
|
||||||
}
|
|
||||||
|
|||||||
71
steamcache/steamcache_test.go
Normal file
71
steamcache/steamcache_test.go
Normal file
@@ -0,0 +1,71 @@
|
|||||||
|
package steamcache
|
||||||
|
|
||||||
|
import (
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
"testing"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestCaching(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
td := t.TempDir()
|
||||||
|
|
||||||
|
os.WriteFile(filepath.Join(td, "key2"), []byte("value2"), 0644)
|
||||||
|
|
||||||
|
sc := New("localhost:8080", "1GB", 10, "1GB", 100, td, "", false)
|
||||||
|
|
||||||
|
sc.dirty = true
|
||||||
|
sc.LogStats()
|
||||||
|
|
||||||
|
if err := sc.vfs.Set("key", []byte("value")); err != nil {
|
||||||
|
t.Errorf("Set failed: %v", err)
|
||||||
|
}
|
||||||
|
if err := sc.vfs.Set("key1", []byte("value1")); err != nil {
|
||||||
|
t.Errorf("Set failed: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
sc.dirty = true
|
||||||
|
sc.LogStats()
|
||||||
|
|
||||||
|
if sc.diskgc.Size() != 17 {
|
||||||
|
t.Errorf("Size failed: got %d, want %d", sc.diskgc.Size(), 17)
|
||||||
|
}
|
||||||
|
|
||||||
|
if sc.vfs.Size() != 17 {
|
||||||
|
t.Errorf("Size failed: got %d, want %d", sc.vfs.Size(), 17)
|
||||||
|
}
|
||||||
|
|
||||||
|
if d, err := sc.vfs.Get("key"); err != nil {
|
||||||
|
t.Errorf("Get failed: %v", err)
|
||||||
|
} else if string(d) != "value" {
|
||||||
|
t.Errorf("Get failed: got %s, want %s", d, "value")
|
||||||
|
}
|
||||||
|
|
||||||
|
if d, err := sc.vfs.Get("key1"); err != nil {
|
||||||
|
t.Errorf("Get failed: %v", err)
|
||||||
|
} else if string(d) != "value1" {
|
||||||
|
t.Errorf("Get failed: got %s, want %s", d, "value1")
|
||||||
|
}
|
||||||
|
|
||||||
|
if d, err := sc.vfs.Get("key2"); err != nil {
|
||||||
|
t.Errorf("Get failed: %v", err)
|
||||||
|
} else if string(d) != "value2" {
|
||||||
|
t.Errorf("Get failed: got %s, want %s", d, "value2")
|
||||||
|
}
|
||||||
|
|
||||||
|
if sc.diskgc.Size() != 17 {
|
||||||
|
t.Errorf("Size failed: got %d, want %d", sc.diskgc.Size(), 17)
|
||||||
|
}
|
||||||
|
|
||||||
|
if sc.vfs.Size() != 17 {
|
||||||
|
t.Errorf("Size failed: got %d, want %d", sc.vfs.Size(), 17)
|
||||||
|
}
|
||||||
|
|
||||||
|
sc.memory.Delete("key2")
|
||||||
|
os.Remove(filepath.Join(td, "key2"))
|
||||||
|
|
||||||
|
if _, err := sc.vfs.Get("key2"); err == nil {
|
||||||
|
t.Errorf("Get failed: got nil, want error")
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -1,11 +1,13 @@
|
|||||||
package disk
|
package disk
|
||||||
|
|
||||||
import (
|
import (
|
||||||
|
"fmt"
|
||||||
"os"
|
"os"
|
||||||
"path/filepath"
|
"path/filepath"
|
||||||
"s1d3sw1ped/SteamCache2/steamcache/logger"
|
"s1d3sw1ped/SteamCache2/steamcache/logger"
|
||||||
"s1d3sw1ped/SteamCache2/vfs"
|
"s1d3sw1ped/SteamCache2/vfs"
|
||||||
"s1d3sw1ped/SteamCache2/vfs/vfserror"
|
"s1d3sw1ped/SteamCache2/vfs/vfserror"
|
||||||
|
"strings"
|
||||||
"sync"
|
"sync"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
@@ -82,6 +84,8 @@ func (d *DiskFS) init() {
|
|||||||
Str("name", d.Name()).
|
Str("name", d.Name()).
|
||||||
Str("root", d.root).
|
Str("root", d.root).
|
||||||
Str("capacity", units.HumanSize(float64(d.capacity))).
|
Str("capacity", units.HumanSize(float64(d.capacity))).
|
||||||
|
Str("size", units.HumanSize(float64(d.Size()))).
|
||||||
|
Str("files", fmt.Sprint(len(d.info))).
|
||||||
Str("duration", time.Since(tstart).String()).
|
Str("duration", time.Since(tstart).String()).
|
||||||
Msg("init")
|
Msg("init")
|
||||||
}
|
}
|
||||||
@@ -105,7 +109,8 @@ func (d *DiskFS) walk(path string) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
d.mu.Lock()
|
d.mu.Lock()
|
||||||
k := npath[len(d.root)+1:]
|
k := strings.ReplaceAll(npath[len(d.root)+1:], "\\", "/")
|
||||||
|
logger.Logger.Debug().Str("name", k).Str("root", d.root).Msg("walk")
|
||||||
d.info[k] = vfs.NewFileInfoFromOS(info, k)
|
d.info[k] = vfs.NewFileInfoFromOS(info, k)
|
||||||
d.mu.Unlock()
|
d.mu.Unlock()
|
||||||
|
|
||||||
@@ -124,9 +129,10 @@ func (d *DiskFS) Name() string {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func (d *DiskFS) Size() int64 {
|
func (d *DiskFS) Size() int64 {
|
||||||
var size int64
|
|
||||||
d.mu.Lock()
|
d.mu.Lock()
|
||||||
defer d.mu.Unlock()
|
defer d.mu.Unlock()
|
||||||
|
|
||||||
|
var size int64
|
||||||
for _, v := range d.info {
|
for _, v := range d.info {
|
||||||
size += v.Size()
|
size += v.Size()
|
||||||
}
|
}
|
||||||
@@ -134,24 +140,34 @@ func (d *DiskFS) Size() int64 {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func (d *DiskFS) Set(key string, src []byte) error {
|
func (d *DiskFS) Set(key string, src []byte) error {
|
||||||
|
if key == "" {
|
||||||
|
return vfserror.ErrInvalidKey
|
||||||
|
}
|
||||||
|
if key[0] == '/' {
|
||||||
|
return vfserror.ErrInvalidKey
|
||||||
|
}
|
||||||
|
|
||||||
if d.capacity > 0 {
|
if d.capacity > 0 {
|
||||||
if size := d.Size() + int64(len(src)); size > d.capacity {
|
if size := d.Size() + int64(len(src)); size > d.capacity {
|
||||||
return vfserror.ErrDiskFull
|
return vfserror.ErrDiskFull
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
logger.Logger.Debug().Str("name", key).Str("root", d.root).Msg("set")
|
||||||
|
|
||||||
if _, err := d.Stat(key); err == nil {
|
if _, err := d.Stat(key); err == nil {
|
||||||
|
logger.Logger.Debug().Str("name", key).Str("root", d.root).Msg("delete")
|
||||||
d.Delete(key)
|
d.Delete(key)
|
||||||
}
|
}
|
||||||
|
|
||||||
d.mu.Lock()
|
d.mu.Lock()
|
||||||
defer d.mu.Unlock()
|
defer d.mu.Unlock()
|
||||||
os.MkdirAll(filepath.Join(d.root, filepath.Dir(key)), 0755)
|
os.MkdirAll(d.root+"/"+filepath.Dir(key), 0755)
|
||||||
if err := os.WriteFile(filepath.Join(d.root, key), src, 0644); err != nil {
|
if err := os.WriteFile(d.root+"/"+key, src, 0644); err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
|
||||||
fi, err := os.Stat(filepath.Join(d.root, key))
|
fi, err := os.Stat(d.root + "/" + key)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
panic(err)
|
panic(err)
|
||||||
}
|
}
|
||||||
@@ -163,6 +179,13 @@ func (d *DiskFS) Set(key string, src []byte) error {
|
|||||||
|
|
||||||
// Delete deletes the value of key.
|
// Delete deletes the value of key.
|
||||||
func (d *DiskFS) Delete(key string) error {
|
func (d *DiskFS) Delete(key string) error {
|
||||||
|
if key == "" {
|
||||||
|
return vfserror.ErrInvalidKey
|
||||||
|
}
|
||||||
|
if key[0] == '/' {
|
||||||
|
return vfserror.ErrInvalidKey
|
||||||
|
}
|
||||||
|
|
||||||
_, err := d.Stat(key)
|
_, err := d.Stat(key)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
@@ -170,6 +193,7 @@ func (d *DiskFS) Delete(key string) error {
|
|||||||
|
|
||||||
d.mu.Lock()
|
d.mu.Lock()
|
||||||
defer d.mu.Unlock()
|
defer d.mu.Unlock()
|
||||||
|
|
||||||
delete(d.info, key)
|
delete(d.info, key)
|
||||||
if err := os.Remove(filepath.Join(d.root, key)); err != nil {
|
if err := os.Remove(filepath.Join(d.root, key)); err != nil {
|
||||||
return err
|
return err
|
||||||
@@ -180,6 +204,13 @@ func (d *DiskFS) Delete(key string) error {
|
|||||||
|
|
||||||
// Get gets the value of key and returns it.
|
// Get gets the value of key and returns it.
|
||||||
func (d *DiskFS) Get(key string) ([]byte, error) {
|
func (d *DiskFS) Get(key string) ([]byte, error) {
|
||||||
|
if key == "" {
|
||||||
|
return nil, vfserror.ErrInvalidKey
|
||||||
|
}
|
||||||
|
if key[0] == '/' {
|
||||||
|
return nil, vfserror.ErrInvalidKey
|
||||||
|
}
|
||||||
|
|
||||||
_, err := d.Stat(key)
|
_, err := d.Stat(key)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
@@ -198,25 +229,23 @@ func (d *DiskFS) Get(key string) ([]byte, error) {
|
|||||||
|
|
||||||
// Stat returns the FileInfo of key. If key is not found in the cache, it will stat the file on disk. If the file is not found on disk, it will return vfs.ErrNotFound.
|
// Stat returns the FileInfo of key. If key is not found in the cache, it will stat the file on disk. If the file is not found on disk, it will return vfs.ErrNotFound.
|
||||||
func (d *DiskFS) Stat(key string) (*vfs.FileInfo, error) {
|
func (d *DiskFS) Stat(key string) (*vfs.FileInfo, error) {
|
||||||
d.mu.Lock()
|
if key == "" {
|
||||||
fi, ok := d.info[key]
|
return nil, vfserror.ErrInvalidKey
|
||||||
d.mu.Unlock() // unlock before statting the file
|
}
|
||||||
|
if key[0] == '/' {
|
||||||
if !ok {
|
return nil, vfserror.ErrInvalidKey
|
||||||
fii, err := os.Stat(filepath.Join(d.root, key))
|
|
||||||
if err != nil {
|
|
||||||
return nil, vfserror.ErrNotFound
|
|
||||||
}
|
|
||||||
|
|
||||||
d.mu.Lock() // relock to update the info map
|
|
||||||
defer d.mu.Unlock() // nothing else needs to unlock before returning
|
|
||||||
|
|
||||||
d.info[key] = vfs.NewFileInfoFromOS(fii, key)
|
|
||||||
fi = d.info[key]
|
|
||||||
// fallthrough to return fi with shiny new info
|
|
||||||
}
|
}
|
||||||
|
|
||||||
return fi, nil
|
logger.Logger.Debug().Str("name", key).Str("root", d.root).Msg("stat")
|
||||||
|
|
||||||
|
d.mu.Lock()
|
||||||
|
defer d.mu.Unlock()
|
||||||
|
|
||||||
|
if fi, ok := d.info[key]; !ok {
|
||||||
|
return nil, vfserror.ErrNotFound
|
||||||
|
} else {
|
||||||
|
return fi, nil
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func (m *DiskFS) StatAll() []*vfs.FileInfo {
|
func (m *DiskFS) StatAll() []*vfs.FileInfo {
|
||||||
|
|||||||
@@ -85,3 +85,62 @@ func TestInit(t *testing.T) {
|
|||||||
t.Errorf("Stat failed: got %s, want %s", s.Name(), "key")
|
t.Errorf("Stat failed: got %s, want %s", s.Name(), "key")
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestDiskSizeDiscrepancy(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
td := t.TempDir()
|
||||||
|
|
||||||
|
assumedSize := int64(6 + 5 + 6) // 6 + 5 + 6 bytes for key, key1, key2
|
||||||
|
os.WriteFile(filepath.Join(td, "key2"), []byte("value2"), 0644)
|
||||||
|
|
||||||
|
m := New(td, 1024)
|
||||||
|
if 6 != m.Size() {
|
||||||
|
t.Errorf("Size failed: got %d, want %d", m.Size(), 6)
|
||||||
|
}
|
||||||
|
|
||||||
|
if err := m.Set("key", []byte("value")); err != nil {
|
||||||
|
t.Errorf("Set failed: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
if err := m.Set("key1", []byte("value1")); err != nil {
|
||||||
|
t.Errorf("Set failed: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
if assumedSize != m.Size() {
|
||||||
|
t.Errorf("Size failed: got %d, want %d", m.Size(), assumedSize)
|
||||||
|
}
|
||||||
|
|
||||||
|
if d, err := m.Get("key"); err != nil {
|
||||||
|
t.Errorf("Get failed: %v", err)
|
||||||
|
} else if string(d) != "value" {
|
||||||
|
t.Errorf("Get failed: got %s, want %s", d, "value")
|
||||||
|
}
|
||||||
|
|
||||||
|
if d, err := m.Get("key1"); err != nil {
|
||||||
|
t.Errorf("Get failed: %v", err)
|
||||||
|
} else if string(d) != "value1" {
|
||||||
|
t.Errorf("Get failed: got %s, want %s", d, "value1")
|
||||||
|
}
|
||||||
|
|
||||||
|
m = New(td, 1024)
|
||||||
|
|
||||||
|
if assumedSize != m.Size() {
|
||||||
|
t.Errorf("Size failed: got %d, want %d", m.Size(), assumedSize)
|
||||||
|
}
|
||||||
|
|
||||||
|
if d, err := m.Get("key"); err != nil {
|
||||||
|
t.Errorf("Get failed: %v", err)
|
||||||
|
} else if string(d) != "value" {
|
||||||
|
t.Errorf("Get failed: got %s, want %s", d, "value")
|
||||||
|
}
|
||||||
|
|
||||||
|
if d, err := m.Get("key1"); err != nil {
|
||||||
|
t.Errorf("Get failed: %v", err)
|
||||||
|
} else if string(d) != "value1" {
|
||||||
|
t.Errorf("Get failed: got %s, want %s", d, "value1")
|
||||||
|
}
|
||||||
|
|
||||||
|
if assumedSize != m.Size() {
|
||||||
|
t.Errorf("Size failed: got %d, want %d", m.Size(), assumedSize)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|||||||
@@ -12,9 +12,9 @@ type FileInfo struct {
|
|||||||
ATime time.Time
|
ATime time.Time
|
||||||
}
|
}
|
||||||
|
|
||||||
func NewFileInfo(name string, size int64, modTime time.Time) *FileInfo {
|
func NewFileInfo(key string, size int64, modTime time.Time) *FileInfo {
|
||||||
return &FileInfo{
|
return &FileInfo{
|
||||||
name: name,
|
name: key,
|
||||||
size: size,
|
size: size,
|
||||||
MTime: modTime,
|
MTime: modTime,
|
||||||
ATime: time.Now(),
|
ATime: time.Now(),
|
||||||
|
|||||||
@@ -73,9 +73,8 @@ func (m *MemoryFS) Set(key string, src []byte) error {
|
|||||||
int64(len(src)),
|
int64(len(src)),
|
||||||
time.Now(),
|
time.Now(),
|
||||||
),
|
),
|
||||||
data: make([]byte, len(src)),
|
data: src,
|
||||||
}
|
}
|
||||||
copy(m.files[key].data, src)
|
|
||||||
|
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -3,6 +3,9 @@ package vfserror
|
|||||||
import "errors"
|
import "errors"
|
||||||
|
|
||||||
var (
|
var (
|
||||||
|
// ErrInvalidKey is returned when a key is invalid.
|
||||||
|
ErrInvalidKey = errors.New("vfs: invalid key")
|
||||||
|
|
||||||
// ErrUnreachable is returned when a code path is unreachable.
|
// ErrUnreachable is returned when a code path is unreachable.
|
||||||
ErrUnreachable = errors.New("unreachable")
|
ErrUnreachable = errors.New("unreachable")
|
||||||
|
|
||||||
|
|||||||
Reference in New Issue
Block a user