feat: add upstream and verbose flags to command line interface

feat: add upstream support allowing to chain cache servers if needed
fix: tweaked garbage collection to be better

steamcache/gc.go

@@ -1,6 +1,7 @@
 package steamcache
 
 import (
+	"runtime/debug"
 	"s1d3sw1ped/SteamCache2/vfs"
 	"s1d3sw1ped/SteamCache2/vfs/cachestate"
 	"time"
@@ -8,9 +9,13 @@ import (
 	"golang.org/x/exp/rand"
 )
 
+func init() {
+	// Set the GC percentage to 50%. This is a good balance between performance and memory usage.
+	debug.SetGCPercent(50)
+}
+
 // RandomGC randomly deletes files until we've reclaimed enough space.
 func randomgc(vfss vfs.VFS, size uint) (uint, uint) {
-
 	// Randomly delete files until we've reclaimed enough space.
 	random := func(vfss vfs.VFS, stats []*vfs.FileInfo) int64 {
 		randfile := stats[rand.Intn(len(stats))]
@@ -29,16 +34,16 @@ func randomgc(vfss vfs.VFS, size uint) (uint, uint) {
 
 	stats := vfss.StatAll()
 	for {
-		reclaimed += random(vfss, stats)
-		deletions++
 		if reclaimed >= targetreclaim {
 			break
 		}
+		reclaimed += random(vfss, stats)
+		deletions++
 	}
 
 	return uint(reclaimed), uint(deletions)
 }
 
 func cachehandler(fi *vfs.FileInfo, cs cachestate.CacheState) bool {
-	return time.Since(fi.AccessTime()) < time.Minute*10 // Put files in the cache if they've been accessed twice in the last 10 minutes
+	return time.Since(fi.AccessTime()) < time.Second*10 // Put hot files in the fast vfs if equipped
 }

steamcache/logger/logger.go

@@ -6,4 +6,8 @@ import (
 	"github.com/rs/zerolog"
 )
 
+func init() {
+	zerolog.SetGlobalLevel(zerolog.InfoLevel)
+}
+
 var Logger = zerolog.New(zerolog.ConsoleWriter{Out: os.Stderr}).With().Timestamp().Logger()

steamcache/steamcache.go

@@ -6,6 +6,7 @@ import (
 	"net/http"
 	"net/url"
 	"os"
+	"runtime"
 	"s1d3sw1ped/SteamCache2/steamcache/avgcachestate"
 	"s1d3sw1ped/SteamCache2/steamcache/logger"
 	"s1d3sw1ped/SteamCache2/version"
@@ -20,12 +21,17 @@ import (
 	"sync"
 	"time"
 
+	pprof "net/http/pprof"
+
 	"github.com/docker/go-units"
 )
 
 type SteamCache struct {
-	address string
-	vfs     vfs.VFS
+	pprof    bool
+	address  string
+	upstream string
+
+	vfs vfs.VFS
 
 	memory *memory.MemoryFS
 	disk   *disk.DiskFS
@@ -39,7 +45,7 @@ type SteamCache struct {
 	mu    sync.Mutex
 }
 
-func New(address string, memorySize string, memoryMultiplier int, diskSize string, diskMultiplier int, diskPath string) *SteamCache {
+func New(address string, memorySize string, memoryMultiplier int, diskSize string, diskMultiplier int, diskPath, upstream string, pprof bool) *SteamCache {
 	memorysize, err := units.FromHumanSize(memorySize)
 	if err != nil {
 		panic(err)
@@ -92,8 +98,10 @@ func New(address string, memorySize string, memoryMultiplier int, diskSize strin
 	}
 
 	sc := &SteamCache{
-		address: address,
-		vfs:     syncfs.New(c),
+		pprof:    pprof,
+		upstream: upstream,
+		address:  address,
+		vfs:      syncfs.New(c),
 
 		memory: m,
 		disk:   d,
@@ -116,6 +124,15 @@ func New(address string, memorySize string, memoryMultiplier int, diskSize strin
 func (sc *SteamCache) Run() {
 	logger.Logger.Info().Str("address", sc.address).Str("version", version.Version).Msg("listening")
 
+	if sc.upstream != "" {
+		_, err := http.Get(sc.upstream)
+		if err != nil {
+			logger.Logger.Error().Err(err).Str("upstream", sc.upstream).Msg("Failed to connect to upstream server")
+			os.Exit(1)
+		}
+		logger.Logger.Info().Str("upstream", sc.upstream).Msg("connected")
+	}
+
 	sc.mu.Lock()
 	sc.dirty = true
 	sc.mu.Unlock()
@@ -182,6 +199,15 @@ func (sc *SteamCache) LogStats() {
 				Msg("disk_gc")
 		}
 
+		// log golang Garbage Collection stats
+		var m runtime.MemStats
+		runtime.ReadMemStats(&m)
+
+		logger.Logger.Info().
+			Str("alloc", units.HumanSize(float64(m.Alloc))).
+			Str("sys", units.HumanSize(float64(m.Sys))).
+			Msg("go_gc")
+
 		logger.Logger.Info().
 			Str("hitrate", fmt.Sprintf("%.2f%%", sc.hits.Avg()*100)).
 			Msg("cache")
@@ -191,6 +217,14 @@ func (sc *SteamCache) LogStats() {
 }
 
 func (sc *SteamCache) ServeHTTP(w http.ResponseWriter, r *http.Request) {
+	if sc.pprof && r.URL.Path == "/debug/pprof/" {
+		pprof.Index(w, r)
+		return
+	} else if sc.pprof && strings.HasPrefix(r.URL.Path, "/debug/pprof/") {
+		pprof.Handler(strings.TrimPrefix(r.URL.Path, "/debug/pprof/")).ServeHTTP(w, r)
+		return
+	}
+
 	if r.Method != http.MethodGet {
 		http.Error(w, "Only GET method is supported", http.StatusMethodNotAllowed)
 		return
@@ -203,54 +237,68 @@ func (sc *SteamCache) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 		return
 	}
 
-	if r.Header.Get("User-Agent") != "Valve/Steam HTTP Client 1.0" {
-		http.Error(w, "Only Valve/Steam HTTP Client 1.0 is supported", http.StatusForbidden)
-		return
-	}
-
-	if strings.Contains(r.URL.String(), "manifest") {
-		w.Header().Add("X-LanCache-Processed-By", "SteamCache2")
-		forward(w, r)
-		return
-	}
-
-	// tstart := time.Now()
-	// defer func() {
-	// logger.Logger.Info().Str("method", r.Method).Str("url", r.URL.String()).Str("status", w.Header().Get("X-LanCache-Status")).Dur("duration", time.Since(tstart)).Msg("Request")
-	// }()
-
 	sc.mu.Lock()
 	sc.dirty = true
 	sc.mu.Unlock()
 
 	w.Header().Add("X-LanCache-Processed-By", "SteamCache2") // SteamPrefill uses this header to determine if the request was processed by the cache maybe steam uses it too
 
-	cacheKey := r.URL.String()
-
-	// if vfs is also a vfs.GetSer, we can use it to get the cache state
+	cacheKey := strings.ReplaceAll(r.URL.String()[1:], "\\", "/") // replace all backslashes with forward slashes shouldn't be necessary but just in case
+	if cacheKey == "" {
+		http.Error(w, "Invalid URL", http.StatusBadRequest)
+		return
+	}
 
 	data, err := sc.vfs.Get(cacheKey)
 	if err == nil {
 		sc.hits.Add(cachestate.CacheStateHit)
 		w.Header().Add("X-LanCache-Status", "HIT")
 		w.Write(data)
+		logger.Logger.Debug().Str("key", r.URL.String()).Msg("cache")
 		return
 	}
 
-	htt := "http://"
-	if r.Header.Get("X-Sls-Https") == "enable" {
-		htt = "https://"
+	var req *http.Request
+	if sc.upstream != "" { // if an upstream server is configured, proxy the request to the upstream server
+		ur, err := url.JoinPath(sc.upstream, r.URL.String())
+		if err != nil {
+			http.Error(w, "Failed to join URL path", http.StatusInternalServerError)
+			return
+		}
+
+		req, err = http.NewRequest(http.MethodGet, ur, nil)
+		if err != nil {
+			http.Error(w, "Failed to create request", http.StatusInternalServerError)
+			return
+		}
+		req.Host = r.Host
+		logger.Logger.Debug().Str("key", cacheKey).Str("host", sc.upstream).Msg("upstream")
+	} else { // if no upstream server is configured, proxy the request to the host specified in the request
+		host := r.Host
+		if r.Header.Get("X-Sls-Https") == "enable" {
+			host = "https://" + host
+		} else {
+			host = "http://" + host
+		}
+
+		ur, err := url.JoinPath(host, r.URL.String())
+		if err != nil {
+			http.Error(w, "Failed to join URL path", http.StatusInternalServerError)
+			return
+		}
+
+		req, err = http.NewRequest(http.MethodGet, ur, nil)
+		if err != nil {
+			http.Error(w, "Failed to create request", http.StatusInternalServerError)
+			return
+		}
+
+		logger.Logger.Debug().Str("key", cacheKey).Str("host", host).Msg("forward")
 	}
 
-	base := htt + r.Host
-
-	hosturl, err := url.JoinPath(base, cacheKey)
-	if err != nil {
-		http.Error(w, "Failed to join URL path", http.StatusInternalServerError)
-		return
-	}
-
-	resp, err := http.Get(hosturl)
+	req.Header.Add("X-Sls-Https", r.Header.Get("X-Sls-Https"))
+	req.Header.Add("User-Agent", r.Header.Get("User-Agent"))
+	resp, err := http.DefaultClient.Do(req)
 	if err != nil {
 		http.Error(w, "Failed to fetch the requested URL", http.StatusInternalServerError)
 		return
@@ -273,40 +321,3 @@ func (sc *SteamCache) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 	w.Header().Add("X-LanCache-Status", "MISS")
 	w.Write(body)
 }
-
-func forward(w http.ResponseWriter, r *http.Request) {
-	htt := "http://"
-	if r.Header.Get("X-Sls-Https") == "enable" {
-		htt = "https://"
-	}
-
-	base := htt + r.Host
-
-	cacheKey := r.URL.String()
-
-	hosturl, err := url.JoinPath(base, cacheKey)
-	if err != nil {
-		http.Error(w, "Failed to join URL path", http.StatusInternalServerError)
-		return
-	}
-
-	resp, err := http.Get(hosturl)
-	if err != nil {
-		http.Error(w, "Failed to fetch the requested URL", http.StatusInternalServerError)
-		return
-	}
-	defer resp.Body.Close()
-
-	if resp.StatusCode != http.StatusOK {
-		http.Error(w, "Failed to fetch the requested URL", resp.StatusCode)
-		return
-	}
-
-	body, err := io.ReadAll(resp.Body)
-	if err != nil {
-		http.Error(w, "Failed to read response body", http.StatusInternalServerError)
-		return
-	}
-
-	w.Write(body)
-}

steamcache/steamcache_test.go

@@ -0,0 +1,71 @@
+package steamcache
+
+import (
+	"os"
+	"path/filepath"
+	"testing"
+)
+
+func TestCaching(t *testing.T) {
+	t.Parallel()
+
+	td := t.TempDir()
+
+	os.WriteFile(filepath.Join(td, "key2"), []byte("value2"), 0644)
+
+	sc := New("localhost:8080", "1GB", 10, "1GB", 100, td, "", false)
+
+	sc.dirty = true
+	sc.LogStats()
+
+	if err := sc.vfs.Set("key", []byte("value")); err != nil {
+		t.Errorf("Set failed: %v", err)
+	}
+	if err := sc.vfs.Set("key1", []byte("value1")); err != nil {
+		t.Errorf("Set failed: %v", err)
+	}
+
+	sc.dirty = true
+	sc.LogStats()
+
+	if sc.diskgc.Size() != 17 {
+		t.Errorf("Size failed: got %d, want %d", sc.diskgc.Size(), 17)
+	}
+
+	if sc.vfs.Size() != 17 {
+		t.Errorf("Size failed: got %d, want %d", sc.vfs.Size(), 17)
+	}
+
+	if d, err := sc.vfs.Get("key"); err != nil {
+		t.Errorf("Get failed: %v", err)
+	} else if string(d) != "value" {
+		t.Errorf("Get failed: got %s, want %s", d, "value")
+	}
+
+	if d, err := sc.vfs.Get("key1"); err != nil {
+		t.Errorf("Get failed: %v", err)
+	} else if string(d) != "value1" {
+		t.Errorf("Get failed: got %s, want %s", d, "value1")
+	}
+
+	if d, err := sc.vfs.Get("key2"); err != nil {
+		t.Errorf("Get failed: %v", err)
+	} else if string(d) != "value2" {
+		t.Errorf("Get failed: got %s, want %s", d, "value2")
+	}
+
+	if sc.diskgc.Size() != 17 {
+		t.Errorf("Size failed: got %d, want %d", sc.diskgc.Size(), 17)
+	}
+
+	if sc.vfs.Size() != 17 {
+		t.Errorf("Size failed: got %d, want %d", sc.vfs.Size(), 17)
+	}
+
+	sc.memory.Delete("key2")
+	os.Remove(filepath.Join(td, "key2"))
+
+	if _, err := sc.vfs.Get("key2"); err == nil {
+		t.Errorf("Get failed: got nil, want error")
+	}
+}