feat: add upstream and verbose flags to command line interface

feat: add upstream support allowing to chain cache servers if needed
fix: tweaked garbage collection to be better

steamcache/steamcache.go

@@ -6,6 +6,7 @@ import (
 	"net/http"
 	"net/url"
 	"os"
+	"runtime"
 	"s1d3sw1ped/SteamCache2/steamcache/avgcachestate"
 	"s1d3sw1ped/SteamCache2/steamcache/logger"
 	"s1d3sw1ped/SteamCache2/version"
@@ -20,12 +21,17 @@ import (
 	"sync"
 	"time"
 
+	pprof "net/http/pprof"
+
 	"github.com/docker/go-units"
 )
 
 type SteamCache struct {
-	address string
-	vfs     vfs.VFS
+	pprof    bool
+	address  string
+	upstream string
+
+	vfs vfs.VFS
 
 	memory *memory.MemoryFS
 	disk   *disk.DiskFS
@@ -39,7 +45,7 @@ type SteamCache struct {
 	mu    sync.Mutex
 }
 
-func New(address string, memorySize string, memoryMultiplier int, diskSize string, diskMultiplier int, diskPath string) *SteamCache {
+func New(address string, memorySize string, memoryMultiplier int, diskSize string, diskMultiplier int, diskPath, upstream string, pprof bool) *SteamCache {
 	memorysize, err := units.FromHumanSize(memorySize)
 	if err != nil {
 		panic(err)
@@ -92,8 +98,10 @@ func New(address string, memorySize string, memoryMultiplier int, diskSize strin
 	}
 
 	sc := &SteamCache{
-		address: address,
-		vfs:     syncfs.New(c),
+		pprof:    pprof,
+		upstream: upstream,
+		address:  address,
+		vfs:      syncfs.New(c),
 
 		memory: m,
 		disk:   d,
@@ -116,6 +124,15 @@ func New(address string, memorySize string, memoryMultiplier int, diskSize strin
 func (sc *SteamCache) Run() {
 	logger.Logger.Info().Str("address", sc.address).Str("version", version.Version).Msg("listening")
 
+	if sc.upstream != "" {
+		_, err := http.Get(sc.upstream)
+		if err != nil {
+			logger.Logger.Error().Err(err).Str("upstream", sc.upstream).Msg("Failed to connect to upstream server")
+			os.Exit(1)
+		}
+		logger.Logger.Info().Str("upstream", sc.upstream).Msg("connected")
+	}
+
 	sc.mu.Lock()
 	sc.dirty = true
 	sc.mu.Unlock()
@@ -182,6 +199,15 @@ func (sc *SteamCache) LogStats() {
 				Msg("disk_gc")
 		}
 
+		// log golang Garbage Collection stats
+		var m runtime.MemStats
+		runtime.ReadMemStats(&m)
+
+		logger.Logger.Info().
+			Str("alloc", units.HumanSize(float64(m.Alloc))).
+			Str("sys", units.HumanSize(float64(m.Sys))).
+			Msg("go_gc")
+
 		logger.Logger.Info().
 			Str("hitrate", fmt.Sprintf("%.2f%%", sc.hits.Avg()*100)).
 			Msg("cache")
@@ -191,6 +217,14 @@ func (sc *SteamCache) LogStats() {
 }
 
 func (sc *SteamCache) ServeHTTP(w http.ResponseWriter, r *http.Request) {
+	if sc.pprof && r.URL.Path == "/debug/pprof/" {
+		pprof.Index(w, r)
+		return
+	} else if sc.pprof && strings.HasPrefix(r.URL.Path, "/debug/pprof/") {
+		pprof.Handler(strings.TrimPrefix(r.URL.Path, "/debug/pprof/")).ServeHTTP(w, r)
+		return
+	}
+
 	if r.Method != http.MethodGet {
 		http.Error(w, "Only GET method is supported", http.StatusMethodNotAllowed)
 		return
@@ -203,54 +237,68 @@ func (sc *SteamCache) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 		return
 	}
 
-	if r.Header.Get("User-Agent") != "Valve/Steam HTTP Client 1.0" {
-		http.Error(w, "Only Valve/Steam HTTP Client 1.0 is supported", http.StatusForbidden)
-		return
-	}
-
-	if strings.Contains(r.URL.String(), "manifest") {
-		w.Header().Add("X-LanCache-Processed-By", "SteamCache2")
-		forward(w, r)
-		return
-	}
-
-	// tstart := time.Now()
-	// defer func() {
-	// logger.Logger.Info().Str("method", r.Method).Str("url", r.URL.String()).Str("status", w.Header().Get("X-LanCache-Status")).Dur("duration", time.Since(tstart)).Msg("Request")
-	// }()
-
 	sc.mu.Lock()
 	sc.dirty = true
 	sc.mu.Unlock()
 
 	w.Header().Add("X-LanCache-Processed-By", "SteamCache2") // SteamPrefill uses this header to determine if the request was processed by the cache maybe steam uses it too
 
-	cacheKey := r.URL.String()
-
-	// if vfs is also a vfs.GetSer, we can use it to get the cache state
+	cacheKey := strings.ReplaceAll(r.URL.String()[1:], "\\", "/") // replace all backslashes with forward slashes shouldn't be necessary but just in case
+	if cacheKey == "" {
+		http.Error(w, "Invalid URL", http.StatusBadRequest)
+		return
+	}
 
 	data, err := sc.vfs.Get(cacheKey)
 	if err == nil {
 		sc.hits.Add(cachestate.CacheStateHit)
 		w.Header().Add("X-LanCache-Status", "HIT")
 		w.Write(data)
+		logger.Logger.Debug().Str("key", r.URL.String()).Msg("cache")
 		return
 	}
 
-	htt := "http://"
-	if r.Header.Get("X-Sls-Https") == "enable" {
-		htt = "https://"
+	var req *http.Request
+	if sc.upstream != "" { // if an upstream server is configured, proxy the request to the upstream server
+		ur, err := url.JoinPath(sc.upstream, r.URL.String())
+		if err != nil {
+			http.Error(w, "Failed to join URL path", http.StatusInternalServerError)
+			return
+		}
+
+		req, err = http.NewRequest(http.MethodGet, ur, nil)
+		if err != nil {
+			http.Error(w, "Failed to create request", http.StatusInternalServerError)
+			return
+		}
+		req.Host = r.Host
+		logger.Logger.Debug().Str("key", cacheKey).Str("host", sc.upstream).Msg("upstream")
+	} else { // if no upstream server is configured, proxy the request to the host specified in the request
+		host := r.Host
+		if r.Header.Get("X-Sls-Https") == "enable" {
+			host = "https://" + host
+		} else {
+			host = "http://" + host
+		}
+
+		ur, err := url.JoinPath(host, r.URL.String())
+		if err != nil {
+			http.Error(w, "Failed to join URL path", http.StatusInternalServerError)
+			return
+		}
+
+		req, err = http.NewRequest(http.MethodGet, ur, nil)
+		if err != nil {
+			http.Error(w, "Failed to create request", http.StatusInternalServerError)
+			return
+		}
+
+		logger.Logger.Debug().Str("key", cacheKey).Str("host", host).Msg("forward")
 	}
 
-	base := htt + r.Host
-
-	hosturl, err := url.JoinPath(base, cacheKey)
-	if err != nil {
-		http.Error(w, "Failed to join URL path", http.StatusInternalServerError)
-		return
-	}
-
-	resp, err := http.Get(hosturl)
+	req.Header.Add("X-Sls-Https", r.Header.Get("X-Sls-Https"))
+	req.Header.Add("User-Agent", r.Header.Get("User-Agent"))
+	resp, err := http.DefaultClient.Do(req)
 	if err != nil {
 		http.Error(w, "Failed to fetch the requested URL", http.StatusInternalServerError)
 		return
@@ -273,40 +321,3 @@ func (sc *SteamCache) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 	w.Header().Add("X-LanCache-Status", "MISS")
 	w.Write(body)
 }
-
-func forward(w http.ResponseWriter, r *http.Request) {
-	htt := "http://"
-	if r.Header.Get("X-Sls-Https") == "enable" {
-		htt = "https://"
-	}
-
-	base := htt + r.Host
-
-	cacheKey := r.URL.String()
-
-	hosturl, err := url.JoinPath(base, cacheKey)
-	if err != nil {
-		http.Error(w, "Failed to join URL path", http.StatusInternalServerError)
-		return
-	}
-
-	resp, err := http.Get(hosturl)
-	if err != nil {
-		http.Error(w, "Failed to fetch the requested URL", http.StatusInternalServerError)
-		return
-	}
-	defer resp.Body.Close()
-
-	if resp.StatusCode != http.StatusOK {
-		http.Error(w, "Failed to fetch the requested URL", resp.StatusCode)
-		return
-	}
-
-	body, err := io.ReadAll(resp.Body)
-	if err != nil {
-		http.Error(w, "Failed to read response body", http.StatusInternalServerError)
-		return
-	}
-
-	w.Write(body)
-}