Enhance garbage collection and caching functionality

- Updated .gitignore to include all .exe files and ensure .smashignore is tracked.
- Expanded README.md with advanced configuration options for garbage collection algorithms, detailing available algorithms and use cases.
- Modified launch.json to include memory and disk garbage collection flags for better configuration.
- Refactored root.go to introduce memoryGC and diskGC flags for garbage collection algorithms.
- Implemented hash extraction and verification in steamcache.go to ensure data integrity during caching.
- Added new tests in steamcache_test.go for hash extraction and verification, ensuring correctness of caching behavior.
- Enhanced garbage collection strategies in gc.go, introducing LFU, FIFO, Largest, Smallest, and Hybrid algorithms with corresponding metrics.
- Updated caching logic to conditionally cache responses based on hash verification results.

steamcache/steamcache.go

@@ -3,17 +3,23 @@ package steamcache
 
 import (
 	"context"
+	"crypto/sha1"
+	"encoding/hex"
+	"fmt"
 	"io"
 	"net"
 	"net/http"
 	"net/url"
 	"os"
+	"path/filepath"
+	"regexp"
 	"s1d3sw1ped/SteamCache2/steamcache/logger"
 	"s1d3sw1ped/SteamCache2/vfs"
 	"s1d3sw1ped/SteamCache2/vfs/cache"
 	"s1d3sw1ped/SteamCache2/vfs/disk"
 	"s1d3sw1ped/SteamCache2/vfs/gc"
 	"s1d3sw1ped/SteamCache2/vfs/memory"
+	"sort"
 	"strings"
 	"sync"
 	"time"
@@ -24,6 +30,14 @@ import (
 	"github.com/prometheus/client_golang/prometheus/promhttp"
 )
 
+// min returns the minimum of two integers
+func min(a, b int) int {
+	if a < b {
+		return a
+	}
+	return b
+}
+
 var (
 	requestsTotal = promauto.NewCounterVec(
 		prometheus.CounterOpts{
@@ -41,15 +55,97 @@ var (
 		[]string{"status"},
 	)
 
-	responseTime = promauto.NewHistogram(
+	responseTime = promauto.NewHistogramVec(
 		prometheus.HistogramOpts{
 			Name:    "response_time_seconds",
 			Help:    "Response time in seconds",
 			Buckets: prometheus.DefBuckets,
 		},
+		[]string{"cache_status"},
 	)
 )
 
+// hashVerificationTotal tracks hash verification attempts
+var hashVerificationTotal = promauto.NewCounterVec(
+	prometheus.CounterOpts{
+		Name: "hash_verification_total",
+		Help: "Total hash verification attempts",
+	},
+	[]string{"result"},
+)
+
+// extractHashFromFilename extracts a hash from a filename if present
+// Steam depot files often have hashes in their names like: filename_hash.ext
+func extractHashFromFilename(filename string) (string, bool) {
+	// Common patterns for Steam depot files with hashes
+	patterns := []*regexp.Regexp{
+		regexp.MustCompile(`^([a-fA-F0-9]{40})$`),  // Standalone SHA1 hash (40 hex chars)
+		regexp.MustCompile(`^([a-fA-F0-9]{40})\.`), // SHA1 hash with extension
+	}
+
+	for _, pattern := range patterns {
+		if matches := pattern.FindStringSubmatch(filename); len(matches) > 1 {
+			return strings.ToLower(matches[1]), true
+		}
+	}
+
+	// Debug: log when we don't find a hash pattern
+	if strings.Contains(filename, "manifest") {
+		logger.Logger.Debug().
+			Str("filename", filename).
+			Msg("No hash pattern found in manifest filename")
+	}
+
+	return "", false
+}
+
+// calculateFileHash calculates the SHA1 hash of the given data
+func calculateFileHash(data []byte) string {
+	hash := sha1.Sum(data)
+	return hex.EncodeToString(hash[:])
+}
+
+// calculateResponseHash calculates the SHA1 hash of the full HTTP response
+func calculateResponseHash(resp *http.Response, bodyData []byte) string {
+	hash := sha1.New()
+
+	// Include status line
+	statusLine := fmt.Sprintf("HTTP/1.1 %d %s\n", resp.StatusCode, resp.Status)
+	hash.Write([]byte(statusLine))
+
+	// Include headers (sorted for consistency)
+	headers := make([]string, 0, len(resp.Header))
+	for key, values := range resp.Header {
+		for _, value := range values {
+			headers = append(headers, fmt.Sprintf("%s: %s\n", key, value))
+		}
+	}
+	sort.Strings(headers)
+	for _, header := range headers {
+		hash.Write([]byte(header))
+	}
+
+	// Include empty line between headers and body
+	hash.Write([]byte("\n"))
+
+	// Include body
+	hash.Write(bodyData)
+
+	return hex.EncodeToString(hash.Sum(nil))
+}
+
+// verifyFileHash verifies that the file content matches the expected hash
+func verifyFileHash(data []byte, expectedHash string) bool {
+	actualHash := calculateFileHash(data)
+	return strings.EqualFold(actualHash, expectedHash)
+}
+
+// verifyResponseHash verifies that the full HTTP response matches the expected hash
+func verifyResponseHash(resp *http.Response, bodyData []byte, expectedHash string) bool {
+	actualHash := calculateResponseHash(resp, bodyData)
+	return strings.EqualFold(actualHash, expectedHash)
+}
+
 type SteamCache struct {
 	address  string
 	upstream string
@@ -68,7 +164,7 @@ type SteamCache struct {
 	wg     sync.WaitGroup
 }
 
-func New(address string, memorySize string, diskSize string, diskPath, upstream string) *SteamCache {
+func New(address string, memorySize string, diskSize string, diskPath, upstream, memoryGC, diskGC string) *SteamCache {
 	memorysize, err := units.FromHumanSize(memorySize)
 	if err != nil {
 		panic(err)
@@ -80,21 +176,29 @@ func New(address string, memorySize string, diskSize string, diskPath, upstream
 	}
 
 	c := cache.New(
-		gc.PromotionDecider,
+		gc.AdaptivePromotionDeciderFunc,
 	)
 
 	var m *memory.MemoryFS
 	var mgc *gc.GCFS
 	if memorysize > 0 {
 		m = memory.New(memorysize)
-		mgc = gc.New(m, gc.LRUGC)
+		memoryGCAlgo := gc.GCAlgorithm(memoryGC)
+		if memoryGCAlgo == "" {
+			memoryGCAlgo = gc.LRU // default to LRU
+		}
+		mgc = gc.New(m, gc.GetGCAlgorithm(memoryGCAlgo))
 	}
 
 	var d *disk.DiskFS
 	var dgc *gc.GCFS
 	if disksize > 0 {
 		d = disk.New(diskPath, disksize)
-		dgc = gc.New(d, gc.LRUGC)
+		diskGCAlgo := gc.GCAlgorithm(diskGC)
+		if diskGCAlgo == "" {
+			diskGCAlgo = gc.LRU // default to LRU
+		}
+		dgc = gc.New(d, gc.GetGCAlgorithm(diskGCAlgo))
 	}
 
 	// configure the cache to match the specified mode (memory only, disk only, or memory and disk) based on the provided sizes
@@ -152,6 +256,14 @@ func New(address string, memorySize string, diskSize string, diskPath, upstream
 		},
 	}
 
+	// Log GC algorithm configuration
+	if m != nil {
+		logger.Logger.Info().Str("memory_gc", memoryGC).Msg("Memory cache GC algorithm configured")
+	}
+	if d != nil {
+		logger.Logger.Info().Str("disk_gc", diskGC).Msg("Disk cache GC algorithm configured")
+	}
+
 	if d != nil {
 		if d.Size() > d.Capacity() {
 			gc.LRUGC(d, uint(d.Size()-d.Capacity()))
@@ -223,7 +335,6 @@ func (sc *SteamCache) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 		path := strings.Split(r.URL.String(), "?")[0]
 
 		tstart := time.Now()
-		defer func() { responseTime.Observe(time.Since(tstart).Seconds()) }()
 
 		cacheKey := strings.ReplaceAll(path[1:], "\\", "/") // replace all backslashes with forward slashes shouldn't be necessary but just in case
 
@@ -252,6 +363,7 @@ func (sc *SteamCache) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 
 			requestsTotal.WithLabelValues(r.Method, "200").Inc()
 			cacheStatusTotal.WithLabelValues("HIT").Inc()
+			responseTime.WithLabelValues("HIT").Observe(time.Since(tstart).Seconds())
 
 			return
 		}
@@ -328,27 +440,95 @@ func (sc *SteamCache) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 
 		size := resp.ContentLength
 
-		// this is sortof not needed as we should always be able to get a writer from the cache as long as the gc is able to reclaim enough space aka the file is not bigger than the disk can handle
-		ww := w.(io.Writer)                        // default writer to write to the response writer
-		writer, _ := sc.vfs.Create(cacheKey, size) // create a writer to write to the cache
-		if writer != nil {                         // if the writer is not nil, it means the cache is writable
-			defer writer.Close()           // close the writer when done
-			ww = io.MultiWriter(w, writer) // write to both the response writer and the cache writer
+		// Read the entire response body into memory for hash verification
+		bodyData, err := io.ReadAll(resp.Body)
+		if err != nil {
+			requestsTotal.WithLabelValues(r.Method, "500").Inc()
+			logger.Logger.Error().Err(err).Str("url", req.URL.String()).Msg("Failed to read response body")
+			http.Error(w, "Failed to read response body", http.StatusInternalServerError)
+			return
 		}
 
-		w.Header().Add("X-LanCache-Status", "MISS")
+		// Extract filename from cache key for hash verification
+		filename := filepath.Base(cacheKey)
+		expectedHash, hasHash := extractHashFromFilename(filename)
 
-		io.Copy(ww, resp.Body)
+		// Debug logging for manifest files
+		if strings.Contains(cacheKey, "manifest") {
+			logger.Logger.Debug().
+				Str("key", cacheKey).
+				Str("filename", filename).
+				Bool("hasHash", hasHash).
+				Str("expectedHash", expectedHash).
+				Int64("content_length_header", resp.ContentLength).
+				Int("actual_content_length", len(bodyData)).
+				Msg("Manifest file hash verification debug")
+		}
+
+		// Hash verification using Steam's X-Content-Sha header and content length verification
+		hashVerified := true
+		if hasHash {
+			// Get the hash from Steam's X-Content-Sha header
+			steamHash := resp.Header.Get("X-Content-Sha")
+
+			// Verify using Steam's hash
+			if strings.EqualFold(steamHash, expectedHash) {
+				hashVerificationTotal.WithLabelValues("success").Inc()
+			} else {
+				hashVerificationTotal.WithLabelValues("failed").Inc()
+				logger.Logger.Error().
+					Str("key", cacheKey).
+					Str("expected_hash", expectedHash).
+					Str("steam_hash", steamHash).
+					Int("content_length", len(bodyData)).
+					Msg("Steam hash verification failed - Steam's hash doesn't match filename")
+				hashVerified = false
+			}
+		} else {
+			hashVerificationTotal.WithLabelValues("no_hash").Inc()
+		}
+
+		// Always verify content length as an additional safety check
+		if resp.ContentLength > 0 && int64(len(bodyData)) != resp.ContentLength {
+			hashVerificationTotal.WithLabelValues("content_length_failed").Inc()
+			logger.Logger.Error().
+				Str("key", cacheKey).
+				Int("actual_content_length", len(bodyData)).
+				Int64("expected_content_length", resp.ContentLength).
+				Msg("Content length verification failed")
+			hashVerified = false
+		} else if resp.ContentLength > 0 {
+			hashVerificationTotal.WithLabelValues("content_length_success").Inc()
+		}
+
+		// Write to response (always serve the file)
+		w.Header().Add("X-LanCache-Status", "MISS")
+		w.Write(bodyData)
+
+		// Only cache the file if hash verification passed (or no hash was present)
+		if hashVerified {
+			writer, _ := sc.vfs.Create(cacheKey, size)
+			if writer != nil {
+				defer writer.Close()
+				writer.Write(bodyData)
+			}
+		} else {
+			logger.Logger.Warn().
+				Str("key", cacheKey).
+				Msg("File served but not cached due to hash verification failure")
+		}
 
 		logger.Logger.Info().
 			Str("key", cacheKey).
 			Str("host", r.Host).
 			Str("status", "MISS").
+			Bool("hash_verified", hasHash).
 			Dur("duration", time.Since(tstart)).
 			Msg("request")
 
 		requestsTotal.WithLabelValues(r.Method, "200").Inc()
 		cacheStatusTotal.WithLabelValues("MISS").Inc()
+		responseTime.WithLabelValues("MISS").Observe(time.Since(tstart).Seconds())
 
 		return
 	}