Enhance file metadata tracking and garbage collection logic

- Added AccessCount field to FileInfo struct for improved tracking of file access frequency.
- Updated NewFileInfo and NewFileInfoFromOS functions to initialize AccessCount.
- Modified DiskFS and MemoryFS to preserve and increment AccessCount during file operations.
- Enhanced garbage collection methods (LRU, LFU, FIFO, Largest, Smallest, Hybrid) to utilize AccessCount for more effective space reclamation.

.gitignore

@@ -1,5 +1,5 @@
 dist/
 tmp/
-__*.exe
+*.exe
 .smashed.txt
-.smashignore
+.smashignore

.goreleaser.yaml

@@ -2,11 +2,17 @@ version: 2
 
 before:
   hooks:
-    - go mod tidy
+    - go mod tidy -v
 
 builds:
-  - ldflags:
+  - id: default
+    binary: steamcache2
+    ldflags:
+      - -s
+      - -w
+      - -extldflags "-static"
       - -X s1d3sw1ped/SteamCache2/version.Version={{.Version}}
+      - -X s1d3sw1ped/SteamCache2/version.Date={{.Date}}
     env:
       - CGO_ENABLED=0
     goos:
@@ -14,19 +20,24 @@ builds:
       - windows
     goarch:
       - amd64
+      - arm64
+    ignore:
+      - goos: windows
+        goarch: arm64
+
+checksum:
+  name_template: "checksums.txt"
 
 archives:
-  - formats: tar.gz
-    name_template: >- 
-      {{ .ProjectName }}_
-      {{- title .Os }}_
-      {{- if eq .Arch "amd64" }}x86_64
-      {{- else if eq .Arch "386" }}i386
-      {{- else }}{{ .Arch }}{{ end }}
-      {{- if .Arm }}v{{ .Arm }}{{ end }}
+  - id: default
+    name_template: "{{ .ProjectName }}-{{ .Os }}-{{ .Arch }}"
+    formats: tar.gz
     format_overrides:
       - goos: windows
         formats: zip
+    files:
+      - README.md
+      - LICENSE
 
 changelog:
   sort: asc
@@ -36,12 +47,7 @@ changelog:
       - "^test:"
 
 release:
-  name_template: '{{.ProjectName}}-{{.Version}}'
-  footer: >-
-  
-    ---
-
-    Released by [GoReleaser](https://github.com/goreleaser/goreleaser).
+  name_template: "{{ .ProjectName }}-{{ .Version }}"
 
 gitea_urls:
   api: https://git.s1d3sw1ped.com/api/v1

.vscode/launch.json

@@ -17,7 +17,14 @@
                 "10G",
                 "--disk-path",
                 "tmp/disk",
-                "--verbose",
+                "--memory-gc",
+                "lfu",
+                "--disk-gc",
+                "lru",
+                "--log-level",
+                "debug",
+                // "--upstream",
+                // "http://192.168.2.5:80",
             ],
         },
         {
@@ -31,7 +38,12 @@
                 "10G",
                 "--disk-path",
                 "tmp/disk",
-                "--verbose",
+                "--disk-gc",
+                "hybrid",
+                "--log-level",
+                "debug",
+                // "--upstream",
+                // "http://192.168.2.5:80",
             ],
         },
         {
@@ -43,7 +55,12 @@
             "args": [
                 "--memory",
                 "1G",
-                "--verbose",
+                "--memory-gc",
+                "lru",
+                "--log-level",
+                "debug",
+                // "--upstream",
+                // "http://192.168.2.5:80",
             ],
         }
     ]

README.md

@@ -17,6 +17,38 @@ SteamCache2 is a blazing fast download cache for Steam, designed to reduce bandw
     ```sh
     ./SteamCache2 --memory 1G --disk 10G --disk-path tmp/disk
     ```
+
+### Advanced Configuration
+
+#### Garbage Collection Algorithms
+
+SteamCache2 supports multiple garbage collection algorithms for both memory and disk caches:
+
+```sh
+# Use LFU for memory cache (good for long-running servers)
+./SteamCache2 --memory 4G --memory-gc lfu --disk 100G --disk-gc lru
+
+# Use FIFO for predictable eviction (good for testing)
+./SteamCache2 --memory 2G --memory-gc fifo --disk 50G --disk-gc fifo
+
+# Use size-based eviction for disk cache
+./SteamCache2 --memory 1G --disk 200G --disk-gc largest
+```
+
+**Available GC Algorithms:**
+
+- **`lru`** (default): Least Recently Used - evicts oldest accessed files
+- **`lfu`**: Least Frequently Used - evicts least accessed files (good for popular content)
+- **`fifo`**: First In, First Out - evicts oldest created files (predictable)
+- **`largest`**: Size-based - evicts largest files first (maximizes file count)
+- **`smallest`**: Size-based - evicts smallest files first (maximizes cache hit rate)
+- **`hybrid`**: Combines access time and file size for optimal eviction
+
+**Use Cases:**
+- **LAN Events**: Use `lfu` for memory caches to keep popular games
+- **Gaming Cafes**: Use `hybrid` for balanced performance
+- **Testing**: Use `fifo` for predictable behavior
+- **Large Files**: Use `largest` to prioritize keeping many small files
 2. Configure your DNS:
     - If your on Windows and don't want a whole network implementation (THIS)[#windows-hosts-file-override]
 

cmd/root.go

@@ -15,12 +15,13 @@ import (
 var (
 	threads int
 
-	memory           string
-	memorymultiplier int
-	disk             string
-	diskmultiplier   int
-	diskpath         string
-	upstream         string
+	memory   string
+	disk     string
+	diskpath string
+	upstream string
+
+	memoryGC string
+	diskGC   string
 
 	logLevel  string
 	logFormat string
@@ -55,7 +56,7 @@ var rootCmd = &cobra.Command{
 		logger.Logger = zerolog.New(writer).With().Timestamp().Logger()
 
 		logger.Logger.Info().
-			Msg("SteamCache2 " + version.Version + " starting...")
+			Msg("SteamCache2 " + version.Version + " " + version.Date + " starting...")
 
 		address := ":80"
 
@@ -69,11 +70,11 @@ var rootCmd = &cobra.Command{
 		sc := steamcache.New(
 			address,
 			memory,
-			memorymultiplier,
 			disk,
-			diskmultiplier,
 			diskpath,
 			upstream,
+			memoryGC,
+			diskGC,
 		)
 
 		logger.Logger.Info().
@@ -99,13 +100,14 @@ func init() {
 	rootCmd.Flags().IntVarP(&threads, "threads", "t", runtime.GOMAXPROCS(-1), "Number of worker threads to use for processing requests")
 
 	rootCmd.Flags().StringVarP(&memory, "memory", "m", "0", "The size of the memory cache")
-	rootCmd.Flags().IntVarP(&memorymultiplier, "memory-gc", "M", 10, "The gc value for the memory cache")
 	rootCmd.Flags().StringVarP(&disk, "disk", "d", "0", "The size of the disk cache")
-	rootCmd.Flags().IntVarP(&diskmultiplier, "disk-gc", "D", 100, "The gc value for the disk cache")
 	rootCmd.Flags().StringVarP(&diskpath, "disk-path", "p", "", "The path to the disk cache")
 
 	rootCmd.Flags().StringVarP(&upstream, "upstream", "u", "", "The upstream server to proxy requests overrides the host header from the client but forwards the original host header to the upstream server")
 
+	rootCmd.Flags().StringVarP(&memoryGC, "memory-gc", "", "lru", "Memory cache GC algorithm: lru, lfu, fifo, largest, smallest, hybrid")
+	rootCmd.Flags().StringVarP(&diskGC, "disk-gc", "", "lru", "Disk cache GC algorithm: lru, lfu, fifo, largest, smallest, hybrid")
+
 	rootCmd.Flags().StringVarP(&logLevel, "log-level", "l", "info", "Logging level: debug, info, error")
 	rootCmd.Flags().StringVarP(&logFormat, "log-format", "f", "console", "Logging format: json, console")
 }

cmd/version.go

@@ -15,7 +15,7 @@ var versionCmd = &cobra.Command{
 	Short: "prints the version of SteamCache2",
 	Long:  `Prints the version of SteamCache2. This command is useful for checking the version of the application.`,
 	Run: func(cmd *cobra.Command, args []string) {
-		fmt.Fprintln(os.Stderr, "SteamCache2", version.Version)
+		fmt.Fprintln(os.Stderr, "SteamCache2", version.Version, version.Date)
 	},
 }
 

steamcache/steamcache.go

@@ -2,28 +2,45 @@
 package steamcache
 
 import (
+	"bufio"
 	"context"
+	"crypto/sha1"
+	"encoding/hex"
+	"fmt"
 	"io"
 	"net"
 	"net/http"
 	"net/url"
 	"os"
+	"path/filepath"
+	"regexp"
 	"s1d3sw1ped/SteamCache2/steamcache/logger"
 	"s1d3sw1ped/SteamCache2/vfs"
 	"s1d3sw1ped/SteamCache2/vfs/cache"
 	"s1d3sw1ped/SteamCache2/vfs/disk"
 	"s1d3sw1ped/SteamCache2/vfs/gc"
 	"s1d3sw1ped/SteamCache2/vfs/memory"
+	"sort"
 	"strings"
 	"sync"
 	"time"
 
+	"bytes"
+
 	"github.com/docker/go-units"
 	"github.com/prometheus/client_golang/prometheus"
 	"github.com/prometheus/client_golang/prometheus/promauto"
 	"github.com/prometheus/client_golang/prometheus/promhttp"
 )
 
+// min returns the minimum of two integers
+func min(a, b int) int {
+	if a < b {
+		return a
+	}
+	return b
+}
+
 var (
 	requestsTotal = promauto.NewCounterVec(
 		prometheus.CounterOpts{
@@ -41,15 +58,110 @@ var (
 		[]string{"status"},
 	)
 
-	responseTime = promauto.NewHistogram(
+	responseTime = promauto.NewHistogramVec(
 		prometheus.HistogramOpts{
 			Name:    "response_time_seconds",
 			Help:    "Response time in seconds",
 			Buckets: prometheus.DefBuckets,
 		},
+		[]string{"cache_status"},
 	)
 )
 
+// hashVerificationTotal tracks hash verification attempts
+var hashVerificationTotal = promauto.NewCounterVec(
+	prometheus.CounterOpts{
+		Name: "hash_verification_total",
+		Help: "Total hash verification attempts",
+	},
+	[]string{"result"},
+)
+
+// extractHashFromFilename extracts a hash from a filename if present
+// Steam depot files often have hashes in their names like: filename_hash.ext
+func extractHashFromFilename(filename string) (string, bool) {
+	// Common patterns for Steam depot files with hashes
+	patterns := []*regexp.Regexp{
+		regexp.MustCompile(`^([a-fA-F0-9]{40})$`),  // Standalone SHA1 hash (40 hex chars)
+		regexp.MustCompile(`^([a-fA-F0-9]{40})\.`), // SHA1 hash with extension
+	}
+
+	for _, pattern := range patterns {
+		if matches := pattern.FindStringSubmatch(filename); len(matches) > 1 {
+			return strings.ToLower(matches[1]), true
+		}
+	}
+
+	// Debug: log when we don't find a hash pattern
+	if strings.Contains(filename, "manifest") {
+		logger.Logger.Debug().
+			Str("filename", filename).
+			Msg("No hash pattern found in manifest filename")
+	}
+
+	return "", false
+}
+
+// calculateFileHash calculates the SHA1 hash of the given data
+func calculateFileHash(data []byte) string {
+	hash := sha1.Sum(data)
+	return hex.EncodeToString(hash[:])
+}
+
+// calculateResponseHash calculates the SHA1 hash of the full HTTP response
+func calculateResponseHash(resp *http.Response, bodyData []byte) string {
+	hash := sha1.New()
+
+	// Include status line
+	statusLine := fmt.Sprintf("HTTP/1.1 %d %s\n", resp.StatusCode, resp.Status)
+	hash.Write([]byte(statusLine))
+
+	// Include headers (sorted for consistency)
+	headers := make([]string, 0, len(resp.Header))
+	for key, values := range resp.Header {
+		for _, value := range values {
+			headers = append(headers, fmt.Sprintf("%s: %s\n", key, value))
+		}
+	}
+	sort.Strings(headers)
+	for _, header := range headers {
+		hash.Write([]byte(header))
+	}
+
+	// Include empty line between headers and body
+	hash.Write([]byte("\n"))
+
+	// Include body
+	hash.Write(bodyData)
+
+	return hex.EncodeToString(hash.Sum(nil))
+}
+
+// verifyFileHash verifies that the file content matches the expected hash
+func verifyFileHash(data []byte, expectedHash string) bool {
+	actualHash := calculateFileHash(data)
+	return strings.EqualFold(actualHash, expectedHash)
+}
+
+// verifyResponseHash verifies that the full HTTP response matches the expected hash
+func verifyResponseHash(resp *http.Response, bodyData []byte, expectedHash string) bool {
+	actualHash := calculateResponseHash(resp, bodyData)
+	return strings.EqualFold(actualHash, expectedHash)
+}
+
+var hopByHopHeaders = map[string]struct{}{
+	"Connection":          {},
+	"Keep-Alive":          {},
+	"Proxy-Authenticate":  {},
+	"Proxy-Authorization": {},
+	"TE":                  {},
+	"Trailer":             {},
+	"Transfer-Encoding":   {},
+	"Upgrade":             {},
+	"Date":                {},
+	"Server":              {},
+}
+
 type SteamCache struct {
 	address  string
 	upstream string
@@ -68,7 +180,7 @@ type SteamCache struct {
 	wg     sync.WaitGroup
 }
 
-func New(address string, memorySize string, memoryMultiplier int, diskSize string, diskMultiplier int, diskPath, upstream string) *SteamCache {
+func New(address string, memorySize string, diskSize string, diskPath, upstream, memoryGC, diskGC string) *SteamCache {
 	memorysize, err := units.FromHumanSize(memorySize)
 	if err != nil {
 		panic(err)
@@ -80,21 +192,29 @@ func New(address string, memorySize string, memoryMultiplier int, diskSize strin
 	}
 
 	c := cache.New(
-		gc.PromotionDecider,
+		gc.AdaptivePromotionDeciderFunc,
 	)
 
 	var m *memory.MemoryFS
 	var mgc *gc.GCFS
 	if memorysize > 0 {
 		m = memory.New(memorysize)
-		mgc = gc.New(m, memoryMultiplier, gc.LRUGC)
+		memoryGCAlgo := gc.GCAlgorithm(memoryGC)
+		if memoryGCAlgo == "" {
+			memoryGCAlgo = gc.LRU // default to LRU
+		}
+		mgc = gc.New(m, gc.GetGCAlgorithm(memoryGCAlgo))
 	}
 
 	var d *disk.DiskFS
 	var dgc *gc.GCFS
 	if disksize > 0 {
 		d = disk.New(diskPath, disksize)
-		dgc = gc.New(d, diskMultiplier, gc.LRUGC)
+		diskGCAlgo := gc.GCAlgorithm(diskGC)
+		if diskGCAlgo == "" {
+			diskGCAlgo = gc.LRU // default to LRU
+		}
+		dgc = gc.New(d, gc.GetGCAlgorithm(diskGCAlgo))
 	}
 
 	// configure the cache to match the specified mode (memory only, disk only, or memory and disk) based on the provided sizes
@@ -118,21 +238,23 @@ func New(address string, memorySize string, memoryMultiplier int, diskSize strin
 	}
 
 	transport := &http.Transport{
-		MaxIdleConns:        100,
-		MaxIdleConnsPerHost: 10,
-		IdleConnTimeout:     90 * time.Second,
+		MaxIdleConns:        200,               // Increased from 100
+		MaxIdleConnsPerHost: 50,                // Increased from 10
+		IdleConnTimeout:     120 * time.Second, // Increased from 90s
 		DialContext: (&net.Dialer{
 			Timeout:   30 * time.Second,
 			KeepAlive: 30 * time.Second,
 		}).DialContext,
-		TLSHandshakeTimeout:   10 * time.Second,
-		ResponseHeaderTimeout: 10 * time.Second,
-		ExpectContinueTimeout: 1 * time.Second,
+		TLSHandshakeTimeout:   15 * time.Second, // Increased from 10s
+		ResponseHeaderTimeout: 30 * time.Second, // Increased from 10s
+		ExpectContinueTimeout: 5 * time.Second,  // Increased from 1s
+		DisableCompression:    true,             // Steam doesn't use compression
+		ForceAttemptHTTP2:     true,             // Enable HTTP/2 if available
 	}
 
 	client := &http.Client{
 		Transport: transport,
-		Timeout:   60 * time.Second,
+		Timeout:   120 * time.Second, // Increased from 60s
 	}
 
 	sc := &SteamCache{
@@ -145,16 +267,27 @@ func New(address string, memorySize string, memoryMultiplier int, diskSize strin
 		diskgc:   dgc,
 		client:   client,
 		server: &http.Server{
-			Addr:         address,
-			ReadTimeout:  5 * time.Second,
-			WriteTimeout: 10 * time.Second,
-			IdleTimeout:  120 * time.Second,
+			Addr:              address,
+			ReadTimeout:       30 * time.Second,  // Increased
+			WriteTimeout:      60 * time.Second,  // Increased
+			IdleTimeout:       120 * time.Second, // Good for keep-alive
+			ReadHeaderTimeout: 10 * time.Second,  // New, for header attacks
+			MaxHeaderBytes:    1 << 20,           // 1MB, optional
 		},
 	}
 
+	// Log GC algorithm configuration
+	if m != nil {
+		logger.Logger.Info().Str("memory_gc", memoryGC).Msg("Memory cache GC algorithm configured")
+	}
+	if d != nil {
+		logger.Logger.Info().Str("disk_gc", diskGC).Msg("Disk cache GC algorithm configured")
+	}
+
 	if d != nil {
 		if d.Size() > d.Capacity() {
-			gc.LRUGC(d, uint(d.Size()-d.Capacity()))
+			gcHandler := gc.GetGCAlgorithm(gc.GCAlgorithm(diskGC))
+			gcHandler(d, uint(d.Size()-d.Capacity()))
 		}
 	}
 
@@ -165,7 +298,7 @@ func (sc *SteamCache) Run() {
 	if sc.upstream != "" {
 		resp, err := sc.client.Get(sc.upstream)
 		if err != nil || resp.StatusCode != http.StatusOK {
-			logger.Logger.Error().Err(err).Str("upstream", sc.upstream).Msg("Failed to connect to upstream server")
+			logger.Logger.Error().Err(err).Int("status_code", resp.StatusCode).Str("upstream", sc.upstream).Msg("Failed to connect to upstream server")
 			os.Exit(1)
 		}
 		resp.Body.Close()
@@ -198,11 +331,6 @@ func (sc *SteamCache) Shutdown() {
 }
 
 func (sc *SteamCache) ServeHTTP(w http.ResponseWriter, r *http.Request) {
-	if r.URL.Path == "/metrics" {
-		promhttp.Handler().ServeHTTP(w, r)
-		return
-	}
-
 	if r.Method != http.MethodGet {
 		requestsTotal.WithLabelValues(r.Method, "405").Inc()
 		logger.Logger.Warn().Str("method", r.Method).Msg("Only GET method is supported")
@@ -210,6 +338,11 @@ func (sc *SteamCache) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 		return
 	}
 
+	if r.URL.Path == "/" {
+		w.WriteHeader(http.StatusOK) // this is used by steamcache2's upstream verification at startup
+		return
+	}
+
 	if r.URL.String() == "/lancache-heartbeat" {
 		w.Header().Add("X-LanCache-Processed-By", "SteamCache2")
 		w.WriteHeader(http.StatusNoContent)
@@ -217,44 +350,69 @@ func (sc *SteamCache) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 		return
 	}
 
-	if strings.HasPrefix(r.URL.String(), "/depot/") {
-		tstart := time.Now()
-		defer func() { responseTime.Observe(time.Since(tstart).Seconds()) }()
+	if r.URL.Path == "/metrics" {
+		promhttp.Handler().ServeHTTP(w, r)
+		return
+	}
 
-		cacheKey := strings.ReplaceAll(r.URL.String()[1:], "\\", "/") // replace all backslashes with forward slashes shouldn't be necessary but just in case
+	if strings.HasPrefix(r.URL.String(), "/depot/") {
+		// trim the query parameters from the URL path
+		// this is necessary because the cache key should not include query parameters
+		path := strings.Split(r.URL.String(), "?")[0]
+
+		tstart := time.Now()
+
+		cacheKey := strings.ReplaceAll(path[1:], "\\", "/") // replace all backslashes with forward slashes shouldn't be necessary but just in case
 
 		if cacheKey == "" {
 			requestsTotal.WithLabelValues(r.Method, "400").Inc()
-			logger.Logger.Warn().Str("url", r.URL.String()).Msg("Invalid URL")
+			logger.Logger.Warn().Str("url", path).Msg("Invalid URL")
 			http.Error(w, "Invalid URL", http.StatusBadRequest)
 			return
 		}
 
 		w.Header().Add("X-LanCache-Processed-By", "SteamCache2") // SteamPrefill uses this header to determine if the request was processed by the cache maybe steam uses it too
 
-		reader, err := sc.vfs.Open(cacheKey)
+		cachePath := cacheKey // You may want to add a .http or .cache extension for clarity
+
+		// Try to serve from cache
+		file, err := sc.vfs.Open(cachePath)
 		if err == nil {
-			defer reader.Close()
-			w.Header().Add("X-LanCache-Status", "HIT")
-
-			io.Copy(w, reader)
-
-			logger.Logger.Info().
-				Str("key", cacheKey).
-				Str("host", r.Host).
-				Str("status", "HIT").
-				Dur("duration", time.Since(tstart)).
-				Msg("request")
-
-			requestsTotal.WithLabelValues(r.Method, "200").Inc()
-			cacheStatusTotal.WithLabelValues("HIT").Inc()
-
-			return
+			defer file.Close()
+			buf := bufio.NewReader(file)
+			resp, err := http.ReadResponse(buf, nil)
+			if err == nil {
+				// Remove hop-by-hop and server-specific headers
+				for k, vv := range resp.Header {
+					if _, skip := hopByHopHeaders[http.CanonicalHeaderKey(k)]; skip {
+						continue
+					}
+					for _, v := range vv {
+						w.Header().Add(k, v)
+					}
+				}
+				// Add our own headers
+				w.Header().Set("X-LanCache-Status", "HIT")
+				w.Header().Set("X-LanCache-Processed-By", "SteamCache2")
+				w.WriteHeader(resp.StatusCode)
+				io.Copy(w, resp.Body)
+				resp.Body.Close()
+				logger.Logger.Info().
+					Str("key", cacheKey).
+					Str("host", r.Host).
+					Str("status", "HIT").
+					Dur("duration", time.Since(tstart)).
+					Msg("request")
+				requestsTotal.WithLabelValues(r.Method, "200").Inc()
+				cacheStatusTotal.WithLabelValues("HIT").Inc()
+				responseTime.WithLabelValues("HIT").Observe(time.Since(tstart).Seconds())
+				return
+			}
 		}
 
 		var req *http.Request
 		if sc.upstream != "" { // if an upstream server is configured, proxy the request to the upstream server
-			ur, err := url.JoinPath(sc.upstream, r.URL.String())
+			ur, err := url.JoinPath(sc.upstream, path)
 			if err != nil {
 				requestsTotal.WithLabelValues(r.Method, "500").Inc()
 				logger.Logger.Error().Err(err).Str("upstream", sc.upstream).Msg("Failed to join URL path")
@@ -278,7 +436,7 @@ func (sc *SteamCache) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 				host = "http://" + host
 			}
 
-			ur, err := url.JoinPath(host, r.URL.String())
+			ur, err := url.JoinPath(host, path)
 			if err != nil {
 				requestsTotal.WithLabelValues(r.Method, "500").Inc()
 				logger.Logger.Error().Err(err).Str("host", host).Msg("Failed to join URL path")
@@ -322,18 +480,84 @@ func (sc *SteamCache) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 		}
 		defer resp.Body.Close()
 
-		size := resp.ContentLength
-
-		writer, err := sc.vfs.Create(cacheKey, size)
+		// Read the entire response body into memory for hash verification
+		bodyData, err := io.ReadAll(resp.Body)
 		if err != nil {
-			http.Error(w, err.Error(), http.StatusInternalServerError)
+			requestsTotal.WithLabelValues(r.Method, "500").Inc()
+			logger.Logger.Error().Err(err).Str("url", req.URL.String()).Msg("Failed to read response body")
+			http.Error(w, "Failed to read response body", http.StatusInternalServerError)
 			return
 		}
-		defer writer.Close()
 
-		w.Header().Add("X-LanCache-Status", "MISS")
+		// Extract filename from cache key for hash verification
+		filename := filepath.Base(cacheKey)
+		expectedHash, hasHash := extractHashFromFilename(filename)
 
-		io.Copy(io.MultiWriter(w, writer), resp.Body)
+		// Hash verification using Steam's X-Content-Sha header and content length verification
+		hashVerified := true
+		if hasHash {
+			// Get the hash from Steam's X-Content-Sha header
+			steamHash := resp.Header.Get("X-Content-Sha")
+
+			// Verify using Steam's hash
+			if strings.EqualFold(steamHash, expectedHash) {
+				hashVerificationTotal.WithLabelValues("success").Inc()
+			} else {
+				hashVerificationTotal.WithLabelValues("failed").Inc()
+				logger.Logger.Error().
+					Str("key", cacheKey).
+					Str("expected_hash", expectedHash).
+					Str("steam_hash", steamHash).
+					Int("content_length", len(bodyData)).
+					Msg("Steam hash verification failed - Steam's hash doesn't match filename")
+				hashVerified = false
+			}
+		} else {
+			hashVerificationTotal.WithLabelValues("no_hash").Inc()
+		}
+
+		// Always verify content length as an additional safety check
+		if resp.ContentLength > 0 && int64(len(bodyData)) != resp.ContentLength {
+			hashVerificationTotal.WithLabelValues("content_length_failed").Inc()
+			logger.Logger.Error().
+				Str("key", cacheKey).
+				Int("actual_content_length", len(bodyData)).
+				Int64("expected_content_length", resp.ContentLength).
+				Msg("Content length verification failed")
+			hashVerified = false
+		} else if resp.ContentLength > 0 {
+			hashVerificationTotal.WithLabelValues("content_length_success").Inc()
+		}
+
+		// Write to response (always serve the file)
+		// Remove hop-by-hop and server-specific headers
+		for k, vv := range resp.Header {
+			if _, skip := hopByHopHeaders[http.CanonicalHeaderKey(k)]; skip {
+				continue
+			}
+			for _, v := range vv {
+				w.Header().Add(k, v)
+			}
+		}
+		// Add our own headers
+		w.Header().Set("X-LanCache-Status", "MISS")
+		w.Header().Set("X-LanCache-Processed-By", "SteamCache2")
+		w.Write(bodyData)
+
+		// Only cache the file if hash verification passed (or no hash was present)
+		if hashVerified {
+			writer, _ := sc.vfs.Create(cachePath, int64(0)) // size is not known in advance
+			if writer != nil {
+				defer writer.Close()
+				// Write the full HTTP response to cache
+				resp.Body = io.NopCloser(bytes.NewReader(bodyData)) // Reset body for writing
+				resp.Write(writer)
+			}
+		} else {
+			logger.Logger.Warn().
+				Str("key", cacheKey).
+				Msg("File served but not cached due to hash verification failure")
+		}
 
 		logger.Logger.Info().
 			Str("key", cacheKey).
@@ -344,6 +568,7 @@ func (sc *SteamCache) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 
 		requestsTotal.WithLabelValues(r.Method, "200").Inc()
 		cacheStatusTotal.WithLabelValues("MISS").Inc()
+		responseTime.WithLabelValues("MISS").Observe(time.Since(tstart).Seconds())
 
 		return
 	}

steamcache/steamcache_test.go

@@ -3,6 +3,7 @@ package steamcache
 
 import (
 	"io"
+	"net/http"
 	"os"
 	"path/filepath"
 	"testing"
@@ -13,7 +14,7 @@ func TestCaching(t *testing.T) {
 
 	os.WriteFile(filepath.Join(td, "key2"), []byte("value2"), 0644)
 
-	sc := New("localhost:8080", "1G", 10, "1G", 100, td, "")
+	sc := New("localhost:8080", "1G", "1G", td, "", "lru", "lru")
 
 	w, err := sc.vfs.Create("key", 5)
 	if err != nil {
@@ -84,7 +85,7 @@ func TestCaching(t *testing.T) {
 }
 
 func TestCacheMissAndHit(t *testing.T) {
-	sc := New("localhost:8080", "0", 0, "1G", 100, t.TempDir(), "")
+	sc := New("localhost:8080", "0", "1G", t.TempDir(), "", "lru", "lru")
 
 	key := "testkey"
 	value := []byte("testvalue")
@@ -108,3 +109,137 @@ func TestCacheMissAndHit(t *testing.T) {
 		t.Errorf("expected %s, got %s", value, got)
 	}
 }
+
+func TestHashExtraction(t *testing.T) {
+	// Test the specific key from the user's issue
+	testCases := []struct {
+		filename       string
+		expectedHash   string
+		shouldHaveHash bool
+	}{
+		{
+			filename:       "e89c81a1a926eb4732e146bc806491da8a7d89ca",
+			expectedHash:   "e89c81a1a926eb4732e146bc806491da8a7d89ca",
+			shouldHaveHash: true, // Now it should work with the new standalone hash pattern
+		},
+		{
+			filename:       "chunk_e89c81a1a926eb4732e146bc806491da8a7d89ca",
+			expectedHash:   "",
+			shouldHaveHash: false, // No longer supported with simplified patterns
+		},
+		{
+			filename:       "file.e89c81a1a926eb4732e146bc806491da8a7d89ca.chunk",
+			expectedHash:   "",
+			shouldHaveHash: false, // No longer supported with simplified patterns
+		},
+		{
+			filename:       "chunk_abc123def456",
+			expectedHash:   "",
+			shouldHaveHash: false, // Not 40 chars
+		},
+	}
+
+	for _, tc := range testCases {
+		hash, hasHash := extractHashFromFilename(tc.filename)
+		if hasHash != tc.shouldHaveHash {
+			t.Errorf("filename: %s, expected hasHash: %v, got: %v", tc.filename, tc.shouldHaveHash, hasHash)
+		}
+		if hasHash && hash != tc.expectedHash {
+			t.Errorf("filename: %s, expected hash: %s, got: %s", tc.filename, tc.expectedHash, hash)
+		}
+	}
+}
+
+func TestHashCalculation(t *testing.T) {
+	// Test data
+	testData := []byte("Hello, World!")
+
+	// Calculate hash
+	hash := calculateFileHash(testData)
+
+	// Expected SHA1 hash of "Hello, World!"
+	expectedHash := "0a0a9f2a6772942557ab5355d76af442f8f65e01"
+
+	if hash != expectedHash {
+		t.Errorf("Hash calculation failed: expected %s, got %s", expectedHash, hash)
+	}
+
+	// Test verification
+	if !verifyFileHash(testData, expectedHash) {
+		t.Error("Hash verification failed for correct hash")
+	}
+
+	if verifyFileHash(testData, "wronghash") {
+		t.Error("Hash verification passed for wrong hash")
+	}
+}
+
+func TestHashVerificationWithRealData(t *testing.T) {
+	// Test with some real data to ensure our hash calculation is correct
+	testCases := []struct {
+		data     string
+		expected string
+	}{
+		{"", "da39a3ee5e6b4b0d3255bfef95601890afd80709"},              // SHA1 of empty string
+		{"test", "a94a8fe5ccb19ba61c4c0873d391e987982fbbd3"},          // SHA1 of "test"
+		{"Hello, World!", "0a0a9f2a6772942557ab5355d76af442f8f65e01"}, // SHA1 of "Hello, World!"
+	}
+
+	for _, tc := range testCases {
+		data := []byte(tc.data)
+		hash := calculateFileHash(data)
+		if hash != tc.expected {
+			t.Errorf("Hash calculation failed for '%s': expected %s, got %s", tc.data, tc.expected, hash)
+		}
+
+		if !verifyFileHash(data, tc.expected) {
+			t.Errorf("Hash verification failed for '%s'", tc.data)
+		}
+	}
+}
+
+func TestResponseHashCalculation(t *testing.T) {
+	// Create a mock HTTP response
+	resp := &http.Response{
+		StatusCode: 200,
+		Status:     "200 OK",
+		Header: http.Header{
+			"Content-Type":   []string{"application/octet-stream"},
+			"Content-Length": []string{"13"},
+			"Cache-Control":  []string{"public, max-age=3600"},
+		},
+	}
+
+	bodyData := []byte("Hello, World!")
+	
+	// Calculate response hash
+	responseHash := calculateResponseHash(resp, bodyData)
+	
+	// The hash should be different from just the body hash
+	bodyHash := calculateFileHash(bodyData)
+	
+	if responseHash == bodyHash {
+		t.Error("Response hash should be different from body hash when headers are present")
+	}
+	
+	// Test that the same response produces the same hash
+	responseHash2 := calculateResponseHash(resp, bodyData)
+	if responseHash != responseHash2 {
+		t.Error("Response hash should be consistent for the same response")
+	}
+	
+	// Test with different headers
+	resp2 := &http.Response{
+		StatusCode: 200,
+		Status:     "200 OK",
+		Header: http.Header{
+			"Content-Type":   []string{"text/plain"},
+			"Content-Length": []string{"13"},
+		},
+	}
+	
+	responseHash3 := calculateResponseHash(resp2, bodyData)
+	if responseHash == responseHash3 {
+		t.Error("Response hash should be different for different headers")
+	}
+}

version/version.go

@@ -1,10 +1,16 @@
 // version/version.go
 package version
 
+import "time"
+
 var Version string
+var Date string
 
 func init() {
 	if Version == "" {
 		Version = "0.0.0-dev"
 	}
+	if Date == "" {
+		Date = time.Now().Format("2006-01-02 15:04:05")
+	}
 }

vfs/cache/cache.go

@@ -6,6 +6,7 @@ import (
 	"io"
 	"s1d3sw1ped/SteamCache2/vfs"
 	"s1d3sw1ped/SteamCache2/vfs/cachestate"
+	"s1d3sw1ped/SteamCache2/vfs/gc"
 	"s1d3sw1ped/SteamCache2/vfs/vfserror"
 	"sync"
 )
@@ -98,6 +99,10 @@ func (c *CacheFS) Open(key string) (io.ReadCloser, error) {
 	switch state {
 	case cachestate.CacheStateHit:
 		// if c.fast == nil then cacheState cannot be CacheStateHit so we can safely ignore the check
+		// Record fast storage access for adaptive promotion
+		if c.fast != nil {
+			gc.RecordFastStorageAccess()
+		}
 		return c.fast.Open(key)
 	case cachestate.CacheStateMiss:
 		slowReader, err := c.slow.Open(key)

vfs/disk/disk.go

@@ -242,10 +242,12 @@ func (d *DiskFS) Create(key string, size int64) (io.WriteCloser, error) {
 
 	// Check again after lock
 	d.mu.Lock()
+	var accessCount int64 = 0
 	if fi, exists := d.info[key]; exists {
 		d.size -= fi.Size()
 		d.LRU.Remove(key)
 		delete(d.info, key)
+		accessCount = fi.AccessCount // preserve access count if overwriting
 		path := filepath.Join(d.root, key)
 		os.Remove(path) // Ignore error, as file might not exist or other issues
 	}
@@ -274,6 +276,7 @@ func (d *DiskFS) Create(key string, size int64) (io.WriteCloser, error) {
 
 			d.mu.Lock()
 			finfo := vfs.NewFileInfoFromOS(fi, key)
+			finfo.AccessCount = accessCount
 			d.info[key] = finfo
 			d.LRU.Add(key, finfo)
 			d.size += n
@@ -381,6 +384,7 @@ func (d *DiskFS) Open(key string) (io.ReadCloser, error) {
 		return nil, vfserror.ErrNotFound
 	}
 	fi.ATime = time.Now()
+	fi.AccessCount++ // Increment access count for LFU
 	d.LRU.MoveToFront(key)
 	d.mu.Unlock()
 

vfs/fileinfo.go

@@ -7,27 +7,30 @@ import (
 )
 
 type FileInfo struct {
-	name  string
-	size  int64
-	MTime time.Time
-	ATime time.Time
+	name        string
+	size        int64
+	MTime       time.Time
+	ATime       time.Time
+	AccessCount int64 // Number of times the file has been accessed
 }
 
 func NewFileInfo(key string, size int64, modTime time.Time) *FileInfo {
 	return &FileInfo{
-		name:  key,
-		size:  size,
-		MTime: modTime,
-		ATime: time.Now(),
+		name:        key,
+		size:        size,
+		MTime:       modTime,
+		ATime:       time.Now(),
+		AccessCount: 0,
 	}
 }
 
 func NewFileInfoFromOS(f os.FileInfo, key string) *FileInfo {
 	return &FileInfo{
-		name:  key,
-		size:  f.Size(),
-		MTime: f.ModTime(),
-		ATime: time.Now(),
+		name:        key,
+		size:        f.Size(),
+		MTime:       f.ModTime(),
+		ATime:       time.Now(),
+		AccessCount: 0,
 	}
 }
 

vfs/gc/gc.go

@@ -10,63 +10,360 @@ import (
 	"s1d3sw1ped/SteamCache2/vfs/disk"
 	"s1d3sw1ped/SteamCache2/vfs/memory"
 	"s1d3sw1ped/SteamCache2/vfs/vfserror"
+	"sort"
+	"sync"
 	"time"
+
+	"github.com/prometheus/client_golang/prometheus"
+	"github.com/prometheus/client_golang/prometheus/promauto"
+)
+
+var (
+	// ErrInsufficientSpace is returned when there are no files to delete in the VFS.
+	ErrInsufficientSpace = fmt.Errorf("no files to delete")
+)
+
+// Prometheus metrics for adaptive promotion
+var (
+	promotionThresholds = promauto.NewGaugeVec(
+		prometheus.GaugeOpts{
+			Name: "promotion_thresholds_bytes",
+			Help: "Current promotion thresholds in bytes",
+		},
+		[]string{"threshold_type"},
+	)
+
+	promotionWindows = promauto.NewGaugeVec(
+		prometheus.GaugeOpts{
+			Name: "promotion_windows_seconds",
+			Help: "Current promotion time windows in seconds",
+		},
+		[]string{"window_type"},
+	)
+
+	promotionStats = promauto.NewGaugeVec(
+		prometheus.GaugeOpts{
+			Name: "promotion_stats",
+			Help: "Promotion statistics",
+		},
+		[]string{"metric_type"},
+	)
+
+	promotionAdaptations = promauto.NewCounterVec(
+		prometheus.CounterOpts{
+			Name: "promotion_adaptations_total",
+			Help: "Total number of promotion threshold adaptations",
+		},
+		[]string{"direction"},
+	)
+)
+
+// GCAlgorithm represents different garbage collection strategies
+type GCAlgorithm string
+
+const (
+	LRU      GCAlgorithm = "lru"
+	LFU      GCAlgorithm = "lfu"
+	FIFO     GCAlgorithm = "fifo"
+	Largest  GCAlgorithm = "largest"
+	Smallest GCAlgorithm = "smallest"
+	Hybrid   GCAlgorithm = "hybrid"
 )
 
 // LRUGC deletes files in LRU order until enough space is reclaimed.
-func LRUGC(vfss vfs.VFS, size uint) {
-	attempts := 0
-	deletions := 0
-	var reclaimed uint
+func LRUGC(vfss vfs.VFS, size uint) error {
+	logger.Logger.Debug().Uint("target", size).Msg("Attempting to reclaim space using LRU GC")
 
-	for reclaimed < size {
-		if attempts > 10 {
-			logger.Logger.Debug().
-				Int("attempts", attempts).
-				Msg("GC: Too many attempts to reclaim space, giving up")
-			return
-		}
-		attempts++
+	var reclaimed uint // reclaimed space in bytes
+	deleted := false
+
+	for {
 		switch fs := vfss.(type) {
 		case *disk.DiskFS:
 			fi := fs.LRU.Back()
 			if fi == nil {
-				break
+				if deleted {
+					logger.Logger.Debug().Uint("target", size).Uint("achieved", reclaimed).Msg("Reclaimed enough space using LRU GC (at least one file deleted)")
+					return nil
+				}
+				return ErrInsufficientSpace // No files to delete
 			}
 			sz := uint(fi.Size())
 			err := fs.Delete(fi.Name())
 			if err != nil {
-				continue
+				continue // If delete fails, try the next file
 			}
 			reclaimed += sz
-			deletions++
+			deleted = true
 		case *memory.MemoryFS:
 			fi := fs.LRU.Back()
 			if fi == nil {
-				break
+				if deleted {
+					logger.Logger.Debug().Uint("target", size).Uint("achieved", reclaimed).Msg("Reclaimed enough space using LRU GC (at least one file deleted)")
+					return nil
+				}
+				return ErrInsufficientSpace // No files to delete
 			}
 			sz := uint(fi.Size())
 			err := fs.Delete(fi.Name())
 			if err != nil {
-				continue
+				continue // If delete fails, try the next file
 			}
 			reclaimed += sz
-			deletions++
+			deleted = true
 		default:
-			// Fallback to old method if not supported
-			stats := vfss.StatAll()
-			if len(stats) == 0 {
-				break
-			}
-			fi := stats[0] // Assume sorted or pick first
-			sz := uint(fi.Size())
-			err := vfss.Delete(fi.Name())
-			if err != nil {
-				continue
-			}
-			reclaimed += sz
-			deletions++
+			panic("unreachable: unsupported VFS type for LRU GC") // panic if the VFS is not disk or memory
 		}
+
+		if deleted && (size == 0 || reclaimed >= size) {
+			logger.Logger.Debug().Uint("target", size).Uint("achieved", reclaimed).Msg("Reclaimed enough space using LRU GC (at least one file deleted)")
+			return nil // stop if enough space is reclaimed or at least one file deleted for size==0
+		}
+	}
+}
+
+// LFUGC deletes files in LFU (Least Frequently Used) order until enough space is reclaimed.
+func LFUGC(vfss vfs.VFS, size uint) error {
+	logger.Logger.Debug().Uint("target", size).Msg("Attempting to reclaim space using LFU GC")
+
+	files := getAllFiles(vfss)
+	if len(files) == 0 {
+		return ErrInsufficientSpace
+	}
+
+	sort.Slice(files, func(i, j int) bool {
+		return files[i].AccessCount < files[j].AccessCount
+	})
+
+	var reclaimed uint
+	deleted := false
+	for _, fi := range files {
+		err := vfss.Delete(fi.Name)
+		if err != nil {
+			continue
+		}
+		reclaimed += uint(fi.Size)
+		deleted = true
+		if deleted && (size == 0 || reclaimed >= size) {
+			logger.Logger.Debug().Uint("target", size).Uint("achieved", reclaimed).Msg("Reclaimed enough space using LFU GC (at least one file deleted)")
+			return nil
+		}
+	}
+
+	if deleted {
+		logger.Logger.Debug().Uint("target", size).Uint("achieved", reclaimed).Msg("Reclaimed enough space using LFU GC (at least one file deleted)")
+		return nil
+	}
+	return ErrInsufficientSpace
+}
+
+// FIFOGC deletes files in FIFO (First In, First Out) order until enough space is reclaimed.
+func FIFOGC(vfss vfs.VFS, size uint) error {
+	logger.Logger.Debug().Uint("target", size).Msg("Attempting to reclaim space using FIFO GC")
+
+	files := getAllFiles(vfss)
+	if len(files) == 0 {
+		return ErrInsufficientSpace
+	}
+
+	sort.Slice(files, func(i, j int) bool {
+		return files[i].MTime.Before(files[j].MTime)
+	})
+
+	var reclaimed uint
+	deleted := false
+	for _, fi := range files {
+		err := vfss.Delete(fi.Name)
+		if err != nil {
+			continue
+		}
+		reclaimed += uint(fi.Size)
+		deleted = true
+		if deleted && (size == 0 || reclaimed >= size) {
+			logger.Logger.Debug().Uint("target", size).Uint("achieved", reclaimed).Msg("Reclaimed enough space using FIFO GC (at least one file deleted)")
+			return nil
+		}
+	}
+
+	if deleted {
+		logger.Logger.Debug().Uint("target", size).Uint("achieved", reclaimed).Msg("Reclaimed enough space using FIFO GC (at least one file deleted)")
+		return nil
+	}
+	return ErrInsufficientSpace
+}
+
+// LargestGC deletes the largest files first until enough space is reclaimed.
+func LargestGC(vfss vfs.VFS, size uint) error {
+	logger.Logger.Debug().Uint("target", size).Msg("Attempting to reclaim space using Largest GC")
+
+	files := getAllFiles(vfss)
+	if len(files) == 0 {
+		return ErrInsufficientSpace
+	}
+
+	sort.Slice(files, func(i, j int) bool {
+		return files[i].Size > files[j].Size
+	})
+
+	var reclaimed uint
+	deleted := false
+	for _, fi := range files {
+		err := vfss.Delete(fi.Name)
+		if err != nil {
+			continue
+		}
+		reclaimed += uint(fi.Size)
+		deleted = true
+		if deleted && (size == 0 || reclaimed >= size) {
+			logger.Logger.Debug().Uint("target", size).Uint("achieved", reclaimed).Msg("Reclaimed enough space using Largest GC (at least one file deleted)")
+			return nil
+		}
+	}
+
+	if deleted {
+		logger.Logger.Debug().Uint("target", size).Uint("achieved", reclaimed).Msg("Reclaimed enough space using Largest GC (at least one file deleted)")
+		return nil
+	}
+	return ErrInsufficientSpace
+}
+
+// SmallestGC deletes the smallest files first until enough space is reclaimed.
+func SmallestGC(vfss vfs.VFS, size uint) error {
+	logger.Logger.Debug().Uint("target", size).Msg("Attempting to reclaim space using Smallest GC")
+
+	files := getAllFiles(vfss)
+	if len(files) == 0 {
+		return ErrInsufficientSpace
+	}
+
+	sort.Slice(files, func(i, j int) bool {
+		return files[i].Size < files[j].Size
+	})
+
+	var reclaimed uint
+	deleted := false
+	for _, fi := range files {
+		err := vfss.Delete(fi.Name)
+		if err != nil {
+			continue
+		}
+		reclaimed += uint(fi.Size)
+		deleted = true
+		if deleted && (size == 0 || reclaimed >= size) {
+			logger.Logger.Debug().Uint("target", size).Uint("achieved", reclaimed).Msg("Reclaimed enough space using Smallest GC (at least one file deleted)")
+			return nil
+		}
+	}
+
+	if deleted {
+		logger.Logger.Debug().Uint("target", size).Uint("achieved", reclaimed).Msg("Reclaimed enough space using Smallest GC (at least one file deleted)")
+		return nil
+	}
+	return ErrInsufficientSpace
+}
+
+// HybridGC combines LRU and size-based eviction with a scoring system.
+func HybridGC(vfss vfs.VFS, size uint) error {
+	logger.Logger.Debug().Uint("target", size).Msg("Attempting to reclaim space using Hybrid GC")
+
+	files := getAllFiles(vfss)
+	if len(files) == 0 {
+		return ErrInsufficientSpace
+	}
+
+	now := time.Now()
+	for i := range files {
+		timeSinceAccess := now.Sub(files[i].ATime).Seconds()
+		sizeMB := float64(files[i].Size) / (1024 * 1024)
+		files[i].HybridScore = timeSinceAccess * sizeMB
+	}
+
+	sort.Slice(files, func(i, j int) bool {
+		return files[i].HybridScore < files[j].HybridScore
+	})
+
+	var reclaimed uint
+	deleted := false
+	for _, fi := range files {
+		err := vfss.Delete(fi.Name)
+		if err != nil {
+			continue
+		}
+		reclaimed += uint(fi.Size)
+		deleted = true
+		if deleted && (size == 0 || reclaimed >= size) {
+			logger.Logger.Debug().Uint("target", size).Uint("achieved", reclaimed).Msg("Reclaimed enough space using Hybrid GC (at least one file deleted)")
+			return nil
+		}
+	}
+
+	if deleted {
+		logger.Logger.Debug().Uint("target", size).Uint("achieved", reclaimed).Msg("Reclaimed enough space using Hybrid GC (at least one file deleted)")
+		return nil
+	}
+	return ErrInsufficientSpace
+}
+
+// fileInfoWithMetadata extends FileInfo with additional metadata for GC algorithms
+type fileInfoWithMetadata struct {
+	Name        string
+	Size        int64
+	MTime       time.Time
+	ATime       time.Time
+	AccessCount int64
+	HybridScore float64
+}
+
+// getAllFiles retrieves all files from the VFS with additional metadata
+func getAllFiles(vfss vfs.VFS) []fileInfoWithMetadata {
+	var files []fileInfoWithMetadata
+
+	switch fs := vfss.(type) {
+	case *disk.DiskFS:
+		allFiles := fs.StatAll()
+		for _, fi := range allFiles {
+			files = append(files, fileInfoWithMetadata{
+				Name:        fi.Name(),
+				Size:        fi.Size(),
+				MTime:       fi.ModTime(),
+				ATime:       fi.AccessTime(),
+				AccessCount: fi.AccessCount,
+			})
+		}
+	case *memory.MemoryFS:
+		allFiles := fs.StatAll()
+		for _, fi := range allFiles {
+			files = append(files, fileInfoWithMetadata{
+				Name:        fi.Name(),
+				Size:        fi.Size(),
+				MTime:       fi.ModTime(),
+				ATime:       fi.AccessTime(),
+				AccessCount: fi.AccessCount,
+			})
+		}
+	}
+
+	return files
+}
+
+// GetGCAlgorithm returns the appropriate GC function based on the algorithm name
+func GetGCAlgorithm(algorithm GCAlgorithm) GCHandlerFunc {
+	switch algorithm {
+	case LRU:
+		return LRUGC
+	case LFU:
+		return LFUGC
+	case FIFO:
+		return FIFOGC
+	case Largest:
+		return LargestGC
+	case Smallest:
+		return SmallestGC
+	case Hybrid:
+		return HybridGC
+	default:
+		logger.Logger.Warn().Str("algorithm", string(algorithm)).Msg("Unknown GC algorithm, falling back to LRU")
+		return LRUGC
 	}
 }
 
@@ -74,28 +371,346 @@ func PromotionDecider(fi *vfs.FileInfo, cs cachestate.CacheState) bool {
 	return time.Since(fi.AccessTime()) < time.Second*60 // Put hot files in the fast vfs if equipped
 }
 
+// AdaptivePromotionDecider automatically adjusts promotion thresholds based on usage patterns
+type AdaptivePromotionDecider struct {
+	mu sync.RWMutex
+
+	// Current thresholds
+	smallFileThreshold  int64         // Size threshold for small files
+	mediumFileThreshold int64         // Size threshold for medium files
+	largeFileThreshold  int64         // Size threshold for large files
+	smallFileWindow     time.Duration // Time window for small files
+	mediumFileWindow    time.Duration // Time window for medium files
+	largeFileWindow     time.Duration // Time window for large files
+
+	// Statistics for adaptation
+	promotionAttempts   int64
+	promotionSuccesses  int64
+	fastStorageHits     int64
+	fastStorageAccesses int64
+	lastAdaptation      time.Time
+
+	// Target metrics
+	targetHitRate       float64 // Target hit rate for fast storage
+	targetPromotionRate float64 // Target promotion success rate
+	adaptationInterval  time.Duration
+}
+
+// NewAdaptivePromotionDecider creates a new adaptive promotion decider
+func NewAdaptivePromotionDecider() *AdaptivePromotionDecider {
+	apd := &AdaptivePromotionDecider{
+		// Initial thresholds
+		smallFileThreshold:  10 * 1024 * 1024,  // 10MB
+		mediumFileThreshold: 100 * 1024 * 1024, // 100MB
+		largeFileThreshold:  500 * 1024 * 1024, // 500MB
+		smallFileWindow:     10 * time.Minute,
+		mediumFileWindow:    2 * time.Minute,
+		largeFileWindow:     30 * time.Second,
+
+		// Target metrics
+		targetHitRate:       0.8, // 80% hit rate
+		targetPromotionRate: 0.7, // 70% promotion success rate
+		adaptationInterval:  5 * time.Minute,
+	}
+
+	// Initialize Prometheus metrics
+	apd.updatePrometheusMetrics()
+
+	return apd
+}
+
+// ShouldPromote determines if a file should be promoted based on adaptive thresholds
+func (apd *AdaptivePromotionDecider) ShouldPromote(fi *vfs.FileInfo, cs cachestate.CacheState) bool {
+	apd.mu.Lock()
+	defer apd.mu.Unlock()
+
+	// Check if it's time to adapt thresholds
+	if time.Since(apd.lastAdaptation) > apd.adaptationInterval {
+		apd.adaptThresholds()
+	}
+
+	size := fi.Size()
+	timeSinceAccess := time.Since(fi.AccessTime())
+
+	// Record promotion attempt
+	apd.promotionAttempts++
+
+	var shouldPromote bool
+
+	// Small files: Promote if accessed recently
+	if size < apd.smallFileThreshold {
+		shouldPromote = timeSinceAccess < apd.smallFileWindow
+	} else if size < apd.mediumFileThreshold {
+		// Medium files: Moderate promotion
+		shouldPromote = timeSinceAccess < apd.mediumFileWindow
+	} else if size < apd.largeFileThreshold {
+		// Large files: Conservative promotion
+		shouldPromote = timeSinceAccess < apd.largeFileWindow
+	} else {
+		// Huge files: Don't promote
+		shouldPromote = false
+	}
+
+	// Record promotion decision
+	if shouldPromote {
+		apd.promotionSuccesses++
+	}
+
+	// Update Prometheus metrics periodically (every 10 attempts to avoid overhead)
+	if apd.promotionAttempts%10 == 0 {
+		apd.updatePrometheusMetrics()
+	}
+
+	return shouldPromote
+}
+
+// RecordFastStorageAccess records when fast storage is accessed
+func (apd *AdaptivePromotionDecider) RecordFastStorageAccess() {
+	apd.mu.Lock()
+	defer apd.mu.Unlock()
+	apd.fastStorageAccesses++
+
+	// Update Prometheus metrics periodically
+	if apd.fastStorageAccesses%10 == 0 {
+		apd.updatePrometheusMetrics()
+	}
+}
+
+// RecordFastStorageHit records when fast storage has a hit
+func (apd *AdaptivePromotionDecider) RecordFastStorageHit() {
+	apd.mu.Lock()
+	defer apd.mu.Unlock()
+	apd.fastStorageHits++
+
+	// Update Prometheus metrics periodically
+	if apd.fastStorageHits%10 == 0 {
+		apd.updatePrometheusMetrics()
+	}
+}
+
+// adaptThresholds adjusts thresholds based on current performance
+func (apd *AdaptivePromotionDecider) adaptThresholds() {
+	if apd.promotionAttempts < 10 || apd.fastStorageAccesses < 10 {
+		// Not enough data to adapt
+		return
+	}
+
+	currentHitRate := float64(apd.fastStorageHits) / float64(apd.fastStorageAccesses)
+	currentPromotionRate := float64(apd.promotionSuccesses) / float64(apd.promotionAttempts)
+
+	logger.Logger.Debug().
+		Float64("hit_rate", currentHitRate).
+		Float64("promotion_rate", currentPromotionRate).
+		Float64("target_hit_rate", apd.targetHitRate).
+		Float64("target_promotion_rate", apd.targetPromotionRate).
+		Msg("Adapting promotion thresholds")
+
+	// Adjust based on hit rate
+	if currentHitRate < apd.targetHitRate {
+		// Hit rate too low - be more aggressive with promotion
+		apd.adjustThresholdsMoreAggressive()
+	} else if currentHitRate > apd.targetHitRate+0.1 {
+		// Hit rate too high - be more conservative
+		apd.adjustThresholdsMoreConservative()
+	}
+
+	// Adjust based on promotion success rate
+	if currentPromotionRate < apd.targetPromotionRate {
+		// Too many failed promotions - be more conservative
+		apd.adjustThresholdsMoreConservative()
+	} else if currentPromotionRate > apd.targetPromotionRate+0.1 {
+		// High promotion success - can be more aggressive
+		apd.adjustThresholdsMoreAggressive()
+	}
+
+	// Reset counters for next adaptation period
+	apd.promotionAttempts = 0
+	apd.promotionSuccesses = 0
+	apd.fastStorageHits = 0
+	apd.fastStorageAccesses = 0
+	apd.lastAdaptation = time.Now()
+
+	logger.Logger.Info().
+		Int64("small_threshold_mb", apd.smallFileThreshold/(1024*1024)).
+		Int64("medium_threshold_mb", apd.mediumFileThreshold/(1024*1024)).
+		Int64("large_threshold_mb", apd.largeFileThreshold/(1024*1024)).
+		Dur("small_window", apd.smallFileWindow).
+		Dur("medium_window", apd.mediumFileWindow).
+		Dur("large_window", apd.largeFileWindow).
+		Msg("Updated promotion thresholds")
+}
+
+// updatePrometheusMetrics updates all Prometheus metrics with current values
+func (apd *AdaptivePromotionDecider) updatePrometheusMetrics() {
+	// Update threshold metrics
+	promotionThresholds.WithLabelValues("small").Set(float64(apd.smallFileThreshold))
+	promotionThresholds.WithLabelValues("medium").Set(float64(apd.mediumFileThreshold))
+	promotionThresholds.WithLabelValues("large").Set(float64(apd.largeFileThreshold))
+
+	// Update window metrics
+	promotionWindows.WithLabelValues("small").Set(apd.smallFileWindow.Seconds())
+	promotionWindows.WithLabelValues("medium").Set(apd.mediumFileWindow.Seconds())
+	promotionWindows.WithLabelValues("large").Set(apd.largeFileWindow.Seconds())
+
+	// Update statistics metrics
+	hitRate := 0.0
+	if apd.fastStorageAccesses > 0 {
+		hitRate = float64(apd.fastStorageHits) / float64(apd.fastStorageAccesses)
+	}
+	promotionRate := 0.0
+	if apd.promotionAttempts > 0 {
+		promotionRate = float64(apd.promotionSuccesses) / float64(apd.promotionAttempts)
+	}
+
+	promotionStats.WithLabelValues("hit_rate").Set(hitRate)
+	promotionStats.WithLabelValues("promotion_rate").Set(promotionRate)
+	promotionStats.WithLabelValues("promotion_attempts").Set(float64(apd.promotionAttempts))
+	promotionStats.WithLabelValues("promotion_successes").Set(float64(apd.promotionSuccesses))
+	promotionStats.WithLabelValues("fast_storage_accesses").Set(float64(apd.fastStorageAccesses))
+	promotionStats.WithLabelValues("fast_storage_hits").Set(float64(apd.fastStorageHits))
+}
+
+// adjustThresholdsMoreAggressive makes promotion more aggressive
+func (apd *AdaptivePromotionDecider) adjustThresholdsMoreAggressive() {
+	// Increase size thresholds (promote larger files)
+	apd.smallFileThreshold = minInt64(apd.smallFileThreshold*11/10, 50*1024*1024)    // Max 50MB
+	apd.mediumFileThreshold = minInt64(apd.mediumFileThreshold*11/10, 200*1024*1024) // Max 200MB
+	apd.largeFileThreshold = minInt64(apd.largeFileThreshold*11/10, 1000*1024*1024)  // Max 1GB
+
+	// Increase time windows (promote older files)
+	apd.smallFileWindow = minDuration(apd.smallFileWindow*11/10, 20*time.Minute)
+	apd.mediumFileWindow = minDuration(apd.mediumFileWindow*11/10, 5*time.Minute)
+	apd.largeFileWindow = minDuration(apd.largeFileWindow*11/10, 2*time.Minute)
+
+	// Record adaptation in Prometheus
+	promotionAdaptations.WithLabelValues("aggressive").Inc()
+
+	// Update Prometheus metrics
+	apd.updatePrometheusMetrics()
+}
+
+// adjustThresholdsMoreConservative makes promotion more conservative
+func (apd *AdaptivePromotionDecider) adjustThresholdsMoreConservative() {
+	// Decrease size thresholds (promote smaller files)
+	apd.smallFileThreshold = maxInt64(apd.smallFileThreshold*9/10, 5*1024*1024)    // Min 5MB
+	apd.mediumFileThreshold = maxInt64(apd.mediumFileThreshold*9/10, 50*1024*1024) // Min 50MB
+	apd.largeFileThreshold = maxInt64(apd.largeFileThreshold*9/10, 200*1024*1024)  // Min 200MB
+
+	// Decrease time windows (promote only recent files)
+	apd.smallFileWindow = maxDuration(apd.smallFileWindow*9/10, 5*time.Minute)
+	apd.mediumFileWindow = maxDuration(apd.mediumFileWindow*9/10, 1*time.Minute)
+	apd.largeFileWindow = maxDuration(apd.largeFileWindow*9/10, 15*time.Second)
+
+	// Record adaptation in Prometheus
+	promotionAdaptations.WithLabelValues("conservative").Inc()
+
+	// Update Prometheus metrics
+	apd.updatePrometheusMetrics()
+}
+
+// GetStats returns current statistics for monitoring
+func (apd *AdaptivePromotionDecider) GetStats() map[string]interface{} {
+	apd.mu.RLock()
+	defer apd.mu.RUnlock()
+
+	hitRate := 0.0
+	if apd.fastStorageAccesses > 0 {
+		hitRate = float64(apd.fastStorageHits) / float64(apd.fastStorageAccesses)
+	}
+
+	promotionRate := 0.0
+	if apd.promotionAttempts > 0 {
+		promotionRate = float64(apd.promotionSuccesses) / float64(apd.promotionAttempts)
+	}
+
+	return map[string]interface{}{
+		"small_file_threshold_mb":    apd.smallFileThreshold / (1024 * 1024),
+		"medium_file_threshold_mb":   apd.mediumFileThreshold / (1024 * 1024),
+		"large_file_threshold_mb":    apd.largeFileThreshold / (1024 * 1024),
+		"small_file_window_minutes":  apd.smallFileWindow.Minutes(),
+		"medium_file_window_minutes": apd.mediumFileWindow.Minutes(),
+		"large_file_window_seconds":  apd.largeFileWindow.Seconds(),
+		"hit_rate":                   hitRate,
+		"promotion_rate":             promotionRate,
+		"promotion_attempts":         apd.promotionAttempts,
+		"promotion_successes":        apd.promotionSuccesses,
+		"fast_storage_accesses":      apd.fastStorageAccesses,
+		"fast_storage_hits":          apd.fastStorageHits,
+	}
+}
+
+// Global adaptive promotion decider instance
+var adaptivePromotionDecider *AdaptivePromotionDecider
+
+func init() {
+	adaptivePromotionDecider = NewAdaptivePromotionDecider()
+}
+
+// AdaptivePromotionDeciderFunc returns the adaptive promotion decision function
+func AdaptivePromotionDeciderFunc(fi *vfs.FileInfo, cs cachestate.CacheState) bool {
+	return adaptivePromotionDecider.ShouldPromote(fi, cs)
+}
+
+// RecordFastStorageAccess records fast storage access for adaptation
+func RecordFastStorageAccess() {
+	adaptivePromotionDecider.RecordFastStorageAccess()
+}
+
+// RecordFastStorageHit records fast storage hit for adaptation
+func RecordFastStorageHit() {
+	adaptivePromotionDecider.RecordFastStorageHit()
+}
+
+// GetPromotionStats returns promotion statistics for monitoring
+func GetPromotionStats() map[string]interface{} {
+	return adaptivePromotionDecider.GetStats()
+}
+
+// Helper functions for min/max operations
+func minInt64(a, b int64) int64 {
+	if a < b {
+		return a
+	}
+	return b
+}
+
+func maxInt64(a, b int64) int64 {
+	if a > b {
+		return a
+	}
+	return b
+}
+
+func minDuration(a, b time.Duration) time.Duration {
+	if a < b {
+		return a
+	}
+	return b
+}
+
+func maxDuration(a, b time.Duration) time.Duration {
+	if a > b {
+		return a
+	}
+	return b
+}
+
 // Ensure GCFS implements VFS.
 var _ vfs.VFS = (*GCFS)(nil)
 
 // GCFS is a virtual file system that calls a GC handler when the disk is full. The GC handler is responsible for freeing up space on the disk. The GCFS is a wrapper around another VFS.
 type GCFS struct {
 	vfs.VFS
-	multiplier int
 
-	// protected by mu
 	gcHanderFunc GCHandlerFunc
 }
 
 // GCHandlerFunc is a function that is called when the disk is full and the GCFS needs to free up space. It is passed the VFS and the size of the file that needs to be written. Its up to the implementation to free up space. How much space is freed is also up to the implementation.
-type GCHandlerFunc func(vfs vfs.VFS, size uint)
+type GCHandlerFunc func(vfs vfs.VFS, size uint) error
 
-func New(vfs vfs.VFS, multiplier int, gcHandlerFunc GCHandlerFunc) *GCFS {
-	if multiplier <= 0 {
-		multiplier = 1 // if the multiplier is less than or equal to 0 set it to 1 will be slow but the user can set it to a higher value if they want
-	}
+func New(vfs vfs.VFS, gcHandlerFunc GCHandlerFunc) *GCFS {
 	return &GCFS{
 		VFS:          vfs,
-		multiplier:   multiplier,
 		gcHanderFunc: gcHandlerFunc,
 	}
 }
@@ -103,11 +718,24 @@ func New(vfs vfs.VFS, multiplier int, gcHandlerFunc GCHandlerFunc) *GCFS {
 // Create overrides the Create method of the VFS interface. It tries to create the key, if it fails due to disk full error, it calls the GC handler and tries again. If it still fails it returns the error.
 func (g *GCFS) Create(key string, size int64) (io.WriteCloser, error) {
 	w, err := g.VFS.Create(key, size) // try to create the key
-
-	// if it fails due to disk full error, call the GC handler and try again in a loop that will continue until it succeeds or the error is not disk full
-	if err == vfserror.ErrDiskFull && g.gcHanderFunc != nil { // if the error is disk full and there is a GC handler
-		g.gcHanderFunc(g.VFS, uint(size*int64(g.multiplier))) // call the GC handler
+	for err == vfserror.ErrDiskFull && g.gcHanderFunc != nil {
+		errGC := g.gcHanderFunc(g.VFS, uint(size)) // call the GC handler
+		if errGC == ErrInsufficientSpace {
+			return nil, errGC // if the GC handler returns no files to delete, return the error
+		}
 		w, err = g.VFS.Create(key, size)
+		if err == vfserror.ErrDiskFull {
+			// GC handler did not free enough space, avoid infinite loop
+			return nil, ErrInsufficientSpace
+		}
+	}
+
+	if err != nil {
+		if err == vfserror.ErrDiskFull {
+			logger.Logger.Error().Str("key", key).Int64("size", size).Msg("Failed to create file due to disk full, even after GC")
+		} else {
+			logger.Logger.Error().Str("key", key).Int64("size", size).Err(err).Msg("Failed to create file")
+		}
 	}
 
 	return w, err

vfs/gc/gc_test.go

@@ -2,72 +2,41 @@
 package gc
 
 import (
-	"errors"
-	"fmt"
-	"s1d3sw1ped/SteamCache2/vfs/memory"
-	"s1d3sw1ped/SteamCache2/vfs/vfserror"
 	"testing"
 )
 
-func TestGCOnFull(t *testing.T) {
-	m := memory.New(10)
-	gc := New(m, 2, LRUGC)
+func TestGetGCAlgorithm(t *testing.T) {
+	tests := []struct {
+		name      string
+		algorithm GCAlgorithm
+		expected  bool // true if we expect a non-nil function
+	}{
+		{"LRU", LRU, true},
+		{"LFU", LFU, true},
+		{"FIFO", FIFO, true},
+		{"Largest", Largest, true},
+		{"Smallest", Smallest, true},
+		{"Hybrid", Hybrid, true},
+		{"Unknown", "unknown", true}, // should fall back to LRU
+		{"Empty", "", true},          // should fall back to LRU
+	}
 
-	for i := 0; i < 5; i++ {
-		w, err := gc.Create(fmt.Sprintf("key%d", i), 2)
-		if err != nil {
-			t.Fatalf("Create failed: %v", err)
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			fn := GetGCAlgorithm(tt.algorithm)
+			if fn == nil {
+				t.Errorf("GetGCAlgorithm(%s) returned nil, expected non-nil function", tt.algorithm)
+			}
+		})
+	}
+}
+
+func TestGCAlgorithmConstants(t *testing.T) {
+	expectedAlgorithms := []GCAlgorithm{LRU, LFU, FIFO, Largest, Smallest, Hybrid}
+
+	for _, algo := range expectedAlgorithms {
+		if algo == "" {
+			t.Errorf("GC algorithm constant is empty")
 		}
-		w.Write([]byte("ab"))
-		w.Close()
-	}
-
-	// Cache full at 10 bytes
-	w, err := gc.Create("key5", 2)
-	if err != nil {
-		t.Fatalf("Create failed: %v", err)
-	}
-	w.Write([]byte("cd"))
-	w.Close()
-
-	if gc.Size() > 10 {
-		t.Errorf("Size exceeded: %d > 10", gc.Size())
-	}
-
-	// Check if older keys were evicted
-	_, err = m.Open("key0")
-	if err == nil {
-		t.Error("Expected key0 to be evicted")
-	}
-}
-
-func TestNoGCNeeded(t *testing.T) {
-	m := memory.New(20)
-	gc := New(m, 2, LRUGC)
-
-	for i := 0; i < 5; i++ {
-		w, err := gc.Create(fmt.Sprintf("key%d", i), 2)
-		if err != nil {
-			t.Fatalf("Create failed: %v", err)
-		}
-		w.Write([]byte("ab"))
-		w.Close()
-	}
-
-	if gc.Size() != 10 {
-		t.Errorf("Size: got %d, want 10", gc.Size())
-	}
-}
-
-func TestGCInsufficientSpace(t *testing.T) {
-	m := memory.New(5)
-	gc := New(m, 1, LRUGC)
-
-	w, err := gc.Create("key0", 10)
-	if err == nil {
-		w.Close()
-		t.Error("Expected ErrDiskFull")
-	} else if !errors.Is(err, vfserror.ErrDiskFull) {
-		t.Errorf("Unexpected error: %v", err)
 	}
 }

vfs/memory/memory.go

@@ -169,11 +169,14 @@ func (m *MemoryFS) Create(key string, size int64) (io.WriteCloser, error) {
 		onClose: func() error {
 			data := buf.Bytes()
 			m.mu.Lock()
+			var accessCount int64 = 0
 			if f, exists := m.files[key]; exists {
 				m.size -= int64(len(f.data))
 				m.LRU.Remove(key)
+				accessCount = f.fileinfo.AccessCount // preserve access count if overwriting
 			}
 			fi := vfs.NewFileInfo(key, int64(len(data)), time.Now())
+			fi.AccessCount = accessCount
 			m.files[key] = &file{
 				fileinfo: fi,
 				data:     data,
@@ -232,6 +235,7 @@ func (m *MemoryFS) Open(key string) (io.ReadCloser, error) {
 		return nil, vfserror.ErrNotFound
 	}
 	f.fileinfo.ATime = time.Now()
+	f.fileinfo.AccessCount++ // Increment access count for LFU
 	m.LRU.MoveToFront(key)
 	dataCopy := make([]byte, len(f.data))
 	copy(dataCopy, f.data)