Refactor configuration management and enhance build process

- Introduced a YAML-based configuration system, allowing for automatic generation of a default `config.yaml` file.
- Updated the application to load configuration settings from the YAML file, improving flexibility and ease of use.
- Added a Makefile to streamline development tasks, including running the application, testing, and managing dependencies.
- Enhanced `.gitignore` to include build artifacts and configuration files.
- Removed unused Prometheus metrics and related code to simplify the codebase.
- Updated dependencies in `go.mod` and `go.sum` for improved functionality and performance.

steamcache/steamcache.go

@@ -12,8 +12,6 @@ import (
 	"net/http"
 	"net/url"
 	"os"
-	"path/filepath"
-	"regexp"
 	"s1d3sw1ped/SteamCache2/steamcache/logger"
 	"s1d3sw1ped/SteamCache2/vfs"
 	"s1d3sw1ped/SteamCache2/vfs/cache"
@@ -28,9 +26,6 @@ import (
 	"bytes"
 
 	"github.com/docker/go-units"
-	"github.com/prometheus/client_golang/prometheus"
-	"github.com/prometheus/client_golang/prometheus/promauto"
-	"github.com/prometheus/client_golang/prometheus/promhttp"
 )
 
 // min returns the minimum of two integers
@@ -41,67 +36,65 @@ func min(a, b int) int {
 	return b
 }
 
-var (
-	requestsTotal = promauto.NewCounterVec(
-		prometheus.CounterOpts{
-			Name: "http_requests_total",
-			Help: "Total number of HTTP requests",
-		},
-		[]string{"method", "status"},
-	)
+// Removed Prometheus metrics - keeping this comment for reference:
+// requestsTotal, cacheStatusTotal, responseTime, hashVerificationTotal
 
-	cacheStatusTotal = promauto.NewCounterVec(
-		prometheus.CounterOpts{
-			Name: "cache_status_total",
-			Help: "Total cache status counts",
-		},
-		[]string{"status"},
-	)
-
-	responseTime = promauto.NewHistogramVec(
-		prometheus.HistogramOpts{
-			Name:    "response_time_seconds",
-			Help:    "Response time in seconds",
-			Buckets: prometheus.DefBuckets,
-		},
-		[]string{"cache_status"},
-	)
-)
-
-// hashVerificationTotal tracks hash verification attempts
-var hashVerificationTotal = promauto.NewCounterVec(
-	prometheus.CounterOpts{
-		Name: "hash_verification_total",
-		Help: "Total hash verification attempts",
-	},
-	[]string{"result"},
-)
-
-// extractHashFromFilename extracts a hash from a filename if present
-// Steam depot files often have hashes in their names like: filename_hash.ext
-func extractHashFromFilename(filename string) (string, bool) {
-	// Common patterns for Steam depot files with hashes
-	patterns := []*regexp.Regexp{
-		regexp.MustCompile(`^([a-fA-F0-9]{40})$`),  // Standalone SHA1 hash (40 hex chars)
-		regexp.MustCompile(`^([a-fA-F0-9]{40})\.`), // SHA1 hash with extension
+// extractHashFromSteamPath extracts a hash from Steam depot URLs
+// Handles patterns like: /depot/123/chunk/abcdef... or /depot/123/manifest/456/789/hash
+func extractHashFromSteamPath(path string) (string, bool) {
+	// Remove leading slash
+	if strings.HasPrefix(path, "/") {
+		path = path[1:]
 	}
 
-	for _, pattern := range patterns {
-		if matches := pattern.FindStringSubmatch(filename); len(matches) > 1 {
-			return strings.ToLower(matches[1]), true
+	parts := strings.Split(path, "/")
+	if len(parts) < 3 {
+		return "", false
+	}
+
+	// Handle chunk files: depot/{id}/chunk/{hash}
+	if len(parts) >= 4 && parts[0] == "depot" && parts[2] == "chunk" {
+		hash := parts[3]
+		// Validate it's a 40-character hex hash
+		if len(hash) == 40 && isHexString(hash) {
+			return strings.ToLower(hash), true
 		}
 	}
 
-	// Debug: log when we don't find a hash pattern
-	if strings.Contains(filename, "manifest") {
-		logger.Logger.Debug().
-			Str("filename", filename).
-			Msg("No hash pattern found in manifest filename")
+	// Handle manifest files: depot/{id}/manifest/{manifest_id}/{version}/{hash}
+	if len(parts) >= 6 && parts[0] == "depot" && parts[2] == "manifest" {
+		hash := parts[5]
+		// Note: Manifest hashes can be shorter than 40 characters
+		if len(hash) >= 10 && isHexString(hash) {
+			return strings.ToLower(hash), true
+		}
 	}
 
 	return "", false
 }
 
+// isHexString checks if a string contains only hexadecimal characters
+func isHexString(s string) bool {
+	for _, r := range s {
+		if !((r >= '0' && r <= '9') || (r >= 'a' && r <= 'f') || (r >= 'A' && r <= 'F')) {
+			return false
+		}
+	}
+	return true
+}
+
+// generateSteamCacheKey converts Steam depot paths to simplified cache keys
+// Input: /depot/1684171/chunk/0016cfc5019b8baa6026aa1cce93e685d6e06c6e
+// Output: steam/0016cfc5019b8baa6026aa1cce93e685d6e06c6e
+func generateSteamCacheKey(urlPath string) string {
+	if hash, ok := extractHashFromSteamPath(urlPath); ok {
+		return "steam/" + hash
+	}
+
+	// Return empty string for unsupported depot URLs
+	return ""
+}
+
 // calculateFileHash calculates the SHA1 hash of the given data
 func calculateFileHash(data []byte) string {
 	hash := sha1.Sum(data)
@@ -162,6 +155,78 @@ var hopByHopHeaders = map[string]struct{}{
 	"Server":              {},
 }
 
+var (
+	// Request coalescing structures
+	coalescedRequests   = make(map[string]*coalescedRequest)
+	coalescedRequestsMu sync.RWMutex
+)
+
+type coalescedRequest struct {
+	responseChan chan *http.Response
+	errorChan    chan error
+	waitingCount int
+	done         bool
+	mu           sync.Mutex
+}
+
+func newCoalescedRequest() *coalescedRequest {
+	return &coalescedRequest{
+		responseChan: make(chan *http.Response, 1),
+		errorChan:    make(chan error, 1),
+		waitingCount: 1,
+		done:         false,
+	}
+}
+
+func (cr *coalescedRequest) addWaiter() {
+	cr.mu.Lock()
+	defer cr.mu.Unlock()
+	cr.waitingCount++
+}
+
+func (cr *coalescedRequest) complete(resp *http.Response, err error) {
+	cr.mu.Lock()
+	defer cr.mu.Unlock()
+	if cr.done {
+		return
+	}
+	cr.done = true
+
+	if err != nil {
+		select {
+		case cr.errorChan <- err:
+		default:
+		}
+	} else {
+		select {
+		case cr.responseChan <- resp:
+		default:
+		}
+	}
+}
+
+// getOrCreateCoalescedRequest gets an existing coalesced request or creates a new one
+func getOrCreateCoalescedRequest(cacheKey string) (*coalescedRequest, bool) {
+	coalescedRequestsMu.Lock()
+	defer coalescedRequestsMu.Unlock()
+
+	if cr, exists := coalescedRequests[cacheKey]; exists {
+		cr.addWaiter()
+		return cr, false
+	}
+
+	cr := newCoalescedRequest()
+	coalescedRequests[cacheKey] = cr
+	return cr, true
+}
+
+// removeCoalescedRequest removes a completed coalesced request
+func removeCoalescedRequest(cacheKey string) {
+	coalescedRequestsMu.Lock()
+	defer coalescedRequestsMu.Unlock()
+	delete(coalescedRequests, cacheKey)
+}
+
 type SteamCache struct {
 	address  string
 	upstream string
@@ -191,9 +256,7 @@ func New(address string, memorySize string, diskSize string, diskPath, upstream,
 		panic(err)
 	}
 
-	c := cache.New(
-		gc.AdaptivePromotionDeciderFunc,
-	)
+	c := cache.New()
 
 	var m *memory.MemoryFS
 	var mgc *gc.GCFS
@@ -203,7 +266,7 @@ func New(address string, memorySize string, diskSize string, diskPath, upstream,
 		if memoryGCAlgo == "" {
 			memoryGCAlgo = gc.LRU // default to LRU
 		}
-		mgc = gc.New(m, gc.GetGCAlgorithm(memoryGCAlgo))
+		mgc = gc.New(m, memoryGCAlgo)
 	}
 
 	var d *disk.DiskFS
@@ -214,7 +277,7 @@ func New(address string, memorySize string, diskSize string, diskPath, upstream,
 		if diskGCAlgo == "" {
 			diskGCAlgo = gc.LRU // default to LRU
 		}
-		dgc = gc.New(d, gc.GetGCAlgorithm(diskGCAlgo))
+		dgc = gc.New(d, diskGCAlgo)
 	}
 
 	// configure the cache to match the specified mode (memory only, disk only, or memory and disk) based on the provided sizes
@@ -332,7 +395,6 @@ func (sc *SteamCache) Shutdown() {
 
 func (sc *SteamCache) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 	if r.Method != http.MethodGet {
-		requestsTotal.WithLabelValues(r.Method, "405").Inc()
 		logger.Logger.Warn().Str("method", r.Method).Msg("Only GET method is supported")
 		http.Error(w, "Only GET method is supported", http.StatusMethodNotAllowed)
 		return
@@ -350,23 +412,18 @@ func (sc *SteamCache) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 		return
 	}
 
-	if r.URL.Path == "/metrics" {
-		promhttp.Handler().ServeHTTP(w, r)
-		return
-	}
-
 	if strings.HasPrefix(r.URL.String(), "/depot/") {
 		// trim the query parameters from the URL path
 		// this is necessary because the cache key should not include query parameters
-		path := strings.Split(r.URL.String(), "?")[0]
+		urlPath := strings.Split(r.URL.String(), "?")[0]
 
 		tstart := time.Now()
 
-		cacheKey := strings.ReplaceAll(path[1:], "\\", "/") // replace all backslashes with forward slashes shouldn't be necessary but just in case
+		// Generate simplified Steam cache key: steam/{hash}
+		cacheKey := generateSteamCacheKey(urlPath)
 
 		if cacheKey == "" {
-			requestsTotal.WithLabelValues(r.Method, "400").Inc()
-			logger.Logger.Warn().Str("url", path).Msg("Invalid URL")
+			logger.Logger.Warn().Str("url", urlPath).Msg("Invalid URL")
 			http.Error(w, "Invalid URL", http.StatusBadRequest)
 			return
 		}
@@ -403,18 +460,67 @@ func (sc *SteamCache) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 					Str("status", "HIT").
 					Dur("duration", time.Since(tstart)).
 					Msg("request")
-				requestsTotal.WithLabelValues(r.Method, "200").Inc()
-				cacheStatusTotal.WithLabelValues("HIT").Inc()
-				responseTime.WithLabelValues("HIT").Observe(time.Since(tstart).Seconds())
 				return
 			}
 		}
 
+		// Check for coalesced request (another client already downloading this)
+		coalescedReq, isNew := getOrCreateCoalescedRequest(cacheKey)
+		if !isNew {
+			// Wait for the existing download to complete
+			logger.Logger.Debug().
+				Str("key", cacheKey).
+				Int("waiting_clients", coalescedReq.waitingCount).
+				Msg("Joining coalesced request")
+
+			select {
+			case resp := <-coalescedReq.responseChan:
+				// Use the downloaded response
+				defer resp.Body.Close()
+				bodyData, err := io.ReadAll(resp.Body)
+				if err != nil {
+					logger.Logger.Error().Err(err).Str("key", cacheKey).Msg("Failed to read coalesced response body")
+					http.Error(w, "Failed to read response body", http.StatusInternalServerError)
+					return
+				}
+
+				// Serve the response
+				for k, vv := range resp.Header {
+					if _, skip := hopByHopHeaders[http.CanonicalHeaderKey(k)]; skip {
+						continue
+					}
+					for _, v := range vv {
+						w.Header().Add(k, v)
+					}
+				}
+				w.Header().Set("X-LanCache-Status", "HIT-COALESCED")
+				w.Header().Set("X-LanCache-Processed-By", "SteamCache2")
+				w.WriteHeader(resp.StatusCode)
+				w.Write(bodyData)
+
+				logger.Logger.Info().
+					Str("key", cacheKey).
+					Str("host", r.Host).
+					Str("status", "HIT-COALESCED").
+					Dur("duration", time.Since(tstart)).
+					Msg("request")
+
+				return
+
+			case err := <-coalescedReq.errorChan:
+				logger.Logger.Error().Err(err).Str("key", cacheKey).Msg("Coalesced request failed")
+				http.Error(w, "Upstream request failed", http.StatusInternalServerError)
+				return
+			}
+		}
+
+		// Remove coalesced request when done
+		defer removeCoalescedRequest(cacheKey)
+
 		var req *http.Request
 		if sc.upstream != "" { // if an upstream server is configured, proxy the request to the upstream server
-			ur, err := url.JoinPath(sc.upstream, path)
+			ur, err := url.JoinPath(sc.upstream, urlPath)
 			if err != nil {
-				requestsTotal.WithLabelValues(r.Method, "500").Inc()
 				logger.Logger.Error().Err(err).Str("upstream", sc.upstream).Msg("Failed to join URL path")
 				http.Error(w, "Failed to join URL path", http.StatusInternalServerError)
 				return
@@ -422,7 +528,6 @@ func (sc *SteamCache) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 
 			req, err = http.NewRequest(http.MethodGet, ur, nil)
 			if err != nil {
-				requestsTotal.WithLabelValues(r.Method, "500").Inc()
 				logger.Logger.Error().Err(err).Str("upstream", sc.upstream).Msg("Failed to create request")
 				http.Error(w, "Failed to create request", http.StatusInternalServerError)
 				return
@@ -436,9 +541,8 @@ func (sc *SteamCache) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 				host = "http://" + host
 			}
 
-			ur, err := url.JoinPath(host, path)
+			ur, err := url.JoinPath(host, urlPath)
 			if err != nil {
-				requestsTotal.WithLabelValues(r.Method, "500").Inc()
 				logger.Logger.Error().Err(err).Str("host", host).Msg("Failed to join URL path")
 				http.Error(w, "Failed to join URL path", http.StatusInternalServerError)
 				return
@@ -446,7 +550,6 @@ func (sc *SteamCache) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 
 			req, err = http.NewRequest(http.MethodGet, ur, nil)
 			if err != nil {
-				requestsTotal.WithLabelValues(r.Method, "500").Inc()
 				logger.Logger.Error().Err(err).Str("host", host).Msg("Failed to create request")
 				http.Error(w, "Failed to create request", http.StatusInternalServerError)
 				return
@@ -473,8 +576,13 @@ func (sc *SteamCache) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 			}
 		}
 		if err != nil || resp.StatusCode != http.StatusOK {
-			requestsTotal.WithLabelValues(r.Method, "500 upstream host "+r.Host).Inc()
 			logger.Logger.Error().Err(err).Str("url", req.URL.String()).Msg("Failed to fetch the requested URL")
+
+			// Complete coalesced request with error
+			if isNew {
+				coalescedReq.complete(nil, err)
+			}
+
 			http.Error(w, "Failed to fetch the requested URL", http.StatusInternalServerError)
 			return
 		}
@@ -483,15 +591,24 @@ func (sc *SteamCache) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 		// Read the entire response body into memory for hash verification
 		bodyData, err := io.ReadAll(resp.Body)
 		if err != nil {
-			requestsTotal.WithLabelValues(r.Method, "500").Inc()
 			logger.Logger.Error().Err(err).Str("url", req.URL.String()).Msg("Failed to read response body")
+
+			// Complete coalesced request with error
+			if isNew {
+				coalescedReq.complete(nil, err)
+			}
+
 			http.Error(w, "Failed to read response body", http.StatusInternalServerError)
 			return
 		}
 
-		// Extract filename from cache key for hash verification
-		filename := filepath.Base(cacheKey)
-		expectedHash, hasHash := extractHashFromFilename(filename)
+		// Extract hash from cache key for verification
+		var expectedHash string
+		var hasHash bool
+		if strings.HasPrefix(cacheKey, "steam/") {
+			expectedHash = cacheKey[6:]       // Remove "steam/" prefix
+			hasHash = len(expectedHash) == 64 // SHA-256 hashes are 64 characters
+		}
 
 		// Hash verification using Steam's X-Content-Sha header and content length verification
 		hashVerified := true
@@ -501,9 +618,8 @@ func (sc *SteamCache) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 
 			// Verify using Steam's hash
 			if strings.EqualFold(steamHash, expectedHash) {
-				hashVerificationTotal.WithLabelValues("success").Inc()
+				// Hash verification succeeded
 			} else {
-				hashVerificationTotal.WithLabelValues("failed").Inc()
 				logger.Logger.Error().
 					Str("key", cacheKey).
 					Str("expected_hash", expectedHash).
@@ -513,20 +629,17 @@ func (sc *SteamCache) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 				hashVerified = false
 			}
 		} else {
-			hashVerificationTotal.WithLabelValues("no_hash").Inc()
+			// No hash to verify
 		}
 
 		// Always verify content length as an additional safety check
 		if resp.ContentLength > 0 && int64(len(bodyData)) != resp.ContentLength {
-			hashVerificationTotal.WithLabelValues("content_length_failed").Inc()
 			logger.Logger.Error().
 				Str("key", cacheKey).
 				Int("actual_content_length", len(bodyData)).
 				Int64("expected_content_length", resp.ContentLength).
 				Msg("Content length verification failed")
 			hashVerified = false
-		} else if resp.ContentLength > 0 {
-			hashVerificationTotal.WithLabelValues("content_length_success").Inc()
 		}
 
 		// Write to response (always serve the file)
@@ -544,6 +657,22 @@ func (sc *SteamCache) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 		w.Header().Set("X-LanCache-Processed-By", "SteamCache2")
 		w.Write(bodyData)
 
+		// Complete coalesced request for waiting clients
+		if isNew {
+			// Create a new response for coalesced clients
+			coalescedResp := &http.Response{
+				StatusCode: resp.StatusCode,
+				Status:     resp.Status,
+				Header:     make(http.Header),
+				Body:       io.NopCloser(bytes.NewReader(bodyData)),
+			}
+			// Copy headers
+			for k, vv := range resp.Header {
+				coalescedResp.Header[k] = vv
+			}
+			coalescedReq.complete(coalescedResp, nil)
+		}
+
 		// Only cache the file if hash verification passed (or no hash was present)
 		if hashVerified {
 			writer, _ := sc.vfs.Create(cachePath, int64(0)) // size is not known in advance
@@ -566,10 +695,6 @@ func (sc *SteamCache) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 			Dur("duration", time.Since(tstart)).
 			Msg("request")
 
-		requestsTotal.WithLabelValues(r.Method, "200").Inc()
-		cacheStatusTotal.WithLabelValues("MISS").Inc()
-		responseTime.WithLabelValues("MISS").Observe(time.Since(tstart).Seconds())
-
 		return
 	}
 
@@ -585,7 +710,6 @@ func (sc *SteamCache) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 		return
 	}
 
-	requestsTotal.WithLabelValues(r.Method, "404").Inc()
 	logger.Logger.Warn().Str("url", r.URL.String()).Msg("Not found")
 	http.Error(w, "Not found", http.StatusNotFound)
 }