Refactor validation process and update configuration examples

- Replaced the `validate-with-prefill.sh` script with a streamlined `make validate` command for improved usability.
- Updated `validate-config.yaml` to clarify cache management instructions and garbage collection algorithms.
- Enhanced comments to provide better guidance on upstream configurations and their implications for caching setups.

.gitea/workflows/test-pr.yaml

@@ -21,4 +21,4 @@ jobs:
       - run: go install golang.org/x/vuln/cmd/govulncheck@latest
       - run: govulncheck ./...
       - run: go test -race -v -shuffle=on -coverprofile=coverage.out -timeout=5m ./...
-      - run: go tool cover -func=coverage.out | tail -10  # basic coverage report (P2-04)
+      - run: go tool cover -func=coverage.out | tail -10  # basic coverage report

Makefile

@@ -4,21 +4,21 @@ run: ## Run the application (cross-platform; uses go run for dev on Linux/macOS/
 run-debug: ## Run the application with debug logging (cross-platform)
 	@go run . --log-level debug
 
-build: deps ## Build a snapshot of the application for the current platform (uses -short for fast feedback)
+build: deps lint ## Build a snapshot of the application for the current platform (uses -short for fast feedback)
 	@go test -short -v ./...
 	@goreleaser build --single-target --snapshot --clean
 
-test: deps ## Run all tests
+test: deps lint ## Run all tests
 	@go test -shuffle=on -timeout=5m -v ./...
 
-test-race: deps ## Run all tests with the race detector
+test-race: deps lint ## Run all tests with the race detector
 	@go test -race -shuffle=on -timeout=5m -v ./...
 
 lint: deps check-review-labels ## Run golangci-lint + review label hygiene check
 	@golangci-lint run ./...
 
 check-review-labels: ## Fail if temporary review labels (P0-01, T1, I3, R2, etc.) are found in source
-	@! grep -rnE '\b[A-Z][0-9][^a-zA-Z]' --include='*.go' . 2>/dev/null | grep -v 'G[0-9]\{3\}' || (echo "Error: Found temporary review labels (P*, T*, I*, etc.) in source. See AGENTS.md for the rule." && exit 1)
+	@! grep -rnE '\b[A-Z][0-9][^a-zA-Z]' --include='*.go' --include='*.md' --include='*.yaml' --include='*.sh' --exclude='AGENTS.md' . 2>/dev/null | grep -v 'G[0-9]\{3\}' || (echo "Error: Found temporary review labels (P*, T*, I*, etc.) in source. See AGENTS.md for the rule." && exit 1)
 
 deps: ## Download dependencies
 	@go mod tidy
@@ -26,6 +26,9 @@ deps: ## Download dependencies
 clean: ## Remove build artifacts and test cache
 	@rm -rf bin/ dist/ *.test coverage.out steamcache2
 
+clean-disk: ## Remove disk cache
+	@rm -rf validate-disk/
+
 bench: deps ## Run all benchmarks (MemoryFS + DiskFS variants, including all eviction strategies)
 	@echo "Running MemoryFS benchmarks..."
 	@go test -bench=. -benchmem -run=^$ -benchtime=1s ./vfs/memory
@@ -38,7 +41,7 @@ setcap: build ## Explicitly set cap_net_bind_service on the (just-built) binary
 	@sudo setcap 'cap_net_bind_service=+ep' dist/default_linux_amd64_v1/steamcache2
 	@echo "Done. You should now be able to run 'make run-validation' as your normal user (no root)."
 
-validate run-validation: build ## Start steamcache2 on :80 with small test caches (foreground)
+validate run-validation: build clean-disk ## Start steamcache2 on :80 with small test caches (foreground)
 	@echo "=== Starting steamcache2 in validation mode ==="
 	@echo "Port 80 + small memory/disk caches (for exercising disk tier, GC, etc.)"
 	@echo "Press Ctrl-C to stop the server."
@@ -59,17 +62,6 @@ validate run-validation: build ## Start steamcache2 on :80 with small test cache
 	fi; \
 	exec "$$BINARY" --config docs/examples/validate-config.yaml --log-level info
 
-validate-check: ## Quick post-benchmark sanity check against a running steamcache2 (default port 80, override with PORT=xxxx)
-	@echo "=== steamcache2 Full Function Validation Report ==="
-	@PORT="$${PORT:-80}"; \
-	echo "Server: http://localhost:$$PORT"; \
-	curl -s --max-time 5 "http://localhost:$$PORT/metrics" || echo "(could not reach /metrics on port $$PORT - is the server running?)"; \
-	echo ""; \
-	echo "Tip: also inspect recent server logs for errors, coalesced hits, and disk activity."
-
-prefill: ## Download latest SteamPrefill into bin/steam-prefill/SteamPrefill (gitignored)
-	@./scripts/download-prefill.sh
-
 validate-kill: ## Kill leftover steamcache2 processes (safer, checks process name)
 	@echo "Looking for steamcache2 processes on common validation ports (80 is primary)..."
 	@for port in 80 8040 8080; do \
@@ -95,6 +87,11 @@ validate-kill: ## Kill leftover steamcache2 processes (safer, checks process nam
 	done
 	@echo "Validation server cleanup complete."
 
+prefill: ## Download latest SteamPrefill into bin/steam-prefill/SteamPrefill (gitignored)
+	@./scripts/download-prefill.sh
+
+
+
 
 
 help: ## Show this help message
@@ -109,9 +106,9 @@ help: ## Show this help message
 	@echo "  check-review-labels  Fail on temporary review labels (P*, T*, I*, R*, etc.)"
 	@echo "  deps               Download dependencies"
 	@echo "  clean              Remove build/test artifacts"
+	@echo "  clean-disk         Remove disk cache"
 	@echo "  bench              Run low-level VFS microbenchmarks"
-	@echo "  validate / run-validation  Start server on :80 (builds, auto-setcaps fresh binary, then runs as normal user)"
+	@echo "  validate / run-validation  Start server on :80 (builds, auto-setcaps fresh binary, then runs as normal user, cleans disk cache first)"
 	@echo "  setcap               Explicitly set cap on current build (for port 80 use outside validate)"
-	@echo "  validate-check     Quick /metrics report after running a workload"
 	@echo "  validate-kill      Kill leftover steamcache2 processes (safer)"
-	@echo "  prefill            Download latest SteamPrefill into bin/steam-prefill/ (for use with run-validation)"
+	@echo "  prefill            Download latest SteamPrefill into bin/steam-prefill/SteamPrefill (gitignored)"

README.md

@@ -70,26 +70,7 @@ This gives you:
 - Excellent `benchmark setup` / `benchmark run` workflow with warmup, randomization, and mixed chunk sizes
 - The ability to validate a **just-built binary** end-to-end (caching, coalescing, Range support, memory+disk tiers, GC/eviction, metrics, special endpoints, startup validation, etc.)
 
-#### Quick Start
-
-```bash
-# 1. Build the binary (this also runs short tests)
-make build
-
-# 2. Start a validation-oriented instance (small caches so disk tier + GC get exercised)
-#    Uses port 80 by default; the script will automatically set the needed
-#    capability on the binary via sudo setcap if it is missing.
-./scripts/validate-with-prefill.sh
-
-# 3. In another terminal (or on another machine), create a workload once if you haven't already,
-#    then drive it through your local steamcache2.
-#    Note: when using a non-80 port you may need to give SteamPrefill the full address.
-./scripts/validate-with-prefill.sh   # (shows the exact commands with the correct port)
-```
-
-When the benchmark finishes, press Ctrl-C in the first terminal to cleanly stop the server.
-
-#### Simple validation server (recommended for manual testing)
+#### Validation server (recommended)
 
 For easy validation with external tools (SteamPrefill, etc.), use:
 
@@ -140,7 +121,7 @@ Look for:
 
 #### The Validation Config
 
-The script uses [docs/examples/validate-config.yaml](docs/examples/validate-config.yaml). It enables both memory and disk tiers at modest sizes (128 MB / 512 MB) with conservative concurrency. Edit or copy it if you need larger caches for bigger workloads.
+The recommended validation config is at [docs/examples/validate-config.yaml](docs/examples/validate-config.yaml). It enables both memory and disk tiers at modest sizes (128 MB / 512 MB) with conservative concurrency. Edit or copy it if you need larger caches for bigger workloads.
 
 #### What Gets Validated
 
@@ -194,7 +175,7 @@ SteamCache2 uses a YAML configuration file (`config.yaml`) for all settings. Her
 # Server configuration
 listen_address: :80
 
-# P1 hardening (see Security Hardening section)
+# Hardening (see Security Hardening section)
 max_object_size: "0"   # 0=unlimited; set e.g. "256MB" for response size DoS protection
 trusted_proxies: []    # empty = safe (ignore XFF for rate limit); set CIDRs for trusted proxies
 
@@ -222,7 +203,7 @@ upstream: "https://steam.cdn.com"
 ```
 
 #### Startup Validation
-As of P0, `steamcache2` performs strict validation on startup (after loading config + CLI overrides, before creating the cache). Invalid configs cause immediate clean failure (no default written, no panic):
+`steamcache2` performs strict validation on startup (after loading config + CLI overrides, before creating the cache). Invalid configs cause immediate clean failure (no default written, no panic):
 
 - Negative `max_concurrent_requests` / `max_requests_per_client`: "negative concurrency not allowed"
 - Invalid `gc_algorithm` (memory): "invalid memory gc algorithm: badvalue"
@@ -236,12 +217,12 @@ Error: Invalid configuration: invalid memory gc algorithm: foo. Please fix the c
 
 See `config.Validate()` and `steamcache.New` error paths. This ensures the LAN appliance fails fast on misconfig.
 
-#### Security Hardening (P1)
-- `max_object_size` (default "0" = unlimited): set e.g. "256MB" or "512MB" to reject oversized upstream responses with HTTP 413 before buffering/ReadAll. Prevents OOM DoS from large or malicious responses (P1-01). Large legitimate Steam files still served if under limit.
-- `trusted_proxies`: CIDR list (default empty). When empty (safe default), X-Forwarded-For and client IP spoofing are ignored for rate limiting — always uses `r.RemoteAddr` only. When set (e.g. your reverse proxy CIDR), uses correct "rightmost untrusted" extraction. Prevents bypass of `max_requests_per_client` (P1-02). Documented for LAN proxy setups only.
-- These + P0 validation make steamcache2 safe-by-default for LAN exposure.
+#### Security Hardening
+- `max_object_size` (default "0" = unlimited): set e.g. "256MB" or "512MB" to reject oversized upstream responses with HTTP 413 before buffering/ReadAll. Prevents OOM DoS from large or malicious responses. Large legitimate Steam files still served if under limit.
+- `trusted_proxies`: CIDR list (default empty). When empty (safe default), X-Forwarded-For and client IP spoofing are ignored for rate limiting — always uses `r.RemoteAddr` only. When set (e.g. your reverse proxy CIDR), uses correct "rightmost untrusted" extraction. Prevents bypass of `max_requests_per_client`. Documented for LAN proxy setups only.
+- These + the startup validation make steamcache2 safe-by-default for LAN exposure.
 
-#### Migration / Breaking Changes (P1)
+#### Migration / Breaking Changes
 - `New()` public signature gained 2 required trailing params (`maxObjectSize`, `trustedProxies`). Direct callers (rare; most use config or NewWithOptions) must update.
 - Recommended: migrate to `NewWithOptions(Options{...})` (non-breaking) or rely on YAML config + cmd/root.go.
 - No behavior change for existing configs (defaults preserve prior semantics).

docs/examples/validate-config.yaml

@@ -22,7 +22,7 @@
 #
 # Usage (typical dev workflow):
 #   make build
-#   ./scripts/validate-with-prefill.sh
+#   make validate
 #   # In another terminal:
 #   SteamPrefill benchmark run -c 20 ...
 #
@@ -43,12 +43,15 @@ trusted_proxies: ["127.0.0.0/8"]
 cache:
   memory:
     size: 1GB
-    gc_algorithm: largest
+    gc_algorithm: hybrid
   disk:
     size: 2GB
-    path: ./validate-disk     # ephemeral; clean between runs if you want a fresh test
+    path: ./validate-disk     # cleaned between runs by make validate or make clean-disk
     gc_algorithm: hybrid      # recommended for disk in the project README
 
 # Empty upstream = use Host header from the client (SteamPrefill / real Steam clients).
-# This matches the common "DNS points lancache.steamcontent.com at the cache" setup.
+# Allows for chaining steamcache2 instances if needed.
+# For example, for a lan party you could have a small fast ram only cache at each table pointing to a larger slower disk cache in the back somewhere
+# It would reduce the amount of bandwidth needed to the internet and the amount needed to each table
+# just as a little reminder there is no authentication so this is not a good idea for a public cache just out on the internet.
 upstream: ""

scripts/validate-with-prefill.sh

@@ -1,182 +0,0 @@
-#!/usr/bin/env bash
-#
-# validate-with-prefill.sh
-#
-# Thin glue script to make it trivial for developers to validate complete
-# steamcache2 functionality using the external SteamPrefill (lancacheprefill)
-# tool as the realistic client simulator.
-#
-# Usage:
-#   1. make build
-#   2. ./scripts/validate-with-prefill.sh
-#      (Automatically kills any leftover steamcache2 on the target port first.)
-#   3. In another terminal (or on another machine), run the printed
-#      SteamPrefill benchmark commands (the script tells you the exact address/port).
-#   4. After the benchmark finishes, run the suggested metrics check.
-#   5. Ctrl-C here to cleanly stop the steamcache2 instance.
-#
-# This script + the accompanying validate-config.yaml + README docs are the
-# entire "couple little scripts to hook steamcache2 and lancacheprefill together"
-# implementation. No Go code, no new dependencies, stays outside go test / bench.
-#
-set -euo pipefail
-
-# --- Locate the built steamcache2 binary (produced by "make build") ---
-BINARY=""
-for candidate in \
-    "dist/default_linux_amd64_v1/steamcache2" \
-    "dist/steamcache2" \
-    "./steamcache2" \
-    "steamcache2"
-do
-    if [[ -x "$candidate" ]]; then
-        BINARY="$candidate"
-        break
-    fi
-done
-
-if [[ -z "$BINARY" ]]; then
-    echo "ERROR: Could not find a built steamcache2 binary."
-    echo "Run 'make build' first (or place the binary in one of the searched locations)."
-    exit 1
-fi
-
-echo "Using steamcache2 binary: $BINARY"
-
-# --- Validation config (small dual-tier so disk + GC get real exercise) ---
-# Source of truth lives in docs/examples/ (safe from "make clean").
-# The script will also accept an explicit path via STEAMCACHE2_VALIDATE_CONFIG.
-VALIDATE_CONFIG="${STEAMCACHE2_VALIDATE_CONFIG:-docs/examples/validate-config.yaml}"
-if [[ ! -f "$VALIDATE_CONFIG" ]]; then
-    echo "ERROR: Validation config not found at: $VALIDATE_CONFIG"
-    echo "Set STEAMCACHE2_VALIDATE_CONFIG=/path/to/your-config.yaml or place a copy at docs/examples/validate-config.yaml"
-    exit 1
-fi
-
-# Extract the listen port from the config (supports ":80", "127.0.0.1:80", etc.)
-# Falls back to 80 if we can't parse it.
-PORT=$(grep -E '^\s*listen_address:' "$VALIDATE_CONFIG" | head -1 | sed -E 's/.*:([0-9]+).*/\1/' || true)
-if [[ -z "$PORT" || ! "$PORT" =~ ^[0-9]+$ ]]; then
-    PORT=80
-fi
-
-SERVER_URL="http://localhost:${PORT}"
-
-# For privileged ports (<1024, i.e. the default :80) we require the
-# cap_net_bind_service capability. We apply it automatically here (via sudo
-# setcap) on the binary we are about to run. This happens after any build
-# so the cap is never "lost" when the binary is rebuilt.
-if [ "$PORT" -lt 1024 ] && [ "$(id -u)" -ne 0 ]; then
-    if ! command -v getcap >/dev/null 2>&1 || ! getcap "$BINARY" 2>/dev/null | grep -q "cap_net_bind_service"; then
-        echo "Setting cap_net_bind_service on the binary (sudo may prompt)..."
-        if ! sudo setcap 'cap_net_bind_service=+ep' "$BINARY"; then
-            echo "ERROR: Failed to set capability."
-            echo "Run 'make setcap' manually, then retry."
-            exit 1
-        fi
-    fi
-fi
-
-# Safely kill only steamcache2 processes listening on this specific port.
-# We look up PIDs on the port, check their actual process name/command,
-# and only kill via PID if it looks like steamcache2.
-echo "Checking for leftover steamcache2 processes on port ${PORT}..."
-
-kill_steamcache_on_port() {
-    local port=$1
-    local pids=""
-
-    # Try ss first (modern, usually available)
-    if command -v ss >/dev/null 2>&1; then
-        pids=$(ss -tlnp 2>/dev/null | grep ":${port} " | sed -n 's/.*pid=\([0-9]*\).*/\1/p' | sort -u)
-    fi
-
-    # Fallback to lsof
-    if [[ -z "$pids" ]] && command -v lsof >/dev/null 2>&1; then
-        pids=$(lsof -ti :${port} 2>/dev/null | sort -u)
-    fi
-
-    if [[ -z "$pids" ]]; then
-        return 0
-    fi
-
-    for pid in $pids; do
-        # Get process name and command line
-        local proc_name
-        local cmdline
-        proc_name=$(ps -p "$pid" -o comm= 2>/dev/null || true)
-        cmdline=$(ps -p "$pid" -o cmd= 2>/dev/null || true)
-
-        # Check if this looks like a steamcache2 process
-        if echo "$proc_name $cmdline" | grep -qi "steamcache"; then
-            echo "  → Found steamcache2 on port ${port} (PID $pid, name: ${proc_name:-unknown})"
-            kill -TERM "$pid" 2>/dev/null || true
-            sleep 0.3
-            # If still alive, force kill
-            if kill -0 "$pid" 2>/dev/null; then
-                kill -9 "$pid" 2>/dev/null || true
-            fi
-            echo "    Killed PID $pid"
-        else
-            echo "  → Skipping PID $pid on port ${port} (not steamcache2: ${proc_name:-$cmdline})"
-        fi
-    done
-}
-
-kill_steamcache_on_port "$PORT"
-sleep 0.5
-
-# --- Launch the server in the background ---
-echo "Starting steamcache2 with validation config (small caches for disk/GC testing)..."
-"$BINARY" --config "$VALIDATE_CONFIG" --log-level info &
-SERVER_PID=$!
-
-# Ensure we always clean up the child on exit / Ctrl-C / error
-cleanup() {
-    echo ""
-    echo "Stopping steamcache2 (pid $SERVER_PID)..."
-    if kill "$SERVER_PID" 2>/dev/null; then
-        wait "$SERVER_PID" 2>/dev/null || true
-    fi
-    echo "Server stopped."
-}
-trap cleanup EXIT INT TERM
-
-# Give the server a moment to bind and pass its own startup checks
-sleep 2
-
-# Basic readiness probe using the actual configured port
-if ! curl -s --max-time 3 "${SERVER_URL}/" >/dev/null 2>&1; then
-    echo "WARNING: Server did not respond quickly on ${SERVER_URL}/"
-    echo "         It may still be starting or bound to a different address."
-    echo "         Check the server logs above. You can still try the SteamPrefill commands."
-fi
-
-if [[ "${VALIDATE_QUIET:-}" != "1" ]]; then
-echo ""
-echo "======================================================================"
-echo "steamcache2 is running (validation mode) on ${SERVER_URL}"
-echo ""
-echo "In another terminal (or on a machine that can reach this one), run:"
-echo ""
-echo "    # One-time workload creation (run on a machine with SteamPrefill + Steam):"
-echo "    SteamPrefill benchmark setup --preset LargeChunks"
-echo "    # (or --use-selected, --all, --appid ..., or your own preset)"
-echo ""
-echo "    # Copy the generated workload file to this machine if needed."
-echo ""
-echo "    # Then run the actual benchmark (this is the realistic client simulator):"
-echo "    SteamPrefill benchmark run -c 20 -i 3"
-echo ""
-echo "After the benchmark completes, you can inspect the cache with:"
-echo "    curl -s ${SERVER_URL}/metrics | cat"
-echo ""
-echo "Or run:  make validate-check   (if the Makefile target exists)"
-echo ""
-echo "When you are finished, press Ctrl-C in this window to stop the server cleanly."
-echo "======================================================================"
-echo ""
-fi
-
-# Wait for the background server (or for the user to Ctrl-C)
-wait $SERVER_PID || true

steamcache/metrics/metrics.go

@@ -24,8 +24,6 @@ type Metrics struct {
 	TotalBytesServed  int64
 	TotalBytesSaved   int64 // bytes served from cache instead of being re-downloaded from upstream
 
-
-
 	// Cache metrics
 	MemoryCacheSize int64
 	DiskCacheSize   int64
@@ -194,8 +192,8 @@ func (m *Metrics) GetStats() *Stats {
 		RateLimited:        atomic.LoadInt64(&m.RateLimited),
 		HitRate:            hitRate,
 		AvgResponseTime:    avgResponseTime,
-		TotalBytesServed: atomic.LoadInt64(&m.TotalBytesServed),
-		TotalBytesSaved:  atomic.LoadInt64(&m.TotalBytesSaved),
+		TotalBytesServed:   atomic.LoadInt64(&m.TotalBytesServed),
+		TotalBytesSaved:    atomic.LoadInt64(&m.TotalBytesSaved),
 		MemoryCacheSize:    atomic.LoadInt64(&m.MemoryCacheSize),
 		DiskCacheSize:      atomic.LoadInt64(&m.DiskCacheSize),
 		MemoryCacheHits:    atomic.LoadInt64(&m.MemoryCacheHits),
@@ -242,18 +240,17 @@ func (m *Metrics) Reset() {
 
 // Stats represents a snapshot of metrics
 type Stats struct {
-	TotalRequests      int64
-	CacheHits          int64
-	CacheMisses        int64
-	CacheCoalesced     int64
-	Errors             int64
-	RateLimited        int64
-	HitRate            float64
-	AvgResponseTime    time.Duration
-	TotalBytesServed  int64
-	TotalBytesSaved   int64
-	MemoryCacheSize   int64
-
+	TotalRequests    int64
+	CacheHits        int64
+	CacheMisses      int64
+	CacheCoalesced   int64
+	Errors           int64
+	RateLimited      int64
+	HitRate          float64
+	AvgResponseTime  time.Duration
+	TotalBytesServed int64
+	TotalBytesSaved  int64
+	MemoryCacheSize  int64
 
 	DiskCacheSize      int64
 	MemoryCacheHits    int64