From 04f55535a5922fb549bab6114586ce2b90dddb18 Mon Sep 17 00:00:00 2001 From: Justin Harms Date: Thu, 28 May 2026 21:06:28 -0500 Subject: [PATCH] Refactor validation process and update configuration examples - Replaced the `validate-with-prefill.sh` script with a streamlined `make validate` command for improved usability. - Updated `validate-config.yaml` to clarify cache management instructions and garbage collection algorithms. - Enhanced comments to provide better guidance on upstream configurations and their implications for caching setups. --- docs/examples/validate-config.yaml | 11 +- scripts/validate-with-prefill.sh | 182 ----------------------------- 2 files changed, 7 insertions(+), 186 deletions(-) delete mode 100755 scripts/validate-with-prefill.sh diff --git a/docs/examples/validate-config.yaml b/docs/examples/validate-config.yaml index 0eafeeb..9546661 100644 --- a/docs/examples/validate-config.yaml +++ b/docs/examples/validate-config.yaml @@ -22,7 +22,7 @@ # # Usage (typical dev workflow): # make build -# ./scripts/validate-with-prefill.sh +# make validate # # In another terminal: # SteamPrefill benchmark run -c 20 ... # @@ -43,12 +43,15 @@ trusted_proxies: ["127.0.0.0/8"] cache: memory: size: 1GB - gc_algorithm: largest + gc_algorithm: hybrid disk: size: 2GB - path: ./validate-disk # ephemeral; clean between runs if you want a fresh test + path: ./validate-disk # cleaned between runs by make validate or make clean-disk gc_algorithm: hybrid # recommended for disk in the project README # Empty upstream = use Host header from the client (SteamPrefill / real Steam clients). -# This matches the common "DNS points lancache.steamcontent.com at the cache" setup. +# Allows for chaining steamcache2 instances if needed. +# For example, for a lan party you could have a small fast ram only cache at each table pointing to a larger slower disk cache in the back somewhere +# It would reduce the amount of bandwidth needed to the internet and the amount needed to each table +# just as a little reminder there is no authentication so this is not a good idea for a public cache just out on the internet. upstream: "" diff --git a/scripts/validate-with-prefill.sh b/scripts/validate-with-prefill.sh deleted file mode 100755 index 8ba8f08..0000000 --- a/scripts/validate-with-prefill.sh +++ /dev/null @@ -1,182 +0,0 @@ -#!/usr/bin/env bash -# -# validate-with-prefill.sh -# -# Thin glue script to make it trivial for developers to validate complete -# steamcache2 functionality using the external SteamPrefill (lancacheprefill) -# tool as the realistic client simulator. -# -# Usage: -# 1. make build -# 2. ./scripts/validate-with-prefill.sh -# (Automatically kills any leftover steamcache2 on the target port first.) -# 3. In another terminal (or on another machine), run the printed -# SteamPrefill benchmark commands (the script tells you the exact address/port). -# 4. After the benchmark finishes, run the suggested metrics check. -# 5. Ctrl-C here to cleanly stop the steamcache2 instance. -# -# This script + the accompanying validate-config.yaml + README docs are the -# entire "couple little scripts to hook steamcache2 and lancacheprefill together" -# implementation. No Go code, no new dependencies, stays outside go test / bench. -# -set -euo pipefail - -# --- Locate the built steamcache2 binary (produced by "make build") --- -BINARY="" -for candidate in \ - "dist/default_linux_amd64_v1/steamcache2" \ - "dist/steamcache2" \ - "./steamcache2" \ - "steamcache2" -do - if [[ -x "$candidate" ]]; then - BINARY="$candidate" - break - fi -done - -if [[ -z "$BINARY" ]]; then - echo "ERROR: Could not find a built steamcache2 binary." - echo "Run 'make build' first (or place the binary in one of the searched locations)." - exit 1 -fi - -echo "Using steamcache2 binary: $BINARY" - -# --- Validation config (small dual-tier so disk + GC get real exercise) --- -# Source of truth lives in docs/examples/ (safe from "make clean"). -# The script will also accept an explicit path via STEAMCACHE2_VALIDATE_CONFIG. -VALIDATE_CONFIG="${STEAMCACHE2_VALIDATE_CONFIG:-docs/examples/validate-config.yaml}" -if [[ ! -f "$VALIDATE_CONFIG" ]]; then - echo "ERROR: Validation config not found at: $VALIDATE_CONFIG" - echo "Set STEAMCACHE2_VALIDATE_CONFIG=/path/to/your-config.yaml or place a copy at docs/examples/validate-config.yaml" - exit 1 -fi - -# Extract the listen port from the config (supports ":80", "127.0.0.1:80", etc.) -# Falls back to 80 if we can't parse it. -PORT=$(grep -E '^\s*listen_address:' "$VALIDATE_CONFIG" | head -1 | sed -E 's/.*:([0-9]+).*/\1/' || true) -if [[ -z "$PORT" || ! "$PORT" =~ ^[0-9]+$ ]]; then - PORT=80 -fi - -SERVER_URL="http://localhost:${PORT}" - -# For privileged ports (<1024, i.e. the default :80) we require the -# cap_net_bind_service capability. We apply it automatically here (via sudo -# setcap) on the binary we are about to run. This happens after any build -# so the cap is never "lost" when the binary is rebuilt. -if [ "$PORT" -lt 1024 ] && [ "$(id -u)" -ne 0 ]; then - if ! command -v getcap >/dev/null 2>&1 || ! getcap "$BINARY" 2>/dev/null | grep -q "cap_net_bind_service"; then - echo "Setting cap_net_bind_service on the binary (sudo may prompt)..." - if ! sudo setcap 'cap_net_bind_service=+ep' "$BINARY"; then - echo "ERROR: Failed to set capability." - echo "Run 'make setcap' manually, then retry." - exit 1 - fi - fi -fi - -# Safely kill only steamcache2 processes listening on this specific port. -# We look up PIDs on the port, check their actual process name/command, -# and only kill via PID if it looks like steamcache2. -echo "Checking for leftover steamcache2 processes on port ${PORT}..." - -kill_steamcache_on_port() { - local port=$1 - local pids="" - - # Try ss first (modern, usually available) - if command -v ss >/dev/null 2>&1; then - pids=$(ss -tlnp 2>/dev/null | grep ":${port} " | sed -n 's/.*pid=\([0-9]*\).*/\1/p' | sort -u) - fi - - # Fallback to lsof - if [[ -z "$pids" ]] && command -v lsof >/dev/null 2>&1; then - pids=$(lsof -ti :${port} 2>/dev/null | sort -u) - fi - - if [[ -z "$pids" ]]; then - return 0 - fi - - for pid in $pids; do - # Get process name and command line - local proc_name - local cmdline - proc_name=$(ps -p "$pid" -o comm= 2>/dev/null || true) - cmdline=$(ps -p "$pid" -o cmd= 2>/dev/null || true) - - # Check if this looks like a steamcache2 process - if echo "$proc_name $cmdline" | grep -qi "steamcache"; then - echo " → Found steamcache2 on port ${port} (PID $pid, name: ${proc_name:-unknown})" - kill -TERM "$pid" 2>/dev/null || true - sleep 0.3 - # If still alive, force kill - if kill -0 "$pid" 2>/dev/null; then - kill -9 "$pid" 2>/dev/null || true - fi - echo " Killed PID $pid" - else - echo " → Skipping PID $pid on port ${port} (not steamcache2: ${proc_name:-$cmdline})" - fi - done -} - -kill_steamcache_on_port "$PORT" -sleep 0.5 - -# --- Launch the server in the background --- -echo "Starting steamcache2 with validation config (small caches for disk/GC testing)..." -"$BINARY" --config "$VALIDATE_CONFIG" --log-level info & -SERVER_PID=$! - -# Ensure we always clean up the child on exit / Ctrl-C / error -cleanup() { - echo "" - echo "Stopping steamcache2 (pid $SERVER_PID)..." - if kill "$SERVER_PID" 2>/dev/null; then - wait "$SERVER_PID" 2>/dev/null || true - fi - echo "Server stopped." -} -trap cleanup EXIT INT TERM - -# Give the server a moment to bind and pass its own startup checks -sleep 2 - -# Basic readiness probe using the actual configured port -if ! curl -s --max-time 3 "${SERVER_URL}/" >/dev/null 2>&1; then - echo "WARNING: Server did not respond quickly on ${SERVER_URL}/" - echo " It may still be starting or bound to a different address." - echo " Check the server logs above. You can still try the SteamPrefill commands." -fi - -if [[ "${VALIDATE_QUIET:-}" != "1" ]]; then -echo "" -echo "======================================================================" -echo "steamcache2 is running (validation mode) on ${SERVER_URL}" -echo "" -echo "In another terminal (or on a machine that can reach this one), run:" -echo "" -echo " # One-time workload creation (run on a machine with SteamPrefill + Steam):" -echo " SteamPrefill benchmark setup --preset LargeChunks" -echo " # (or --use-selected, --all, --appid ..., or your own preset)" -echo "" -echo " # Copy the generated workload file to this machine if needed." -echo "" -echo " # Then run the actual benchmark (this is the realistic client simulator):" -echo " SteamPrefill benchmark run -c 20 -i 3" -echo "" -echo "After the benchmark completes, you can inspect the cache with:" -echo " curl -s ${SERVER_URL}/metrics | cat" -echo "" -echo "Or run: make validate-check (if the Makefile target exists)" -echo "" -echo "When you are finished, press Ctrl-C in this window to stop the server cleanly." -echo "======================================================================" -echo "" -fi - -# Wait for the background server (or for the user to Ctrl-C) -wait $SERVER_PID || true \ No newline at end of file