Refactor validation process and update configuration examples
- Replaced the `validate-with-prefill.sh` script with a streamlined `make validate` command for improved usability. - Updated `validate-config.yaml` to clarify cache management instructions and garbage collection algorithms. - Enhanced comments to provide better guidance on upstream configurations and their implications for caching setups.
This commit is contained in:
@@ -22,7 +22,7 @@
|
||||
#
|
||||
# Usage (typical dev workflow):
|
||||
# make build
|
||||
# ./scripts/validate-with-prefill.sh
|
||||
# make validate
|
||||
# # In another terminal:
|
||||
# SteamPrefill benchmark run -c 20 ...
|
||||
#
|
||||
@@ -43,12 +43,15 @@ trusted_proxies: ["127.0.0.0/8"]
|
||||
cache:
|
||||
memory:
|
||||
size: 1GB
|
||||
gc_algorithm: largest
|
||||
gc_algorithm: hybrid
|
||||
disk:
|
||||
size: 2GB
|
||||
path: ./validate-disk # ephemeral; clean between runs if you want a fresh test
|
||||
path: ./validate-disk # cleaned between runs by make validate or make clean-disk
|
||||
gc_algorithm: hybrid # recommended for disk in the project README
|
||||
|
||||
# Empty upstream = use Host header from the client (SteamPrefill / real Steam clients).
|
||||
# This matches the common "DNS points lancache.steamcontent.com at the cache" setup.
|
||||
# Allows for chaining steamcache2 instances if needed.
|
||||
# For example, for a lan party you could have a small fast ram only cache at each table pointing to a larger slower disk cache in the back somewhere
|
||||
# It would reduce the amount of bandwidth needed to the internet and the amount needed to each table
|
||||
# just as a little reminder there is no authentication so this is not a good idea for a public cache just out on the internet.
|
||||
upstream: ""
|
||||
|
||||
@@ -1,182 +0,0 @@
|
||||
#!/usr/bin/env bash
|
||||
#
|
||||
# validate-with-prefill.sh
|
||||
#
|
||||
# Thin glue script to make it trivial for developers to validate complete
|
||||
# steamcache2 functionality using the external SteamPrefill (lancacheprefill)
|
||||
# tool as the realistic client simulator.
|
||||
#
|
||||
# Usage:
|
||||
# 1. make build
|
||||
# 2. ./scripts/validate-with-prefill.sh
|
||||
# (Automatically kills any leftover steamcache2 on the target port first.)
|
||||
# 3. In another terminal (or on another machine), run the printed
|
||||
# SteamPrefill benchmark commands (the script tells you the exact address/port).
|
||||
# 4. After the benchmark finishes, run the suggested metrics check.
|
||||
# 5. Ctrl-C here to cleanly stop the steamcache2 instance.
|
||||
#
|
||||
# This script + the accompanying validate-config.yaml + README docs are the
|
||||
# entire "couple little scripts to hook steamcache2 and lancacheprefill together"
|
||||
# implementation. No Go code, no new dependencies, stays outside go test / bench.
|
||||
#
|
||||
set -euo pipefail
|
||||
|
||||
# --- Locate the built steamcache2 binary (produced by "make build") ---
|
||||
BINARY=""
|
||||
for candidate in \
|
||||
"dist/default_linux_amd64_v1/steamcache2" \
|
||||
"dist/steamcache2" \
|
||||
"./steamcache2" \
|
||||
"steamcache2"
|
||||
do
|
||||
if [[ -x "$candidate" ]]; then
|
||||
BINARY="$candidate"
|
||||
break
|
||||
fi
|
||||
done
|
||||
|
||||
if [[ -z "$BINARY" ]]; then
|
||||
echo "ERROR: Could not find a built steamcache2 binary."
|
||||
echo "Run 'make build' first (or place the binary in one of the searched locations)."
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo "Using steamcache2 binary: $BINARY"
|
||||
|
||||
# --- Validation config (small dual-tier so disk + GC get real exercise) ---
|
||||
# Source of truth lives in docs/examples/ (safe from "make clean").
|
||||
# The script will also accept an explicit path via STEAMCACHE2_VALIDATE_CONFIG.
|
||||
VALIDATE_CONFIG="${STEAMCACHE2_VALIDATE_CONFIG:-docs/examples/validate-config.yaml}"
|
||||
if [[ ! -f "$VALIDATE_CONFIG" ]]; then
|
||||
echo "ERROR: Validation config not found at: $VALIDATE_CONFIG"
|
||||
echo "Set STEAMCACHE2_VALIDATE_CONFIG=/path/to/your-config.yaml or place a copy at docs/examples/validate-config.yaml"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Extract the listen port from the config (supports ":80", "127.0.0.1:80", etc.)
|
||||
# Falls back to 80 if we can't parse it.
|
||||
PORT=$(grep -E '^\s*listen_address:' "$VALIDATE_CONFIG" | head -1 | sed -E 's/.*:([0-9]+).*/\1/' || true)
|
||||
if [[ -z "$PORT" || ! "$PORT" =~ ^[0-9]+$ ]]; then
|
||||
PORT=80
|
||||
fi
|
||||
|
||||
SERVER_URL="http://localhost:${PORT}"
|
||||
|
||||
# For privileged ports (<1024, i.e. the default :80) we require the
|
||||
# cap_net_bind_service capability. We apply it automatically here (via sudo
|
||||
# setcap) on the binary we are about to run. This happens after any build
|
||||
# so the cap is never "lost" when the binary is rebuilt.
|
||||
if [ "$PORT" -lt 1024 ] && [ "$(id -u)" -ne 0 ]; then
|
||||
if ! command -v getcap >/dev/null 2>&1 || ! getcap "$BINARY" 2>/dev/null | grep -q "cap_net_bind_service"; then
|
||||
echo "Setting cap_net_bind_service on the binary (sudo may prompt)..."
|
||||
if ! sudo setcap 'cap_net_bind_service=+ep' "$BINARY"; then
|
||||
echo "ERROR: Failed to set capability."
|
||||
echo "Run 'make setcap' manually, then retry."
|
||||
exit 1
|
||||
fi
|
||||
fi
|
||||
fi
|
||||
|
||||
# Safely kill only steamcache2 processes listening on this specific port.
|
||||
# We look up PIDs on the port, check their actual process name/command,
|
||||
# and only kill via PID if it looks like steamcache2.
|
||||
echo "Checking for leftover steamcache2 processes on port ${PORT}..."
|
||||
|
||||
kill_steamcache_on_port() {
|
||||
local port=$1
|
||||
local pids=""
|
||||
|
||||
# Try ss first (modern, usually available)
|
||||
if command -v ss >/dev/null 2>&1; then
|
||||
pids=$(ss -tlnp 2>/dev/null | grep ":${port} " | sed -n 's/.*pid=\([0-9]*\).*/\1/p' | sort -u)
|
||||
fi
|
||||
|
||||
# Fallback to lsof
|
||||
if [[ -z "$pids" ]] && command -v lsof >/dev/null 2>&1; then
|
||||
pids=$(lsof -ti :${port} 2>/dev/null | sort -u)
|
||||
fi
|
||||
|
||||
if [[ -z "$pids" ]]; then
|
||||
return 0
|
||||
fi
|
||||
|
||||
for pid in $pids; do
|
||||
# Get process name and command line
|
||||
local proc_name
|
||||
local cmdline
|
||||
proc_name=$(ps -p "$pid" -o comm= 2>/dev/null || true)
|
||||
cmdline=$(ps -p "$pid" -o cmd= 2>/dev/null || true)
|
||||
|
||||
# Check if this looks like a steamcache2 process
|
||||
if echo "$proc_name $cmdline" | grep -qi "steamcache"; then
|
||||
echo " → Found steamcache2 on port ${port} (PID $pid, name: ${proc_name:-unknown})"
|
||||
kill -TERM "$pid" 2>/dev/null || true
|
||||
sleep 0.3
|
||||
# If still alive, force kill
|
||||
if kill -0 "$pid" 2>/dev/null; then
|
||||
kill -9 "$pid" 2>/dev/null || true
|
||||
fi
|
||||
echo " Killed PID $pid"
|
||||
else
|
||||
echo " → Skipping PID $pid on port ${port} (not steamcache2: ${proc_name:-$cmdline})"
|
||||
fi
|
||||
done
|
||||
}
|
||||
|
||||
kill_steamcache_on_port "$PORT"
|
||||
sleep 0.5
|
||||
|
||||
# --- Launch the server in the background ---
|
||||
echo "Starting steamcache2 with validation config (small caches for disk/GC testing)..."
|
||||
"$BINARY" --config "$VALIDATE_CONFIG" --log-level info &
|
||||
SERVER_PID=$!
|
||||
|
||||
# Ensure we always clean up the child on exit / Ctrl-C / error
|
||||
cleanup() {
|
||||
echo ""
|
||||
echo "Stopping steamcache2 (pid $SERVER_PID)..."
|
||||
if kill "$SERVER_PID" 2>/dev/null; then
|
||||
wait "$SERVER_PID" 2>/dev/null || true
|
||||
fi
|
||||
echo "Server stopped."
|
||||
}
|
||||
trap cleanup EXIT INT TERM
|
||||
|
||||
# Give the server a moment to bind and pass its own startup checks
|
||||
sleep 2
|
||||
|
||||
# Basic readiness probe using the actual configured port
|
||||
if ! curl -s --max-time 3 "${SERVER_URL}/" >/dev/null 2>&1; then
|
||||
echo "WARNING: Server did not respond quickly on ${SERVER_URL}/"
|
||||
echo " It may still be starting or bound to a different address."
|
||||
echo " Check the server logs above. You can still try the SteamPrefill commands."
|
||||
fi
|
||||
|
||||
if [[ "${VALIDATE_QUIET:-}" != "1" ]]; then
|
||||
echo ""
|
||||
echo "======================================================================"
|
||||
echo "steamcache2 is running (validation mode) on ${SERVER_URL}"
|
||||
echo ""
|
||||
echo "In another terminal (or on a machine that can reach this one), run:"
|
||||
echo ""
|
||||
echo " # One-time workload creation (run on a machine with SteamPrefill + Steam):"
|
||||
echo " SteamPrefill benchmark setup --preset LargeChunks"
|
||||
echo " # (or --use-selected, --all, --appid ..., or your own preset)"
|
||||
echo ""
|
||||
echo " # Copy the generated workload file to this machine if needed."
|
||||
echo ""
|
||||
echo " # Then run the actual benchmark (this is the realistic client simulator):"
|
||||
echo " SteamPrefill benchmark run -c 20 -i 3"
|
||||
echo ""
|
||||
echo "After the benchmark completes, you can inspect the cache with:"
|
||||
echo " curl -s ${SERVER_URL}/metrics | cat"
|
||||
echo ""
|
||||
echo "Or run: make validate-check (if the Makefile target exists)"
|
||||
echo ""
|
||||
echo "When you are finished, press Ctrl-C in this window to stop the server cleanly."
|
||||
echo "======================================================================"
|
||||
echo ""
|
||||
fi
|
||||
|
||||
# Wait for the background server (or for the user to Ctrl-C)
|
||||
wait $SERVER_PID || true
|
||||
Reference in New Issue
Block a user