Refactor validation process and update configuration examples
- Replaced the `validate-with-prefill.sh` script with a streamlined `make validate` command for improved usability. - Updated `validate-config.yaml` to clarify cache management instructions and garbage collection algorithms. - Enhanced comments to provide better guidance on upstream configurations and their implications for caching setups.
This commit is contained in:
@@ -22,7 +22,7 @@
|
|||||||
#
|
#
|
||||||
# Usage (typical dev workflow):
|
# Usage (typical dev workflow):
|
||||||
# make build
|
# make build
|
||||||
# ./scripts/validate-with-prefill.sh
|
# make validate
|
||||||
# # In another terminal:
|
# # In another terminal:
|
||||||
# SteamPrefill benchmark run -c 20 ...
|
# SteamPrefill benchmark run -c 20 ...
|
||||||
#
|
#
|
||||||
@@ -43,12 +43,15 @@ trusted_proxies: ["127.0.0.0/8"]
|
|||||||
cache:
|
cache:
|
||||||
memory:
|
memory:
|
||||||
size: 1GB
|
size: 1GB
|
||||||
gc_algorithm: largest
|
gc_algorithm: hybrid
|
||||||
disk:
|
disk:
|
||||||
size: 2GB
|
size: 2GB
|
||||||
path: ./validate-disk # ephemeral; clean between runs if you want a fresh test
|
path: ./validate-disk # cleaned between runs by make validate or make clean-disk
|
||||||
gc_algorithm: hybrid # recommended for disk in the project README
|
gc_algorithm: hybrid # recommended for disk in the project README
|
||||||
|
|
||||||
# Empty upstream = use Host header from the client (SteamPrefill / real Steam clients).
|
# Empty upstream = use Host header from the client (SteamPrefill / real Steam clients).
|
||||||
# This matches the common "DNS points lancache.steamcontent.com at the cache" setup.
|
# Allows for chaining steamcache2 instances if needed.
|
||||||
|
# For example, for a lan party you could have a small fast ram only cache at each table pointing to a larger slower disk cache in the back somewhere
|
||||||
|
# It would reduce the amount of bandwidth needed to the internet and the amount needed to each table
|
||||||
|
# just as a little reminder there is no authentication so this is not a good idea for a public cache just out on the internet.
|
||||||
upstream: ""
|
upstream: ""
|
||||||
|
|||||||
@@ -1,182 +0,0 @@
|
|||||||
#!/usr/bin/env bash
|
|
||||||
#
|
|
||||||
# validate-with-prefill.sh
|
|
||||||
#
|
|
||||||
# Thin glue script to make it trivial for developers to validate complete
|
|
||||||
# steamcache2 functionality using the external SteamPrefill (lancacheprefill)
|
|
||||||
# tool as the realistic client simulator.
|
|
||||||
#
|
|
||||||
# Usage:
|
|
||||||
# 1. make build
|
|
||||||
# 2. ./scripts/validate-with-prefill.sh
|
|
||||||
# (Automatically kills any leftover steamcache2 on the target port first.)
|
|
||||||
# 3. In another terminal (or on another machine), run the printed
|
|
||||||
# SteamPrefill benchmark commands (the script tells you the exact address/port).
|
|
||||||
# 4. After the benchmark finishes, run the suggested metrics check.
|
|
||||||
# 5. Ctrl-C here to cleanly stop the steamcache2 instance.
|
|
||||||
#
|
|
||||||
# This script + the accompanying validate-config.yaml + README docs are the
|
|
||||||
# entire "couple little scripts to hook steamcache2 and lancacheprefill together"
|
|
||||||
# implementation. No Go code, no new dependencies, stays outside go test / bench.
|
|
||||||
#
|
|
||||||
set -euo pipefail
|
|
||||||
|
|
||||||
# --- Locate the built steamcache2 binary (produced by "make build") ---
|
|
||||||
BINARY=""
|
|
||||||
for candidate in \
|
|
||||||
"dist/default_linux_amd64_v1/steamcache2" \
|
|
||||||
"dist/steamcache2" \
|
|
||||||
"./steamcache2" \
|
|
||||||
"steamcache2"
|
|
||||||
do
|
|
||||||
if [[ -x "$candidate" ]]; then
|
|
||||||
BINARY="$candidate"
|
|
||||||
break
|
|
||||||
fi
|
|
||||||
done
|
|
||||||
|
|
||||||
if [[ -z "$BINARY" ]]; then
|
|
||||||
echo "ERROR: Could not find a built steamcache2 binary."
|
|
||||||
echo "Run 'make build' first (or place the binary in one of the searched locations)."
|
|
||||||
exit 1
|
|
||||||
fi
|
|
||||||
|
|
||||||
echo "Using steamcache2 binary: $BINARY"
|
|
||||||
|
|
||||||
# --- Validation config (small dual-tier so disk + GC get real exercise) ---
|
|
||||||
# Source of truth lives in docs/examples/ (safe from "make clean").
|
|
||||||
# The script will also accept an explicit path via STEAMCACHE2_VALIDATE_CONFIG.
|
|
||||||
VALIDATE_CONFIG="${STEAMCACHE2_VALIDATE_CONFIG:-docs/examples/validate-config.yaml}"
|
|
||||||
if [[ ! -f "$VALIDATE_CONFIG" ]]; then
|
|
||||||
echo "ERROR: Validation config not found at: $VALIDATE_CONFIG"
|
|
||||||
echo "Set STEAMCACHE2_VALIDATE_CONFIG=/path/to/your-config.yaml or place a copy at docs/examples/validate-config.yaml"
|
|
||||||
exit 1
|
|
||||||
fi
|
|
||||||
|
|
||||||
# Extract the listen port from the config (supports ":80", "127.0.0.1:80", etc.)
|
|
||||||
# Falls back to 80 if we can't parse it.
|
|
||||||
PORT=$(grep -E '^\s*listen_address:' "$VALIDATE_CONFIG" | head -1 | sed -E 's/.*:([0-9]+).*/\1/' || true)
|
|
||||||
if [[ -z "$PORT" || ! "$PORT" =~ ^[0-9]+$ ]]; then
|
|
||||||
PORT=80
|
|
||||||
fi
|
|
||||||
|
|
||||||
SERVER_URL="http://localhost:${PORT}"
|
|
||||||
|
|
||||||
# For privileged ports (<1024, i.e. the default :80) we require the
|
|
||||||
# cap_net_bind_service capability. We apply it automatically here (via sudo
|
|
||||||
# setcap) on the binary we are about to run. This happens after any build
|
|
||||||
# so the cap is never "lost" when the binary is rebuilt.
|
|
||||||
if [ "$PORT" -lt 1024 ] && [ "$(id -u)" -ne 0 ]; then
|
|
||||||
if ! command -v getcap >/dev/null 2>&1 || ! getcap "$BINARY" 2>/dev/null | grep -q "cap_net_bind_service"; then
|
|
||||||
echo "Setting cap_net_bind_service on the binary (sudo may prompt)..."
|
|
||||||
if ! sudo setcap 'cap_net_bind_service=+ep' "$BINARY"; then
|
|
||||||
echo "ERROR: Failed to set capability."
|
|
||||||
echo "Run 'make setcap' manually, then retry."
|
|
||||||
exit 1
|
|
||||||
fi
|
|
||||||
fi
|
|
||||||
fi
|
|
||||||
|
|
||||||
# Safely kill only steamcache2 processes listening on this specific port.
|
|
||||||
# We look up PIDs on the port, check their actual process name/command,
|
|
||||||
# and only kill via PID if it looks like steamcache2.
|
|
||||||
echo "Checking for leftover steamcache2 processes on port ${PORT}..."
|
|
||||||
|
|
||||||
kill_steamcache_on_port() {
|
|
||||||
local port=$1
|
|
||||||
local pids=""
|
|
||||||
|
|
||||||
# Try ss first (modern, usually available)
|
|
||||||
if command -v ss >/dev/null 2>&1; then
|
|
||||||
pids=$(ss -tlnp 2>/dev/null | grep ":${port} " | sed -n 's/.*pid=\([0-9]*\).*/\1/p' | sort -u)
|
|
||||||
fi
|
|
||||||
|
|
||||||
# Fallback to lsof
|
|
||||||
if [[ -z "$pids" ]] && command -v lsof >/dev/null 2>&1; then
|
|
||||||
pids=$(lsof -ti :${port} 2>/dev/null | sort -u)
|
|
||||||
fi
|
|
||||||
|
|
||||||
if [[ -z "$pids" ]]; then
|
|
||||||
return 0
|
|
||||||
fi
|
|
||||||
|
|
||||||
for pid in $pids; do
|
|
||||||
# Get process name and command line
|
|
||||||
local proc_name
|
|
||||||
local cmdline
|
|
||||||
proc_name=$(ps -p "$pid" -o comm= 2>/dev/null || true)
|
|
||||||
cmdline=$(ps -p "$pid" -o cmd= 2>/dev/null || true)
|
|
||||||
|
|
||||||
# Check if this looks like a steamcache2 process
|
|
||||||
if echo "$proc_name $cmdline" | grep -qi "steamcache"; then
|
|
||||||
echo " → Found steamcache2 on port ${port} (PID $pid, name: ${proc_name:-unknown})"
|
|
||||||
kill -TERM "$pid" 2>/dev/null || true
|
|
||||||
sleep 0.3
|
|
||||||
# If still alive, force kill
|
|
||||||
if kill -0 "$pid" 2>/dev/null; then
|
|
||||||
kill -9 "$pid" 2>/dev/null || true
|
|
||||||
fi
|
|
||||||
echo " Killed PID $pid"
|
|
||||||
else
|
|
||||||
echo " → Skipping PID $pid on port ${port} (not steamcache2: ${proc_name:-$cmdline})"
|
|
||||||
fi
|
|
||||||
done
|
|
||||||
}
|
|
||||||
|
|
||||||
kill_steamcache_on_port "$PORT"
|
|
||||||
sleep 0.5
|
|
||||||
|
|
||||||
# --- Launch the server in the background ---
|
|
||||||
echo "Starting steamcache2 with validation config (small caches for disk/GC testing)..."
|
|
||||||
"$BINARY" --config "$VALIDATE_CONFIG" --log-level info &
|
|
||||||
SERVER_PID=$!
|
|
||||||
|
|
||||||
# Ensure we always clean up the child on exit / Ctrl-C / error
|
|
||||||
cleanup() {
|
|
||||||
echo ""
|
|
||||||
echo "Stopping steamcache2 (pid $SERVER_PID)..."
|
|
||||||
if kill "$SERVER_PID" 2>/dev/null; then
|
|
||||||
wait "$SERVER_PID" 2>/dev/null || true
|
|
||||||
fi
|
|
||||||
echo "Server stopped."
|
|
||||||
}
|
|
||||||
trap cleanup EXIT INT TERM
|
|
||||||
|
|
||||||
# Give the server a moment to bind and pass its own startup checks
|
|
||||||
sleep 2
|
|
||||||
|
|
||||||
# Basic readiness probe using the actual configured port
|
|
||||||
if ! curl -s --max-time 3 "${SERVER_URL}/" >/dev/null 2>&1; then
|
|
||||||
echo "WARNING: Server did not respond quickly on ${SERVER_URL}/"
|
|
||||||
echo " It may still be starting or bound to a different address."
|
|
||||||
echo " Check the server logs above. You can still try the SteamPrefill commands."
|
|
||||||
fi
|
|
||||||
|
|
||||||
if [[ "${VALIDATE_QUIET:-}" != "1" ]]; then
|
|
||||||
echo ""
|
|
||||||
echo "======================================================================"
|
|
||||||
echo "steamcache2 is running (validation mode) on ${SERVER_URL}"
|
|
||||||
echo ""
|
|
||||||
echo "In another terminal (or on a machine that can reach this one), run:"
|
|
||||||
echo ""
|
|
||||||
echo " # One-time workload creation (run on a machine with SteamPrefill + Steam):"
|
|
||||||
echo " SteamPrefill benchmark setup --preset LargeChunks"
|
|
||||||
echo " # (or --use-selected, --all, --appid ..., or your own preset)"
|
|
||||||
echo ""
|
|
||||||
echo " # Copy the generated workload file to this machine if needed."
|
|
||||||
echo ""
|
|
||||||
echo " # Then run the actual benchmark (this is the realistic client simulator):"
|
|
||||||
echo " SteamPrefill benchmark run -c 20 -i 3"
|
|
||||||
echo ""
|
|
||||||
echo "After the benchmark completes, you can inspect the cache with:"
|
|
||||||
echo " curl -s ${SERVER_URL}/metrics | cat"
|
|
||||||
echo ""
|
|
||||||
echo "Or run: make validate-check (if the Makefile target exists)"
|
|
||||||
echo ""
|
|
||||||
echo "When you are finished, press Ctrl-C in this window to stop the server cleanly."
|
|
||||||
echo "======================================================================"
|
|
||||||
echo ""
|
|
||||||
fi
|
|
||||||
|
|
||||||
# Wait for the background server (or for the user to Ctrl-C)
|
|
||||||
wait $SERVER_PID || true
|
|
||||||
Reference in New Issue
Block a user