Enhance Makefile and documentation for validation workflow
- Added new targets in the Makefile for validation, including `run-validation`, `validate-check`, and `validate-kill`, to streamline the testing process with external tools like SteamPrefill. - Introduced a `setcap` target to manage necessary capabilities for running the server on port 80 without root access. - Updated README.md to include detailed instructions for validating functionality, including quick start guides and troubleshooting tips. - Improved .gitignore to exclude validation artifacts and logs, ensuring a cleaner repository.
This commit is contained in:
Executable
+155
@@ -0,0 +1,155 @@
|
||||
#!/usr/bin/env bash
|
||||
#
|
||||
# download-prefill.sh
|
||||
#
|
||||
# Downloads the latest (or specific) release of SteamPrefill
|
||||
# (https://github.com/tpill90/steam-lancache-prefill) into
|
||||
# bin/steam-prefill/SteamPrefill
|
||||
#
|
||||
# Usage:
|
||||
# ./scripts/download-prefill.sh
|
||||
#
|
||||
# Environment:
|
||||
# PREFILL_VERSION - Pin a specific version tag (e.g. v3.4.2)
|
||||
# PREFILL_FORCE - Set to any non-empty value to re-download even if present
|
||||
#
|
||||
set -euo pipefail
|
||||
|
||||
DEST_DIR="bin/steam-prefill"
|
||||
TARGET="$DEST_DIR/SteamPrefill"
|
||||
|
||||
mkdir -p "$DEST_DIR"
|
||||
|
||||
if [[ -x "$TARGET" && -z "${PREFILL_FORCE:-}" ]]; then
|
||||
echo "SteamPrefill already present at $TARGET"
|
||||
echo "Run with PREFILL_FORCE=1 to re-download."
|
||||
exit 0
|
||||
fi
|
||||
|
||||
VERSION="${PREFILL_VERSION:-}"
|
||||
|
||||
OS=$(uname -s | tr '[:upper:]' '[:lower:]')
|
||||
ARCH=$(uname -m)
|
||||
|
||||
case "$ARCH" in
|
||||
x86_64|amd64) ARCH_NAME="x64" ;;
|
||||
aarch64|arm64) ARCH_NAME="arm64" ;;
|
||||
*) echo "Unsupported architecture: $ARCH"; exit 1 ;;
|
||||
esac
|
||||
|
||||
case "$OS" in
|
||||
linux) OS_NAME="linux" ;;
|
||||
darwin) OS_NAME="osx" ;;
|
||||
*) echo "Unsupported OS: $OS"; exit 1 ;;
|
||||
esac
|
||||
|
||||
echo "Resolving SteamPrefill version..."
|
||||
|
||||
if [[ -z "$VERSION" ]]; then
|
||||
# Follow the /latest redirect to discover the current tag
|
||||
LATEST_URL=$(curl -sIL -o /dev/null -w '%{url_effective}' \
|
||||
"https://github.com/tpill90/steam-lancache-prefill/releases/latest" 2>/dev/null || true)
|
||||
|
||||
if [[ "$LATEST_URL" =~ /tag/([^/?#]+) ]]; then
|
||||
VERSION="${BASH_REMATCH[1]}"
|
||||
else
|
||||
echo "Failed to resolve latest version from GitHub redirect."
|
||||
exit 1
|
||||
fi
|
||||
fi
|
||||
|
||||
echo "Downloading SteamPrefill $VERSION for ${OS_NAME}-${ARCH_NAME}..."
|
||||
|
||||
rm -f "$TARGET" "$TARGET.tmp" 2>/dev/null || true
|
||||
|
||||
DOWNLOADED=0
|
||||
|
||||
# Preferred: query the GitHub API for the exact asset list (most reliable)
|
||||
API_URL="https://api.github.com/repos/tpill90/steam-lancache-prefill/releases/tags/${VERSION}"
|
||||
ASSET_URL=""
|
||||
|
||||
if command -v jq >/dev/null 2>&1; then
|
||||
echo " Querying GitHub API for assets..."
|
||||
ASSET_NAME=$(curl -fsSL "$API_URL" 2>/dev/null | jq -r --arg os "$OS_NAME" --arg arch "$ARCH_NAME" '
|
||||
.assets[]
|
||||
| select(.name | ascii_downcase | contains($os))
|
||||
| select(.name | ascii_downcase | contains($arch))
|
||||
| .name
|
||||
' | head -1)
|
||||
|
||||
if [[ -n "$ASSET_NAME" ]]; then
|
||||
ASSET_URL="https://github.com/tpill90/steam-lancache-prefill/releases/download/${VERSION}/${ASSET_NAME}"
|
||||
echo " Found asset via API: $ASSET_NAME"
|
||||
fi
|
||||
fi
|
||||
|
||||
# Fallback: try common name patterns if API or jq not available
|
||||
if [[ -z "$ASSET_URL" ]]; then
|
||||
echo " Trying common asset name patterns..."
|
||||
CANDIDATES=(
|
||||
"SteamPrefill-${VERSION}-${OS_NAME}-${ARCH_NAME}.zip"
|
||||
"SteamPrefill-${VERSION}-${OS_NAME}-${ARCH_NAME}"
|
||||
"SteamPrefill-${OS_NAME}-${ARCH_NAME}.zip"
|
||||
"SteamPrefill-${OS_NAME}-${ARCH_NAME}"
|
||||
"SteamPrefill-linux-${ARCH_NAME}.zip"
|
||||
"SteamPrefill-linux-${ARCH_NAME}"
|
||||
)
|
||||
|
||||
for name in "${CANDIDATES[@]}"; do
|
||||
URL="https://github.com/tpill90/steam-lancache-prefill/releases/download/${VERSION}/${name}"
|
||||
echo " Trying $name ..."
|
||||
if curl -fI -s --retry 2 "$URL" >/dev/null 2>&1; then
|
||||
ASSET_URL="$URL"
|
||||
ASSET_NAME="$name"
|
||||
break
|
||||
fi
|
||||
done
|
||||
fi
|
||||
|
||||
if [[ -n "$ASSET_URL" ]]; then
|
||||
echo "Downloading $ASSET_NAME ..."
|
||||
if curl -fL --retry 3 --retry-delay 2 -A "Mozilla/5.0 (compatible; SteamPrefill-Downloader)" \
|
||||
--progress-bar -o "$TARGET.tmp" "$ASSET_URL"; then
|
||||
|
||||
echo "Download complete."
|
||||
|
||||
if [[ "$ASSET_NAME" == *.zip ]]; then
|
||||
echo "Extracting..."
|
||||
if ! command -v unzip >/dev/null 2>&1; then
|
||||
echo "Error: unzip is required for this release."
|
||||
rm -f "$TARGET.tmp"
|
||||
exit 1
|
||||
fi
|
||||
unzip -o -q "$TARGET.tmp" -d "$DEST_DIR"
|
||||
FOUND=$(find "$DEST_DIR" -type f -name "SteamPrefill" | head -1)
|
||||
if [[ -n "$FOUND" ]]; then
|
||||
mv "$FOUND" "$TARGET"
|
||||
fi
|
||||
rm -f "$TARGET.tmp"
|
||||
find "$DEST_DIR" -mindepth 1 -maxdepth 1 -type d -name "SteamPrefill*" -exec rm -rf {} + 2>/dev/null || true
|
||||
else
|
||||
mv "$TARGET.tmp" "$TARGET"
|
||||
fi
|
||||
|
||||
chmod +x "$TARGET"
|
||||
DOWNLOADED=1
|
||||
fi
|
||||
fi
|
||||
|
||||
if [[ $DOWNLOADED -eq 0 ]]; then
|
||||
echo ""
|
||||
echo "Failed to download a matching asset for $VERSION."
|
||||
echo "You can try pinning a different version:"
|
||||
echo " PREFILL_VERSION=vX.Y.Z ./scripts/download-prefill.sh"
|
||||
echo ""
|
||||
echo "Or download manually from:"
|
||||
echo " https://github.com/tpill90/steam-lancache-prefill/releases/tag/${VERSION}"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo ""
|
||||
echo "Installed SteamPrefill $VERSION → $TARGET"
|
||||
echo ""
|
||||
echo "You can now run it directly, for example:"
|
||||
echo " ./bin/steam-prefill/SteamPrefill --help"
|
||||
echo " ./bin/steam-prefill/SteamPrefill benchmark run ..."
|
||||
Executable
+182
@@ -0,0 +1,182 @@
|
||||
#!/usr/bin/env bash
|
||||
#
|
||||
# validate-with-prefill.sh
|
||||
#
|
||||
# Thin glue script to make it trivial for developers to validate complete
|
||||
# steamcache2 functionality using the external SteamPrefill (lancacheprefill)
|
||||
# tool as the realistic client simulator.
|
||||
#
|
||||
# Usage:
|
||||
# 1. make build
|
||||
# 2. ./scripts/validate-with-prefill.sh
|
||||
# (Automatically kills any leftover steamcache2 on the target port first.)
|
||||
# 3. In another terminal (or on another machine), run the printed
|
||||
# SteamPrefill benchmark commands (the script tells you the exact address/port).
|
||||
# 4. After the benchmark finishes, run the suggested metrics check.
|
||||
# 5. Ctrl-C here to cleanly stop the steamcache2 instance.
|
||||
#
|
||||
# This script + the accompanying validate-config.yaml + README docs are the
|
||||
# entire "couple little scripts to hook steamcache2 and lancacheprefill together"
|
||||
# implementation. No Go code, no new dependencies, stays outside go test / bench.
|
||||
#
|
||||
set -euo pipefail
|
||||
|
||||
# --- Locate the built steamcache2 binary (produced by "make build") ---
|
||||
BINARY=""
|
||||
for candidate in \
|
||||
"dist/default_linux_amd64_v1/steamcache2" \
|
||||
"dist/steamcache2" \
|
||||
"./steamcache2" \
|
||||
"steamcache2"
|
||||
do
|
||||
if [[ -x "$candidate" ]]; then
|
||||
BINARY="$candidate"
|
||||
break
|
||||
fi
|
||||
done
|
||||
|
||||
if [[ -z "$BINARY" ]]; then
|
||||
echo "ERROR: Could not find a built steamcache2 binary."
|
||||
echo "Run 'make build' first (or place the binary in one of the searched locations)."
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo "Using steamcache2 binary: $BINARY"
|
||||
|
||||
# --- Validation config (small dual-tier so disk + GC get real exercise) ---
|
||||
# Source of truth lives in docs/examples/ (safe from "make clean").
|
||||
# The script will also accept an explicit path via STEAMCACHE2_VALIDATE_CONFIG.
|
||||
VALIDATE_CONFIG="${STEAMCACHE2_VALIDATE_CONFIG:-docs/examples/validate-config.yaml}"
|
||||
if [[ ! -f "$VALIDATE_CONFIG" ]]; then
|
||||
echo "ERROR: Validation config not found at: $VALIDATE_CONFIG"
|
||||
echo "Set STEAMCACHE2_VALIDATE_CONFIG=/path/to/your-config.yaml or place a copy at docs/examples/validate-config.yaml"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Extract the listen port from the config (supports ":80", "127.0.0.1:80", etc.)
|
||||
# Falls back to 80 if we can't parse it.
|
||||
PORT=$(grep -E '^\s*listen_address:' "$VALIDATE_CONFIG" | head -1 | sed -E 's/.*:([0-9]+).*/\1/' || true)
|
||||
if [[ -z "$PORT" || ! "$PORT" =~ ^[0-9]+$ ]]; then
|
||||
PORT=80
|
||||
fi
|
||||
|
||||
SERVER_URL="http://localhost:${PORT}"
|
||||
|
||||
# For privileged ports (<1024, i.e. the default :80) we require the
|
||||
# cap_net_bind_service capability. We apply it automatically here (via sudo
|
||||
# setcap) on the binary we are about to run. This happens after any build
|
||||
# so the cap is never "lost" when the binary is rebuilt.
|
||||
if [ "$PORT" -lt 1024 ] && [ "$(id -u)" -ne 0 ]; then
|
||||
if ! command -v getcap >/dev/null 2>&1 || ! getcap "$BINARY" 2>/dev/null | grep -q "cap_net_bind_service"; then
|
||||
echo "Setting cap_net_bind_service on the binary (sudo may prompt)..."
|
||||
if ! sudo setcap 'cap_net_bind_service=+ep' "$BINARY"; then
|
||||
echo "ERROR: Failed to set capability."
|
||||
echo "Run 'make setcap' manually, then retry."
|
||||
exit 1
|
||||
fi
|
||||
fi
|
||||
fi
|
||||
|
||||
# Safely kill only steamcache2 processes listening on this specific port.
|
||||
# We look up PIDs on the port, check their actual process name/command,
|
||||
# and only kill via PID if it looks like steamcache2.
|
||||
echo "Checking for leftover steamcache2 processes on port ${PORT}..."
|
||||
|
||||
kill_steamcache_on_port() {
|
||||
local port=$1
|
||||
local pids=""
|
||||
|
||||
# Try ss first (modern, usually available)
|
||||
if command -v ss >/dev/null 2>&1; then
|
||||
pids=$(ss -tlnp 2>/dev/null | grep ":${port} " | sed -n 's/.*pid=\([0-9]*\).*/\1/p' | sort -u)
|
||||
fi
|
||||
|
||||
# Fallback to lsof
|
||||
if [[ -z "$pids" ]] && command -v lsof >/dev/null 2>&1; then
|
||||
pids=$(lsof -ti :${port} 2>/dev/null | sort -u)
|
||||
fi
|
||||
|
||||
if [[ -z "$pids" ]]; then
|
||||
return 0
|
||||
fi
|
||||
|
||||
for pid in $pids; do
|
||||
# Get process name and command line
|
||||
local proc_name
|
||||
local cmdline
|
||||
proc_name=$(ps -p "$pid" -o comm= 2>/dev/null || true)
|
||||
cmdline=$(ps -p "$pid" -o cmd= 2>/dev/null || true)
|
||||
|
||||
# Check if this looks like a steamcache2 process
|
||||
if echo "$proc_name $cmdline" | grep -qi "steamcache"; then
|
||||
echo " → Found steamcache2 on port ${port} (PID $pid, name: ${proc_name:-unknown})"
|
||||
kill -TERM "$pid" 2>/dev/null || true
|
||||
sleep 0.3
|
||||
# If still alive, force kill
|
||||
if kill -0 "$pid" 2>/dev/null; then
|
||||
kill -9 "$pid" 2>/dev/null || true
|
||||
fi
|
||||
echo " Killed PID $pid"
|
||||
else
|
||||
echo " → Skipping PID $pid on port ${port} (not steamcache2: ${proc_name:-$cmdline})"
|
||||
fi
|
||||
done
|
||||
}
|
||||
|
||||
kill_steamcache_on_port "$PORT"
|
||||
sleep 0.5
|
||||
|
||||
# --- Launch the server in the background ---
|
||||
echo "Starting steamcache2 with validation config (small caches for disk/GC testing)..."
|
||||
"$BINARY" --config "$VALIDATE_CONFIG" --log-level info &
|
||||
SERVER_PID=$!
|
||||
|
||||
# Ensure we always clean up the child on exit / Ctrl-C / error
|
||||
cleanup() {
|
||||
echo ""
|
||||
echo "Stopping steamcache2 (pid $SERVER_PID)..."
|
||||
if kill "$SERVER_PID" 2>/dev/null; then
|
||||
wait "$SERVER_PID" 2>/dev/null || true
|
||||
fi
|
||||
echo "Server stopped."
|
||||
}
|
||||
trap cleanup EXIT INT TERM
|
||||
|
||||
# Give the server a moment to bind and pass its own startup checks
|
||||
sleep 2
|
||||
|
||||
# Basic readiness probe using the actual configured port
|
||||
if ! curl -s --max-time 3 "${SERVER_URL}/" >/dev/null 2>&1; then
|
||||
echo "WARNING: Server did not respond quickly on ${SERVER_URL}/"
|
||||
echo " It may still be starting or bound to a different address."
|
||||
echo " Check the server logs above. You can still try the SteamPrefill commands."
|
||||
fi
|
||||
|
||||
if [[ "${VALIDATE_QUIET:-}" != "1" ]]; then
|
||||
echo ""
|
||||
echo "======================================================================"
|
||||
echo "steamcache2 is running (validation mode) on ${SERVER_URL}"
|
||||
echo ""
|
||||
echo "In another terminal (or on a machine that can reach this one), run:"
|
||||
echo ""
|
||||
echo " # One-time workload creation (run on a machine with SteamPrefill + Steam):"
|
||||
echo " SteamPrefill benchmark setup --preset LargeChunks"
|
||||
echo " # (or --use-selected, --all, --appid ..., or your own preset)"
|
||||
echo ""
|
||||
echo " # Copy the generated workload file to this machine if needed."
|
||||
echo ""
|
||||
echo " # Then run the actual benchmark (this is the realistic client simulator):"
|
||||
echo " SteamPrefill benchmark run -c 20 -i 3"
|
||||
echo ""
|
||||
echo "After the benchmark completes, you can inspect the cache with:"
|
||||
echo " curl -s ${SERVER_URL}/metrics | cat"
|
||||
echo ""
|
||||
echo "Or run: make validate-check (if the Makefile target exists)"
|
||||
echo ""
|
||||
echo "When you are finished, press Ctrl-C in this window to stop the server cleanly."
|
||||
echo "======================================================================"
|
||||
echo ""
|
||||
fi
|
||||
|
||||
# Wait for the background server (or for the user to Ctrl-C)
|
||||
wait $SERVER_PID || true
|
||||
Reference in New Issue
Block a user