Enhance Makefile and documentation for validation workflow
- Added new targets in the Makefile for validation, including `run-validation`, `validate-check`, and `validate-kill`, to streamline the testing process with external tools like SteamPrefill. - Introduced a `setcap` target to manage necessary capabilities for running the server on port 80 without root access. - Updated README.md to include detailed instructions for validating functionality, including quick start guides and troubleshooting tips. - Improved .gitignore to exclude validation artifacts and logs, ensuring a cleaner repository.
This commit is contained in:
+9
-1
@@ -2,11 +2,19 @@
|
||||
/dist/
|
||||
/bin/
|
||||
steamcache2
|
||||
|
||||
# Downloaded SteamPrefill client simulator (auto-managed by make client)
|
||||
/bin/steam-prefill/*
|
||||
!/bin/steam-prefill/.gitkeep
|
||||
/plans/
|
||||
|
||||
#disk cache
|
||||
#validation artifacts
|
||||
/validate-disk/
|
||||
/disk/
|
||||
|
||||
#logs
|
||||
*.log
|
||||
|
||||
#config file
|
||||
/config.yaml
|
||||
|
||||
|
||||
@@ -33,15 +33,85 @@ bench: deps ## Run all benchmarks (MemoryFS + DiskFS variants, including all evi
|
||||
@go test -bench=. -benchmem -run=^$ -benchtime=1s ./vfs/disk
|
||||
@echo "Bench done."
|
||||
|
||||
setcap: build ## Explicitly set cap_net_bind_service on the (just-built) binary for port 80 use outside validate targets
|
||||
@echo "Setting cap_net_bind_service on the binary so it can listen on port 80 as your normal user..."
|
||||
@sudo setcap 'cap_net_bind_service=+ep' dist/default_linux_amd64_v1/steamcache2
|
||||
@echo "Done. You should now be able to run 'make run-validation' as your normal user (no root)."
|
||||
|
||||
validate run-validation: build ## Start steamcache2 on :80 with small test caches (foreground)
|
||||
@echo "=== Starting steamcache2 in validation mode ==="
|
||||
@echo "Port 80 + small memory/disk caches (for exercising disk tier, GC, etc.)"
|
||||
@echo "Press Ctrl-C to stop the server."
|
||||
@echo ""
|
||||
@BINARY=dist/default_linux_amd64_v1/steamcache2; \
|
||||
if [ "$$(id -u)" -ne 0 ] && ! getcap "$$BINARY" 2>/dev/null | grep -q cap_net_bind_service; then \
|
||||
echo "Setting cap_net_bind_service on the freshly built binary (sudo may prompt)..."; \
|
||||
sudo setcap 'cap_net_bind_service=+ep' "$$BINARY" || { \
|
||||
echo "ERROR: setcap failed (or was cancelled)."; \
|
||||
echo "You can run 'make setcap' manually, then retry 'make validate'."; \
|
||||
exit 1; \
|
||||
}; \
|
||||
fi; \
|
||||
if [ "$$(id -u)" -ne 0 ] && ! getcap "$$BINARY" 2>/dev/null | grep -q cap_net_bind_service; then \
|
||||
echo "ERROR: Port 80 still requires the capability after setcap attempt."; \
|
||||
echo "Run 'make setcap' and retry."; \
|
||||
exit 1; \
|
||||
fi; \
|
||||
exec "$$BINARY" --config docs/examples/validate-config.yaml --log-level info
|
||||
|
||||
validate-check: ## Quick post-benchmark sanity check against a running steamcache2 (default port 80, override with PORT=xxxx)
|
||||
@echo "=== steamcache2 Full Function Validation Report ==="
|
||||
@PORT="$${PORT:-80}"; \
|
||||
echo "Server: http://localhost:$$PORT"; \
|
||||
curl -s --max-time 5 "http://localhost:$$PORT/metrics" || echo "(could not reach /metrics on port $$PORT - is the server running?)"; \
|
||||
echo ""; \
|
||||
echo "Tip: also inspect recent server logs for errors, coalesced hits, and disk activity."
|
||||
|
||||
prefill: ## Download latest SteamPrefill into bin/steam-prefill/SteamPrefill (gitignored)
|
||||
@./scripts/download-prefill.sh
|
||||
|
||||
validate-kill: ## Kill leftover steamcache2 processes (safer, checks process name)
|
||||
@echo "Looking for steamcache2 processes on common validation ports (80 is primary)..."
|
||||
@for port in 80 8040 8080; do \
|
||||
pids=""; \
|
||||
if command -v ss >/dev/null 2>&1; then \
|
||||
pids=$$(ss -tlnp 2>/dev/null | grep ":$${port} " | sed -n 's/.*pid=\([0-9]*\).*/\1/p' | sort -u); \
|
||||
fi; \
|
||||
if [ -z "$$pids" ] && command -v lsof >/dev/null 2>&1; then \
|
||||
pids=$$(lsof -ti :$${port} 2>/dev/null | sort -u); \
|
||||
fi; \
|
||||
for pid in $$pids; do \
|
||||
proc=$$(ps -p $$pid -o comm= 2>/dev/null || true); \
|
||||
cmd=$$(ps -p $$pid -o cmd= 2>/dev/null || true); \
|
||||
if echo "$$proc $$cmd" | grep -qi "steamcache"; then \
|
||||
echo " Killing steamcache2 (port $$port, PID $$pid, $$proc)"; \
|
||||
kill -TERM $$pid 2>/dev/null || true; \
|
||||
sleep 0.2; \
|
||||
kill -0 $$pid 2>/dev/null && kill -9 $$pid 2>/dev/null || true; \
|
||||
else \
|
||||
echo " Skipping PID $$pid on port $$port (not steamcache2: $$proc)"; \
|
||||
fi; \
|
||||
done; \
|
||||
done
|
||||
@echo "Validation server cleanup complete."
|
||||
|
||||
|
||||
|
||||
help: ## Show this help message
|
||||
@echo steamcache2 Makefile
|
||||
@echo Available targets:
|
||||
@echo run Run the application (cross-platform via go run)
|
||||
@echo run-debug Run the application with debug logging (cross-platform)
|
||||
@echo build Build the application (goreleaser snapshot)
|
||||
@echo test Run all tests
|
||||
@echo test-race Run all tests with the race detector
|
||||
@echo lint Run golangci-lint + review label check
|
||||
@echo check-review-labels Fail on temporary review labels (P*, T*, I*, R*, etc.)
|
||||
@echo deps Download dependencies
|
||||
@echo clean Remove build/test artifacts
|
||||
@echo "steamcache2 Makefile"
|
||||
@echo "Available targets:"
|
||||
@echo " run Run the application (cross-platform via go run)"
|
||||
@echo " run-debug Run the application with debug logging (cross-platform)"
|
||||
@echo " build Build the application (goreleaser snapshot)"
|
||||
@echo " test Run all tests"
|
||||
@echo " test-race Run all tests with the race detector"
|
||||
@echo " lint Run golangci-lint + review label check"
|
||||
@echo " check-review-labels Fail on temporary review labels (P*, T*, I*, R*, etc.)"
|
||||
@echo " deps Download dependencies"
|
||||
@echo " clean Remove build/test artifacts"
|
||||
@echo " bench Run low-level VFS microbenchmarks"
|
||||
@echo " validate / run-validation Start server on :80 (builds, auto-setcaps fresh binary, then runs as normal user)"
|
||||
@echo " setcap Explicitly set cap on current build (for port 80 use outside validate)"
|
||||
@echo " validate-check Quick /metrics report after running a workload"
|
||||
@echo " validate-kill Kill leftover steamcache2 processes (safer)"
|
||||
@echo " prefill Download latest SteamPrefill into bin/steam-prefill/ (for use with run-validation)"
|
||||
@@ -61,6 +61,113 @@ Run `make help` to see the full list of available commands.
|
||||
|
||||
This is the preferred approach for day-to-day development. Avoid running raw `go test`, `go run`, or `golangci-lint` commands directly for routine tasks.
|
||||
|
||||
### Validating Full Functionality with external tools
|
||||
|
||||
steamcache2 provides a convenient small-cache configuration and helper targets so you can easily validate behavior using external tools such as [SteamPrefill (tpill90/steam-lancache-prefill)](https://github.com/tpill90/steam-lancache-prefill).
|
||||
|
||||
This gives you:
|
||||
- Real Steam manifest + chunk traffic (no reinventing the wheel)
|
||||
- Excellent `benchmark setup` / `benchmark run` workflow with warmup, randomization, and mixed chunk sizes
|
||||
- The ability to validate a **just-built binary** end-to-end (caching, coalescing, Range support, memory+disk tiers, GC/eviction, metrics, special endpoints, startup validation, etc.)
|
||||
|
||||
#### Quick Start
|
||||
|
||||
```bash
|
||||
# 1. Build the binary (this also runs short tests)
|
||||
make build
|
||||
|
||||
# 2. Start a validation-oriented instance (small caches so disk tier + GC get exercised)
|
||||
# Uses port 80 by default; the script will automatically set the needed
|
||||
# capability on the binary via sudo setcap if it is missing.
|
||||
./scripts/validate-with-prefill.sh
|
||||
|
||||
# 3. In another terminal (or on another machine), create a workload once if you haven't already,
|
||||
# then drive it through your local steamcache2.
|
||||
# Note: when using a non-80 port you may need to give SteamPrefill the full address.
|
||||
./scripts/validate-with-prefill.sh # (shows the exact commands with the correct port)
|
||||
```
|
||||
|
||||
When the benchmark finishes, press Ctrl-C in the first terminal to cleanly stop the server.
|
||||
|
||||
#### Simple validation server (recommended for manual testing)
|
||||
|
||||
For easy validation with external tools (SteamPrefill, etc.), use:
|
||||
|
||||
```bash
|
||||
make run-validation
|
||||
# or
|
||||
make validate
|
||||
```
|
||||
|
||||
This starts `steamcache2` on port 80 using a deliberately small memory + disk configuration (good for exercising the disk tier, GC, coalescing, promotions, etc.).
|
||||
|
||||
`make run-validation` (and `make validate`) will automatically ensure the `cap_net_bind_service` capability is set on the binary it just built (one sudo prompt the first time after each rebuild). This keeps the server running as your normal user so the disk cache directory stays owned by you.
|
||||
|
||||
If you want the capability on the binary for other workflows (e.g. `make run`, or running the binary directly on port 80), use the explicit target:
|
||||
|
||||
```bash
|
||||
make setcap
|
||||
```
|
||||
|
||||
When the server is running, point your external SteamPrefill (or other load generator) at it:
|
||||
|
||||
```bash
|
||||
./SteamPrefill benchmark run ...
|
||||
```
|
||||
|
||||
When finished, you can get a quick metrics summary with:
|
||||
|
||||
```bash
|
||||
make validate-check
|
||||
```
|
||||
|
||||
This is the recommended simple workflow. No automatic downloading or running of external tools.
|
||||
|
||||
#### Inspecting the Result
|
||||
|
||||
After a benchmark run you can ask for a quick report:
|
||||
|
||||
```bash
|
||||
make validate-check
|
||||
# or manually:
|
||||
curl -s http://localhost/metrics
|
||||
```
|
||||
|
||||
Look for:
|
||||
- High cache hit rate after the warmup pass
|
||||
- Non-zero `coalesced` and `disk` activity
|
||||
- Zero unexpected errors
|
||||
|
||||
#### The Validation Config
|
||||
|
||||
The script uses [docs/examples/validate-config.yaml](docs/examples/validate-config.yaml). It enables both memory and disk tiers at modest sizes (128 MB / 512 MB) with conservative concurrency. Edit or copy it if you need larger caches for bigger workloads.
|
||||
|
||||
#### What Gets Validated
|
||||
|
||||
Running a realistic SteamPrefill benchmark workload through a built steamcache2 exercises the complete public surface that matters for production use:
|
||||
- Steam User-Agent detection and depot/manifest/chunk URL patterns
|
||||
- Full MISS → cache write → HIT (and HIT-COALESCED) paths
|
||||
- Range request handling from cached full responses
|
||||
- Request coalescing under concurrent load
|
||||
- Memory tier + disk tier interaction (including async disk attach)
|
||||
- Garbage collection and eviction under pressure
|
||||
- Metrics and special endpoints (`/`, `/lancache-heartbeat`, `/metrics`)
|
||||
- Per-client and global rate limiting (with trusted proxy handling)
|
||||
- Startup configuration validation and upstream behavior
|
||||
- Clean shutdown hygiene
|
||||
|
||||
This is the closest practical equivalent to "run the thing real clients will run and make sure nothing is broken."
|
||||
|
||||
#### Troubleshooting
|
||||
|
||||
- **Low hit rate on first run**: Normal. The first `benchmark run` is the warmup that populates the cache.
|
||||
- **Want to test real disk I/O (not RAM cache)**: Make sure your workload size (shown by `benchmark setup`) is larger than the total RAM on the machine running steamcache2.
|
||||
- **Server won't start or bind on port 80 as non-root**: `make run-validation` and `make validate` automatically run `setcap` on the binary they just built. If it still fails, run `make setcap` explicitly and retry. The server always runs as your normal user (no root) so the disk cache directory ownership stays correct.
|
||||
- **SteamPrefill not found**: Install it yourself from its GitHub releases. Then use `make validate` to start the server with small caches and point SteamPrefill at it manually.
|
||||
- **SteamPrefill won't use server as cache properly**: SteamPrefill has some bad autodetectiong functions sometimes it works when the server is resolvable from localhost or 127.0.0.1 other times you have to fully override the dns for the proper dns name lancache.steamcontent.com to point to 127.0.0.1 i don't recommend doing it unless your okay with having to undo and redo it depending on if your running the server or not its a pain.
|
||||
|
||||
See also the SteamPrefill documentation for `benchmark setup` and `benchmark run` options.
|
||||
|
||||
### Command Line Flags
|
||||
|
||||
While most configuration is done via the YAML file, some runtime options are still available as command-line flags:
|
||||
|
||||
@@ -0,0 +1,51 @@
|
||||
# validate-config.yaml
|
||||
#
|
||||
# Small dual-tier configuration intended for full-function validation of a
|
||||
# built steamcache2 binary using realistic Steam client workloads driven by
|
||||
# the external SteamPrefill (https://github.com/tpill90/steam-lancache-prefill)
|
||||
# "benchmark" commands.
|
||||
#
|
||||
# Why these values?
|
||||
# - Both memory and disk tiers are enabled but deliberately small.
|
||||
# A modest benchmark workload (a few hundred MB to low GB) will exercise:
|
||||
# * Memory tier promotions
|
||||
# * Disk tier attach + writes
|
||||
# * GC / eviction pressure on at least one tier
|
||||
# - Conservative concurrency limits suitable for a developer laptop.
|
||||
# - trusted_proxies set for 127.0.0.0/8 so that an external benchmark tool
|
||||
# can simulate multiple distinct clients via X-Forwarded-For if desired.
|
||||
# - upstream left empty: the server will use the incoming Host header
|
||||
# (exactly what happens when you point SteamPrefill at your Lancache IP).
|
||||
#
|
||||
# Usage (typical dev workflow):
|
||||
# make build
|
||||
# ./scripts/validate-with-prefill.sh
|
||||
# # In another terminal:
|
||||
# SteamPrefill benchmark run -c 20 ...
|
||||
#
|
||||
# After the benchmark run, inspect with:
|
||||
# curl -s http://localhost/metrics
|
||||
#
|
||||
# Tweak sizes upward if you want to run very large workloads while still
|
||||
# exercising the disk tier (workload >> RAM is ideal for real disk testing).
|
||||
|
||||
listen_address: :80
|
||||
|
||||
max_concurrent_requests: 100
|
||||
max_requests_per_client: 10
|
||||
|
||||
max_object_size: "0" # unlimited for validation (real Steam files can be large)
|
||||
trusted_proxies: ["127.0.0.0/8"]
|
||||
|
||||
cache:
|
||||
memory:
|
||||
size: 128MB
|
||||
gc_algorithm: lru # lru is somewhat recommeded in the project README and is better for validation since its not just using the same again
|
||||
disk:
|
||||
size: 512MB
|
||||
path: ./validate-disk # ephemeral; clean between runs if you want a fresh test
|
||||
gc_algorithm: hybrid # recommended for disk in the project README
|
||||
|
||||
# Empty upstream = use Host header from the client (SteamPrefill / real Steam clients).
|
||||
# This matches the common "DNS points lancache.steamcontent.com at the cache" setup.
|
||||
upstream: ""
|
||||
Executable
+155
@@ -0,0 +1,155 @@
|
||||
#!/usr/bin/env bash
|
||||
#
|
||||
# download-prefill.sh
|
||||
#
|
||||
# Downloads the latest (or specific) release of SteamPrefill
|
||||
# (https://github.com/tpill90/steam-lancache-prefill) into
|
||||
# bin/steam-prefill/SteamPrefill
|
||||
#
|
||||
# Usage:
|
||||
# ./scripts/download-prefill.sh
|
||||
#
|
||||
# Environment:
|
||||
# PREFILL_VERSION - Pin a specific version tag (e.g. v3.4.2)
|
||||
# PREFILL_FORCE - Set to any non-empty value to re-download even if present
|
||||
#
|
||||
set -euo pipefail
|
||||
|
||||
DEST_DIR="bin/steam-prefill"
|
||||
TARGET="$DEST_DIR/SteamPrefill"
|
||||
|
||||
mkdir -p "$DEST_DIR"
|
||||
|
||||
if [[ -x "$TARGET" && -z "${PREFILL_FORCE:-}" ]]; then
|
||||
echo "SteamPrefill already present at $TARGET"
|
||||
echo "Run with PREFILL_FORCE=1 to re-download."
|
||||
exit 0
|
||||
fi
|
||||
|
||||
VERSION="${PREFILL_VERSION:-}"
|
||||
|
||||
OS=$(uname -s | tr '[:upper:]' '[:lower:]')
|
||||
ARCH=$(uname -m)
|
||||
|
||||
case "$ARCH" in
|
||||
x86_64|amd64) ARCH_NAME="x64" ;;
|
||||
aarch64|arm64) ARCH_NAME="arm64" ;;
|
||||
*) echo "Unsupported architecture: $ARCH"; exit 1 ;;
|
||||
esac
|
||||
|
||||
case "$OS" in
|
||||
linux) OS_NAME="linux" ;;
|
||||
darwin) OS_NAME="osx" ;;
|
||||
*) echo "Unsupported OS: $OS"; exit 1 ;;
|
||||
esac
|
||||
|
||||
echo "Resolving SteamPrefill version..."
|
||||
|
||||
if [[ -z "$VERSION" ]]; then
|
||||
# Follow the /latest redirect to discover the current tag
|
||||
LATEST_URL=$(curl -sIL -o /dev/null -w '%{url_effective}' \
|
||||
"https://github.com/tpill90/steam-lancache-prefill/releases/latest" 2>/dev/null || true)
|
||||
|
||||
if [[ "$LATEST_URL" =~ /tag/([^/?#]+) ]]; then
|
||||
VERSION="${BASH_REMATCH[1]}"
|
||||
else
|
||||
echo "Failed to resolve latest version from GitHub redirect."
|
||||
exit 1
|
||||
fi
|
||||
fi
|
||||
|
||||
echo "Downloading SteamPrefill $VERSION for ${OS_NAME}-${ARCH_NAME}..."
|
||||
|
||||
rm -f "$TARGET" "$TARGET.tmp" 2>/dev/null || true
|
||||
|
||||
DOWNLOADED=0
|
||||
|
||||
# Preferred: query the GitHub API for the exact asset list (most reliable)
|
||||
API_URL="https://api.github.com/repos/tpill90/steam-lancache-prefill/releases/tags/${VERSION}"
|
||||
ASSET_URL=""
|
||||
|
||||
if command -v jq >/dev/null 2>&1; then
|
||||
echo " Querying GitHub API for assets..."
|
||||
ASSET_NAME=$(curl -fsSL "$API_URL" 2>/dev/null | jq -r --arg os "$OS_NAME" --arg arch "$ARCH_NAME" '
|
||||
.assets[]
|
||||
| select(.name | ascii_downcase | contains($os))
|
||||
| select(.name | ascii_downcase | contains($arch))
|
||||
| .name
|
||||
' | head -1)
|
||||
|
||||
if [[ -n "$ASSET_NAME" ]]; then
|
||||
ASSET_URL="https://github.com/tpill90/steam-lancache-prefill/releases/download/${VERSION}/${ASSET_NAME}"
|
||||
echo " Found asset via API: $ASSET_NAME"
|
||||
fi
|
||||
fi
|
||||
|
||||
# Fallback: try common name patterns if API or jq not available
|
||||
if [[ -z "$ASSET_URL" ]]; then
|
||||
echo " Trying common asset name patterns..."
|
||||
CANDIDATES=(
|
||||
"SteamPrefill-${VERSION}-${OS_NAME}-${ARCH_NAME}.zip"
|
||||
"SteamPrefill-${VERSION}-${OS_NAME}-${ARCH_NAME}"
|
||||
"SteamPrefill-${OS_NAME}-${ARCH_NAME}.zip"
|
||||
"SteamPrefill-${OS_NAME}-${ARCH_NAME}"
|
||||
"SteamPrefill-linux-${ARCH_NAME}.zip"
|
||||
"SteamPrefill-linux-${ARCH_NAME}"
|
||||
)
|
||||
|
||||
for name in "${CANDIDATES[@]}"; do
|
||||
URL="https://github.com/tpill90/steam-lancache-prefill/releases/download/${VERSION}/${name}"
|
||||
echo " Trying $name ..."
|
||||
if curl -fI -s --retry 2 "$URL" >/dev/null 2>&1; then
|
||||
ASSET_URL="$URL"
|
||||
ASSET_NAME="$name"
|
||||
break
|
||||
fi
|
||||
done
|
||||
fi
|
||||
|
||||
if [[ -n "$ASSET_URL" ]]; then
|
||||
echo "Downloading $ASSET_NAME ..."
|
||||
if curl -fL --retry 3 --retry-delay 2 -A "Mozilla/5.0 (compatible; SteamPrefill-Downloader)" \
|
||||
--progress-bar -o "$TARGET.tmp" "$ASSET_URL"; then
|
||||
|
||||
echo "Download complete."
|
||||
|
||||
if [[ "$ASSET_NAME" == *.zip ]]; then
|
||||
echo "Extracting..."
|
||||
if ! command -v unzip >/dev/null 2>&1; then
|
||||
echo "Error: unzip is required for this release."
|
||||
rm -f "$TARGET.tmp"
|
||||
exit 1
|
||||
fi
|
||||
unzip -o -q "$TARGET.tmp" -d "$DEST_DIR"
|
||||
FOUND=$(find "$DEST_DIR" -type f -name "SteamPrefill" | head -1)
|
||||
if [[ -n "$FOUND" ]]; then
|
||||
mv "$FOUND" "$TARGET"
|
||||
fi
|
||||
rm -f "$TARGET.tmp"
|
||||
find "$DEST_DIR" -mindepth 1 -maxdepth 1 -type d -name "SteamPrefill*" -exec rm -rf {} + 2>/dev/null || true
|
||||
else
|
||||
mv "$TARGET.tmp" "$TARGET"
|
||||
fi
|
||||
|
||||
chmod +x "$TARGET"
|
||||
DOWNLOADED=1
|
||||
fi
|
||||
fi
|
||||
|
||||
if [[ $DOWNLOADED -eq 0 ]]; then
|
||||
echo ""
|
||||
echo "Failed to download a matching asset for $VERSION."
|
||||
echo "You can try pinning a different version:"
|
||||
echo " PREFILL_VERSION=vX.Y.Z ./scripts/download-prefill.sh"
|
||||
echo ""
|
||||
echo "Or download manually from:"
|
||||
echo " https://github.com/tpill90/steam-lancache-prefill/releases/tag/${VERSION}"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo ""
|
||||
echo "Installed SteamPrefill $VERSION → $TARGET"
|
||||
echo ""
|
||||
echo "You can now run it directly, for example:"
|
||||
echo " ./bin/steam-prefill/SteamPrefill --help"
|
||||
echo " ./bin/steam-prefill/SteamPrefill benchmark run ..."
|
||||
Executable
+182
@@ -0,0 +1,182 @@
|
||||
#!/usr/bin/env bash
|
||||
#
|
||||
# validate-with-prefill.sh
|
||||
#
|
||||
# Thin glue script to make it trivial for developers to validate complete
|
||||
# steamcache2 functionality using the external SteamPrefill (lancacheprefill)
|
||||
# tool as the realistic client simulator.
|
||||
#
|
||||
# Usage:
|
||||
# 1. make build
|
||||
# 2. ./scripts/validate-with-prefill.sh
|
||||
# (Automatically kills any leftover steamcache2 on the target port first.)
|
||||
# 3. In another terminal (or on another machine), run the printed
|
||||
# SteamPrefill benchmark commands (the script tells you the exact address/port).
|
||||
# 4. After the benchmark finishes, run the suggested metrics check.
|
||||
# 5. Ctrl-C here to cleanly stop the steamcache2 instance.
|
||||
#
|
||||
# This script + the accompanying validate-config.yaml + README docs are the
|
||||
# entire "couple little scripts to hook steamcache2 and lancacheprefill together"
|
||||
# implementation. No Go code, no new dependencies, stays outside go test / bench.
|
||||
#
|
||||
set -euo pipefail
|
||||
|
||||
# --- Locate the built steamcache2 binary (produced by "make build") ---
|
||||
BINARY=""
|
||||
for candidate in \
|
||||
"dist/default_linux_amd64_v1/steamcache2" \
|
||||
"dist/steamcache2" \
|
||||
"./steamcache2" \
|
||||
"steamcache2"
|
||||
do
|
||||
if [[ -x "$candidate" ]]; then
|
||||
BINARY="$candidate"
|
||||
break
|
||||
fi
|
||||
done
|
||||
|
||||
if [[ -z "$BINARY" ]]; then
|
||||
echo "ERROR: Could not find a built steamcache2 binary."
|
||||
echo "Run 'make build' first (or place the binary in one of the searched locations)."
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo "Using steamcache2 binary: $BINARY"
|
||||
|
||||
# --- Validation config (small dual-tier so disk + GC get real exercise) ---
|
||||
# Source of truth lives in docs/examples/ (safe from "make clean").
|
||||
# The script will also accept an explicit path via STEAMCACHE2_VALIDATE_CONFIG.
|
||||
VALIDATE_CONFIG="${STEAMCACHE2_VALIDATE_CONFIG:-docs/examples/validate-config.yaml}"
|
||||
if [[ ! -f "$VALIDATE_CONFIG" ]]; then
|
||||
echo "ERROR: Validation config not found at: $VALIDATE_CONFIG"
|
||||
echo "Set STEAMCACHE2_VALIDATE_CONFIG=/path/to/your-config.yaml or place a copy at docs/examples/validate-config.yaml"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Extract the listen port from the config (supports ":80", "127.0.0.1:80", etc.)
|
||||
# Falls back to 80 if we can't parse it.
|
||||
PORT=$(grep -E '^\s*listen_address:' "$VALIDATE_CONFIG" | head -1 | sed -E 's/.*:([0-9]+).*/\1/' || true)
|
||||
if [[ -z "$PORT" || ! "$PORT" =~ ^[0-9]+$ ]]; then
|
||||
PORT=80
|
||||
fi
|
||||
|
||||
SERVER_URL="http://localhost:${PORT}"
|
||||
|
||||
# For privileged ports (<1024, i.e. the default :80) we require the
|
||||
# cap_net_bind_service capability. We apply it automatically here (via sudo
|
||||
# setcap) on the binary we are about to run. This happens after any build
|
||||
# so the cap is never "lost" when the binary is rebuilt.
|
||||
if [ "$PORT" -lt 1024 ] && [ "$(id -u)" -ne 0 ]; then
|
||||
if ! command -v getcap >/dev/null 2>&1 || ! getcap "$BINARY" 2>/dev/null | grep -q "cap_net_bind_service"; then
|
||||
echo "Setting cap_net_bind_service on the binary (sudo may prompt)..."
|
||||
if ! sudo setcap 'cap_net_bind_service=+ep' "$BINARY"; then
|
||||
echo "ERROR: Failed to set capability."
|
||||
echo "Run 'make setcap' manually, then retry."
|
||||
exit 1
|
||||
fi
|
||||
fi
|
||||
fi
|
||||
|
||||
# Safely kill only steamcache2 processes listening on this specific port.
|
||||
# We look up PIDs on the port, check their actual process name/command,
|
||||
# and only kill via PID if it looks like steamcache2.
|
||||
echo "Checking for leftover steamcache2 processes on port ${PORT}..."
|
||||
|
||||
kill_steamcache_on_port() {
|
||||
local port=$1
|
||||
local pids=""
|
||||
|
||||
# Try ss first (modern, usually available)
|
||||
if command -v ss >/dev/null 2>&1; then
|
||||
pids=$(ss -tlnp 2>/dev/null | grep ":${port} " | sed -n 's/.*pid=\([0-9]*\).*/\1/p' | sort -u)
|
||||
fi
|
||||
|
||||
# Fallback to lsof
|
||||
if [[ -z "$pids" ]] && command -v lsof >/dev/null 2>&1; then
|
||||
pids=$(lsof -ti :${port} 2>/dev/null | sort -u)
|
||||
fi
|
||||
|
||||
if [[ -z "$pids" ]]; then
|
||||
return 0
|
||||
fi
|
||||
|
||||
for pid in $pids; do
|
||||
# Get process name and command line
|
||||
local proc_name
|
||||
local cmdline
|
||||
proc_name=$(ps -p "$pid" -o comm= 2>/dev/null || true)
|
||||
cmdline=$(ps -p "$pid" -o cmd= 2>/dev/null || true)
|
||||
|
||||
# Check if this looks like a steamcache2 process
|
||||
if echo "$proc_name $cmdline" | grep -qi "steamcache"; then
|
||||
echo " → Found steamcache2 on port ${port} (PID $pid, name: ${proc_name:-unknown})"
|
||||
kill -TERM "$pid" 2>/dev/null || true
|
||||
sleep 0.3
|
||||
# If still alive, force kill
|
||||
if kill -0 "$pid" 2>/dev/null; then
|
||||
kill -9 "$pid" 2>/dev/null || true
|
||||
fi
|
||||
echo " Killed PID $pid"
|
||||
else
|
||||
echo " → Skipping PID $pid on port ${port} (not steamcache2: ${proc_name:-$cmdline})"
|
||||
fi
|
||||
done
|
||||
}
|
||||
|
||||
kill_steamcache_on_port "$PORT"
|
||||
sleep 0.5
|
||||
|
||||
# --- Launch the server in the background ---
|
||||
echo "Starting steamcache2 with validation config (small caches for disk/GC testing)..."
|
||||
"$BINARY" --config "$VALIDATE_CONFIG" --log-level info &
|
||||
SERVER_PID=$!
|
||||
|
||||
# Ensure we always clean up the child on exit / Ctrl-C / error
|
||||
cleanup() {
|
||||
echo ""
|
||||
echo "Stopping steamcache2 (pid $SERVER_PID)..."
|
||||
if kill "$SERVER_PID" 2>/dev/null; then
|
||||
wait "$SERVER_PID" 2>/dev/null || true
|
||||
fi
|
||||
echo "Server stopped."
|
||||
}
|
||||
trap cleanup EXIT INT TERM
|
||||
|
||||
# Give the server a moment to bind and pass its own startup checks
|
||||
sleep 2
|
||||
|
||||
# Basic readiness probe using the actual configured port
|
||||
if ! curl -s --max-time 3 "${SERVER_URL}/" >/dev/null 2>&1; then
|
||||
echo "WARNING: Server did not respond quickly on ${SERVER_URL}/"
|
||||
echo " It may still be starting or bound to a different address."
|
||||
echo " Check the server logs above. You can still try the SteamPrefill commands."
|
||||
fi
|
||||
|
||||
if [[ "${VALIDATE_QUIET:-}" != "1" ]]; then
|
||||
echo ""
|
||||
echo "======================================================================"
|
||||
echo "steamcache2 is running (validation mode) on ${SERVER_URL}"
|
||||
echo ""
|
||||
echo "In another terminal (or on a machine that can reach this one), run:"
|
||||
echo ""
|
||||
echo " # One-time workload creation (run on a machine with SteamPrefill + Steam):"
|
||||
echo " SteamPrefill benchmark setup --preset LargeChunks"
|
||||
echo " # (or --use-selected, --all, --appid ..., or your own preset)"
|
||||
echo ""
|
||||
echo " # Copy the generated workload file to this machine if needed."
|
||||
echo ""
|
||||
echo " # Then run the actual benchmark (this is the realistic client simulator):"
|
||||
echo " SteamPrefill benchmark run -c 20 -i 3"
|
||||
echo ""
|
||||
echo "After the benchmark completes, you can inspect the cache with:"
|
||||
echo " curl -s ${SERVER_URL}/metrics | cat"
|
||||
echo ""
|
||||
echo "Or run: make validate-check (if the Makefile target exists)"
|
||||
echo ""
|
||||
echo "When you are finished, press Ctrl-C in this window to stop the server cleanly."
|
||||
echo "======================================================================"
|
||||
echo ""
|
||||
fi
|
||||
|
||||
# Wait for the background server (or for the user to Ctrl-C)
|
||||
wait $SERVER_PID || true
|
||||
Reference in New Issue
Block a user