Compare commits
45 Commits
c3464d692e
..
1.0.25
| Author | SHA1 | Date | |
|---|---|---|---|
| 8eb31143e5 | |||
| 337741e06e | |||
| 2c8ec276e7 | |||
| 144a68dcd7 | |||
| 8cebc1f96c | |||
| 12ea3ee4f6 | |||
| a500f51f17 | |||
| ac2d36f1ad | |||
| c43bfba568 | |||
| 3411defd51 | |||
| 71d5106777 | |||
| 8b1b229539 | |||
| ff1ab31327 | |||
| 85e14bc8af | |||
| 3f5175b482 | |||
| 04d1c6c368 | |||
| 523a9a4782 | |||
| 8e09c89e24 | |||
| a2ac13d317 | |||
| 5d006ac44f | |||
| acd006d4a2 | |||
| 30a695458e | |||
| e8bcf0ddbd | |||
| f497e71ef0 | |||
| 0198e8990b | |||
| 2a2cd8d393 | |||
| 81b3a7df53 | |||
| 8e8e877533 | |||
| d63d7b4d3c | |||
| 35d698a232 | |||
| 36b613b6cd | |||
| 79f02d9868 | |||
| 19497eba0c | |||
| c7a2312994 | |||
| fbb084d824 | |||
| e7d4a19c3f | |||
| 0c54ef3404 | |||
| 3d3c74fdb2 | |||
| 04f55535a5 | |||
| 05640bb549 | |||
| e4be82cddf | |||
| 60b2c3e514 | |||
| 099e5347d5 | |||
| b7e3a0da86 | |||
| 3fd72705fc |
@@ -7,6 +7,8 @@ on:
|
|||||||
jobs:
|
jobs:
|
||||||
release:
|
release:
|
||||||
runs-on: ubuntu-latest
|
runs-on: ubuntu-latest
|
||||||
|
permissions:
|
||||||
|
contents: write
|
||||||
steps:
|
steps:
|
||||||
- uses: actions/checkout@main
|
- uses: actions/checkout@main
|
||||||
with:
|
with:
|
||||||
@@ -21,4 +23,6 @@ jobs:
|
|||||||
version: 'latest'
|
version: 'latest'
|
||||||
args: release
|
args: release
|
||||||
env:
|
env:
|
||||||
GITEA_TOKEN: ${{secrets.RELEASE_TOKEN}}
|
GITEA_TOKEN: ${{ secrets.GITEA_TOKEN }}
|
||||||
|
GITHUB_TOKEN: ${{ secrets.GITEA_TOKEN }}
|
||||||
|
GORELEASER_FORCE_TOKEN: gitea
|
||||||
|
|||||||
@@ -1,24 +1,38 @@
|
|||||||
name: PR Check
|
name: CI
|
||||||
on:
|
on:
|
||||||
- pull_request
|
pull_request:
|
||||||
|
push:
|
||||||
|
branches:
|
||||||
|
- main
|
||||||
|
paths-ignore:
|
||||||
|
- '**.md'
|
||||||
|
- 'CONTRIBUTING.md'
|
||||||
|
|
||||||
jobs:
|
jobs:
|
||||||
check-and-test:
|
check-and-test:
|
||||||
runs-on: ubuntu-latest
|
runs-on: ubuntu-latest
|
||||||
steps:
|
steps:
|
||||||
- uses: actions/checkout@main
|
- uses: actions/checkout@v4
|
||||||
- uses: actions/setup-go@main
|
- uses: actions/setup-go@v5
|
||||||
with:
|
with:
|
||||||
go-version-file: 'go.mod'
|
go-version-file: 'go.mod'
|
||||||
- run: go mod tidy
|
- run: go mod tidy
|
||||||
- run: go build ./...
|
- run: go build ./...
|
||||||
- run: go vet ./...
|
- run: go vet ./...
|
||||||
- name: golangci-lint
|
- name: golangci-lint
|
||||||
uses: golangci/golangci-lint-action@v4
|
uses: golangci/golangci-lint-action@v8
|
||||||
with:
|
with:
|
||||||
version: latest
|
version: v2.13.2
|
||||||
args: --timeout=5m
|
args: --timeout=5m
|
||||||
|
- run: go test -race -v -shuffle=on -coverprofile=coverage.out -timeout=5m ./...
|
||||||
|
- run: go tool cover -func=coverage.out | tail -10 # basic coverage report
|
||||||
|
|
||||||
|
vulncheck:
|
||||||
|
runs-on: ubuntu-latest
|
||||||
|
steps:
|
||||||
|
- uses: actions/checkout@v4
|
||||||
|
- uses: actions/setup-go@v5
|
||||||
|
with:
|
||||||
|
go-version-file: 'go.mod'
|
||||||
- run: go install golang.org/x/vuln/cmd/govulncheck@latest
|
- run: go install golang.org/x/vuln/cmd/govulncheck@latest
|
||||||
- run: govulncheck ./...
|
- run: govulncheck ./...
|
||||||
- run: go test -race -v -shuffle=on -coverprofile=coverage.out -timeout=5m ./...
|
|
||||||
- run: go tool cover -func=coverage.out | tail -10 # basic coverage report (P2-04)
|
|
||||||
+9
-1
@@ -2,11 +2,19 @@
|
|||||||
/dist/
|
/dist/
|
||||||
/bin/
|
/bin/
|
||||||
steamcache2
|
steamcache2
|
||||||
|
|
||||||
|
# Downloaded SteamPrefill client simulator (auto-managed by make client)
|
||||||
|
/bin/steam-prefill/*
|
||||||
|
!/bin/steam-prefill/.gitkeep
|
||||||
/plans/
|
/plans/
|
||||||
|
|
||||||
#disk cache
|
#validation artifacts
|
||||||
|
/validate-disk/
|
||||||
/disk/
|
/disk/
|
||||||
|
|
||||||
|
#logs
|
||||||
|
*.log
|
||||||
|
|
||||||
#config file
|
#config file
|
||||||
/config.yaml
|
/config.yaml
|
||||||
|
|
||||||
|
|||||||
+33
-25
@@ -1,58 +1,55 @@
|
|||||||
# .golangci.yml - steamcache2 lint config
|
# .golangci.yml - steamcache2 lint config (golangci-lint v2)
|
||||||
# Philosophy: enable reasonable linters by default (golangci curated set + key additions)
|
# Philosophy: enable reasonable linters by default (golangci curated set + key additions)
|
||||||
# then use most specific suppressions possible (source //nosec with justification,
|
# then use most specific suppressions possible (source //nosec with justification,
|
||||||
# _ = discard for errcheck on unavoidable client writes, narrow exclude-rules only for tests).
|
# _ = discard for errcheck on unavoidable client writes, narrow exclude-rules only for tests).
|
||||||
# This makes remaining accepted issues visible and actionable in the code.
|
# This makes remaining accepted issues visible and actionable in the code.
|
||||||
# Run with: make lint (or golangci-lint run ./...)
|
# Run with: make lint (or golangci-lint run ./...)
|
||||||
# Install: go install github.com/golangci/golangci-lint/cmd/golangci-lint@latest
|
version: "2"
|
||||||
|
|
||||||
run:
|
run:
|
||||||
timeout: 5m
|
timeout: 5m
|
||||||
modules-download-mode: readonly
|
modules-download-mode: readonly
|
||||||
|
|
||||||
linters:
|
linters:
|
||||||
# No disable-all: use golangci defaults (errcheck, govet, ineffassign, staticcheck, unused, gosimple, etc.)
|
# No default: none — use golangci defaults (errcheck, govet, ineffassign, staticcheck, unused, etc.)
|
||||||
# Explicitly enable the non-default linters we require for this LAN cache proxy.
|
# Explicitly enable the non-default linters we require for this LAN cache proxy.
|
||||||
enable:
|
enable:
|
||||||
- gosec # security checks (re-audited; see source //nosec for justified cases)
|
- gosec # security checks (re-audited; see source //nosec for justified cases)
|
||||||
- misspell # documentation hygiene
|
- misspell # documentation hygiene
|
||||||
- goimports # import formatting (enforced)
|
settings:
|
||||||
# gofmt covered via linter or goimports; errcheck/govet etc. from defaults
|
|
||||||
|
|
||||||
linters-settings:
|
|
||||||
errcheck:
|
errcheck:
|
||||||
check-type-assertions: false
|
check-type-assertions: false
|
||||||
check-blank: false
|
check-blank: false
|
||||||
|
# gosec: keep source-level //nosec for G104/G115/G301/G304/G306.
|
||||||
|
# G704/G705 are new taint-analysis rules (SSRF/XSS) not present in v1.64.8;
|
||||||
|
# a CDN cache proxy forwards upstream URLs and response bodies by design.
|
||||||
gosec:
|
gosec:
|
||||||
# Broad global excludes removed (G104/G115/G301/G304/G306).
|
excludes:
|
||||||
# - G301 addressed by switching cache MkdirAll to 0700 (least privilege for CDN content).
|
- G704
|
||||||
# - Remaining justified cases documented with precise //nosec (or #nosec) + comments at the call sites.
|
- G705
|
||||||
# - G104 largely eliminated by errcheck + explicit _ = handling (or defer wrappers).
|
# v1 staticcheck checks: ["all"] meant SA* only. v2 merged stylecheck (ST*)
|
||||||
|
# and quickfix (QF*) into staticcheck; keep the previous SA*+gosimple set.
|
||||||
staticcheck:
|
staticcheck:
|
||||||
checks: ["all"] # SA1019 exclusion removed (no deprecated API usages in tree)
|
checks:
|
||||||
|
- all
|
||||||
|
- "-ST*"
|
||||||
|
- "-QF*"
|
||||||
govet:
|
govet:
|
||||||
enable-all: true
|
enable-all: true
|
||||||
disable:
|
disable:
|
||||||
- fieldalignment # performance tuning not a priority for this proxy appliance
|
- fieldalignment # performance tuning not a priority for this proxy appliance
|
||||||
- shadow # common idiomatic "err" redeclarations in error-handling chains (large ServeHTTP, root, parse funcs); enabling adds noise with no real bugs; would require scope refactor for little gain
|
- shadow # common idiomatic "err" redeclarations in error-handling chains (large ServeHTTP, root, parse funcs); enabling adds noise with no real bugs; would require scope refactor for little gain
|
||||||
|
exclusions:
|
||||||
# Old global errcheck disable + aspirational "re-enable after refactors" comments deleted.
|
generated: lax
|
||||||
# errcheck is now on via defaults. Unavoidable cases handled at source with _ = or (rarely) narrow rules.
|
paths:
|
||||||
|
|
||||||
issues:
|
|
||||||
max-issues-per-linter: 0
|
|
||||||
max-same-issues: 0
|
|
||||||
exclude-use-default: false
|
|
||||||
exclude-dirs:
|
|
||||||
- dist
|
- dist
|
||||||
- bin
|
- bin
|
||||||
exclude-rules:
|
rules:
|
||||||
- path: _test\.go
|
- path: _test\.go
|
||||||
linters:
|
linters:
|
||||||
- errcheck
|
- errcheck
|
||||||
- gosec # tests often use weak patterns intentionally (e.g. error injection, temp files)
|
- gosec # tests often use weak patterns intentionally (e.g. error injection, temp files)
|
||||||
# NOTE: narrow SA9003 exclude retained only for the one remaining intentional empty branch in test (best-effort status check; main assert is metrics side-effect).
|
# NOTE: narrow SA9003 exclude retained only for the one remaining intentional empty branch in test (best-effort status check; main assert is metrics side-effect).
|
||||||
# The config one was a truly redundant check (already errored above); deleted surgically in Fix Round 1 (Issue 1), eliminating its exclude-rule.
|
|
||||||
- path: steamcache/steamcache_test.go
|
- path: steamcache/steamcache_test.go
|
||||||
linters:
|
linters:
|
||||||
- staticcheck
|
- staticcheck
|
||||||
@@ -82,5 +79,16 @@ issues:
|
|||||||
linters:
|
linters:
|
||||||
- gosec
|
- gosec
|
||||||
text: "G304"
|
text: "G304"
|
||||||
# Predictive/* rules deleted: vfs/predictive/ removed in commit 0dbb2e0; rules were stale/dead.
|
|
||||||
# All other suppressions use source-level //nosec (gosec) or _= (errcheck) for precision and visibility.
|
formatters:
|
||||||
|
enable:
|
||||||
|
- goimports
|
||||||
|
exclusions:
|
||||||
|
generated: lax
|
||||||
|
paths:
|
||||||
|
- dist
|
||||||
|
- bin
|
||||||
|
|
||||||
|
issues:
|
||||||
|
max-issues-per-linter: 0
|
||||||
|
max-same-issues: 0
|
||||||
|
|||||||
@@ -0,0 +1,40 @@
|
|||||||
|
# Contributing
|
||||||
|
|
||||||
|
## Propose changes
|
||||||
|
|
||||||
|
Open a pull request against `develop`. Keep the default branch for releases and
|
||||||
|
stable tips; land work on `develop` first.
|
||||||
|
|
||||||
|
Point at an existing issue when one fits. Prefer a short issue that states the
|
||||||
|
symptom or request before a large PR.
|
||||||
|
|
||||||
|
## Commits
|
||||||
|
|
||||||
|
Subject form:
|
||||||
|
|
||||||
|
```
|
||||||
|
area: Imperative summary
|
||||||
|
```
|
||||||
|
|
||||||
|
- **Area** is a real package, directory, or subsystem token (`ci:`, `docs:`,
|
||||||
|
Go package name). Not a lone filename.
|
||||||
|
- **Imperative** mood: Fix, Add, Remove — not "Fixed" or "This patch…".
|
||||||
|
- No trailing period. Aim ≤ ~70–75 characters for the whole subject.
|
||||||
|
- Not conventional-commits (`feat:` / `fix:` / `chore:` as types).
|
||||||
|
|
||||||
|
Body explains **why**. Establish the problem, then say what you are doing.
|
||||||
|
One logical change per commit; split fix and cleanup.
|
||||||
|
|
||||||
|
## Pull requests
|
||||||
|
|
||||||
|
Title matches the primary commit subject.
|
||||||
|
|
||||||
|
- **What** changed
|
||||||
|
- **Why** (problem and impact)
|
||||||
|
- **Test** (concrete steps; "CI green" alone is weak)
|
||||||
|
|
||||||
|
## Issues and closing
|
||||||
|
|
||||||
|
Cite leftover issues by **full URL**. Gitea closes issues when `#N` appears in
|
||||||
|
merge text, so do not put `#N` in the merge message unless that issue is actually
|
||||||
|
done. Use `Fixes #N` / `Closes #N` only when the leftover work is finished.
|
||||||
@@ -4,21 +4,21 @@ run: ## Run the application (cross-platform; uses go run for dev on Linux/macOS/
|
|||||||
run-debug: ## Run the application with debug logging (cross-platform)
|
run-debug: ## Run the application with debug logging (cross-platform)
|
||||||
@go run . --log-level debug
|
@go run . --log-level debug
|
||||||
|
|
||||||
build: deps ## Build a snapshot of the application for the current platform (uses -short for fast feedback)
|
build: deps lint ## Build a snapshot of the application for the current platform (uses -short for fast feedback)
|
||||||
@go test -short -v ./...
|
@go test -short -v ./...
|
||||||
@goreleaser build --single-target --snapshot --clean
|
@goreleaser build --single-target --snapshot --clean
|
||||||
|
|
||||||
test: deps ## Run all tests
|
test: deps lint ## Run all tests
|
||||||
@go test -shuffle=on -timeout=5m -v ./...
|
@go test -shuffle=on -timeout=5m -v ./...
|
||||||
|
|
||||||
test-race: deps ## Run all tests with the race detector
|
test-race: deps lint ## Run all tests with the race detector
|
||||||
@go test -race -shuffle=on -timeout=5m -v ./...
|
@go test -race -shuffle=on -timeout=5m -v ./...
|
||||||
|
|
||||||
lint: deps check-review-labels ## Run golangci-lint + review label hygiene check
|
lint: deps check-review-labels ## Run golangci-lint + review label hygiene check
|
||||||
@golangci-lint run ./...
|
@golangci-lint run ./...
|
||||||
|
|
||||||
check-review-labels: ## Fail if temporary review labels (P0-01, T1, I3, R2, etc.) are found in source
|
check-review-labels: ## Fail if temporary review labels (P0-01, T1, I3, R2, etc.) are found in source
|
||||||
@! grep -rnE '\b[A-Z][0-9][^a-zA-Z]' --include='*.go' . 2>/dev/null | grep -v 'G[0-9]\{3\}' || (echo "Error: Found temporary review labels (P*, T*, I*, etc.) in source. See AGENTS.md for the rule." && exit 1)
|
@! grep -rnE '\b[A-Z][0-9][^a-zA-Z]' --include='*.go' --include='*.md' --include='*.yaml' --include='*.sh' --exclude='AGENTS.md' . 2>/dev/null | grep -v 'G[0-9]\{3\}' || (echo "Error: Found temporary review labels (P*, T*, I*, etc.) in source. See AGENTS.md for the rule." && exit 1)
|
||||||
|
|
||||||
deps: ## Download dependencies
|
deps: ## Download dependencies
|
||||||
@go mod tidy
|
@go mod tidy
|
||||||
@@ -26,6 +26,9 @@ deps: ## Download dependencies
|
|||||||
clean: ## Remove build artifacts and test cache
|
clean: ## Remove build artifacts and test cache
|
||||||
@rm -rf bin/ dist/ *.test coverage.out steamcache2
|
@rm -rf bin/ dist/ *.test coverage.out steamcache2
|
||||||
|
|
||||||
|
clean-disk: ## Remove disk cache
|
||||||
|
@rm -rf validate-disk/
|
||||||
|
|
||||||
bench: deps ## Run all benchmarks (MemoryFS + DiskFS variants, including all eviction strategies)
|
bench: deps ## Run all benchmarks (MemoryFS + DiskFS variants, including all eviction strategies)
|
||||||
@echo "Running MemoryFS benchmarks..."
|
@echo "Running MemoryFS benchmarks..."
|
||||||
@go test -bench=. -benchmem -run=^$ -benchtime=1s ./vfs/memory
|
@go test -bench=. -benchmem -run=^$ -benchtime=1s ./vfs/memory
|
||||||
@@ -33,15 +36,104 @@ bench: deps ## Run all benchmarks (MemoryFS + DiskFS variants, including all evi
|
|||||||
@go test -bench=. -benchmem -run=^$ -benchtime=1s ./vfs/disk
|
@go test -bench=. -benchmem -run=^$ -benchtime=1s ./vfs/disk
|
||||||
@echo "Bench done."
|
@echo "Bench done."
|
||||||
|
|
||||||
|
setcap: build ## Explicitly set cap_net_bind_service on the (just-built) binary for port 80 use outside validate targets
|
||||||
|
@echo "Setting cap_net_bind_service on the binary so it can listen on port 80 as your normal user..."
|
||||||
|
@sudo setcap 'cap_net_bind_service=+ep' dist/default_linux_amd64_v1/steamcache2
|
||||||
|
@echo "Done. You should now be able to run 'make run-validation' as your normal user (no root)."
|
||||||
|
|
||||||
|
validate run-validation: build clean-disk ## Start steamcache2 on :80 with small test caches (foreground)
|
||||||
|
@echo "=== Starting steamcache2 in validation mode ==="
|
||||||
|
@echo "Port 80 + small memory/disk caches (for exercising disk tier, GC, etc.)"
|
||||||
|
@echo "Press Ctrl-C to stop the server."
|
||||||
|
@echo ""
|
||||||
|
@BINARY=dist/default_linux_amd64_v1/steamcache2; \
|
||||||
|
if [ "$$(id -u)" -ne 0 ] && ! getcap "$$BINARY" 2>/dev/null | grep -q cap_net_bind_service; then \
|
||||||
|
echo "Setting cap_net_bind_service on the freshly built binary (sudo may prompt)..."; \
|
||||||
|
sudo setcap 'cap_net_bind_service=+ep' "$$BINARY" || { \
|
||||||
|
echo "ERROR: setcap failed (or was cancelled)."; \
|
||||||
|
echo "You can run 'make setcap' manually, then retry 'make validate'."; \
|
||||||
|
exit 1; \
|
||||||
|
}; \
|
||||||
|
fi; \
|
||||||
|
if [ "$$(id -u)" -ne 0 ] && ! getcap "$$BINARY" 2>/dev/null | grep -q cap_net_bind_service; then \
|
||||||
|
echo "ERROR: Port 80 still requires the capability after setcap attempt."; \
|
||||||
|
echo "Run 'make setcap' and retry."; \
|
||||||
|
exit 1; \
|
||||||
|
fi; \
|
||||||
|
exec "$$BINARY" --config docs/examples/validate-config.yaml --log-level info
|
||||||
|
|
||||||
|
validate-check: ## Curl local /metrics (full dump + hit/miss + upstream/write/rate fields) and /lancache-heartbeat (default :80)
|
||||||
|
@echo "=== http://localhost/metrics ==="
|
||||||
|
@metrics=$$(curl -sf --max-time 5 http://localhost/metrics) || { \
|
||||||
|
echo "ERROR: could not fetch http://localhost/metrics"; \
|
||||||
|
echo "Is steamcache2 running on the default listen address :80?"; \
|
||||||
|
exit 1; \
|
||||||
|
}; \
|
||||||
|
printf '%s\n' "$$metrics"; \
|
||||||
|
echo ""; \
|
||||||
|
echo "=== hit/miss + upstream/write/rate fields ==="; \
|
||||||
|
printf '%s\n' "$$metrics" | grep -E '^(total_requests|cache_hits|cache_misses|hit_rate|memory_cache_hits|disk_cache_hits|errors|upstream_errors|cache_write_failures|rate_limited) ' || true; \
|
||||||
|
echo ""; \
|
||||||
|
echo "=== http://localhost/lancache-heartbeat (GET; expect 204 + X-LanCache-Processed-By: SteamCache2) ==="; \
|
||||||
|
hb=$$(curl -sD - -o /dev/null --max-time 5 http://localhost/lancache-heartbeat) || { \
|
||||||
|
echo "ERROR: could not fetch http://localhost/lancache-heartbeat"; \
|
||||||
|
echo "Is steamcache2 running on the default listen address :80?"; \
|
||||||
|
exit 1; \
|
||||||
|
}; \
|
||||||
|
printf '%s\n' "$$hb"; \
|
||||||
|
echo "$$hb" | grep -q '204' && echo "$$hb" | grep -qi 'X-LanCache-Processed-By' || { \
|
||||||
|
echo "ERROR: expected HTTP 204 and X-LanCache-Processed-By on /lancache-heartbeat"; \
|
||||||
|
exit 1; \
|
||||||
|
}
|
||||||
|
|
||||||
|
validate-kill: ## Kill leftover steamcache2 processes (safer, checks process name)
|
||||||
|
@echo "Looking for steamcache2 processes on common validation ports (80 is primary)..."
|
||||||
|
@for port in 80 8040 8080; do \
|
||||||
|
pids=""; \
|
||||||
|
if command -v ss >/dev/null 2>&1; then \
|
||||||
|
pids=$$(ss -tlnp 2>/dev/null | grep ":$${port} " | sed -n 's/.*pid=\([0-9]*\).*/\1/p' | sort -u); \
|
||||||
|
fi; \
|
||||||
|
if [ -z "$$pids" ] && command -v lsof >/dev/null 2>&1; then \
|
||||||
|
pids=$$(lsof -ti :$${port} 2>/dev/null | sort -u); \
|
||||||
|
fi; \
|
||||||
|
for pid in $$pids; do \
|
||||||
|
proc=$$(ps -p $$pid -o comm= 2>/dev/null || true); \
|
||||||
|
cmd=$$(ps -p $$pid -o cmd= 2>/dev/null || true); \
|
||||||
|
if echo "$$proc $$cmd" | grep -qi "steamcache"; then \
|
||||||
|
echo " Killing steamcache2 (port $$port, PID $$pid, $$proc)"; \
|
||||||
|
kill -TERM $$pid 2>/dev/null || true; \
|
||||||
|
sleep 0.2; \
|
||||||
|
kill -0 $$pid 2>/dev/null && kill -9 $$pid 2>/dev/null || true; \
|
||||||
|
else \
|
||||||
|
echo " Skipping PID $$pid on port $$port (not steamcache2: $$proc)"; \
|
||||||
|
fi; \
|
||||||
|
done; \
|
||||||
|
done
|
||||||
|
@echo "Validation server cleanup complete."
|
||||||
|
|
||||||
|
prefill: ## Download latest SteamPrefill into bin/steam-prefill/SteamPrefill (gitignored)
|
||||||
|
@./scripts/download-prefill.sh
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
help: ## Show this help message
|
help: ## Show this help message
|
||||||
@echo steamcache2 Makefile
|
@echo "steamcache2 Makefile"
|
||||||
@echo Available targets:
|
@echo "Available targets:"
|
||||||
@echo run Run the application (cross-platform via go run)
|
@echo " run Run the application (cross-platform via go run)"
|
||||||
@echo run-debug Run the application with debug logging (cross-platform)
|
@echo " run-debug Run the application with debug logging (cross-platform)"
|
||||||
@echo build Build the application (goreleaser snapshot)
|
@echo " build Build the application (goreleaser snapshot)"
|
||||||
@echo test Run all tests
|
@echo " test Run all tests"
|
||||||
@echo test-race Run all tests with the race detector
|
@echo " test-race Run all tests with the race detector"
|
||||||
@echo lint Run golangci-lint + review label check
|
@echo " lint Run golangci-lint + review label check"
|
||||||
@echo check-review-labels Fail on temporary review labels (P*, T*, I*, R*, etc.)
|
@echo " check-review-labels Fail on temporary review labels (P*, T*, I*, R*, etc.)"
|
||||||
@echo deps Download dependencies
|
@echo " deps Download dependencies"
|
||||||
@echo clean Remove build/test artifacts
|
@echo " clean Remove build/test artifacts"
|
||||||
|
@echo " clean-disk Remove disk cache"
|
||||||
|
@echo " bench Run low-level VFS microbenchmarks"
|
||||||
|
@echo " validate / run-validation Start server on :80 (builds, auto-setcaps fresh binary, then runs as normal user, cleans disk cache first)"
|
||||||
|
@echo " validate-check Curl local /metrics (full dump + hit/miss + upstream/write/rate fields) and /lancache-heartbeat (default :80)"
|
||||||
|
@echo " setcap Explicitly set cap on current build (for port 80 use outside validate)"
|
||||||
|
@echo " validate-kill Kill leftover steamcache2 processes (safer)"
|
||||||
|
@echo " prefill Download latest SteamPrefill into bin/steam-prefill/SteamPrefill (gitignored)"
|
||||||
|
|||||||
@@ -10,8 +10,8 @@ SteamCache2 is a blazing fast download cache for Steam, designed to reduce bandw
|
|||||||
- Reduces bandwidth usage
|
- Reduces bandwidth usage
|
||||||
- Easy to set up and configure aside from dns stuff to trick Steam into using it
|
- Easy to set up and configure aside from dns stuff to trick Steam into using it
|
||||||
- Supports multiple clients
|
- Supports multiple clients
|
||||||
- **NEW:** YAML configuration system with automatic config generation
|
- YAML configuration with automatic config generation on first run
|
||||||
- **NEW:** Simple Makefile for development workflow
|
- Makefile for development and validation workflows
|
||||||
- Cross-platform builds (Linux, macOS, Windows)
|
- Cross-platform builds (Linux, macOS, Windows)
|
||||||
|
|
||||||
## Quick Start
|
## Quick Start
|
||||||
@@ -34,7 +34,10 @@ SteamCache2 is a blazing fast download cache for Steam, designed to reduce bandw
|
|||||||
|
|
||||||
The application will automatically create a `config.yaml` file with default settings and exit, allowing you to customize it.
|
The application will automatically create a `config.yaml` file with default settings and exit, allowing you to customize it.
|
||||||
|
|
||||||
3. **Edit the configuration** (`config.yaml`):
|
3. **Edit the configuration** (`config.yaml`) for a real Steam front-door:
|
||||||
|
|
||||||
|
Leave `upstream` empty. With an empty upstream, steamcache2 fetches from the request `Host` (the same pattern SteamPrefill / real Steam clients use when DNS points them at your cache). Do **not** paste a fake host like `https://steam.cdn.com` — that is not a Steam CDN and will not put you in front of Steam.
|
||||||
|
|
||||||
```yaml
|
```yaml
|
||||||
listen_address: :80
|
listen_address: :80
|
||||||
cache:
|
cache:
|
||||||
@@ -45,14 +48,65 @@ SteamCache2 is a blazing fast download cache for Steam, designed to reduce bandw
|
|||||||
size: 10GB
|
size: 10GB
|
||||||
path: ./disk
|
path: ./disk
|
||||||
gc_algorithm: hybrid
|
gc_algorithm: hybrid
|
||||||
upstream: "https://steam.cdn.com" # Set your upstream server
|
# Empty upstream = use the client Host as the origin (Steam CDN names only).
|
||||||
|
upstream: ""
|
||||||
```
|
```
|
||||||
|
|
||||||
4. **Run the application again:**
|
4. **Point Steam (or SteamPrefill) at this cache** before you expect hits:
|
||||||
|
|
||||||
|
- **LAN DNS:** resolve `lancache.steamcontent.com` (and other Steam content names your clients use) to this server's LAN IP.
|
||||||
|
- **Single Windows PC:** add a hosts override — see [Windows Hosts File Override](#windows-hosts-file-override) (`<cache-ip> lancache.steamcontent.com`).
|
||||||
|
- Restart Steam (or your prefill tool) after DNS/hosts changes.
|
||||||
|
|
||||||
|
Empty-upstream direct fetch only allows Steam CDN host suffixes (`steamcontent.com`, `steampowered.com`, `steamstatic.com`). Literal IPs and unrelated hosts are rejected.
|
||||||
|
|
||||||
|
5. **Run the application again:**
|
||||||
```bash
|
```bash
|
||||||
make run # or ./steamcache2
|
make run # or ./steamcache2
|
||||||
```
|
```
|
||||||
|
|
||||||
|
### Quick check: is it caching?
|
||||||
|
|
||||||
|
After steamcache2 is running (default `listen_address: :80`) and has seen a little Steam traffic — a game download, a short SteamPrefill pass, or any cacheable request — confirm hits vs misses from the existing endpoints. You do not need a full benchmark or log diving.
|
||||||
|
|
||||||
|
```bash
|
||||||
|
make validate-check
|
||||||
|
# or, manually:
|
||||||
|
curl -s http://localhost/metrics
|
||||||
|
curl -s -i http://localhost/lancache-heartbeat
|
||||||
|
```
|
||||||
|
|
||||||
|
`make validate-check` prints the full `/metrics` dump, highlights hit/miss plus `upstream_errors` / `cache_write_failures` / `rate_limited`, and curls `/lancache-heartbeat`. Read these fields:
|
||||||
|
|
||||||
|
| Field | Meaning |
|
||||||
|
| --- | --- |
|
||||||
|
| `cache_hits` / `cache_misses` / `hit_rate` | Whether later requests were served from cache |
|
||||||
|
| `range_cache` / `range_upstream` | Range GETs served as 206 from a cached object vs after a full upstream fetch |
|
||||||
|
| `memory_cache_hits` / `disk_cache_hits` | Which tier served the hits |
|
||||||
|
| `total_requests` / `errors` | Volume and failures |
|
||||||
|
| `upstream_errors` / `cache_write_failures` / `rate_limited` | Upstream pipe, cache write, and rate-limit pressure (Quick check highlights these next to hit/miss) |
|
||||||
|
| `disk_tier_ready` | `0` while disk slow-tier attach pending; `1` when attached, or when no disk configured (N/A — not waiting) |
|
||||||
|
|
||||||
|
A first pass through new content is mostly misses (`hit_rate` near 0). Repeat the same content and `cache_hits` / `hit_rate` should rise.
|
||||||
|
|
||||||
|
Cache entries are keyed by depot object path (not the CDN `Host` header), so when Steam rotates CDN hostnames for the same depot path, hits still climb across the aliases.
|
||||||
|
|
||||||
|
Steam clients lean on Range requests. When an object is already cached, a Range GET is served locally as 206 from that full object (`range_cache`). On a Range miss the cache still fetches and stores the full upstream body, then returns the requested byte range as 206 (`range_upstream`).
|
||||||
|
|
||||||
|
To confirm the process is up (HTTP 204 and `X-LanCache-Processed-By: SteamCache2`):
|
||||||
|
|
||||||
|
```bash
|
||||||
|
curl -s -i http://localhost/lancache-heartbeat
|
||||||
|
```
|
||||||
|
|
||||||
|
Use GET (`curl -i`), not HEAD (`curl -I`): the server only accepts GET.
|
||||||
|
|
||||||
|
Heartbeat also returns `X-SteamCache-Disk-Tier: pending|ready|disabled` (`disabled` = memory-only / no disk; `pending`/`ready` = disk configured attach state).
|
||||||
|
|
||||||
|
These are the cache process's own `/metrics` and `/lancache-heartbeat` endpoints. There is no separate metrics daemon.
|
||||||
|
|
||||||
|
If you changed `listen_address`, point curl at that host:port instead. For a full SteamPrefill validation workflow (small caches, coalescing, GC), see [Validating Full Functionality](#validating-full-functionality-with-external-tools).
|
||||||
|
|
||||||
### Development Workflow
|
### Development Workflow
|
||||||
|
|
||||||
Use `make` for the majority of common development tasks. The Makefile handles running tests, linting, hygiene checks, building, running the application, and other routine boilerplate work.
|
Use `make` for the majority of common development tasks. The Makefile handles running tests, linting, hygiene checks, building, running the application, and other routine boilerplate work.
|
||||||
@@ -61,6 +115,100 @@ Run `make help` to see the full list of available commands.
|
|||||||
|
|
||||||
This is the preferred approach for day-to-day development. Avoid running raw `go test`, `go run`, or `golangci-lint` commands directly for routine tasks.
|
This is the preferred approach for day-to-day development. Avoid running raw `go test`, `go run`, or `golangci-lint` commands directly for routine tasks.
|
||||||
|
|
||||||
|
### Validating Full Functionality with external tools
|
||||||
|
|
||||||
|
steamcache2 provides a convenient small-cache configuration and helper targets so you can easily validate behavior using external tools such as [SteamPrefill (tpill90/steam-lancache-prefill)](https://github.com/tpill90/steam-lancache-prefill).
|
||||||
|
|
||||||
|
This gives you:
|
||||||
|
- Real Steam manifest + chunk traffic (no reinventing the wheel)
|
||||||
|
- Excellent `benchmark setup` / `benchmark run` workflow with warmup, randomization, and mixed chunk sizes
|
||||||
|
- The ability to validate a **just-built binary** end-to-end (caching, coalescing, Range support, memory+disk tiers, GC/eviction, metrics, special endpoints, startup validation, etc.)
|
||||||
|
|
||||||
|
#### Validation server (recommended)
|
||||||
|
|
||||||
|
For easy validation with external tools (SteamPrefill, etc.), use:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
make run-validation
|
||||||
|
# or
|
||||||
|
make validate
|
||||||
|
```
|
||||||
|
|
||||||
|
This starts `steamcache2` on port 80 using a deliberately small memory + disk configuration (good for exercising the disk tier, GC, coalescing, promotions, etc.).
|
||||||
|
|
||||||
|
`make run-validation` (and `make validate`) will automatically ensure the `cap_net_bind_service` capability is set on the binary it just built (one sudo prompt the first time after each rebuild). This keeps the server running as your normal user so the disk cache directory stays owned by you.
|
||||||
|
|
||||||
|
If you want the capability on the binary for other workflows (e.g. `make run`, or running the binary directly on port 80), use the explicit target:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
make setcap
|
||||||
|
```
|
||||||
|
|
||||||
|
When the server is running, point your external SteamPrefill (or other load generator) at it:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
./SteamPrefill benchmark run ...
|
||||||
|
```
|
||||||
|
|
||||||
|
When finished, you can get a quick metrics + heartbeat report with:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
make validate-check
|
||||||
|
# or, manually:
|
||||||
|
curl -s http://localhost/metrics
|
||||||
|
curl -s -i http://localhost/lancache-heartbeat
|
||||||
|
```
|
||||||
|
|
||||||
|
See [Quick check: is it caching?](#quick-check-is-it-caching) for which fields to read. This is the recommended simple workflow. No automatic downloading or running of external tools.
|
||||||
|
|
||||||
|
#### Inspecting the Result
|
||||||
|
|
||||||
|
After a benchmark run you can ask for a quick report:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
make validate-check
|
||||||
|
# or, manually:
|
||||||
|
curl -s http://localhost/metrics
|
||||||
|
curl -s -i http://localhost/lancache-heartbeat
|
||||||
|
```
|
||||||
|
|
||||||
|
`make validate-check` prints the full `/metrics` dump, highlights hit/miss fields, and curls `/lancache-heartbeat`. Look for:
|
||||||
|
- High cache hit rate after the warmup pass (`cache_hits`, `hit_rate`, plus `memory_cache_hits` / `disk_cache_hits`)
|
||||||
|
- Non-zero `coalesced` and `disk` activity
|
||||||
|
- Zero unexpected `errors`, and quiet `upstream_errors` / `cache_write_failures` / `rate_limited`
|
||||||
|
|
||||||
|
Heartbeat should be HTTP 204 with `X-LanCache-Processed-By: SteamCache2`. Use GET (`curl -i`), not HEAD (`curl -I`).
|
||||||
|
|
||||||
|
#### The Validation Config
|
||||||
|
|
||||||
|
The recommended validation config is at [docs/examples/validate-config.yaml](docs/examples/validate-config.yaml). It enables both memory and disk tiers at modest sizes (128 MB / 512 MB) with conservative concurrency. Edit or copy it if you need larger caches for bigger workloads.
|
||||||
|
|
||||||
|
#### What Gets Validated
|
||||||
|
|
||||||
|
Running a realistic SteamPrefill benchmark workload through a built steamcache2 exercises the complete public surface that matters for production use:
|
||||||
|
- Steam User-Agent detection and depot/manifest/chunk URL patterns
|
||||||
|
- Full MISS → cache write → HIT (and HIT-COALESCED) paths
|
||||||
|
- Range request handling from cached full responses (local 206 on HIT via `range_cache`; MISS fetches the full object then serves the requested slice as 206 via `range_upstream`)
|
||||||
|
- Request coalescing under concurrent load
|
||||||
|
- Memory tier + disk tier interaction (including async disk attach)
|
||||||
|
- Garbage collection and eviction under pressure
|
||||||
|
- Metrics and special endpoints (`/`, `/lancache-heartbeat`, `/metrics`)
|
||||||
|
- Per-client and global rate limiting (with trusted proxy handling)
|
||||||
|
- Startup configuration validation and upstream behavior
|
||||||
|
- Clean shutdown hygiene
|
||||||
|
|
||||||
|
This is the closest practical equivalent to "run the thing real clients will run and make sure nothing is broken."
|
||||||
|
|
||||||
|
#### Troubleshooting
|
||||||
|
|
||||||
|
- **Low hit rate on first run**: Normal. The first `benchmark run` is the warmup that populates the cache.
|
||||||
|
- **Want to test real disk I/O (not RAM cache)**: Make sure your workload size (shown by `benchmark setup`) is larger than the total RAM on the machine running steamcache2.
|
||||||
|
- **Server won't start or bind on port 80 as non-root**: `make run-validation` and `make validate` automatically run `setcap` on the binary they just built. If it still fails, run `make setcap` explicitly and retry. The server always runs as your normal user (no root) so the disk cache directory ownership stays correct.
|
||||||
|
- **SteamPrefill not found**: Install it yourself from its GitHub releases. Then use `make validate` to start the server with small caches and point SteamPrefill at it manually.
|
||||||
|
- **SteamPrefill won't use server as cache properly**: SteamPrefill has some bad autodetectiong functions sometimes it works when the server is resolvable from localhost or 127.0.0.1 other times you have to fully override the dns for the proper dns name lancache.steamcontent.com to point to 127.0.0.1 i don't recommend doing it unless your okay with having to undo and redo it depending on if your running the server or not its a pain.
|
||||||
|
|
||||||
|
See also the SteamPrefill documentation for `benchmark setup` and `benchmark run` options.
|
||||||
|
|
||||||
### Command Line Flags
|
### Command Line Flags
|
||||||
|
|
||||||
While most configuration is done via the YAML file, some runtime options are still available as command-line flags:
|
While most configuration is done via the YAML file, some runtime options are still available as command-line flags:
|
||||||
@@ -72,8 +220,9 @@ While most configuration is done via the YAML file, some runtime options are sti
|
|||||||
# Set logging level
|
# Set logging level
|
||||||
./steamcache2 --log-level debug --log-format json
|
./steamcache2 --log-level debug --log-format json
|
||||||
|
|
||||||
# Set number of worker threads
|
# Override concurrency from the CLI (0 = use config.yaml)
|
||||||
./steamcache2 --threads 8
|
./steamcache2 --max-concurrent-requests 8
|
||||||
|
./steamcache2 --max-requests-per-client 4
|
||||||
|
|
||||||
# Show help
|
# Show help
|
||||||
./steamcache2 --help
|
./steamcache2 --help
|
||||||
@@ -87,7 +236,7 @@ SteamCache2 uses a YAML configuration file (`config.yaml`) for all settings. Her
|
|||||||
# Server configuration
|
# Server configuration
|
||||||
listen_address: :80
|
listen_address: :80
|
||||||
|
|
||||||
# P1 hardening (see Security Hardening section)
|
# Hardening (see Security Hardening section)
|
||||||
max_object_size: "0" # 0=unlimited; set e.g. "256MB" for response size DoS protection
|
max_object_size: "0" # 0=unlimited; set e.g. "256MB" for response size DoS protection
|
||||||
trusted_proxies: [] # empty = safe (ignore XFF for rate limit); set CIDRs for trusted proxies
|
trusted_proxies: [] # empty = safe (ignore XFF for rate limit); set CIDRs for trusted proxies
|
||||||
|
|
||||||
@@ -110,12 +259,13 @@ cache:
|
|||||||
gc_algorithm: hybrid
|
gc_algorithm: hybrid
|
||||||
|
|
||||||
# Upstream server configuration
|
# Upstream server configuration
|
||||||
# The upstream server to proxy requests to
|
# Leave empty to fetch from the request Host (Steam CDN names only).
|
||||||
upstream: "https://steam.cdn.com"
|
# Set only when chaining caches (table RAM cache -> room disk cache).
|
||||||
|
upstream: ""
|
||||||
```
|
```
|
||||||
|
|
||||||
#### Startup Validation
|
#### Startup Validation
|
||||||
As of P0, `steamcache2` performs strict validation on startup (after loading config + CLI overrides, before creating the cache). Invalid configs cause immediate clean failure (no default written, no panic):
|
`steamcache2` performs strict validation on startup (after loading config + CLI overrides, before creating the cache). Invalid configs cause immediate clean failure (no default written, no panic):
|
||||||
|
|
||||||
- Negative `max_concurrent_requests` / `max_requests_per_client`: "negative concurrency not allowed"
|
- Negative `max_concurrent_requests` / `max_requests_per_client`: "negative concurrency not allowed"
|
||||||
- Invalid `gc_algorithm` (memory): "invalid memory gc algorithm: badvalue"
|
- Invalid `gc_algorithm` (memory): "invalid memory gc algorithm: badvalue"
|
||||||
@@ -129,12 +279,12 @@ Error: Invalid configuration: invalid memory gc algorithm: foo. Please fix the c
|
|||||||
|
|
||||||
See `config.Validate()` and `steamcache.New` error paths. This ensures the LAN appliance fails fast on misconfig.
|
See `config.Validate()` and `steamcache.New` error paths. This ensures the LAN appliance fails fast on misconfig.
|
||||||
|
|
||||||
#### Security Hardening (P1)
|
#### Security Hardening
|
||||||
- `max_object_size` (default "0" = unlimited): set e.g. "256MB" or "512MB" to reject oversized upstream responses with HTTP 413 before buffering/ReadAll. Prevents OOM DoS from large or malicious responses (P1-01). Large legitimate Steam files still served if under limit.
|
- `max_object_size` (default "0" = unlimited): set e.g. "256MB" or "512MB" to reject oversized upstream responses with HTTP 413 before buffering/ReadAll. Prevents OOM DoS from large or malicious responses. Large legitimate Steam files still served if under limit.
|
||||||
- `trusted_proxies`: CIDR list (default empty). When empty (safe default), X-Forwarded-For and client IP spoofing are ignored for rate limiting — always uses `r.RemoteAddr` only. When set (e.g. your reverse proxy CIDR), uses correct "rightmost untrusted" extraction. Prevents bypass of `max_requests_per_client` (P1-02). Documented for LAN proxy setups only.
|
- `trusted_proxies`: CIDR list (default empty). When empty (safe default), X-Forwarded-For and client IP spoofing are ignored for rate limiting — always uses `r.RemoteAddr` only. When set (e.g. your reverse proxy CIDR), uses correct "rightmost untrusted" extraction. Prevents bypass of `max_requests_per_client`. Documented for LAN proxy setups only.
|
||||||
- These + P0 validation make steamcache2 safe-by-default for LAN exposure.
|
- These + the startup validation make steamcache2 safe-by-default for LAN exposure.
|
||||||
|
|
||||||
#### Migration / Breaking Changes (P1)
|
#### Migration / Breaking Changes
|
||||||
- `New()` public signature gained 2 required trailing params (`maxObjectSize`, `trustedProxies`). Direct callers (rare; most use config or NewWithOptions) must update.
|
- `New()` public signature gained 2 required trailing params (`maxObjectSize`, `trustedProxies`). Direct callers (rare; most use config or NewWithOptions) must update.
|
||||||
- Recommended: migrate to `NewWithOptions(Options{...})` (non-breaking) or rely on YAML config + cmd/root.go.
|
- Recommended: migrate to `NewWithOptions(Options{...})` (non-breaking) or rely on YAML config + cmd/root.go.
|
||||||
- No behavior change for existing configs (defaults preserve prior semantics).
|
- No behavior change for existing configs (defaults preserve prior semantics).
|
||||||
@@ -146,6 +296,9 @@ See `config.Validate()` and `steamcache.New` error paths. This ensures the LAN a
|
|||||||
- The explicit startup guard (reduce size if pre-existing on-disk > cap) runs as the literal last step of bg init, before the barrier opens.
|
- The explicit startup guard (reduce size if pre-existing on-disk > cap) runs as the literal last step of bg init, before the barrier opens.
|
||||||
- Add a note for operators: very large disk caches (tens/hundreds GB with millions files) may show extended "memory-only or no-cache" behavior at startup (seconds to minutes depending on storage speed); this is by design for responsiveness.
|
- Add a note for operators: very large disk caches (tens/hundreds GB with millions files) may show extended "memory-only or no-cache" behavior at startup (seconds to minutes depending on storage speed); this is by design for responsiveness.
|
||||||
- Godoc on `disk.New` and `DiskFS.Size` expanded with the barrier/attach behavior.
|
- Godoc on `disk.New` and `DiskFS.Size` expanded with the barrier/attach behavior.
|
||||||
|
- Startup logs: Info "Disk slow tier attach pending..." then later "Disk slow tier attached (...)" for disk-only and mixed modes.
|
||||||
|
- `/metrics` exposes `disk_tier_ready` 0/1 and stays responsive during attach (GetMetrics does not block on Size while pending).
|
||||||
|
- `/lancache-heartbeat` header `X-SteamCache-Disk-Tier` mirrors that state.
|
||||||
|
|
||||||
#### Garbage Collection Algorithms
|
#### Garbage Collection Algorithms
|
||||||
|
|
||||||
@@ -154,11 +307,11 @@ SteamCache2 supports different garbage collection algorithms for memory and disk
|
|||||||
**Available GC Algorithms:**
|
**Available GC Algorithms:**
|
||||||
|
|
||||||
- **`lru`** (default): Least Recently Used - evicts oldest accessed files
|
- **`lru`** (default): Least Recently Used - evicts oldest accessed files
|
||||||
- **`lfu`**: Least Frequently Used (P1 real impl) - evicts by lowest AccessCount (tiebreak older ATime); uses existing FileInfo counters
|
- **`lfu`**: Least Frequently Used - evicts by lowest AccessCount (tiebreak older ATime); uses existing FileInfo counters
|
||||||
- **`fifo`**: First In, First Out - evicts oldest created files (predictable)
|
- **`fifo`**: First In, First Out - evicts oldest created files (predictable and terrible all in one) don't ever use it
|
||||||
- **`largest`**: Size-based - evicts largest files first (maximizes file count)
|
- **`largest`**: Size-based - evicts largest files first (maximizes small file count) if used on memory greatly improves access time
|
||||||
- **`smallest`**: Size-based - evicts smallest files first (maximizes cache hit rate)
|
- **`smallest`**: Size-based - evicts smallest files first (maximizes large file count) probably best used for disk since there kinda slow with small files
|
||||||
- **`hybrid`**: Recency + frequency hybrid (P1 meaningful) - evicts by lowest time-decayed score (GetTimeDecayedScore combining ATime + AccessCount)
|
- **`hybrid`**: Recency + frequency hybrid - evicts by lowest time-decayed score (GetTimeDecayedScore combining ATime + AccessCount)
|
||||||
|
|
||||||
**Recommended Algorithms by Cache Type:**
|
**Recommended Algorithms by Cache Type:**
|
||||||
|
|
||||||
@@ -166,18 +319,19 @@ SteamCache2 supports different garbage collection algorithms for memory and disk
|
|||||||
- **`lru`** - Best overall performance, good balance of speed and hit rate
|
- **`lru`** - Best overall performance, good balance of speed and hit rate
|
||||||
- **`lfu`** - Excellent for gaming cafes where popular games stay cached
|
- **`lfu`** - Excellent for gaming cafes where popular games stay cached
|
||||||
- **`hybrid`** - Optimal for mixed workloads with varying file sizes
|
- **`hybrid`** - Optimal for mixed workloads with varying file sizes
|
||||||
|
- **`largest`** - Crazy good for access times since disks are slow with lots of tiny files
|
||||||
|
|
||||||
**For Disk Cache (Slow, Large Size):**
|
**For Disk Cache (Slow, Large Size):**
|
||||||
- **`hybrid`** - Recommended for optimal performance, balances speed and storage efficiency
|
- **`hybrid`** - Recommended for optimal performance, balances speed and storage efficiency
|
||||||
- **`largest`** - Good for maximizing number of cached files
|
- **`smallest`** - Good for maximizing linear reads which is the only place spinning disks have performance don't expect too much though steam kinda uses small files
|
||||||
- **`lru`** - Reliable default with good performance
|
- **`lru`** - Reliable default with good performance
|
||||||
|
|
||||||
**Use Cases:**
|
**Use Cases:**
|
||||||
- **Gaming Cafes**: Use `lfu` for memory, `hybrid` for disk
|
- **Gaming Cafes**: Use `largest` for memory, `hybrid` for disk
|
||||||
- **LAN Events**: Use `lfu` for memory, `hybrid` for disk
|
- **LAN Events**: Use `largest` for memory, `hybrid` for disk
|
||||||
- **Home Use**: Use `lru` for memory, `hybrid` for disk
|
- **Home Use**: Use `largest` for memory, `hybrid` for disk
|
||||||
- **Testing**: Use `fifo` for predictable behavior
|
- **Testing**: Use `fifo` for nothing its pointless
|
||||||
- **Large File Storage**: Use `largest` for disk to maximize file count
|
- **Large File Storage**: Use `smallest` for disk get rid of the slow tiny files first
|
||||||
|
|
||||||
### DNS Configuration
|
### DNS Configuration
|
||||||
|
|
||||||
@@ -222,7 +376,7 @@ This will direct any requests to `lancache.steamcontent.com` to your SteamCache2
|
|||||||
|
|
||||||
### Prerequisites
|
### Prerequisites
|
||||||
|
|
||||||
- Go 1.19 or later
|
- Go 1.27.0 or later
|
||||||
- Make (optional, but recommended)
|
- Make (optional, but recommended)
|
||||||
|
|
||||||
### Build Commands
|
### Build Commands
|
||||||
@@ -230,7 +384,7 @@ This will direct any requests to `lancache.steamcontent.com` to your SteamCache2
|
|||||||
```bash
|
```bash
|
||||||
# Clone the repository
|
# Clone the repository
|
||||||
git clone <repository-url>
|
git clone <repository-url>
|
||||||
cd SteamCache2
|
cd steamcache2
|
||||||
|
|
||||||
# Download dependencies
|
# Download dependencies
|
||||||
make deps
|
make deps
|
||||||
@@ -282,6 +436,11 @@ make
|
|||||||
- Consider using a different GC algorithm like `hybrid`
|
- Consider using a different GC algorithm like `hybrid`
|
||||||
- Adjust the disk cache size to match available storage
|
- Adjust the disk cache size to match available storage
|
||||||
|
|
||||||
|
6. **Not sure if it is caching**
|
||||||
|
- Do not start with the full SteamPrefill chapter. Use [Quick check: is it caching?](#quick-check-is-it-caching): `make validate-check` (full `/metrics`, hit/miss fields, and `/lancache-heartbeat`)
|
||||||
|
- A first pass is mostly `cache_misses`; repeating the same content should raise `cache_hits` / `hit_rate`
|
||||||
|
- Confirm the process is up with `curl -s -i http://localhost/lancache-heartbeat` (GET, not HEAD)
|
||||||
|
|
||||||
### Getting Help
|
### Getting Help
|
||||||
|
|
||||||
- Check the logs for detailed error messages
|
- Check the logs for detailed error messages
|
||||||
|
|||||||
@@ -0,0 +1,60 @@
|
|||||||
|
# validate-config.yaml
|
||||||
|
#
|
||||||
|
# Small dual-tier configuration intended for full-function validation of a
|
||||||
|
# built steamcache2 binary using realistic Steam client workloads driven by
|
||||||
|
# the external SteamPrefill (https://github.com/tpill90/steam-lancache-prefill)
|
||||||
|
# "benchmark" commands.
|
||||||
|
#
|
||||||
|
# Why these values?
|
||||||
|
# - Both tiers enabled. Memory is sized large enough to survive the disk attach
|
||||||
|
# window in mixed mode (see steamcache.go: the goroutine that blocks on d.Size()
|
||||||
|
# before SetSlow). With a realistic SteamPrefill benchmark (high rate of unique
|
||||||
|
# ~1MB chunks) the old tiny 128MB mem + 512MB disk caused almost all early
|
||||||
|
# content to live only in memory, get evicted by its GC, and never reach disk.
|
||||||
|
# Result: "never hitting", hit_rate 0, memory_size 0, despite files appearing
|
||||||
|
# on disk for late-arriving chunks. Larger mem + disk makes the validation
|
||||||
|
# actually exercise hits, promotions, disk tier, and GC as intended.
|
||||||
|
# - Conservative concurrency limits suitable for a developer laptop.
|
||||||
|
# - trusted_proxies set for 127.0.0.0/8 so that an external benchmark tool
|
||||||
|
# can simulate multiple distinct clients via X-Forwarded-For if desired.
|
||||||
|
# - upstream left empty: the server will use the incoming Host header
|
||||||
|
# (exactly what happens when you point SteamPrefill at your Lancache IP).
|
||||||
|
#
|
||||||
|
# Usage (typical dev workflow):
|
||||||
|
# make build
|
||||||
|
# make validate
|
||||||
|
# # In another terminal:
|
||||||
|
# SteamPrefill benchmark run -c 20 ...
|
||||||
|
#
|
||||||
|
# After the benchmark run, inspect with:
|
||||||
|
# make validate-check # full /metrics + hit/miss fields + /lancache-heartbeat
|
||||||
|
# # or, manually:
|
||||||
|
# curl -s http://localhost/metrics
|
||||||
|
# curl -s -i http://localhost/lancache-heartbeat # GET, not HEAD
|
||||||
|
#
|
||||||
|
# Tweak sizes upward if you want to run very large workloads while still
|
||||||
|
# exercising the disk tier (workload >> RAM is ideal for real disk testing).
|
||||||
|
|
||||||
|
listen_address: :80
|
||||||
|
|
||||||
|
max_concurrent_requests: 1000
|
||||||
|
max_requests_per_client: 10
|
||||||
|
|
||||||
|
max_object_size: "0" # unlimited for validation (real Steam files can be large)
|
||||||
|
trusted_proxies: ["127.0.0.0/8"]
|
||||||
|
|
||||||
|
cache:
|
||||||
|
memory:
|
||||||
|
size: 1GB
|
||||||
|
gc_algorithm: hybrid
|
||||||
|
disk:
|
||||||
|
size: 2GB
|
||||||
|
path: ./validate-disk # cleaned between runs by make validate or make clean-disk
|
||||||
|
gc_algorithm: hybrid # recommended for disk in the project README
|
||||||
|
|
||||||
|
# Empty upstream = use Host header from the client (SteamPrefill / real Steam clients).
|
||||||
|
# Allows for chaining steamcache2 instances if needed.
|
||||||
|
# For example, for a lan party you could have a small fast ram only cache at each table pointing to a larger slower disk cache in the back somewhere
|
||||||
|
# It would reduce the amount of bandwidth needed to the internet and the amount needed to each table
|
||||||
|
# just as a little reminder there is no authentication so this is not a good idea for a public cache just out on the internet.
|
||||||
|
upstream: ""
|
||||||
@@ -1,6 +1,6 @@
|
|||||||
module s1d3sw1ped/steamcache2
|
module s1d3sw1ped/steamcache2
|
||||||
|
|
||||||
go 1.23.0
|
go 1.27.0
|
||||||
|
|
||||||
require (
|
require (
|
||||||
github.com/docker/go-units v0.5.0
|
github.com/docker/go-units v0.5.0
|
||||||
@@ -8,6 +8,7 @@ require (
|
|||||||
github.com/rs/zerolog v1.33.0
|
github.com/rs/zerolog v1.33.0
|
||||||
github.com/spf13/cobra v1.8.1
|
github.com/spf13/cobra v1.8.1
|
||||||
golang.org/x/sync v0.16.0
|
golang.org/x/sync v0.16.0
|
||||||
|
golang.org/x/sys v0.12.0
|
||||||
gopkg.in/yaml.v3 v3.0.1
|
gopkg.in/yaml.v3 v3.0.1
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -16,5 +17,4 @@ require (
|
|||||||
github.com/mattn/go-colorable v0.1.13 // indirect
|
github.com/mattn/go-colorable v0.1.13 // indirect
|
||||||
github.com/mattn/go-isatty v0.0.19 // indirect
|
github.com/mattn/go-isatty v0.0.19 // indirect
|
||||||
github.com/spf13/pflag v1.0.5 // indirect
|
github.com/spf13/pflag v1.0.5 // indirect
|
||||||
golang.org/x/sys v0.12.0 // indirect
|
|
||||||
)
|
)
|
||||||
|
|||||||
Executable
+155
@@ -0,0 +1,155 @@
|
|||||||
|
#!/usr/bin/env bash
|
||||||
|
#
|
||||||
|
# download-prefill.sh
|
||||||
|
#
|
||||||
|
# Downloads the latest (or specific) release of SteamPrefill
|
||||||
|
# (https://github.com/tpill90/steam-lancache-prefill) into
|
||||||
|
# bin/steam-prefill/SteamPrefill
|
||||||
|
#
|
||||||
|
# Usage:
|
||||||
|
# ./scripts/download-prefill.sh
|
||||||
|
#
|
||||||
|
# Environment:
|
||||||
|
# PREFILL_VERSION - Pin a specific version tag (e.g. v3.4.2)
|
||||||
|
# PREFILL_FORCE - Set to any non-empty value to re-download even if present
|
||||||
|
#
|
||||||
|
set -euo pipefail
|
||||||
|
|
||||||
|
DEST_DIR="bin/steam-prefill"
|
||||||
|
TARGET="$DEST_DIR/SteamPrefill"
|
||||||
|
|
||||||
|
mkdir -p "$DEST_DIR"
|
||||||
|
|
||||||
|
if [[ -x "$TARGET" && -z "${PREFILL_FORCE:-}" ]]; then
|
||||||
|
echo "SteamPrefill already present at $TARGET"
|
||||||
|
echo "Run with PREFILL_FORCE=1 to re-download."
|
||||||
|
exit 0
|
||||||
|
fi
|
||||||
|
|
||||||
|
VERSION="${PREFILL_VERSION:-}"
|
||||||
|
|
||||||
|
OS=$(uname -s | tr '[:upper:]' '[:lower:]')
|
||||||
|
ARCH=$(uname -m)
|
||||||
|
|
||||||
|
case "$ARCH" in
|
||||||
|
x86_64|amd64) ARCH_NAME="x64" ;;
|
||||||
|
aarch64|arm64) ARCH_NAME="arm64" ;;
|
||||||
|
*) echo "Unsupported architecture: $ARCH"; exit 1 ;;
|
||||||
|
esac
|
||||||
|
|
||||||
|
case "$OS" in
|
||||||
|
linux) OS_NAME="linux" ;;
|
||||||
|
darwin) OS_NAME="osx" ;;
|
||||||
|
*) echo "Unsupported OS: $OS"; exit 1 ;;
|
||||||
|
esac
|
||||||
|
|
||||||
|
echo "Resolving SteamPrefill version..."
|
||||||
|
|
||||||
|
if [[ -z "$VERSION" ]]; then
|
||||||
|
# Follow the /latest redirect to discover the current tag
|
||||||
|
LATEST_URL=$(curl -sIL -o /dev/null -w '%{url_effective}' \
|
||||||
|
"https://github.com/tpill90/steam-lancache-prefill/releases/latest" 2>/dev/null || true)
|
||||||
|
|
||||||
|
if [[ "$LATEST_URL" =~ /tag/([^/?#]+) ]]; then
|
||||||
|
VERSION="${BASH_REMATCH[1]}"
|
||||||
|
else
|
||||||
|
echo "Failed to resolve latest version from GitHub redirect."
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
fi
|
||||||
|
|
||||||
|
echo "Downloading SteamPrefill $VERSION for ${OS_NAME}-${ARCH_NAME}..."
|
||||||
|
|
||||||
|
rm -f "$TARGET" "$TARGET.tmp" 2>/dev/null || true
|
||||||
|
|
||||||
|
DOWNLOADED=0
|
||||||
|
|
||||||
|
# Preferred: query the GitHub API for the exact asset list (most reliable)
|
||||||
|
API_URL="https://api.github.com/repos/tpill90/steam-lancache-prefill/releases/tags/${VERSION}"
|
||||||
|
ASSET_URL=""
|
||||||
|
|
||||||
|
if command -v jq >/dev/null 2>&1; then
|
||||||
|
echo " Querying GitHub API for assets..."
|
||||||
|
ASSET_NAME=$(curl -fsSL "$API_URL" 2>/dev/null | jq -r --arg os "$OS_NAME" --arg arch "$ARCH_NAME" '
|
||||||
|
.assets[]
|
||||||
|
| select(.name | ascii_downcase | contains($os))
|
||||||
|
| select(.name | ascii_downcase | contains($arch))
|
||||||
|
| .name
|
||||||
|
' | head -1)
|
||||||
|
|
||||||
|
if [[ -n "$ASSET_NAME" ]]; then
|
||||||
|
ASSET_URL="https://github.com/tpill90/steam-lancache-prefill/releases/download/${VERSION}/${ASSET_NAME}"
|
||||||
|
echo " Found asset via API: $ASSET_NAME"
|
||||||
|
fi
|
||||||
|
fi
|
||||||
|
|
||||||
|
# Fallback: try common name patterns if API or jq not available
|
||||||
|
if [[ -z "$ASSET_URL" ]]; then
|
||||||
|
echo " Trying common asset name patterns..."
|
||||||
|
CANDIDATES=(
|
||||||
|
"SteamPrefill-${VERSION}-${OS_NAME}-${ARCH_NAME}.zip"
|
||||||
|
"SteamPrefill-${VERSION}-${OS_NAME}-${ARCH_NAME}"
|
||||||
|
"SteamPrefill-${OS_NAME}-${ARCH_NAME}.zip"
|
||||||
|
"SteamPrefill-${OS_NAME}-${ARCH_NAME}"
|
||||||
|
"SteamPrefill-linux-${ARCH_NAME}.zip"
|
||||||
|
"SteamPrefill-linux-${ARCH_NAME}"
|
||||||
|
)
|
||||||
|
|
||||||
|
for name in "${CANDIDATES[@]}"; do
|
||||||
|
URL="https://github.com/tpill90/steam-lancache-prefill/releases/download/${VERSION}/${name}"
|
||||||
|
echo " Trying $name ..."
|
||||||
|
if curl -fI -s --retry 2 "$URL" >/dev/null 2>&1; then
|
||||||
|
ASSET_URL="$URL"
|
||||||
|
ASSET_NAME="$name"
|
||||||
|
break
|
||||||
|
fi
|
||||||
|
done
|
||||||
|
fi
|
||||||
|
|
||||||
|
if [[ -n "$ASSET_URL" ]]; then
|
||||||
|
echo "Downloading $ASSET_NAME ..."
|
||||||
|
if curl -fL --retry 3 --retry-delay 2 -A "Mozilla/5.0 (compatible; SteamPrefill-Downloader)" \
|
||||||
|
--progress-bar -o "$TARGET.tmp" "$ASSET_URL"; then
|
||||||
|
|
||||||
|
echo "Download complete."
|
||||||
|
|
||||||
|
if [[ "$ASSET_NAME" == *.zip ]]; then
|
||||||
|
echo "Extracting..."
|
||||||
|
if ! command -v unzip >/dev/null 2>&1; then
|
||||||
|
echo "Error: unzip is required for this release."
|
||||||
|
rm -f "$TARGET.tmp"
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
unzip -o -q "$TARGET.tmp" -d "$DEST_DIR"
|
||||||
|
FOUND=$(find "$DEST_DIR" -type f -name "SteamPrefill" | head -1)
|
||||||
|
if [[ -n "$FOUND" ]]; then
|
||||||
|
mv "$FOUND" "$TARGET"
|
||||||
|
fi
|
||||||
|
rm -f "$TARGET.tmp"
|
||||||
|
find "$DEST_DIR" -mindepth 1 -maxdepth 1 -type d -name "SteamPrefill*" -exec rm -rf {} + 2>/dev/null || true
|
||||||
|
else
|
||||||
|
mv "$TARGET.tmp" "$TARGET"
|
||||||
|
fi
|
||||||
|
|
||||||
|
chmod +x "$TARGET"
|
||||||
|
DOWNLOADED=1
|
||||||
|
fi
|
||||||
|
fi
|
||||||
|
|
||||||
|
if [[ $DOWNLOADED -eq 0 ]]; then
|
||||||
|
echo ""
|
||||||
|
echo "Failed to download a matching asset for $VERSION."
|
||||||
|
echo "You can try pinning a different version:"
|
||||||
|
echo " PREFILL_VERSION=vX.Y.Z ./scripts/download-prefill.sh"
|
||||||
|
echo ""
|
||||||
|
echo "Or download manually from:"
|
||||||
|
echo " https://github.com/tpill90/steam-lancache-prefill/releases/tag/${VERSION}"
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
echo ""
|
||||||
|
echo "Installed SteamPrefill $VERSION → $TARGET"
|
||||||
|
echo ""
|
||||||
|
echo "You can now run it directly, for example:"
|
||||||
|
echo " ./bin/steam-prefill/SteamPrefill --help"
|
||||||
|
echo " ./bin/steam-prefill/SteamPrefill benchmark run ..."
|
||||||
@@ -262,6 +262,13 @@ func (sc *SteamCache) streamCachedResponse(w http.ResponseWriter, r *http.Reques
|
|||||||
// Send the range data
|
// Send the range data
|
||||||
_, _ = w.Write(rangeData) // client write error ignored (disconnect during range body send is not actionable)
|
_, _ = w.Write(rangeData) // client write error ignored (disconnect during range body send is not actionable)
|
||||||
|
|
||||||
|
// Range served from cache: count the range-specific metric and the range
|
||||||
|
// bytes actually written (handleCacheHit skips full-blob byte counting for
|
||||||
|
// Range requests so these are not double-counted).
|
||||||
|
sc.metrics.IncrementRangeCache()
|
||||||
|
sc.metrics.AddBytesServed(int64(len(rangeData)))
|
||||||
|
sc.metrics.AddBytesSaved(int64(len(rangeData)))
|
||||||
|
|
||||||
logger.Logger.Info().
|
logger.Logger.Info().
|
||||||
Str("cache_key", cacheKey).
|
Str("cache_key", cacheKey).
|
||||||
Str("url", r.URL.String()).
|
Str("url", r.URL.String()).
|
||||||
|
|||||||
+68
-10
@@ -56,6 +56,15 @@ func (sc *SteamCache) handleSpecialEndpoints(w http.ResponseWriter, r *http.Requ
|
|||||||
logger.Logger.Debug().
|
logger.Logger.Debug().
|
||||||
Str("client_ip", clientIP).
|
Str("client_ip", clientIP).
|
||||||
Msg("LanCache heartbeat request")
|
Msg("LanCache heartbeat request")
|
||||||
|
diskTier := "disabled"
|
||||||
|
if sc.disk != nil {
|
||||||
|
if sc.metrics.GetDiskTierReady() == 1 {
|
||||||
|
diskTier = "ready"
|
||||||
|
} else {
|
||||||
|
diskTier = "pending"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
w.Header().Add("X-SteamCache-Disk-Tier", diskTier)
|
||||||
w.Header().Add("X-LanCache-Processed-By", "SteamCache2")
|
w.Header().Add("X-LanCache-Processed-By", "SteamCache2")
|
||||||
w.WriteHeader(http.StatusNoContent)
|
w.WriteHeader(http.StatusNoContent)
|
||||||
_, _ = w.Write(nil) // client write error ignored (heartbeat path; nil write is no-op)
|
_, _ = w.Write(nil) // client write error ignored (heartbeat path; nil write is no-op)
|
||||||
@@ -109,7 +118,14 @@ func (sc *SteamCache) handleCacheHit(w http.ResponseWriter, r *http.Request, cac
|
|||||||
// Track cache hit metrics
|
// Track cache hit metrics
|
||||||
sc.metrics.IncrementCacheHits()
|
sc.metrics.IncrementCacheHits()
|
||||||
sc.metrics.AddResponseTime(time.Since(tstart))
|
sc.metrics.AddResponseTime(time.Since(tstart))
|
||||||
|
if r.Header.Get("Range") == "" {
|
||||||
|
// Full-object HIT: count the cached blob served. Range HITs skip
|
||||||
|
// this here — streamCachedResponse counts the range bytes actually
|
||||||
|
// written to the client instead, so BytesServed/Saved reflect the
|
||||||
|
// partial body and are not double-counted.
|
||||||
sc.metrics.AddBytesServed(int64(len(cachedData)))
|
sc.metrics.AddBytesServed(int64(len(cachedData)))
|
||||||
|
sc.metrics.AddBytesSaved(int64(len(cachedData)))
|
||||||
|
}
|
||||||
sc.metrics.IncrementServiceRequests(service.Name)
|
sc.metrics.IncrementServiceRequests(service.Name)
|
||||||
|
|
||||||
logger.Logger.Debug().
|
logger.Logger.Debug().
|
||||||
@@ -197,6 +213,7 @@ func (sc *SteamCache) waitForCoalesced(w http.ResponseWriter, r *http.Request, c
|
|||||||
sc.metrics.IncrementCacheCoalesced()
|
sc.metrics.IncrementCacheCoalesced()
|
||||||
sc.metrics.AddResponseTime(time.Since(tstart))
|
sc.metrics.AddResponseTime(time.Since(tstart))
|
||||||
sc.metrics.AddBytesServed(int64(len(responseData)))
|
sc.metrics.AddBytesServed(int64(len(responseData)))
|
||||||
|
sc.metrics.AddBytesSaved(int64(len(responseData)))
|
||||||
sc.metrics.IncrementServiceRequests(service.Name)
|
sc.metrics.IncrementServiceRequests(service.Name)
|
||||||
|
|
||||||
logger.Logger.Info().
|
logger.Logger.Info().
|
||||||
@@ -233,9 +250,6 @@ func (sc *SteamCache) ServeHTTP(w http.ResponseWriter, r *http.Request) {
|
|||||||
}
|
}
|
||||||
defer sc.requestSemaphore.Release(1)
|
defer sc.requestSemaphore.Release(1)
|
||||||
|
|
||||||
// Track total requests
|
|
||||||
sc.metrics.IncrementTotalRequests()
|
|
||||||
|
|
||||||
// Apply per-client rate limiting
|
// Apply per-client rate limiting
|
||||||
clientLimiter := sc.getOrCreateClientLimiter(clientIP)
|
clientLimiter := sc.getOrCreateClientLimiter(clientIP)
|
||||||
|
|
||||||
@@ -265,9 +279,12 @@ func (sc *SteamCache) ServeHTTP(w http.ResponseWriter, r *http.Request) {
|
|||||||
|
|
||||||
// Check if this is a request from a supported service
|
// Check if this is a request from a supported service
|
||||||
if service, isSupported := sc.detectService(r); isSupported {
|
if service, isSupported := sc.detectService(r); isSupported {
|
||||||
// trim the query parameters from the URL path
|
// Cache key is the path only, never the Host: Steam rotates CDN hostnames
|
||||||
// this is necessary because the cache key should not include query parameters
|
// for the same depot object, so different Host headers (or absolute-form
|
||||||
urlPath := strings.SplitN(r.URL.String(), "?", 2)[0] // trim query for cache key (SplitN makes intent explicit vs Cut + ignored bool)
|
// request targets) for the same path must share one cache entry. r.URL.Path
|
||||||
|
// is the decoded path (query is never part of it); validateURLPath checks
|
||||||
|
// this decoded form and url.JoinPath re-escapes it for the upstream join.
|
||||||
|
urlPath := r.URL.Path
|
||||||
|
|
||||||
// Validate URL path for security
|
// Validate URL path for security
|
||||||
if err := validateURLPath(urlPath); err != nil {
|
if err := validateURLPath(urlPath); err != nil {
|
||||||
@@ -305,6 +322,11 @@ func (sc *SteamCache) ServeHTTP(w http.ResponseWriter, r *http.Request) {
|
|||||||
Str("client_ip", clientIP).
|
Str("client_ip", clientIP).
|
||||||
Msg("Generated cache key")
|
Msg("Generated cache key")
|
||||||
|
|
||||||
|
// Only count real cacheable service traffic toward total_requests / hit_rate.
|
||||||
|
// Special endpoints (/, /metrics, /lancache-heartbeat) and unsupported services
|
||||||
|
// are intentionally excluded so that idle monitoring doesn't dilute the hit rate.
|
||||||
|
sc.metrics.IncrementTotalRequests()
|
||||||
|
|
||||||
if sc.handleCacheHit(w, r, cachePath, cacheKey, urlPath, service, clientIP, tstart) {
|
if sc.handleCacheHit(w, r, cachePath, cacheKey, urlPath, service, clientIP, tstart) {
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
@@ -341,6 +363,18 @@ func (sc *SteamCache) ServeHTTP(w http.ResponseWriter, r *http.Request) {
|
|||||||
req.Host = r.Host
|
req.Host = r.Host
|
||||||
} else { // if no upstream server is configured, proxy the request to the host specified in the request
|
} else { // if no upstream server is configured, proxy the request to the host specified in the request
|
||||||
host := r.Host
|
host := r.Host
|
||||||
|
if !hostAllowedForDirectFetch(host) {
|
||||||
|
logger.Logger.Warn().
|
||||||
|
Str("host", host).
|
||||||
|
Str("client_ip", clientIP).
|
||||||
|
Msg("Rejecting direct-fetch Host (not a Steam CDN name)")
|
||||||
|
sc.metrics.IncrementErrors()
|
||||||
|
if isNew {
|
||||||
|
coalescedReq.complete(nil, fmt.Errorf("host not allowed for direct fetch"))
|
||||||
|
}
|
||||||
|
http.Error(w, "Invalid URL", http.StatusBadRequest)
|
||||||
|
return
|
||||||
|
}
|
||||||
if r.Header.Get("X-Sls-Https") == "enable" {
|
if r.Header.Get("X-Sls-Https") == "enable" {
|
||||||
host = "https://" + host
|
host = "https://" + host
|
||||||
} else {
|
} else {
|
||||||
@@ -533,14 +567,40 @@ func (sc *SteamCache) ServeHTTP(w http.ResponseWriter, r *http.Request) {
|
|||||||
w.Header().Set("X-LanCache-Status", "MISS")
|
w.Header().Set("X-LanCache-Status", "MISS")
|
||||||
w.Header().Set("X-LanCache-Processed-By", "SteamCache2")
|
w.Header().Set("X-LanCache-Processed-By", "SteamCache2")
|
||||||
|
|
||||||
// Stream the response body to client
|
// Stream the response body to client.
|
||||||
|
// Range miss: the Range header was stripped for the upstream fetch (so the
|
||||||
|
// FULL object is cached below); serve the client's requested slice from the
|
||||||
|
// full body as 206, matching the HIT Range path.
|
||||||
|
if rangeHeader := r.Header.Get("Range"); rangeHeader != "" {
|
||||||
|
start, end, totalSize, rangeValid := parseRangeHeader(rangeHeader, int64(len(bodyData)))
|
||||||
|
if !rangeValid {
|
||||||
|
// Invalid range — 416 (consistent with the HIT Range path). Drop the
|
||||||
|
// upstream Content-Length: it describes the full body, which 416 does
|
||||||
|
// not send (a stale CL would hang clients waiting for a body).
|
||||||
|
w.Header().Del("Content-Length")
|
||||||
|
w.Header().Set("Content-Range", fmt.Sprintf("bytes */%d", len(bodyData)))
|
||||||
|
w.WriteHeader(http.StatusRequestedRangeNotSatisfiable)
|
||||||
|
} else {
|
||||||
|
rangeData := bodyData[start : end+1]
|
||||||
|
w.Header().Set("Content-Range", fmt.Sprintf("bytes %d-%d/%d", start, end, totalSize))
|
||||||
|
w.Header().Set("Content-Length", fmt.Sprintf("%d", len(rangeData)))
|
||||||
|
w.Header().Set("Accept-Ranges", "bytes")
|
||||||
|
w.WriteHeader(http.StatusPartialContent)
|
||||||
|
_, _ = w.Write(rangeData) // client write error ignored (disconnect during MISS range body send is not actionable)
|
||||||
|
|
||||||
|
// Range required an upstream fetch (full object) then served as 206
|
||||||
|
sc.metrics.IncrementRangeUpstream()
|
||||||
|
sc.metrics.AddBytesServed(int64(len(rangeData))) // range bytes only, not the full cached object
|
||||||
|
}
|
||||||
|
} else {
|
||||||
w.WriteHeader(resp.StatusCode)
|
w.WriteHeader(resp.StatusCode)
|
||||||
_, _ = w.Write(bodyData) // client write error ignored (disconnect during MISS body send is not actionable)
|
_, _ = w.Write(bodyData) // client write error ignored (disconnect during MISS body send is not actionable)
|
||||||
|
sc.metrics.AddBytesServed(int64(len(bodyData)))
|
||||||
|
}
|
||||||
|
|
||||||
// Track cache miss metrics
|
// Track cache miss metrics
|
||||||
sc.metrics.IncrementCacheMisses()
|
sc.metrics.IncrementCacheMisses()
|
||||||
sc.metrics.AddResponseTime(time.Since(tstart))
|
sc.metrics.AddResponseTime(time.Since(tstart))
|
||||||
sc.metrics.AddBytesServed(int64(len(bodyData)))
|
|
||||||
sc.metrics.IncrementServiceRequests(service.Name)
|
sc.metrics.IncrementServiceRequests(service.Name)
|
||||||
|
|
||||||
// Verify we received the complete file by checking Content-Length
|
// Verify we received the complete file by checking Content-Length
|
||||||
@@ -588,8 +648,6 @@ func (sc *SteamCache) ServeHTTP(w http.ResponseWriter, r *http.Request) {
|
|||||||
sc.metrics.IncrementServiceError("cache_write")
|
sc.metrics.IncrementServiceError("cache_write")
|
||||||
_ = sc.vfs.Delete(cachePath) // best-effort removal of partial corrupt cache entry on write failure; non-fatal. Deferred cacheWriter.Close() (from earlier in block) runs after this on error unwind path (harmless per DiskFS design)
|
_ = sc.vfs.Delete(cachePath) // best-effort removal of partial corrupt cache entry on write failure; non-fatal. Deferred cacheWriter.Close() (from earlier in block) runs after this on error unwind path (harmless per DiskFS design)
|
||||||
} else {
|
} else {
|
||||||
// Track successful cache write
|
|
||||||
sc.metrics.AddBytesCached(int64(len(cacheData)))
|
|
||||||
logger.Logger.Debug().
|
logger.Logger.Debug().
|
||||||
Str("key", cacheKey).
|
Str("key", cacheKey).
|
||||||
Str("url", urlPath).
|
Str("url", urlPath).
|
||||||
|
|||||||
@@ -16,13 +16,15 @@ type Metrics struct {
|
|||||||
CacheHits int64
|
CacheHits int64
|
||||||
CacheMisses int64
|
CacheMisses int64
|
||||||
CacheCoalesced int64
|
CacheCoalesced int64
|
||||||
|
RangeCache int64 // Range requests served as 206 from an already-cached object (HIT)
|
||||||
|
RangeUpstream int64 // Range requests that required an upstream fetch (full object), served as 206
|
||||||
Errors int64
|
Errors int64
|
||||||
RateLimited int64
|
RateLimited int64
|
||||||
|
|
||||||
// Performance metrics
|
// Performance metrics
|
||||||
TotalResponseTime int64 // in nanoseconds
|
TotalResponseTime int64 // in nanoseconds
|
||||||
TotalBytesServed int64
|
TotalBytesServed int64
|
||||||
TotalBytesCached int64
|
TotalBytesSaved int64 // bytes served from cache instead of being re-downloaded from upstream
|
||||||
|
|
||||||
// Cache metrics
|
// Cache metrics
|
||||||
MemoryCacheSize int64
|
MemoryCacheSize int64
|
||||||
@@ -31,6 +33,7 @@ type Metrics struct {
|
|||||||
DiskCacheHits int64
|
DiskCacheHits int64
|
||||||
Promotions int64
|
Promotions int64
|
||||||
Evictions int64
|
Evictions int64
|
||||||
|
DiskTierReady int64 // 0=pending (or unset), 1=ready or no-disk (N/A)
|
||||||
|
|
||||||
// Expanded observability (upstream breakdowns, cache write failures, per-service errors)
|
// Expanded observability (upstream breakdowns, cache write failures, per-service errors)
|
||||||
UpstreamErrors int64
|
UpstreamErrors int64
|
||||||
@@ -78,6 +81,17 @@ func (m *Metrics) IncrementCacheCoalesced() {
|
|||||||
atomic.AddInt64(&m.CacheCoalesced, 1)
|
atomic.AddInt64(&m.CacheCoalesced, 1)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// IncrementRangeCache increments the Range-from-cache counter (HIT served as 206)
|
||||||
|
func (m *Metrics) IncrementRangeCache() {
|
||||||
|
atomic.AddInt64(&m.RangeCache, 1)
|
||||||
|
}
|
||||||
|
|
||||||
|
// IncrementRangeUpstream increments the Range-from-upstream counter (full object
|
||||||
|
// fetched upstream, requested slice served as 206)
|
||||||
|
func (m *Metrics) IncrementRangeUpstream() {
|
||||||
|
atomic.AddInt64(&m.RangeUpstream, 1)
|
||||||
|
}
|
||||||
|
|
||||||
// IncrementErrors increments the error counter
|
// IncrementErrors increments the error counter
|
||||||
func (m *Metrics) IncrementErrors() {
|
func (m *Metrics) IncrementErrors() {
|
||||||
atomic.AddInt64(&m.Errors, 1)
|
atomic.AddInt64(&m.Errors, 1)
|
||||||
@@ -98,9 +112,10 @@ func (m *Metrics) AddBytesServed(bytes int64) {
|
|||||||
atomic.AddInt64(&m.TotalBytesServed, bytes)
|
atomic.AddInt64(&m.TotalBytesServed, bytes)
|
||||||
}
|
}
|
||||||
|
|
||||||
// AddBytesCached adds bytes cached to the total
|
// AddBytesSaved records bytes that were served from the cache instead of being
|
||||||
func (m *Metrics) AddBytesCached(bytes int64) {
|
// fetched again from the upstream (the main value metric for a cache).
|
||||||
atomic.AddInt64(&m.TotalBytesCached, bytes)
|
func (m *Metrics) AddBytesSaved(bytes int64) {
|
||||||
|
atomic.AddInt64(&m.TotalBytesSaved, bytes)
|
||||||
}
|
}
|
||||||
|
|
||||||
// SetMemoryCacheSize sets the current memory cache size
|
// SetMemoryCacheSize sets the current memory cache size
|
||||||
@@ -113,6 +128,17 @@ func (m *Metrics) SetDiskCacheSize(size int64) {
|
|||||||
atomic.StoreInt64(&m.DiskCacheSize, size)
|
atomic.StoreInt64(&m.DiskCacheSize, size)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// SetDiskTierReady sets whether the disk slow tier is attached (1) or still pending (0).
|
||||||
|
// Memory-only (no disk) also uses 1 — meaning "not waiting on disk attach". Reset does not clear this.
|
||||||
|
func (m *Metrics) SetDiskTierReady(ready int64) {
|
||||||
|
atomic.StoreInt64(&m.DiskTierReady, ready)
|
||||||
|
}
|
||||||
|
|
||||||
|
// GetDiskTierReady returns 1 if disk tier is ready (or no disk configured), else 0 while attach pending.
|
||||||
|
func (m *Metrics) GetDiskTierReady() int64 {
|
||||||
|
return atomic.LoadInt64(&m.DiskTierReady)
|
||||||
|
}
|
||||||
|
|
||||||
// IncrementMemoryCacheHits increments memory cache hits
|
// IncrementMemoryCacheHits increments memory cache hits
|
||||||
func (m *Metrics) IncrementMemoryCacheHits() {
|
func (m *Metrics) IncrementMemoryCacheHits() {
|
||||||
atomic.AddInt64(&m.MemoryCacheHits, 1)
|
atomic.AddInt64(&m.MemoryCacheHits, 1)
|
||||||
@@ -187,14 +213,17 @@ func (m *Metrics) GetStats() *Stats {
|
|||||||
CacheHits: cacheHits,
|
CacheHits: cacheHits,
|
||||||
CacheMisses: cacheMisses,
|
CacheMisses: cacheMisses,
|
||||||
CacheCoalesced: atomic.LoadInt64(&m.CacheCoalesced),
|
CacheCoalesced: atomic.LoadInt64(&m.CacheCoalesced),
|
||||||
|
RangeCache: atomic.LoadInt64(&m.RangeCache),
|
||||||
|
RangeUpstream: atomic.LoadInt64(&m.RangeUpstream),
|
||||||
Errors: atomic.LoadInt64(&m.Errors),
|
Errors: atomic.LoadInt64(&m.Errors),
|
||||||
RateLimited: atomic.LoadInt64(&m.RateLimited),
|
RateLimited: atomic.LoadInt64(&m.RateLimited),
|
||||||
HitRate: hitRate,
|
HitRate: hitRate,
|
||||||
AvgResponseTime: avgResponseTime,
|
AvgResponseTime: avgResponseTime,
|
||||||
TotalBytesServed: atomic.LoadInt64(&m.TotalBytesServed),
|
TotalBytesServed: atomic.LoadInt64(&m.TotalBytesServed),
|
||||||
TotalBytesCached: atomic.LoadInt64(&m.TotalBytesCached),
|
TotalBytesSaved: atomic.LoadInt64(&m.TotalBytesSaved),
|
||||||
MemoryCacheSize: atomic.LoadInt64(&m.MemoryCacheSize),
|
MemoryCacheSize: atomic.LoadInt64(&m.MemoryCacheSize),
|
||||||
DiskCacheSize: atomic.LoadInt64(&m.DiskCacheSize),
|
DiskCacheSize: atomic.LoadInt64(&m.DiskCacheSize),
|
||||||
|
DiskTierReady: atomic.LoadInt64(&m.DiskTierReady),
|
||||||
MemoryCacheHits: atomic.LoadInt64(&m.MemoryCacheHits),
|
MemoryCacheHits: atomic.LoadInt64(&m.MemoryCacheHits),
|
||||||
DiskCacheHits: atomic.LoadInt64(&m.DiskCacheHits),
|
DiskCacheHits: atomic.LoadInt64(&m.DiskCacheHits),
|
||||||
Promotions: atomic.LoadInt64(&m.Promotions),
|
Promotions: atomic.LoadInt64(&m.Promotions),
|
||||||
@@ -214,11 +243,13 @@ func (m *Metrics) Reset() {
|
|||||||
atomic.StoreInt64(&m.CacheHits, 0)
|
atomic.StoreInt64(&m.CacheHits, 0)
|
||||||
atomic.StoreInt64(&m.CacheMisses, 0)
|
atomic.StoreInt64(&m.CacheMisses, 0)
|
||||||
atomic.StoreInt64(&m.CacheCoalesced, 0)
|
atomic.StoreInt64(&m.CacheCoalesced, 0)
|
||||||
|
atomic.StoreInt64(&m.RangeCache, 0)
|
||||||
|
atomic.StoreInt64(&m.RangeUpstream, 0)
|
||||||
atomic.StoreInt64(&m.Errors, 0)
|
atomic.StoreInt64(&m.Errors, 0)
|
||||||
atomic.StoreInt64(&m.RateLimited, 0)
|
atomic.StoreInt64(&m.RateLimited, 0)
|
||||||
atomic.StoreInt64(&m.TotalResponseTime, 0)
|
atomic.StoreInt64(&m.TotalResponseTime, 0)
|
||||||
atomic.StoreInt64(&m.TotalBytesServed, 0)
|
atomic.StoreInt64(&m.TotalBytesServed, 0)
|
||||||
atomic.StoreInt64(&m.TotalBytesCached, 0)
|
atomic.StoreInt64(&m.TotalBytesSaved, 0)
|
||||||
atomic.StoreInt64(&m.MemoryCacheHits, 0)
|
atomic.StoreInt64(&m.MemoryCacheHits, 0)
|
||||||
atomic.StoreInt64(&m.DiskCacheHits, 0)
|
atomic.StoreInt64(&m.DiskCacheHits, 0)
|
||||||
atomic.StoreInt64(&m.Promotions, 0)
|
atomic.StoreInt64(&m.Promotions, 0)
|
||||||
@@ -243,14 +274,18 @@ type Stats struct {
|
|||||||
CacheHits int64
|
CacheHits int64
|
||||||
CacheMisses int64
|
CacheMisses int64
|
||||||
CacheCoalesced int64
|
CacheCoalesced int64
|
||||||
|
RangeCache int64
|
||||||
|
RangeUpstream int64
|
||||||
Errors int64
|
Errors int64
|
||||||
RateLimited int64
|
RateLimited int64
|
||||||
HitRate float64
|
HitRate float64
|
||||||
AvgResponseTime time.Duration
|
AvgResponseTime time.Duration
|
||||||
TotalBytesServed int64
|
TotalBytesServed int64
|
||||||
TotalBytesCached int64
|
TotalBytesSaved int64
|
||||||
MemoryCacheSize int64
|
MemoryCacheSize int64
|
||||||
|
|
||||||
DiskCacheSize int64
|
DiskCacheSize int64
|
||||||
|
DiskTierReady int64
|
||||||
MemoryCacheHits int64
|
MemoryCacheHits int64
|
||||||
DiskCacheHits int64
|
DiskCacheHits int64
|
||||||
Promotions int64
|
Promotions int64
|
||||||
@@ -274,6 +309,8 @@ func WriteText(w http.ResponseWriter, stats *Stats) {
|
|||||||
_, _ = fmt.Fprintf(w, "cache_hits %d\n", stats.CacheHits)
|
_, _ = fmt.Fprintf(w, "cache_hits %d\n", stats.CacheHits)
|
||||||
_, _ = fmt.Fprintf(w, "cache_misses %d\n", stats.CacheMisses)
|
_, _ = fmt.Fprintf(w, "cache_misses %d\n", stats.CacheMisses)
|
||||||
_, _ = fmt.Fprintf(w, "cache_coalesced %d\n", stats.CacheCoalesced)
|
_, _ = fmt.Fprintf(w, "cache_coalesced %d\n", stats.CacheCoalesced)
|
||||||
|
_, _ = fmt.Fprintf(w, "range_cache %d\n", stats.RangeCache)
|
||||||
|
_, _ = fmt.Fprintf(w, "range_upstream %d\n", stats.RangeUpstream)
|
||||||
_, _ = fmt.Fprintf(w, "errors %d\n", stats.Errors)
|
_, _ = fmt.Fprintf(w, "errors %d\n", stats.Errors)
|
||||||
_, _ = fmt.Fprintf(w, "rate_limited %d\n", stats.RateLimited)
|
_, _ = fmt.Fprintf(w, "rate_limited %d\n", stats.RateLimited)
|
||||||
_, _ = fmt.Fprintf(w, "upstream_errors %d\n", stats.UpstreamErrors)
|
_, _ = fmt.Fprintf(w, "upstream_errors %d\n", stats.UpstreamErrors)
|
||||||
@@ -291,8 +328,10 @@ func WriteText(w http.ResponseWriter, stats *Stats) {
|
|||||||
_, _ = fmt.Fprintf(w, "hit_rate %.4f\n", stats.HitRate)
|
_, _ = fmt.Fprintf(w, "hit_rate %.4f\n", stats.HitRate)
|
||||||
_, _ = fmt.Fprintf(w, "avg_response_time_ms %.2f\n", float64(stats.AvgResponseTime.Nanoseconds())/1e6)
|
_, _ = fmt.Fprintf(w, "avg_response_time_ms %.2f\n", float64(stats.AvgResponseTime.Nanoseconds())/1e6)
|
||||||
_, _ = fmt.Fprintf(w, "total_bytes_served %d\n", stats.TotalBytesServed)
|
_, _ = fmt.Fprintf(w, "total_bytes_served %d\n", stats.TotalBytesServed)
|
||||||
_, _ = fmt.Fprintf(w, "total_bytes_cached %d\n", stats.TotalBytesCached)
|
_, _ = fmt.Fprintf(w, "total_bytes_saved %d\n", stats.TotalBytesSaved)
|
||||||
|
|
||||||
_, _ = fmt.Fprintf(w, "memory_cache_size %d\n", stats.MemoryCacheSize)
|
_, _ = fmt.Fprintf(w, "memory_cache_size %d\n", stats.MemoryCacheSize)
|
||||||
_, _ = fmt.Fprintf(w, "disk_cache_size %d\n", stats.DiskCacheSize)
|
_, _ = fmt.Fprintf(w, "disk_cache_size %d\n", stats.DiskCacheSize)
|
||||||
|
_, _ = fmt.Fprintf(w, "disk_tier_ready %d\n", stats.DiskTierReady)
|
||||||
_, _ = fmt.Fprintf(w, "uptime_seconds %.2f\n", stats.Uptime.Seconds())
|
_, _ = fmt.Fprintf(w, "uptime_seconds %.2f\n", stats.Uptime.Seconds())
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -0,0 +1,386 @@
|
|||||||
|
// steamcache/range_test.go
|
||||||
|
package steamcache
|
||||||
|
|
||||||
|
import (
|
||||||
|
"io"
|
||||||
|
"net/http"
|
||||||
|
"net/http/httptest"
|
||||||
|
"strings"
|
||||||
|
"sync/atomic"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
)
|
||||||
|
|
||||||
|
// TestRangeHitServedLocally verifies that a Range GET against an already-cached
|
||||||
|
// object is served locally as 206 (no upstream re-fetch) and increments range_cache.
|
||||||
|
func TestRangeHitServedLocally(t *testing.T) {
|
||||||
|
body := []byte("0123456789abcdef") // 16 bytes
|
||||||
|
var upstreamCalls atomic.Int64
|
||||||
|
f := func(w http.ResponseWriter, r *http.Request) {
|
||||||
|
upstreamCalls.Add(1)
|
||||||
|
w.Header().Set("Content-Type", "application/octet-stream")
|
||||||
|
_, _ = w.Write(body)
|
||||||
|
}
|
||||||
|
sc, _ := newTestCacheWithFakeUpstream(t, f, "1MB", "0")
|
||||||
|
srv := newCacheServer(t, sc)
|
||||||
|
c := &http.Client{Timeout: 5 * time.Second}
|
||||||
|
|
||||||
|
// 1) Populate the cache with a full (non-Range) MISS.
|
||||||
|
req, err := http.NewRequest("GET", srv.URL+"/depot/rangetest/chunk", nil)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("NewRequest: %v", err)
|
||||||
|
}
|
||||||
|
req.Header.Set("User-Agent", "Valve/Steam HTTP Client 1.0")
|
||||||
|
resp, err := c.Do(req)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("miss GET: %v", err)
|
||||||
|
}
|
||||||
|
if resp.StatusCode != http.StatusOK {
|
||||||
|
t.Fatalf("miss GET: expected 200, got %d", resp.StatusCode)
|
||||||
|
}
|
||||||
|
_, _ = io.Copy(io.Discard, resp.Body)
|
||||||
|
_ = resp.Body.Close()
|
||||||
|
if got := sc.GetMetrics().CacheMisses; got < 1 {
|
||||||
|
t.Fatalf("expected CacheMisses >= 1 after first GET, got %d", got)
|
||||||
|
}
|
||||||
|
|
||||||
|
// 2) Range GET against the same URL — must be a local 206 HIT.
|
||||||
|
req2, err := http.NewRequest("GET", srv.URL+"/depot/rangetest/chunk", nil)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("NewRequest: %v", err)
|
||||||
|
}
|
||||||
|
req2.Header.Set("User-Agent", "Valve/Steam HTTP Client 1.0")
|
||||||
|
req2.Header.Set("Range", "bytes=4-7")
|
||||||
|
resp2, err := c.Do(req2)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("range GET: %v", err)
|
||||||
|
}
|
||||||
|
data, err := io.ReadAll(resp2.Body)
|
||||||
|
_ = resp2.Body.Close()
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("read range body: %v", err)
|
||||||
|
}
|
||||||
|
if resp2.StatusCode != http.StatusPartialContent {
|
||||||
|
t.Fatalf("range GET: expected 206, got %d", resp2.StatusCode)
|
||||||
|
}
|
||||||
|
if string(data) != "4567" {
|
||||||
|
t.Errorf("range GET: expected body %q, got %q", "4567", data)
|
||||||
|
}
|
||||||
|
if got := resp2.Header.Get("X-LanCache-Status"); got != "HIT" {
|
||||||
|
t.Errorf("range GET: expected X-LanCache-Status HIT, got %q", got)
|
||||||
|
}
|
||||||
|
if got := resp2.Header.Get("Content-Range"); got != "bytes 4-7/16" {
|
||||||
|
t.Errorf("range GET: expected Content-Range bytes 4-7/16, got %q", got)
|
||||||
|
}
|
||||||
|
if got := resp2.Header.Get("Accept-Ranges"); got != "bytes" {
|
||||||
|
t.Errorf("range GET: expected Accept-Ranges bytes, got %q", got)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Upstream must NOT have been hit again.
|
||||||
|
if got := upstreamCalls.Load(); got != 1 {
|
||||||
|
t.Errorf("upstream hit %d times, want exactly 1 (range HIT must be local)", got)
|
||||||
|
}
|
||||||
|
|
||||||
|
// range_cache incremented, range_upstream untouched.
|
||||||
|
stats := sc.GetMetrics()
|
||||||
|
if stats.RangeCache != 1 {
|
||||||
|
t.Errorf("expected RangeCache == 1 after range HIT, got %d", stats.RangeCache)
|
||||||
|
}
|
||||||
|
if stats.RangeUpstream != 0 {
|
||||||
|
t.Errorf("expected RangeUpstream == 0 (no range miss yet), got %d", stats.RangeUpstream)
|
||||||
|
}
|
||||||
|
if stats.CacheHits < 1 {
|
||||||
|
t.Errorf("expected CacheHits >= 1 after range HIT, got %d", stats.CacheHits)
|
||||||
|
}
|
||||||
|
// BytesServed: 16 (full miss body) + 4 (range bytes) = 20.
|
||||||
|
if stats.TotalBytesServed != 20 {
|
||||||
|
t.Errorf("expected TotalBytesServed == 20 (16 + range 4), got %d", stats.TotalBytesServed)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestRangeMissServes206FromFullFetch verifies that a Range GET on a cold key fetches
|
||||||
|
// the FULL object from upstream (Range stripped), caches it, and serves the requested
|
||||||
|
// slice as 206 with range_upstream incremented.
|
||||||
|
func TestRangeMissServes206FromFullFetch(t *testing.T) {
|
||||||
|
body := []byte("0123456789abcdef") // 16 bytes
|
||||||
|
var upstreamCalls atomic.Int64
|
||||||
|
var upstreamSawRange atomic.Bool
|
||||||
|
f := func(w http.ResponseWriter, r *http.Request) {
|
||||||
|
upstreamCalls.Add(1)
|
||||||
|
if r.Header.Get("Range") != "" {
|
||||||
|
upstreamSawRange.Store(true)
|
||||||
|
}
|
||||||
|
w.Header().Set("Content-Type", "application/octet-stream")
|
||||||
|
_, _ = w.Write(body)
|
||||||
|
}
|
||||||
|
sc, _ := newTestCacheWithFakeUpstream(t, f, "1MB", "0")
|
||||||
|
srv := newCacheServer(t, sc)
|
||||||
|
c := &http.Client{Timeout: 5 * time.Second}
|
||||||
|
|
||||||
|
// Range GET on a cold key.
|
||||||
|
req, err := http.NewRequest("GET", srv.URL+"/depot/rangetest/chunk2", nil)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("NewRequest: %v", err)
|
||||||
|
}
|
||||||
|
req.Header.Set("User-Agent", "Valve/Steam HTTP Client 1.0")
|
||||||
|
req.Header.Set("Range", "bytes=0-3")
|
||||||
|
resp, err := c.Do(req)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("range miss GET: %v", err)
|
||||||
|
}
|
||||||
|
data, err := io.ReadAll(resp.Body)
|
||||||
|
_ = resp.Body.Close()
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("read body: %v", err)
|
||||||
|
}
|
||||||
|
if resp.StatusCode != http.StatusPartialContent {
|
||||||
|
t.Fatalf("range miss GET: expected 206, got %d", resp.StatusCode)
|
||||||
|
}
|
||||||
|
if string(data) != "0123" {
|
||||||
|
t.Errorf("range miss GET: expected body %q, got %q", "0123", data)
|
||||||
|
}
|
||||||
|
if got := resp.Header.Get("X-LanCache-Status"); got != "MISS" {
|
||||||
|
t.Errorf("range miss GET: expected X-LanCache-Status MISS, got %q", got)
|
||||||
|
}
|
||||||
|
if got := resp.Header.Get("Content-Range"); got != "bytes 0-3/16" {
|
||||||
|
t.Errorf("range miss GET: expected Content-Range bytes 0-3/16, got %q", got)
|
||||||
|
}
|
||||||
|
if got := resp.Header.Get("Accept-Ranges"); got != "bytes" {
|
||||||
|
t.Errorf("range miss GET: expected Accept-Ranges bytes, got %q", got)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Range must have been stripped for the upstream fetch (full file cached).
|
||||||
|
if upstreamSawRange.Load() {
|
||||||
|
t.Error("upstream received a Range header; Range must be stripped so the full object is cached")
|
||||||
|
}
|
||||||
|
if got := upstreamCalls.Load(); got != 1 {
|
||||||
|
t.Errorf("upstream hit %d times, want exactly 1", got)
|
||||||
|
}
|
||||||
|
|
||||||
|
// range_upstream incremented, range_cache untouched.
|
||||||
|
stats := sc.GetMetrics()
|
||||||
|
if stats.RangeUpstream != 1 {
|
||||||
|
t.Errorf("expected RangeUpstream == 1 after range MISS, got %d", stats.RangeUpstream)
|
||||||
|
}
|
||||||
|
if stats.RangeCache != 0 {
|
||||||
|
t.Errorf("expected RangeCache == 0 (no range hit yet), got %d", stats.RangeCache)
|
||||||
|
}
|
||||||
|
if stats.CacheMisses < 1 {
|
||||||
|
t.Errorf("expected CacheMisses >= 1, got %d", stats.CacheMisses)
|
||||||
|
}
|
||||||
|
// BytesServed for the range miss: only the 4 range bytes, not the 16-byte fetch.
|
||||||
|
if stats.TotalBytesServed != 4 {
|
||||||
|
t.Errorf("expected TotalBytesServed == 4 (range bytes only), got %d", stats.TotalBytesServed)
|
||||||
|
}
|
||||||
|
|
||||||
|
// The FULL object must have been cached: a subsequent full GET is a HIT with the
|
||||||
|
// complete 16-byte body.
|
||||||
|
req3, err := http.NewRequest("GET", srv.URL+"/depot/rangetest/chunk2", nil)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("NewRequest: %v", err)
|
||||||
|
}
|
||||||
|
req3.Header.Set("User-Agent", "Valve/Steam HTTP Client 1.0")
|
||||||
|
resp3, err := c.Do(req3)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("full GET after range miss: %v", err)
|
||||||
|
}
|
||||||
|
fullData, err := io.ReadAll(resp3.Body)
|
||||||
|
_ = resp3.Body.Close()
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("read full body: %v", err)
|
||||||
|
}
|
||||||
|
if resp3.StatusCode != http.StatusOK {
|
||||||
|
t.Fatalf("full GET after range miss: expected 200, got %d", resp3.StatusCode)
|
||||||
|
}
|
||||||
|
if got := resp3.Header.Get("X-LanCache-Status"); got != "HIT" {
|
||||||
|
t.Errorf("full GET after range miss: expected X-LanCache-Status HIT, got %q", got)
|
||||||
|
}
|
||||||
|
if len(fullData) != len(body) || string(fullData) != string(body) {
|
||||||
|
t.Errorf("full GET after range miss: expected full %d-byte body, got %d bytes", len(body), len(fullData))
|
||||||
|
}
|
||||||
|
if got := upstreamCalls.Load(); got != 1 {
|
||||||
|
t.Errorf("upstream hit %d times after HIT, want still 1", got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestRangeMissInvalidRange416 verifies that an unsatisfiable Range on a cold key
|
||||||
|
// fetches upstream, returns 416 (as on the HIT path), and does not count range_upstream.
|
||||||
|
func TestRangeMissInvalidRange416(t *testing.T) {
|
||||||
|
body := []byte("0123456789abcdef") // 16 bytes
|
||||||
|
var upstreamCalls atomic.Int64
|
||||||
|
f := func(w http.ResponseWriter, r *http.Request) {
|
||||||
|
upstreamCalls.Add(1)
|
||||||
|
_, _ = w.Write(body)
|
||||||
|
}
|
||||||
|
sc, _ := newTestCacheWithFakeUpstream(t, f, "1MB", "0")
|
||||||
|
srv := newCacheServer(t, sc)
|
||||||
|
c := &http.Client{Timeout: 5 * time.Second}
|
||||||
|
|
||||||
|
req, err := http.NewRequest("GET", srv.URL+"/depot/rangetest/chunk3", nil)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("NewRequest: %v", err)
|
||||||
|
}
|
||||||
|
req.Header.Set("User-Agent", "Valve/Steam HTTP Client 1.0")
|
||||||
|
req.Header.Set("Range", "bytes=100-200")
|
||||||
|
resp, err := c.Do(req)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("invalid range GET: %v", err)
|
||||||
|
}
|
||||||
|
data, err := io.ReadAll(resp.Body)
|
||||||
|
_ = resp.Body.Close()
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("read body: %v", err)
|
||||||
|
}
|
||||||
|
if resp.StatusCode != http.StatusRequestedRangeNotSatisfiable {
|
||||||
|
t.Fatalf("invalid range GET: expected 416, got %d", resp.StatusCode)
|
||||||
|
}
|
||||||
|
if len(data) != 0 {
|
||||||
|
t.Errorf("invalid range GET: expected empty body, got %d bytes", len(data))
|
||||||
|
}
|
||||||
|
if got := resp.Header.Get("Content-Range"); got != "bytes */16" {
|
||||||
|
t.Errorf("invalid range GET: expected Content-Range bytes */16, got %q", got)
|
||||||
|
}
|
||||||
|
if got := upstreamCalls.Load(); got != 1 {
|
||||||
|
t.Errorf("upstream hit %d times, want exactly 1 (fetch happens, then 416 to client)", got)
|
||||||
|
}
|
||||||
|
stats := sc.GetMetrics()
|
||||||
|
if stats.RangeUpstream != 0 {
|
||||||
|
t.Errorf("expected RangeUpstream == 0 for unsatisfiable range, got %d", stats.RangeUpstream)
|
||||||
|
}
|
||||||
|
if stats.RangeCache != 0 {
|
||||||
|
t.Errorf("expected RangeCache == 0, got %d", stats.RangeCache)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestRangeMetricsWriteText verifies /metrics emits the range_cache and range_upstream
|
||||||
|
// lines with the expected values after a range HIT and a range MISS.
|
||||||
|
func TestRangeMetricsWriteText(t *testing.T) {
|
||||||
|
body := []byte("0123456789abcdef")
|
||||||
|
f := func(w http.ResponseWriter, r *http.Request) {
|
||||||
|
_, _ = w.Write(body)
|
||||||
|
}
|
||||||
|
sc, _ := newTestCacheWithFakeUpstream(t, f, "1MB", "0")
|
||||||
|
srv := newCacheServer(t, sc)
|
||||||
|
c := &http.Client{Timeout: 5 * time.Second}
|
||||||
|
|
||||||
|
get := func(path, rangeHeader string) int {
|
||||||
|
t.Helper()
|
||||||
|
req, err := http.NewRequest("GET", srv.URL+path, nil)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("NewRequest: %v", err)
|
||||||
|
}
|
||||||
|
req.Header.Set("User-Agent", "Valve/Steam HTTP Client 1.0")
|
||||||
|
if rangeHeader != "" {
|
||||||
|
req.Header.Set("Range", rangeHeader)
|
||||||
|
}
|
||||||
|
resp, err := c.Do(req)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("GET %s: %v", path, err)
|
||||||
|
}
|
||||||
|
_, _ = io.Copy(io.Discard, resp.Body)
|
||||||
|
_ = resp.Body.Close()
|
||||||
|
return resp.StatusCode
|
||||||
|
}
|
||||||
|
|
||||||
|
// Range MISS on cold key -> range_upstream; warm it; range HIT -> range_cache.
|
||||||
|
if code := get("/depot/rangetest/wt/1", "bytes=0-3"); code != http.StatusPartialContent {
|
||||||
|
t.Fatalf("range miss: expected 206, got %d", code)
|
||||||
|
}
|
||||||
|
if code := get("/depot/rangetest/wt/2", ""); code != http.StatusOK {
|
||||||
|
t.Fatalf("warm miss: expected 200, got %d", code)
|
||||||
|
}
|
||||||
|
if code := get("/depot/rangetest/wt/2", "bytes=8-11"); code != http.StatusPartialContent {
|
||||||
|
t.Fatalf("range hit: expected 206, got %d", code)
|
||||||
|
}
|
||||||
|
|
||||||
|
req, err := http.NewRequest("GET", srv.URL+"/metrics", nil)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("NewRequest: %v", err)
|
||||||
|
}
|
||||||
|
resp, err := c.Do(req)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("GET /metrics: %v", err)
|
||||||
|
}
|
||||||
|
out, err := io.ReadAll(resp.Body)
|
||||||
|
_ = resp.Body.Close()
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("read /metrics: %v", err)
|
||||||
|
}
|
||||||
|
text := string(out)
|
||||||
|
if !strings.Contains(text, "range_cache 1\n") {
|
||||||
|
t.Errorf("/metrics missing 'range_cache 1' line:\n%s", text)
|
||||||
|
}
|
||||||
|
if !strings.Contains(text, "range_upstream 1\n") {
|
||||||
|
t.Errorf("/metrics missing 'range_upstream 1' line:\n%s", text)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestStreamCachedResponseRange206 is a focused unit test for streamCachedResponse:
|
||||||
|
// valid Range yields 206 with the exact slice + metrics; invalid Range yields 416
|
||||||
|
// with no range metrics.
|
||||||
|
func TestStreamCachedResponseRange206(t *testing.T) {
|
||||||
|
body := []byte("0123456789abcdef") // 16 bytes
|
||||||
|
raw := append([]byte("HTTP/1.1 200 OK\r\nContent-Type: application/octet-stream\r\n\r\n"), body...)
|
||||||
|
serialized, err := serializeRawResponse(raw)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("serialize cache file: %v", err)
|
||||||
|
}
|
||||||
|
cf, err := deserializeCacheFile(serialized)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("build cache file: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
sc, _ := newTestCacheWithFakeUpstream(t, func(w http.ResponseWriter, r *http.Request) {
|
||||||
|
_, _ = w.Write([]byte("x"))
|
||||||
|
}, "1MB", "0")
|
||||||
|
sc.ResetMetrics()
|
||||||
|
|
||||||
|
// Valid range -> 206 + slice + metrics.
|
||||||
|
rec := httptest.NewRecorder()
|
||||||
|
req := httptest.NewRequest("GET", "/depot/rangetest/chunk", nil)
|
||||||
|
req.Header.Set("Range", "bytes=4-7")
|
||||||
|
sc.streamCachedResponse(rec, req, cf, "steam/testkey", "127.0.0.1", time.Now())
|
||||||
|
|
||||||
|
if rec.Code != http.StatusPartialContent {
|
||||||
|
t.Fatalf("expected 206, got %d", rec.Code)
|
||||||
|
}
|
||||||
|
if rec.Body.String() != "4567" {
|
||||||
|
t.Errorf("expected body %q, got %q", "4567", rec.Body.String())
|
||||||
|
}
|
||||||
|
if got := rec.Header().Get("Content-Range"); got != "bytes 4-7/16" {
|
||||||
|
t.Errorf("expected Content-Range bytes 4-7/16, got %q", got)
|
||||||
|
}
|
||||||
|
if got := rec.Header().Get("X-LanCache-Status"); got != "HIT" {
|
||||||
|
t.Errorf("expected X-LanCache-Status HIT, got %q", got)
|
||||||
|
}
|
||||||
|
stats := sc.GetMetrics()
|
||||||
|
if stats.RangeCache != 1 {
|
||||||
|
t.Errorf("expected RangeCache == 1, got %d", stats.RangeCache)
|
||||||
|
}
|
||||||
|
if stats.TotalBytesServed != 4 {
|
||||||
|
t.Errorf("expected TotalBytesServed == 4 (range bytes), got %d", stats.TotalBytesServed)
|
||||||
|
}
|
||||||
|
if stats.TotalBytesSaved != 4 {
|
||||||
|
t.Errorf("expected TotalBytesSaved == 4 (range bytes), got %d", stats.TotalBytesSaved)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Invalid range -> 416, no range metrics, no bytes served.
|
||||||
|
rec2 := httptest.NewRecorder()
|
||||||
|
req2 := httptest.NewRequest("GET", "/depot/rangetest/chunk", nil)
|
||||||
|
req2.Header.Set("Range", "bytes=100-200")
|
||||||
|
sc.streamCachedResponse(rec2, req2, cf, "steam/testkey", "127.0.0.1", time.Now())
|
||||||
|
|
||||||
|
if rec2.Code != http.StatusRequestedRangeNotSatisfiable {
|
||||||
|
t.Fatalf("expected 416, got %d", rec2.Code)
|
||||||
|
}
|
||||||
|
if got := rec2.Header().Get("Content-Range"); got != "bytes */16" {
|
||||||
|
t.Errorf("expected Content-Range bytes */16, got %q", got)
|
||||||
|
}
|
||||||
|
stats = sc.GetMetrics()
|
||||||
|
if stats.RangeCache != 1 {
|
||||||
|
t.Errorf("RangeCache must stay 1 after unsatisfiable range, got %d", stats.RangeCache)
|
||||||
|
}
|
||||||
|
if stats.TotalBytesServed != 4 {
|
||||||
|
t.Errorf("TotalBytesServed must stay 4 after 416, got %d", stats.TotalBytesServed)
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -5,6 +5,7 @@ import (
|
|||||||
"crypto/sha256"
|
"crypto/sha256"
|
||||||
"encoding/hex"
|
"encoding/hex"
|
||||||
"fmt"
|
"fmt"
|
||||||
|
"net"
|
||||||
"net/http"
|
"net/http"
|
||||||
"regexp"
|
"regexp"
|
||||||
"strings"
|
"strings"
|
||||||
@@ -163,3 +164,44 @@ func generateServiceCacheKey(urlPath string, servicePrefix string) (string, erro
|
|||||||
}
|
}
|
||||||
return servicePrefix + "/" + hash, nil
|
return servicePrefix + "/" + hash, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// requestHostName strips a port and brackets from an HTTP Host header.
|
||||||
|
func requestHostName(host string) string {
|
||||||
|
host = strings.TrimSpace(host)
|
||||||
|
if host == "" {
|
||||||
|
return ""
|
||||||
|
}
|
||||||
|
if h, _, err := net.SplitHostPort(host); err == nil {
|
||||||
|
host = h
|
||||||
|
}
|
||||||
|
return strings.Trim(host, "[]")
|
||||||
|
}
|
||||||
|
|
||||||
|
func hostIsLiteralIP(host string) bool {
|
||||||
|
return net.ParseIP(requestHostName(host)) != nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// defaultDirectFetchSuffixes are CDN names Steam actually uses. Applied only when
|
||||||
|
// no configured upstream is set and the request Host is used as the fetch target.
|
||||||
|
var defaultDirectFetchSuffixes = []string{
|
||||||
|
"steamcontent.com",
|
||||||
|
"steampowered.com",
|
||||||
|
"steamstatic.com",
|
||||||
|
}
|
||||||
|
|
||||||
|
// hostAllowedForDirectFetch reports whether Host may be used as an origin when
|
||||||
|
// upstream is empty. Literal IPs are rejected (LAN/metadata SSRF). Names must
|
||||||
|
// be Steam CDN suffixes so a spoofed User-Agent cannot turn the cache into an
|
||||||
|
// open reverse proxy.
|
||||||
|
func hostAllowedForDirectFetch(host string) bool {
|
||||||
|
name := strings.ToLower(requestHostName(host))
|
||||||
|
if name == "" || hostIsLiteralIP(host) {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
for _, suf := range defaultDirectFetchSuffixes {
|
||||||
|
if name == suf || strings.HasSuffix(name, "."+suf) {
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
|||||||
@@ -179,6 +179,17 @@ func New(address string, memorySize string, diskSize string, diskPath, upstream,
|
|||||||
metrics: metrics.NewMetrics(),
|
metrics: metrics.NewMetrics(),
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Wire metrics into TieredCache so promotions are counted.
|
||||||
|
c.SetMetrics(sc.metrics)
|
||||||
|
|
||||||
|
// Wire metrics into the concrete storage tiers (MemoryFS / DiskFS) so per-tier hits and evictions are counted.
|
||||||
|
if m != nil {
|
||||||
|
m.SetMetrics(sc.metrics)
|
||||||
|
}
|
||||||
|
if d != nil {
|
||||||
|
d.SetMetrics(sc.metrics)
|
||||||
|
}
|
||||||
|
|
||||||
// Wire the request processor (constructor injection of interfaces for the owned wrappers + vfs + client + scalars).
|
// Wire the request processor (constructor injection of interfaces for the owned wrappers + vfs + client + scalars).
|
||||||
// Done after all fields including wrappers are set; before attach goroutines (no impact on lifecycle paths).
|
// Done after all fields including wrappers are set; before attach goroutines (no impact on lifecycle paths).
|
||||||
sc.processor = newRequestProcessor(sc)
|
sc.processor = newRequestProcessor(sc)
|
||||||
@@ -189,32 +200,47 @@ func New(address string, memorySize string, diskSize string, diskPath, upstream,
|
|||||||
|
|
||||||
if disksize == 0 && memorysize != 0 {
|
if disksize == 0 && memorysize != 0 {
|
||||||
// memory only mode - no disk
|
// memory only mode - no disk
|
||||||
|
sc.metrics.SetDiskTierReady(1) // no disk — N/A / not pending
|
||||||
c.SetSlow(mgc)
|
c.SetSlow(mgc)
|
||||||
} else if disksize != 0 && memorysize == 0 {
|
} else if disksize != 0 && memorysize == 0 {
|
||||||
// disk only mode: delay attach until disk ready (pure-proxy during scan; Create returns ErrNotFound until slow tier Set)
|
// disk only mode: delay attach until disk ready (pure-proxy during scan; Create returns ErrNotFound until slow tier Set)
|
||||||
|
sc.metrics.SetDiskTierReady(0)
|
||||||
|
logger.Logger.Info().Msg("Disk slow tier attach pending; Size barrier in progress")
|
||||||
sc.wg.Add(1)
|
sc.wg.Add(1)
|
||||||
go func() {
|
go func() {
|
||||||
defer sc.wg.Done()
|
defer sc.wg.Done()
|
||||||
|
t0 := time.Now()
|
||||||
_ = d.Size() // block on barrier per design (all Size callers during window do this; documented)
|
_ = d.Size() // block on barrier per design (all Size callers during window do this; documented)
|
||||||
select {
|
select {
|
||||||
case <-sc.shutdownCh:
|
case <-sc.shutdownCh:
|
||||||
return // Shutdown raced; do not attach or SetSlow after stop
|
return // Shutdown raced; do not attach or SetSlow after stop
|
||||||
default:
|
default:
|
||||||
c.SetSlow(dgc)
|
c.SetSlow(dgc)
|
||||||
|
sc.metrics.SetDiskTierReady(1)
|
||||||
|
logger.Logger.Info().
|
||||||
|
Dur("attach_delay", time.Since(t0)).
|
||||||
|
Msg("Disk slow tier attached (disk-only mode); prior traffic had no disk tier")
|
||||||
}
|
}
|
||||||
}()
|
}()
|
||||||
} else if disksize != 0 && memorysize != 0 {
|
} else if disksize != 0 && memorysize != 0 {
|
||||||
// memory and disk mode: fast mem immediate, disk delayed (mem-only during scan)
|
// memory and disk mode: fast mem immediate, disk delayed (mem-only during scan)
|
||||||
c.SetFast(mgc)
|
c.SetFast(mgc)
|
||||||
|
sc.metrics.SetDiskTierReady(0)
|
||||||
|
logger.Logger.Info().Msg("Disk slow tier attach pending; Size barrier in progress")
|
||||||
sc.wg.Add(1)
|
sc.wg.Add(1)
|
||||||
go func() {
|
go func() {
|
||||||
defer sc.wg.Done()
|
defer sc.wg.Done()
|
||||||
|
t0 := time.Now()
|
||||||
_ = d.Size()
|
_ = d.Size()
|
||||||
select {
|
select {
|
||||||
case <-sc.shutdownCh:
|
case <-sc.shutdownCh:
|
||||||
return
|
return
|
||||||
default:
|
default:
|
||||||
c.SetSlow(dgc)
|
c.SetSlow(dgc)
|
||||||
|
sc.metrics.SetDiskTierReady(1)
|
||||||
|
logger.Logger.Info().
|
||||||
|
Dur("attach_delay", time.Since(t0)).
|
||||||
|
Msg("Disk slow tier attached (mixed mode); prior traffic was memory-only")
|
||||||
}
|
}
|
||||||
}()
|
}()
|
||||||
}
|
}
|
||||||
@@ -311,15 +337,13 @@ func (sc *SteamCache) Shutdown() {
|
|||||||
|
|
||||||
// GetMetrics returns current metrics
|
// GetMetrics returns current metrics
|
||||||
func (sc *SteamCache) GetMetrics() *metrics.Stats {
|
func (sc *SteamCache) GetMetrics() *metrics.Stats {
|
||||||
// Update cache sizes
|
|
||||||
if sc.memory != nil {
|
if sc.memory != nil {
|
||||||
sc.metrics.SetMemoryCacheSize(sc.memory.Size())
|
sc.metrics.SetMemoryCacheSize(sc.memory.Size())
|
||||||
}
|
}
|
||||||
if sc.disk != nil {
|
// Skip disk.Size() while attach pending — Size() blocks on initDone and would hang /metrics.
|
||||||
// Note: blocks on initDone (post-eviction state) for accurate post-attach size during long disk init window.
|
if sc.disk != nil && sc.metrics.GetDiskTierReady() == 1 {
|
||||||
sc.metrics.SetDiskCacheSize(sc.disk.Size())
|
sc.metrics.SetDiskCacheSize(sc.disk.Size())
|
||||||
}
|
}
|
||||||
|
|
||||||
return sc.metrics.GetStats()
|
return sc.metrics.GetStats()
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -342,7 +366,7 @@ func newHTTPTransport() *http.Transport {
|
|||||||
DialContext: (&net.Dialer{
|
DialContext: (&net.Dialer{
|
||||||
Timeout: 10 * time.Second, // Faster connection timeout
|
Timeout: 10 * time.Second, // Faster connection timeout
|
||||||
KeepAlive: 60 * time.Second, // Longer keep-alive
|
KeepAlive: 60 * time.Second, // Longer keep-alive
|
||||||
DualStack: true, // Enable dual-stack (IPv4/IPv6)
|
// Dual-stack Happy Eyeballs is the default since Go 1.12 (DualStack is deprecated).
|
||||||
}).DialContext,
|
}).DialContext,
|
||||||
|
|
||||||
// Timeout optimizations
|
// Timeout optimizations
|
||||||
@@ -372,11 +396,10 @@ func newHTTPClient(transport *http.Transport) *http.Client {
|
|||||||
Timeout: 60 * time.Second, // Optimized timeout for better responsiveness
|
Timeout: 60 * time.Second, // Optimized timeout for better responsiveness
|
||||||
// Add redirect policy for better performance
|
// Add redirect policy for better performance
|
||||||
CheckRedirect: func(req *http.Request, via []*http.Request) error {
|
CheckRedirect: func(req *http.Request, via []*http.Request) error {
|
||||||
// Limit redirects to prevent infinite loops
|
// Do not follow redirects. Steam CDN chunk/manifest fetches are
|
||||||
if len(via) >= 10 {
|
// expected to be 200; following Location would let an origin send
|
||||||
|
// the cache at an arbitrary internal URL.
|
||||||
return http.ErrUseLastResponse
|
return http.ErrUseLastResponse
|
||||||
}
|
|
||||||
return nil
|
|
||||||
},
|
},
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -2,10 +2,12 @@
|
|||||||
package steamcache
|
package steamcache
|
||||||
|
|
||||||
import (
|
import (
|
||||||
|
"bufio"
|
||||||
"bytes"
|
"bytes"
|
||||||
"context"
|
"context"
|
||||||
"fmt"
|
"fmt"
|
||||||
"io"
|
"io"
|
||||||
|
"net"
|
||||||
"net/http"
|
"net/http"
|
||||||
"net/http/httptest"
|
"net/http/httptest"
|
||||||
"os"
|
"os"
|
||||||
@@ -17,6 +19,7 @@ import (
|
|||||||
"s1d3sw1ped/steamcache2/vfs/vfserror"
|
"s1d3sw1ped/steamcache2/vfs/vfserror"
|
||||||
"strings"
|
"strings"
|
||||||
"sync"
|
"sync"
|
||||||
|
"sync/atomic"
|
||||||
"testing"
|
"testing"
|
||||||
"time"
|
"time"
|
||||||
)
|
)
|
||||||
@@ -1057,6 +1060,12 @@ func TestDiskOnlyDelayedAttach(t *testing.T) {
|
|||||||
t.Errorf("during init window, expected ErrNotFound from disk-only tiered Create (no slow), got %v", err)
|
t.Errorf("during init window, expected ErrNotFound from disk-only tiered Create (no slow), got %v", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Disk tier is pending while the attach goroutine is in the Size barrier.
|
||||||
|
// GetMetrics must return quickly (it skips disk.Size() while pending) and report 0.
|
||||||
|
if got := sc.GetMetrics().DiskTierReady; got != 0 {
|
||||||
|
t.Errorf("during pending attach, DiskTierReady=%d, want 0", got)
|
||||||
|
}
|
||||||
|
|
||||||
// Wait the barrier (exercises the attach go's Size wait)
|
// Wait the barrier (exercises the attach go's Size wait)
|
||||||
_ = sc.disk.Size()
|
_ = sc.disk.Size()
|
||||||
|
|
||||||
@@ -1083,6 +1092,91 @@ func TestDiskOnlyDelayedAttach(t *testing.T) {
|
|||||||
} else {
|
} else {
|
||||||
rc.Close()
|
rc.Close()
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// After attach, the disk tier must be marked ready (1)
|
||||||
|
if got := sc.GetMetrics().DiskTierReady; got != 1 {
|
||||||
|
t.Errorf("post-attach DiskTierReady=%d, want 1 (ready)", got)
|
||||||
|
}
|
||||||
|
|
||||||
|
// /metrics text output includes the disk_tier_ready line
|
||||||
|
rec := httptest.NewRecorder()
|
||||||
|
metrics.WriteText(rec, sc.GetMetrics())
|
||||||
|
if !bytes.Contains(rec.Body.Bytes(), []byte("disk_tier_ready 1")) {
|
||||||
|
t.Errorf("WriteText output missing \"disk_tier_ready 1\": %q", rec.Body.String())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestDiskTierSignalMemoryOnly covers memory-only mode: DiskTierReady=1 (N/A, not
|
||||||
|
// waiting on disk attach) and heartbeat header X-SteamCache-Disk-Tier: disabled.
|
||||||
|
func TestDiskTierSignalMemoryOnly(t *testing.T) {
|
||||||
|
sc, err := New("127.0.0.1:0", "1MB", "0", t.TempDir(), "", "lru", "lru", 10, 5, "0", nil)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("New memory-only: %v", err)
|
||||||
|
}
|
||||||
|
t.Cleanup(func() { sc.Shutdown() })
|
||||||
|
|
||||||
|
if got := sc.GetMetrics().DiskTierReady; got != 1 {
|
||||||
|
t.Errorf("DiskTierReady=%d, want 1 (memory-only = N/A/not pending)", got)
|
||||||
|
}
|
||||||
|
|
||||||
|
req := httptest.NewRequest("GET", "/lancache-heartbeat", nil)
|
||||||
|
rec := httptest.NewRecorder()
|
||||||
|
sc.ServeHTTP(rec, req)
|
||||||
|
if rec.Code != http.StatusNoContent {
|
||||||
|
t.Errorf("heartbeat status=%d, want 204", rec.Code)
|
||||||
|
}
|
||||||
|
if got := rec.Header().Get("X-SteamCache-Disk-Tier"); got != "disabled" {
|
||||||
|
t.Errorf("X-SteamCache-Disk-Tier=%q, want disabled", got)
|
||||||
|
}
|
||||||
|
if got := rec.Header().Get("X-LanCache-Processed-By"); got != "SteamCache2" {
|
||||||
|
t.Errorf("X-LanCache-Processed-By=%q, want SteamCache2", got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestDiskTierSignalMixedPendingReady covers mixed mode: DiskTierReady=0 (header
|
||||||
|
// pending) while the disk attach is in the Size barrier, then DiskTierReady=1
|
||||||
|
// (header ready) after the barrier opens and the attach goroutine sets SetSlow.
|
||||||
|
func TestDiskTierSignalMixedPendingReady(t *testing.T) {
|
||||||
|
td := t.TempDir()
|
||||||
|
diskPath := filepath.Join(td, "disk")
|
||||||
|
if err := os.MkdirAll(diskPath, 0755); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
sc, err := New("127.0.0.1:0", "1MB", "10MB", diskPath, "", "lru", "lru", 10, 1, "0", nil)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("New mixed: %v", err)
|
||||||
|
}
|
||||||
|
t.Cleanup(func() { sc.Shutdown() })
|
||||||
|
|
||||||
|
// Immediately in the pending window
|
||||||
|
if got := sc.GetMetrics().DiskTierReady; got != 0 {
|
||||||
|
t.Errorf("immediate DiskTierReady=%d, want 0 (pending)", got)
|
||||||
|
}
|
||||||
|
req := httptest.NewRequest("GET", "/lancache-heartbeat", nil)
|
||||||
|
rec := httptest.NewRecorder()
|
||||||
|
sc.ServeHTTP(rec, req)
|
||||||
|
if got := rec.Header().Get("X-SteamCache-Disk-Tier"); got != "pending" {
|
||||||
|
t.Errorf("heartbeat header=%q, want pending", got)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Wait the barrier, then retry until the attach goroutine flips the flag
|
||||||
|
_ = sc.disk.Size()
|
||||||
|
deadline := time.Now().Add(2 * time.Second)
|
||||||
|
for time.Now().Before(deadline) {
|
||||||
|
if sc.GetMetrics().DiskTierReady == 1 {
|
||||||
|
break
|
||||||
|
}
|
||||||
|
time.Sleep(1 * time.Millisecond)
|
||||||
|
}
|
||||||
|
if got := sc.GetMetrics().DiskTierReady; got != 1 {
|
||||||
|
t.Fatalf("DiskTierReady=%d after barrier, want 1 (ready)", got)
|
||||||
|
}
|
||||||
|
|
||||||
|
rec2 := httptest.NewRecorder()
|
||||||
|
sc.ServeHTTP(rec2, httptest.NewRequest("GET", "/lancache-heartbeat", nil))
|
||||||
|
if got := rec2.Header().Get("X-SteamCache-Disk-Tier"); got != "ready" {
|
||||||
|
t.Errorf("heartbeat header=%q, want ready", got)
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// --- Phase 2: narrow black-box tests for the new wrapper types ---
|
// --- Phase 2: narrow black-box tests for the new wrapper types ---
|
||||||
@@ -1165,3 +1259,193 @@ func TestClientRateLimiter_BlackBox(t *testing.T) {
|
|||||||
t.Error("different clients must have distinct limiters")
|
t.Error("different clients must have distinct limiters")
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestHostAllowedForDirectFetch(t *testing.T) {
|
||||||
|
allowed := []string{
|
||||||
|
"lancache.steamcontent.com",
|
||||||
|
"cache1-iad1.steamcontent.com:443",
|
||||||
|
"steamcontent.com",
|
||||||
|
"content.steampowered.com",
|
||||||
|
"cdn.steamstatic.com",
|
||||||
|
}
|
||||||
|
denied := []string{
|
||||||
|
"",
|
||||||
|
"127.0.0.1",
|
||||||
|
"127.0.0.1:80",
|
||||||
|
"[::1]:80",
|
||||||
|
"192.168.1.1",
|
||||||
|
"169.254.169.254",
|
||||||
|
"evil.example",
|
||||||
|
"example.com",
|
||||||
|
"notsteamcontent.com",
|
||||||
|
}
|
||||||
|
for _, h := range allowed {
|
||||||
|
if !hostAllowedForDirectFetch(h) {
|
||||||
|
t.Errorf("expected allowed: %q", h)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
for _, h := range denied {
|
||||||
|
if hostAllowedForDirectFetch(h) {
|
||||||
|
t.Errorf("expected denied: %q", h)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestDirectFetchRejectsNonSteamHost(t *testing.T) {
|
||||||
|
td := t.TempDir()
|
||||||
|
sc, err := New("127.0.0.1:0", "1MB", "0", td, "", "lru", "lru", 200, 5, "0", nil)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("New: %v", err)
|
||||||
|
}
|
||||||
|
t.Cleanup(func() { sc.Shutdown() })
|
||||||
|
|
||||||
|
req := httptest.NewRequest("GET", "/depot/ssrf/chunk", nil)
|
||||||
|
req.Host = "127.0.0.1"
|
||||||
|
req.Header.Set("User-Agent", "Valve/Steam HTTP Client 1.0")
|
||||||
|
rec := httptest.NewRecorder()
|
||||||
|
sc.ServeHTTP(rec, req)
|
||||||
|
if rec.Code != http.StatusBadRequest {
|
||||||
|
t.Errorf("IP Host: expected 400, got %d", rec.Code)
|
||||||
|
}
|
||||||
|
|
||||||
|
req2 := httptest.NewRequest("GET", "/depot/ssrf/chunk2", nil)
|
||||||
|
req2.Host = "evil.example"
|
||||||
|
req2.Header.Set("User-Agent", "Valve/Steam HTTP Client 1.0")
|
||||||
|
rec2 := httptest.NewRecorder()
|
||||||
|
sc.ServeHTTP(rec2, req2)
|
||||||
|
if rec2.Code != http.StatusBadRequest {
|
||||||
|
t.Errorf("non-CDN Host: expected 400, got %d", rec2.Code)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestCacheKeySharedAcrossCDNHostAliases verifies that one cache entry serves
|
||||||
|
// the same depot object across different Steam CDN host aliases: the key uses
|
||||||
|
// only the request path (never the Host header or an absolute-form target
|
||||||
|
// host), so hits climb instead of re-stamping upstream per host rotation.
|
||||||
|
func TestCacheKeySharedAcrossCDNHostAliases(t *testing.T) {
|
||||||
|
body := []byte("depot chunk body for host-alias keying")
|
||||||
|
var upstreamCalls atomic.Int64
|
||||||
|
f := func(w http.ResponseWriter, r *http.Request) {
|
||||||
|
upstreamCalls.Add(1)
|
||||||
|
w.Header().Set("Content-Type", "application/octet-stream")
|
||||||
|
_, _ = w.Write(body)
|
||||||
|
}
|
||||||
|
sc, _ := newTestCacheWithFakeUpstream(t, f, "1MB", "0")
|
||||||
|
srv := newCacheServer(t, sc)
|
||||||
|
const depotPath = "/depot/1684171/chunk/abc123"
|
||||||
|
const ua = "Valve/Steam HTTP Client 1.0"
|
||||||
|
c := &http.Client{Timeout: 5 * time.Second}
|
||||||
|
|
||||||
|
// 1) MISS under the first CDN alias (origin-form target, Host: cdn1).
|
||||||
|
req1, err := http.NewRequest("GET", srv.URL+depotPath, nil)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
req1.Host = "cdn1.steamcontent.com"
|
||||||
|
req1.Header.Set("User-Agent", ua)
|
||||||
|
resp1, err := c.Do(req1)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("host-alias MISS request: %v", err)
|
||||||
|
}
|
||||||
|
data1, err := io.ReadAll(resp1.Body)
|
||||||
|
resp1.Body.Close()
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if resp1.StatusCode != http.StatusOK {
|
||||||
|
t.Fatalf("host-alias MISS: expected 200, got %d", resp1.StatusCode)
|
||||||
|
}
|
||||||
|
if got := resp1.Header.Get("X-LanCache-Status"); got != "MISS" {
|
||||||
|
t.Fatalf("host-alias MISS: expected X-LanCache-Status MISS, got %q", got)
|
||||||
|
}
|
||||||
|
if !bytes.Equal(data1, body) {
|
||||||
|
t.Fatalf("host-alias MISS: body mismatch: got %q", data1)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Bounded wait for the entry to be visible before hitting the next alias
|
||||||
|
// (the MISS handler streams the body to the client before the VFS write).
|
||||||
|
key, err := generateServiceCacheKey(depotPath, "steam")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
deadline := time.Now().Add(2 * time.Second)
|
||||||
|
for {
|
||||||
|
if rc, e := sc.vfs.Open(key); e == nil {
|
||||||
|
_ = rc.Close()
|
||||||
|
break
|
||||||
|
}
|
||||||
|
if time.Now().After(deadline) {
|
||||||
|
t.Fatalf("cache entry %q not visible after MISS", key)
|
||||||
|
}
|
||||||
|
time.Sleep(5 * time.Millisecond)
|
||||||
|
}
|
||||||
|
|
||||||
|
// 2) Same depot path under the second CDN alias (Host header only) -> HIT.
|
||||||
|
req2, err := http.NewRequest("GET", srv.URL+depotPath, nil)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
req2.Host = "cdn2.steamcontent.com"
|
||||||
|
req2.Header.Set("User-Agent", ua)
|
||||||
|
resp2, err := c.Do(req2)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("second host-alias request: %v", err)
|
||||||
|
}
|
||||||
|
data2, err := io.ReadAll(resp2.Body)
|
||||||
|
resp2.Body.Close()
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if resp2.StatusCode != http.StatusOK {
|
||||||
|
t.Fatalf("second host-alias: expected 200, got %d", resp2.StatusCode)
|
||||||
|
}
|
||||||
|
if got := resp2.Header.Get("X-LanCache-Status"); got != "HIT" {
|
||||||
|
t.Fatalf("second host-alias: expected X-LanCache-Status HIT, got %q", got)
|
||||||
|
}
|
||||||
|
if !bytes.Equal(data2, body) {
|
||||||
|
t.Fatalf("second host-alias: body mismatch: got %q", data2)
|
||||||
|
}
|
||||||
|
|
||||||
|
// 3) Same depot path with an absolute-form target embedding a third CDN
|
||||||
|
// hostname in the URL itself -> still a HIT on the same entry.
|
||||||
|
// (Go's http client always sends origin-form targets, so use raw HTTP.)
|
||||||
|
conn, err := net.Dial("tcp", srv.Listener.Addr().String())
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
defer conn.Close()
|
||||||
|
rawRequest := "GET http://cdn3.steamcontent.com" + depotPath + " HTTP/1.1\r\n" +
|
||||||
|
"Host: cdn3.steamcontent.com\r\n" +
|
||||||
|
"User-Agent: " + ua + "\r\n" +
|
||||||
|
"Connection: close\r\n\r\n"
|
||||||
|
if _, err := conn.Write([]byte(rawRequest)); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
rawReq, err := http.NewRequest("GET", "http://cdn3.steamcontent.com"+depotPath, nil)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
resp3, err := http.ReadResponse(bufio.NewReader(conn), rawReq)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
defer resp3.Body.Close()
|
||||||
|
data3, err := io.ReadAll(resp3.Body)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if resp3.StatusCode != http.StatusOK {
|
||||||
|
t.Fatalf("absolute-form host-alias: expected 200, got %d", resp3.StatusCode)
|
||||||
|
}
|
||||||
|
if got := resp3.Header.Get("X-LanCache-Status"); got != "HIT" {
|
||||||
|
t.Fatalf("absolute-form host-alias: expected X-LanCache-Status HIT, got %q", got)
|
||||||
|
}
|
||||||
|
if !bytes.Equal(data3, body) {
|
||||||
|
t.Fatalf("absolute-form host-alias: body mismatch: got %q", data3)
|
||||||
|
}
|
||||||
|
|
||||||
|
// All three aliases must have shared one upstream fetch.
|
||||||
|
if got := upstreamCalls.Load(); got != 1 {
|
||||||
|
t.Errorf("upstream fetched %d times across host aliases, want 1", got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|||||||
Vendored
+15
-2
@@ -3,6 +3,7 @@ package cache
|
|||||||
|
|
||||||
import (
|
import (
|
||||||
"io"
|
"io"
|
||||||
|
"s1d3sw1ped/steamcache2/steamcache/metrics"
|
||||||
"s1d3sw1ped/steamcache2/vfs"
|
"s1d3sw1ped/steamcache2/vfs"
|
||||||
"s1d3sw1ped/steamcache2/vfs/vfserror"
|
"s1d3sw1ped/steamcache2/vfs/vfserror"
|
||||||
"sync/atomic"
|
"sync/atomic"
|
||||||
@@ -12,6 +13,7 @@ import (
|
|||||||
type TieredCache struct {
|
type TieredCache struct {
|
||||||
fast *atomic.Value // Memory cache (fast) - atomic.Value for lock-free access
|
fast *atomic.Value // Memory cache (fast) - atomic.Value for lock-free access
|
||||||
slow *atomic.Value // Disk cache (slow) - atomic.Value for lock-free access
|
slow *atomic.Value // Disk cache (slow) - atomic.Value for lock-free access
|
||||||
|
metrics *metrics.Metrics
|
||||||
}
|
}
|
||||||
|
|
||||||
// New creates a new tiered cache
|
// New creates a new tiered cache
|
||||||
@@ -22,6 +24,11 @@ func New() *TieredCache {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// SetMetrics allows wiring the top-level metrics collector (called from SteamCache).
|
||||||
|
func (tc *TieredCache) SetMetrics(m *metrics.Metrics) {
|
||||||
|
tc.metrics = m
|
||||||
|
}
|
||||||
|
|
||||||
// SetFast sets the fast (memory) tier atomically
|
// SetFast sets the fast (memory) tier atomically
|
||||||
func (tc *TieredCache) SetFast(vfs vfs.VFS) {
|
func (tc *TieredCache) SetFast(vfs vfs.VFS) {
|
||||||
tc.fast.Store(vfs)
|
tc.fast.Store(vfs)
|
||||||
@@ -179,7 +186,7 @@ func (tc *TieredCache) Capacity() int64 {
|
|||||||
func (tc *TieredCache) promoteToFast(key string, reader io.ReadCloser) {
|
func (tc *TieredCache) promoteToFast(key string, reader io.ReadCloser) {
|
||||||
defer func() { _ = reader.Close() }() // best-effort close; error secondary to promotion attempt (async best-effort path)
|
defer func() { _ = reader.Close() }() // best-effort close; error secondary to promotion attempt (async best-effort path)
|
||||||
|
|
||||||
// Get file info from slow tier to determine size
|
// Size for the space/ReadAll guards comes from a Stat snapshot, not the live in-map FileInfo.
|
||||||
var size int64
|
var size int64
|
||||||
if slow := tc.slow.Load(); slow != nil {
|
if slow := tc.slow.Load(); slow != nil {
|
||||||
if vfs, ok := slow.(vfs.VFS); ok {
|
if vfs, ok := slow.(vfs.VFS); ok {
|
||||||
@@ -203,7 +210,7 @@ func (tc *TieredCache) promoteToFast(key string, reader io.ReadCloser) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// Guard promotion ReadAll using already-fetched size (in addition to space check above)
|
// Guard promotion ReadAll using already-fetched size (in addition to space check above)
|
||||||
if size > 0 && size > (1<<30) { // conservative 1GB hard limit on promotion reads (aligns with typical max_object_size)
|
if size > (1 << 30) { // conservative 1GB hard limit on promotion reads (aligns with typical max_object_size)
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
// Read the entire file content
|
// Read the entire file content
|
||||||
@@ -211,6 +218,8 @@ func (tc *TieredCache) promoteToFast(key string, reader io.ReadCloser) {
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
return // Skip promotion if read fails
|
return // Skip promotion if read fails
|
||||||
}
|
}
|
||||||
|
// Create with the bytes we actually hold so we never reuse a live FileInfo.Size.
|
||||||
|
size = int64(len(content))
|
||||||
|
|
||||||
// Create the file in fast tier
|
// Create the file in fast tier
|
||||||
if fast := tc.fast.Load(); fast != nil {
|
if fast := tc.fast.Load(); fast != nil {
|
||||||
@@ -221,6 +230,10 @@ func (tc *TieredCache) promoteToFast(key string, reader io.ReadCloser) {
|
|||||||
// Failure (e.g. mem pressure, concurrent evict) is non-fatal and does not affect correctness of slow tier.
|
// Failure (e.g. mem pressure, concurrent evict) is non-fatal and does not affect correctness of slow tier.
|
||||||
_, _ = writer.Write(content)
|
_, _ = writer.Write(content)
|
||||||
_ = writer.Close()
|
_ = writer.Close()
|
||||||
|
|
||||||
|
if tc.metrics != nil {
|
||||||
|
tc.metrics.IncrementPromotions()
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
+74
-10
@@ -7,6 +7,7 @@ import (
|
|||||||
"os"
|
"os"
|
||||||
"path/filepath"
|
"path/filepath"
|
||||||
"s1d3sw1ped/steamcache2/steamcache/logger"
|
"s1d3sw1ped/steamcache2/steamcache/logger"
|
||||||
|
"s1d3sw1ped/steamcache2/steamcache/metrics"
|
||||||
"s1d3sw1ped/steamcache2/vfs"
|
"s1d3sw1ped/steamcache2/vfs"
|
||||||
"s1d3sw1ped/steamcache2/vfs/locks"
|
"s1d3sw1ped/steamcache2/vfs/locks"
|
||||||
"s1d3sw1ped/steamcache2/vfs/lru"
|
"s1d3sw1ped/steamcache2/vfs/lru"
|
||||||
@@ -43,6 +44,7 @@ type DiskFS struct {
|
|||||||
// initCloseOnce ensures initDone closed exactly once even on panic in bg populator (panic safety for Issue 1).
|
// initCloseOnce ensures initDone closed exactly once even on panic in bg populator (panic safety for Issue 1).
|
||||||
initCloseOnce sync.Once
|
initCloseOnce sync.Once
|
||||||
startupEvict func(vfs.VFS, uint) uint // passed to New (via gc.GetGCAlgorithm); invoked as last step of bg init if over cap (no post-ctor race)
|
startupEvict func(vfs.VFS, uint) uint // passed to New (via gc.GetGCAlgorithm); invoked as last step of bg init if over cap (no post-ctor race)
|
||||||
|
metrics *metrics.Metrics
|
||||||
}
|
}
|
||||||
|
|
||||||
// shardPath converts a Steam cache key to a sharded directory path to reduce inode pressure
|
// shardPath converts a Steam cache key to a sharded directory path to reduce inode pressure
|
||||||
@@ -134,6 +136,12 @@ func New(root string, capacity int64, evict func(vfs.VFS, uint) uint) (*DiskFS,
|
|||||||
return d, nil
|
return d, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// SetMetrics allows the owner (SteamCache) to inject the metrics collector
|
||||||
|
// so that per-tier hit and eviction counters can be recorded.
|
||||||
|
func (d *DiskFS) SetMetrics(met *metrics.Metrics) {
|
||||||
|
d.metrics = met
|
||||||
|
}
|
||||||
|
|
||||||
// calculateSizeAndPopulateIndex runs in background from New to avoid blocking startup or O(N) RAM for large caches (millions of Steam files).
|
// calculateSizeAndPopulateIndex runs in background from New to avoid blocking startup or O(N) RAM for large caches (millions of Steam files).
|
||||||
// It streams batch inserts (bounded by maxEvictBatch) to keep lock times short and eliminate giant temporary slice.
|
// It streams batch inserts (bounded by maxEvictBatch) to keep lock times short and eliminate giant temporary slice.
|
||||||
// Startup over-capacity eviction (if needed) runs as the very last step (using the evict func passed to New, selected via gc.GetGCAlgorithm).
|
// Startup over-capacity eviction (if needed) runs as the very last step (using the evict func passed to New, selected via gc.GetGCAlgorithm).
|
||||||
@@ -240,15 +248,25 @@ func (d *DiskFS) calculateSizeAndPopulateIndex() {
|
|||||||
|
|
||||||
// insertBatch populates info/LRU under lock for a bounded batch (follows maxEvictBatch pattern for short critical sections).
|
// insertBatch populates info/LRU under lock for a bounded batch (follows maxEvictBatch pattern for short critical sections).
|
||||||
// Size is incremented here only for files actually added (prevents double-count vs. concurrent Create during window).
|
// Size is incremented here only for files actually added (prevents double-count vs. concurrent Create during window).
|
||||||
|
// Fail-closed: re-stat each path under d.mu and skip if the file is gone. Create does not wait on
|
||||||
|
// initDone, so a file the scanner observed can be Evict/Delete'd (info + os.Remove) before this
|
||||||
|
// insert runs. Inserting without a live-file check would resurrect the key in d.info and make
|
||||||
|
// Stat succeed while os.Stat fails.
|
||||||
func (d *DiskFS) insertBatch(batch []discoveredFile) {
|
func (d *DiskFS) insertBatch(batch []discoveredFile) {
|
||||||
d.mu.Lock()
|
d.mu.Lock()
|
||||||
for _, df := range batch {
|
for _, df := range batch {
|
||||||
if _, exists := d.info[df.key]; !exists {
|
if _, exists := d.info[df.key]; exists {
|
||||||
fi := vfs.NewFileInfoFromOS(df.osInfo, df.key)
|
continue
|
||||||
|
}
|
||||||
|
path := d.pathForKey(df.key)
|
||||||
|
st, err := os.Stat(path)
|
||||||
|
if err != nil {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
fi := vfs.NewFileInfoFromOS(st, df.key)
|
||||||
d.info[df.key] = fi
|
d.info[df.key] = fi
|
||||||
d.LRU.Add(df.key, fi)
|
d.LRU.Add(df.key, fi)
|
||||||
d.size += df.size
|
d.size += st.Size()
|
||||||
}
|
|
||||||
}
|
}
|
||||||
d.mu.Unlock()
|
d.mu.Unlock()
|
||||||
}
|
}
|
||||||
@@ -489,8 +507,14 @@ func (d *DiskFS) Open(key string) (io.ReadCloser, error) {
|
|||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
|
|
||||||
// Use memory mapping for large files (>1MB) to improve performance
|
// Use memory mapping for large files to improve performance.
|
||||||
const mmapThreshold = 1024 * 1024 // 1MB
|
// We use 8 MiB as the threshold because:
|
||||||
|
// - Most Steam chunks are ~1 MiB (see current disk cache analysis).
|
||||||
|
// - mmap has non-trivial fixed overhead (page tables, TLB, faults).
|
||||||
|
// - For files < ~4-8 MiB the overhead often outweighs the zero-copy benefit
|
||||||
|
// on mostly sequential access patterns.
|
||||||
|
// - Larger files benefit more from kernel readahead + zero-copy.
|
||||||
|
const mmapThreshold = 8 * 1024 * 1024 // 8 MiB
|
||||||
if fi.Size > mmapThreshold {
|
if fi.Size > mmapThreshold {
|
||||||
// Close the regular file handle
|
// Close the regular file handle
|
||||||
_ = file.Close() // best-effort; mmap path takes over or falls back
|
_ = file.Close() // best-effort; mmap path takes over or falls back
|
||||||
@@ -505,9 +529,21 @@ func (d *DiskFS) Open(key string) (io.ReadCloser, error) {
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
_ = mmapFile.Close() // best-effort close before fallback open
|
_ = mmapFile.Close() // best-effort close before fallback open
|
||||||
// Fallback to regular file reading (intentional 3rd open of same path after mmap failure; pre-existing pattern, no leak)
|
// Fallback to regular file reading (intentional 3rd open of same path after mmap failure; pre-existing pattern, no leak)
|
||||||
|
if d.metrics != nil {
|
||||||
|
d.metrics.IncrementDiskCacheHits()
|
||||||
|
}
|
||||||
return os.Open(path)
|
return os.Open(path)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Hint to the kernel (on supported platforms) that we will access
|
||||||
|
// this mapping sequentially. This enables better readahead.
|
||||||
|
if err := madviseSequential(mapped); err != nil {
|
||||||
|
logger.Logger.Debug().
|
||||||
|
Err(err).
|
||||||
|
Str("key", key).
|
||||||
|
Msg("madvise(MADV_SEQUENTIAL) failed on mmap'd chunk")
|
||||||
|
}
|
||||||
|
|
||||||
return &mmapReadCloser{
|
return &mmapReadCloser{
|
||||||
data: mapped,
|
data: mapped,
|
||||||
file: mmapFile,
|
file: mmapFile,
|
||||||
@@ -515,6 +551,9 @@ func (d *DiskFS) Open(key string) (io.ReadCloser, error) {
|
|||||||
}, nil
|
}, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if d.metrics != nil {
|
||||||
|
d.metrics.IncrementDiskCacheHits()
|
||||||
|
}
|
||||||
return file, nil
|
return file, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -573,7 +612,9 @@ func (d *DiskFS) Delete(key string) error {
|
|||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// Stat returns file information with lazy discovery
|
// Stat returns a snapshot of file information with lazy discovery.
|
||||||
|
// The returned *FileInfo is not the live cache entry; Close may update Size
|
||||||
|
// on the in-map object under d.mu.
|
||||||
func (d *DiskFS) Stat(key string) (*vfs.FileInfo, error) {
|
func (d *DiskFS) Stat(key string) (*vfs.FileInfo, error) {
|
||||||
if key == "" {
|
if key == "" {
|
||||||
return nil, vfserror.ErrInvalidKey
|
return nil, vfserror.ErrInvalidKey
|
||||||
@@ -588,9 +629,10 @@ func (d *DiskFS) Stat(key string) (*vfs.FileInfo, error) {
|
|||||||
keyMu.RLock()
|
keyMu.RLock()
|
||||||
d.mu.RLock()
|
d.mu.RLock()
|
||||||
if fi, ok := d.info[key]; ok {
|
if fi, ok := d.info[key]; ok {
|
||||||
|
snap := fi.Clone()
|
||||||
d.mu.RUnlock()
|
d.mu.RUnlock()
|
||||||
keyMu.RUnlock()
|
keyMu.RUnlock()
|
||||||
return fi, nil
|
return snap, nil
|
||||||
}
|
}
|
||||||
d.mu.RUnlock()
|
d.mu.RUnlock()
|
||||||
keyMu.RUnlock()
|
keyMu.RUnlock()
|
||||||
@@ -610,8 +652,9 @@ func (d *DiskFS) Stat(key string) (*vfs.FileInfo, error) {
|
|||||||
// Double-check after acquiring write lock
|
// Double-check after acquiring write lock
|
||||||
d.mu.Lock()
|
d.mu.Lock()
|
||||||
if fi, ok := d.info[key]; ok {
|
if fi, ok := d.info[key]; ok {
|
||||||
|
snap := fi.Clone()
|
||||||
d.mu.Unlock()
|
d.mu.Unlock()
|
||||||
return fi, nil
|
return snap, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// Re-verify the file still exists on disk under the lock before inserting.
|
// Re-verify the file still exists on disk under the lock before inserting.
|
||||||
@@ -630,9 +673,10 @@ func (d *DiskFS) Stat(key string) (*vfs.FileInfo, error) {
|
|||||||
fi.UpdateAccessBatched(d.timeUpdater)
|
fi.UpdateAccessBatched(d.timeUpdater)
|
||||||
// Note: size not updated on lazy discovery (preserves prior behavior; initial on-disk accounted via bg populate at New time,
|
// Note: size not updated on lazy discovery (preserves prior behavior; initial on-disk accounted via bg populate at New time,
|
||||||
// subsequent files come via Create which accounts size).
|
// subsequent files come via Create which accounts size).
|
||||||
|
snap := fi.Clone()
|
||||||
d.mu.Unlock()
|
d.mu.Unlock()
|
||||||
|
|
||||||
return fi, nil
|
return snap, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// EvictLRU evicts the least recently used files to free up space
|
// EvictLRU evicts the least recently used files to free up space
|
||||||
@@ -677,6 +721,10 @@ func (d *DiskFS) EvictLRU(bytesNeeded uint) uint {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
d.mu.Unlock()
|
d.mu.Unlock()
|
||||||
|
|
||||||
|
if d.metrics != nil && evicted > 0 {
|
||||||
|
d.metrics.IncrementEvictions()
|
||||||
|
}
|
||||||
return evicted
|
return evicted
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -726,6 +774,10 @@ func (d *DiskFS) EvictBySize(bytesNeeded uint, ascending bool) uint {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
d.mu.Unlock()
|
d.mu.Unlock()
|
||||||
|
|
||||||
|
if d.metrics != nil && evicted > 0 {
|
||||||
|
d.metrics.IncrementEvictions()
|
||||||
|
}
|
||||||
return evicted
|
return evicted
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -773,6 +825,10 @@ func (d *DiskFS) EvictFIFO(bytesNeeded uint) uint {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
d.mu.Unlock()
|
d.mu.Unlock()
|
||||||
|
|
||||||
|
if d.metrics != nil && evicted > 0 {
|
||||||
|
d.metrics.IncrementEvictions()
|
||||||
|
}
|
||||||
return evicted
|
return evicted
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -825,6 +881,10 @@ func (d *DiskFS) EvictLFU(bytesNeeded uint) uint {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
d.mu.Unlock()
|
d.mu.Unlock()
|
||||||
|
|
||||||
|
if d.metrics != nil && evicted > 0 {
|
||||||
|
d.metrics.IncrementEvictions()
|
||||||
|
}
|
||||||
return evicted
|
return evicted
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -878,5 +938,9 @@ func (d *DiskFS) EvictHybrid(bytesNeeded uint) uint {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
d.mu.Unlock()
|
d.mu.Unlock()
|
||||||
|
|
||||||
|
if d.metrics != nil && evicted > 0 {
|
||||||
|
d.metrics.IncrementEvictions()
|
||||||
|
}
|
||||||
return evicted
|
return evicted
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -0,0 +1,13 @@
|
|||||||
|
//go:build !windows
|
||||||
|
|
||||||
|
package disk
|
||||||
|
|
||||||
|
import (
|
||||||
|
"golang.org/x/sys/unix"
|
||||||
|
)
|
||||||
|
|
||||||
|
// madviseSequential gives the OS a hint that the memory region will be
|
||||||
|
// accessed sequentially. This is a no-op or best-effort on some platforms.
|
||||||
|
func madviseSequential(b []byte) error {
|
||||||
|
return unix.Madvise(b, unix.MADV_SEQUENTIAL)
|
||||||
|
}
|
||||||
@@ -0,0 +1,11 @@
|
|||||||
|
//go:build windows
|
||||||
|
|
||||||
|
package disk
|
||||||
|
|
||||||
|
// madviseSequential is a no-op on Windows.
|
||||||
|
// Windows file mappings don't have a direct equivalent to MADV_SEQUENTIAL
|
||||||
|
// in the same way. Sequential access hints are better done via
|
||||||
|
// FILE_FLAG_SEQUENTIAL_SCAN at file open time (future improvement possible).
|
||||||
|
func madviseSequential(b []byte) error {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
+66
-33
@@ -371,7 +371,8 @@ func testKey(i int) string {
|
|||||||
// artifacts for victims are immediately gone (no resurrection via lazy discovery in Stat/Open),
|
// artifacts for victims are immediately gone (no resurrection via lazy discovery in Stat/Open),
|
||||||
// and that recreating the same key produces independent content that is not subject to any
|
// and that recreating the same key produces independent content that is not subject to any
|
||||||
// stale eviction unlinks. This exercises the coordinated WLock remove path for DiskFS.
|
// stale eviction unlinks. This exercises the coordinated WLock remove path for DiskFS.
|
||||||
// Uses tolerant checks suitable for raw DiskFS lazy discovery + bg size.
|
// Create does not wait on initDone, so this also covers insertBatch racing with eviction:
|
||||||
|
// gone files must not be re-indexed (Stat present / disk missing).
|
||||||
func TestDiskFS_EvictDiskVisibilityAndRecreateSafety(t *testing.T) {
|
func TestDiskFS_EvictDiskVisibilityAndRecreateSafety(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
td := t.TempDir()
|
td := t.TempDir()
|
||||||
@@ -400,33 +401,11 @@ func TestDiskFS_EvictDiskVisibilityAndRecreateSafety(t *testing.T) {
|
|||||||
_ = d.EvictBySize(1024*1024, true)
|
_ = d.EvictBySize(1024*1024, true)
|
||||||
}
|
}
|
||||||
|
|
||||||
// Consistency check: never have a key absent from Stat but with a file on disk (would indicate
|
// Drain bg population so insertBatch cannot still be in flight when we audit.
|
||||||
// either resurrection risk or orphan). If Stat succeeds, file should exist.
|
_ = d.Size()
|
||||||
// A few retries tolerate the documented lazy discovery + eviction coordination windows under
|
|
||||||
// artificial "force massive eviction then immediate audit" load (especially visible under -race).
|
// Consistency: Stat success iff the file exists on disk. insertBatch must not resurrect
|
||||||
for attempt := 0; attempt < 3; attempt++ {
|
// keys whose backing files were already evicted.
|
||||||
bad := false
|
|
||||||
for _, k := range created {
|
|
||||||
p := d.pathForKey(k)
|
|
||||||
_, statErr := d.Stat(k)
|
|
||||||
_, diskErr := os.Stat(p)
|
|
||||||
if statErr != nil {
|
|
||||||
if !os.IsNotExist(diskErr) {
|
|
||||||
bad = true
|
|
||||||
}
|
|
||||||
} else {
|
|
||||||
if diskErr != nil {
|
|
||||||
bad = true
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if !bad {
|
|
||||||
break
|
|
||||||
}
|
|
||||||
if attempt < 2 {
|
|
||||||
time.Sleep(10 * time.Millisecond)
|
|
||||||
} else {
|
|
||||||
// On final attempt, report the last observed state for the keys
|
|
||||||
for _, k := range created {
|
for _, k := range created {
|
||||||
p := d.pathForKey(k)
|
p := d.pathForKey(k)
|
||||||
_, statErr := d.Stat(k)
|
_, statErr := d.Stat(k)
|
||||||
@@ -435,14 +414,10 @@ func TestDiskFS_EvictDiskVisibilityAndRecreateSafety(t *testing.T) {
|
|||||||
if !os.IsNotExist(diskErr) {
|
if !os.IsNotExist(diskErr) {
|
||||||
t.Errorf("key %s absent via Stat but file lingers on disk at %s (resurrection risk)", k, p)
|
t.Errorf("key %s absent via Stat but file lingers on disk at %s (resurrection risk)", k, p)
|
||||||
}
|
}
|
||||||
} else {
|
} else if diskErr != nil {
|
||||||
if diskErr != nil {
|
|
||||||
t.Errorf("key %s present via Stat but missing on disk: %v", k, diskErr)
|
t.Errorf("key %s present via Stat but missing on disk: %v", k, diskErr)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// Recreate one that is currently absent (or any): must work, and new content must not be
|
// Recreate one that is currently absent (or any): must work, and new content must not be
|
||||||
// subject to stale unlinks (guaranteed by inside-WLock removes on evict + keyMu on Create).
|
// subject to stale unlinks (guaranteed by inside-WLock removes on evict + keyMu on Create).
|
||||||
@@ -464,6 +439,64 @@ func TestDiskFS_EvictDiskVisibilityAndRecreateSafety(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// TestDiskFS_InsertBatchSkipsGoneFiles is the fail-closed contract for bg/lazy index
|
||||||
|
// insert: a discoveredFile whose path was removed (evicted) must not be re-inserted
|
||||||
|
// into d.info. That resurrection is what made Stat succeed while os.Stat failed.
|
||||||
|
func TestDiskFS_InsertBatchSkipsGoneFiles(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
td := t.TempDir()
|
||||||
|
d, err := New(td, 10*1024*1024, nil)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
_ = d.Size() // finish constructor scan so it cannot also index these keys
|
||||||
|
|
||||||
|
liveKey := "live"
|
||||||
|
goneKey := "gone"
|
||||||
|
writeKey := func(key, body string) os.FileInfo {
|
||||||
|
t.Helper()
|
||||||
|
p := d.pathForKey(key)
|
||||||
|
if err := os.MkdirAll(filepath.Dir(p), 0700); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if err := os.WriteFile(p, []byte(body), 0600); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
st, err := os.Stat(p)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
return st
|
||||||
|
}
|
||||||
|
liveInfo := writeKey(liveKey, "still-here")
|
||||||
|
goneInfo := writeKey(goneKey, "about-to-vanish")
|
||||||
|
if err := os.Remove(d.pathForKey(goneKey)); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
|
||||||
|
d.insertBatch([]discoveredFile{
|
||||||
|
{key: liveKey, size: liveInfo.Size(), osInfo: liveInfo},
|
||||||
|
{key: goneKey, size: goneInfo.Size(), osInfo: goneInfo},
|
||||||
|
})
|
||||||
|
|
||||||
|
d.mu.RLock()
|
||||||
|
_, liveExists := d.info[liveKey]
|
||||||
|
_, goneExists := d.info[goneKey]
|
||||||
|
d.mu.RUnlock()
|
||||||
|
if !liveExists {
|
||||||
|
t.Errorf("insertBatch skipped live key %s", liveKey)
|
||||||
|
}
|
||||||
|
if goneExists {
|
||||||
|
t.Errorf("insertBatch resurrected gone key %s", goneKey)
|
||||||
|
}
|
||||||
|
if _, err := d.Stat(goneKey); err == nil {
|
||||||
|
t.Errorf("Stat succeeded for gone key %s", goneKey)
|
||||||
|
}
|
||||||
|
if _, err := os.Stat(d.pathForKey(liveKey)); err != nil {
|
||||||
|
t.Errorf("live key %s missing on disk: %v", liveKey, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
// TestDiskFS_EvictBoundedLargeN exercises the maxEvictBatch early-break logic (Idea #2)
|
// TestDiskFS_EvictBoundedLargeN exercises the maxEvictBatch early-break logic (Idea #2)
|
||||||
// under a map size >> batch limit. Forces repeated eviction rounds via GC-style pressure
|
// under a map size >> batch limit. Forces repeated eviction rounds via GC-style pressure
|
||||||
// and asserts progress + consistency (no resurrection/orphans). Covers bounded collection
|
// and asserts progress + consistency (no resurrection/orphans). Covers bounded collection
|
||||||
|
|||||||
+35
-2
@@ -5,6 +5,7 @@ import (
|
|||||||
"bytes"
|
"bytes"
|
||||||
"fmt"
|
"fmt"
|
||||||
"io"
|
"io"
|
||||||
|
"s1d3sw1ped/steamcache2/steamcache/metrics"
|
||||||
"s1d3sw1ped/steamcache2/vfs"
|
"s1d3sw1ped/steamcache2/vfs"
|
||||||
"s1d3sw1ped/steamcache2/vfs/locks"
|
"s1d3sw1ped/steamcache2/vfs/locks"
|
||||||
"s1d3sw1ped/steamcache2/vfs/lru"
|
"s1d3sw1ped/steamcache2/vfs/lru"
|
||||||
@@ -33,6 +34,7 @@ type MemoryFS struct {
|
|||||||
keyLocks []sync.Map // Sharded lock pools for better concurrency
|
keyLocks []sync.Map // Sharded lock pools for better concurrency
|
||||||
LRU *lru.LRUList[*types.FileInfo]
|
LRU *lru.LRUList[*types.FileInfo]
|
||||||
timeUpdater *types.BatchedTimeUpdate // Batched time updates for better performance
|
timeUpdater *types.BatchedTimeUpdate // Batched time updates for better performance
|
||||||
|
metrics *metrics.Metrics
|
||||||
}
|
}
|
||||||
|
|
||||||
// New creates a new MemoryFS
|
// New creates a new MemoryFS
|
||||||
@@ -55,6 +57,12 @@ func New(capacity int64) (*MemoryFS, error) {
|
|||||||
}, nil
|
}, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// SetMetrics allows the owner (SteamCache) to inject the metrics collector
|
||||||
|
// so that per-tier hit and eviction counters can be recorded.
|
||||||
|
func (m *MemoryFS) SetMetrics(met *metrics.Metrics) {
|
||||||
|
m.metrics = met
|
||||||
|
}
|
||||||
|
|
||||||
// Name returns the name of this VFS
|
// Name returns the name of this VFS
|
||||||
func (m *MemoryFS) Name() string {
|
func (m *MemoryFS) Name() string {
|
||||||
return "MemoryFS"
|
return "MemoryFS"
|
||||||
@@ -209,6 +217,10 @@ func (m *MemoryFS) Open(key string) (io.ReadCloser, error) {
|
|||||||
// Use zero-copy approach - return reader that reads directly from buffer
|
// Use zero-copy approach - return reader that reads directly from buffer
|
||||||
m.mu.Unlock()
|
m.mu.Unlock()
|
||||||
|
|
||||||
|
if m.metrics != nil {
|
||||||
|
m.metrics.IncrementMemoryCacheHits()
|
||||||
|
}
|
||||||
|
|
||||||
return &memoryReadCloser{
|
return &memoryReadCloser{
|
||||||
buffer: buffer,
|
buffer: buffer,
|
||||||
offset: 0,
|
offset: 0,
|
||||||
@@ -277,7 +289,8 @@ func (m *MemoryFS) Delete(key string) error {
|
|||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// Stat returns file information
|
// Stat returns a snapshot of file information. The returned *FileInfo is not
|
||||||
|
// the live cache entry; Close may update Size on the in-map object under m.mu.
|
||||||
func (m *MemoryFS) Stat(key string) (*types.FileInfo, error) {
|
func (m *MemoryFS) Stat(key string) (*types.FileInfo, error) {
|
||||||
if key == "" {
|
if key == "" {
|
||||||
return nil, vfserror.ErrInvalidKey
|
return nil, vfserror.ErrInvalidKey
|
||||||
@@ -298,7 +311,7 @@ func (m *MemoryFS) Stat(key string) (*types.FileInfo, error) {
|
|||||||
defer m.mu.RUnlock()
|
defer m.mu.RUnlock()
|
||||||
|
|
||||||
if fi, ok := m.info[key]; ok {
|
if fi, ok := m.info[key]; ok {
|
||||||
return fi, nil
|
return fi.Clone(), nil
|
||||||
}
|
}
|
||||||
|
|
||||||
return nil, vfserror.ErrNotFound
|
return nil, vfserror.ErrNotFound
|
||||||
@@ -344,6 +357,10 @@ func (m *MemoryFS) EvictLRU(bytesNeeded uint) uint {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
m.mu.Unlock()
|
m.mu.Unlock()
|
||||||
|
|
||||||
|
if m.metrics != nil && evicted > 0 {
|
||||||
|
m.metrics.IncrementEvictions()
|
||||||
|
}
|
||||||
return evicted
|
return evicted
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -395,6 +412,10 @@ func (m *MemoryFS) EvictBySize(bytesNeeded uint, ascending bool) uint {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
m.mu.Unlock()
|
m.mu.Unlock()
|
||||||
|
|
||||||
|
if m.metrics != nil && evicted > 0 {
|
||||||
|
m.metrics.IncrementEvictions()
|
||||||
|
}
|
||||||
return evicted
|
return evicted
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -442,6 +463,10 @@ func (m *MemoryFS) EvictFIFO(bytesNeeded uint) uint {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
m.mu.Unlock()
|
m.mu.Unlock()
|
||||||
|
|
||||||
|
if m.metrics != nil && evicted > 0 {
|
||||||
|
m.metrics.IncrementEvictions()
|
||||||
|
}
|
||||||
return evicted
|
return evicted
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -494,6 +519,10 @@ func (m *MemoryFS) EvictLFU(bytesNeeded uint) uint {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
m.mu.Unlock()
|
m.mu.Unlock()
|
||||||
|
|
||||||
|
if m.metrics != nil && evicted > 0 {
|
||||||
|
m.metrics.IncrementEvictions()
|
||||||
|
}
|
||||||
return evicted
|
return evicted
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -548,5 +577,9 @@ func (m *MemoryFS) EvictHybrid(bytesNeeded uint) uint {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
m.mu.Unlock()
|
m.mu.Unlock()
|
||||||
|
|
||||||
|
if m.metrics != nil && evicted > 0 {
|
||||||
|
m.metrics.IncrementEvictions()
|
||||||
|
}
|
||||||
return evicted
|
return evicted
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -346,6 +346,45 @@ func TestMemoryFS_ConcurrentCloseAndEvict_RaceFree(t *testing.T) {
|
|||||||
_ = m.LRU.Len()
|
_ = m.LRU.Len()
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestMemoryFS_StatReturnsSnapshot(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
m, err := New(1024)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
w, err := m.Create("k", 10)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if _, err := w.Write([]byte("hello")); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if err := w.Close(); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
|
||||||
|
fi, err := m.Stat("k")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if fi.Size != 5 {
|
||||||
|
t.Fatalf("size %d want 5", fi.Size)
|
||||||
|
}
|
||||||
|
fi.Size = 999
|
||||||
|
fi.AccessCount = 0
|
||||||
|
|
||||||
|
fi2, err := m.Stat("k")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if fi2.Size != 5 {
|
||||||
|
t.Errorf("Stat returned live FileInfo; store size became %d", fi2.Size)
|
||||||
|
}
|
||||||
|
if fi2.AccessCount == 0 {
|
||||||
|
t.Error("Stat returned live FileInfo; AccessCount mutation leaked")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestMemoryFS_EvictVariantsAndErrors(t *testing.T) {
|
func TestMemoryFS_EvictVariantsAndErrors(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
m, err := New(800)
|
m, err := New(800)
|
||||||
|
|||||||
@@ -27,6 +27,17 @@ func NewFileInfo(key string, size int64) *FileInfo {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Clone returns a snapshot copy of fi. Stat returns Clone() so callers can
|
||||||
|
// read Size and other fields without racing Close/Open mutations of the
|
||||||
|
// in-map FileInfo.
|
||||||
|
func (fi *FileInfo) Clone() *FileInfo {
|
||||||
|
if fi == nil {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
cp := *fi
|
||||||
|
return &cp
|
||||||
|
}
|
||||||
|
|
||||||
// NewFileInfoFromOS creates a FileInfo from os.FileInfo
|
// NewFileInfoFromOS creates a FileInfo from os.FileInfo
|
||||||
func NewFileInfoFromOS(info os.FileInfo, key string) *FileInfo {
|
func NewFileInfoFromOS(info os.FileInfo, key string) *FileInfo {
|
||||||
return &FileInfo{
|
return &FileInfo{
|
||||||
|
|||||||
@@ -16,6 +16,34 @@ func TestNewFileInfo(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestFileInfoClone(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
fi := NewFileInfo("k", 42)
|
||||||
|
fi.AccessCount = 7
|
||||||
|
cp := fi.Clone()
|
||||||
|
if cp == fi {
|
||||||
|
t.Fatal("Clone returned the same pointer")
|
||||||
|
}
|
||||||
|
if cp.Key != fi.Key || cp.Size != fi.Size || cp.AccessCount != fi.AccessCount {
|
||||||
|
t.Errorf("Clone mismatch: %+v vs %+v", cp, fi)
|
||||||
|
}
|
||||||
|
if !cp.ATime.Equal(fi.ATime) || !cp.CTime.Equal(fi.CTime) {
|
||||||
|
t.Error("Clone timestamps mismatch")
|
||||||
|
}
|
||||||
|
cp.Size = 99
|
||||||
|
cp.AccessCount = 1
|
||||||
|
if fi.Size != 42 || fi.AccessCount != 7 {
|
||||||
|
t.Error("mutating Clone affected original")
|
||||||
|
}
|
||||||
|
if NewFileInfo("x", 1).Clone() == nil {
|
||||||
|
t.Error("Clone of non-nil was nil")
|
||||||
|
}
|
||||||
|
var none *FileInfo
|
||||||
|
if none.Clone() != nil {
|
||||||
|
t.Error("Clone of nil was non-nil")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestUpdateAccess(t *testing.T) {
|
func TestUpdateAccess(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
fi := NewFileInfo("k", 1)
|
fi := NewFileInfo("k", 1)
|
||||||
|
|||||||
Reference in New Issue
Block a user