Compare commits
30 Commits
b7e3a0da86
..
1.0.22
| Author | SHA1 | Date | |
|---|---|---|---|
| 85e14bc8af | |||
| 3f5175b482 | |||
| 04d1c6c368 | |||
| 523a9a4782 | |||
| 8e09c89e24 | |||
| a2ac13d317 | |||
| 5d006ac44f | |||
| acd006d4a2 | |||
| 30a695458e | |||
| e8bcf0ddbd | |||
| f497e71ef0 | |||
| 0198e8990b | |||
| 2a2cd8d393 | |||
| 81b3a7df53 | |||
| 8e8e877533 | |||
| d63d7b4d3c | |||
| 35d698a232 | |||
| 36b613b6cd | |||
| 79f02d9868 | |||
| 19497eba0c | |||
| c7a2312994 | |||
| fbb084d824 | |||
| e7d4a19c3f | |||
| 0c54ef3404 | |||
| 3d3c74fdb2 | |||
| 04f55535a5 | |||
| 05640bb549 | |||
| e4be82cddf | |||
| 60b2c3e514 | |||
| 099e5347d5 |
@@ -7,6 +7,8 @@ on:
|
||||
jobs:
|
||||
release:
|
||||
runs-on: ubuntu-latest
|
||||
permissions:
|
||||
contents: write
|
||||
steps:
|
||||
- uses: actions/checkout@main
|
||||
with:
|
||||
@@ -21,4 +23,6 @@ jobs:
|
||||
version: 'latest'
|
||||
args: release
|
||||
env:
|
||||
GITEA_TOKEN: ${{secrets.RELEASE_TOKEN}}
|
||||
GITEA_TOKEN: ${{ secrets.GITEA_TOKEN }}
|
||||
GITHUB_TOKEN: ${{ secrets.GITEA_TOKEN }}
|
||||
GORELEASER_FORCE_TOKEN: gitea
|
||||
|
||||
@@ -1,24 +1,38 @@
|
||||
name: PR Check
|
||||
name: CI
|
||||
on:
|
||||
- pull_request
|
||||
pull_request:
|
||||
push:
|
||||
branches:
|
||||
- main
|
||||
paths-ignore:
|
||||
- '**.md'
|
||||
- 'CONTRIBUTING.md'
|
||||
|
||||
jobs:
|
||||
check-and-test:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@main
|
||||
- uses: actions/setup-go@main
|
||||
- uses: actions/checkout@v4
|
||||
- uses: actions/setup-go@v5
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
- run: go mod tidy
|
||||
- run: go build ./...
|
||||
- run: go vet ./...
|
||||
- name: golangci-lint
|
||||
uses: golangci/golangci-lint-action@v4
|
||||
uses: golangci/golangci-lint-action@v8
|
||||
with:
|
||||
version: latest
|
||||
version: v2.13.2
|
||||
args: --timeout=5m
|
||||
- run: go test -race -v -shuffle=on -coverprofile=coverage.out -timeout=5m ./...
|
||||
- run: go tool cover -func=coverage.out | tail -10 # basic coverage report
|
||||
|
||||
vulncheck:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- uses: actions/setup-go@v5
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
- run: go install golang.org/x/vuln/cmd/govulncheck@latest
|
||||
- run: govulncheck ./...
|
||||
- run: go test -race -v -shuffle=on -coverprofile=coverage.out -timeout=5m ./...
|
||||
- run: go tool cover -func=coverage.out | tail -10 # basic coverage report (P2-04)
|
||||
+74
-66
@@ -1,86 +1,94 @@
|
||||
# .golangci.yml - steamcache2 lint config
|
||||
# .golangci.yml - steamcache2 lint config (golangci-lint v2)
|
||||
# Philosophy: enable reasonable linters by default (golangci curated set + key additions)
|
||||
# then use most specific suppressions possible (source //nosec with justification,
|
||||
# _ = discard for errcheck on unavoidable client writes, narrow exclude-rules only for tests).
|
||||
# This makes remaining accepted issues visible and actionable in the code.
|
||||
# Run with: make lint (or golangci-lint run ./...)
|
||||
# Install: go install github.com/golangci/golangci-lint/cmd/golangci-lint@latest
|
||||
version: "2"
|
||||
|
||||
run:
|
||||
timeout: 5m
|
||||
modules-download-mode: readonly
|
||||
|
||||
linters:
|
||||
# No disable-all: use golangci defaults (errcheck, govet, ineffassign, staticcheck, unused, gosimple, etc.)
|
||||
# No default: none — use golangci defaults (errcheck, govet, ineffassign, staticcheck, unused, etc.)
|
||||
# Explicitly enable the non-default linters we require for this LAN cache proxy.
|
||||
enable:
|
||||
- gosec # security checks (re-audited; see source //nosec for justified cases)
|
||||
- misspell # documentation hygiene
|
||||
- goimports # import formatting (enforced)
|
||||
# gofmt covered via linter or goimports; errcheck/govet etc. from defaults
|
||||
settings:
|
||||
errcheck:
|
||||
check-type-assertions: false
|
||||
check-blank: false
|
||||
# gosec: keep source-level //nosec for G104/G115/G301/G304/G306.
|
||||
# G704/G705 are new taint-analysis rules (SSRF/XSS) not present in v1.64.8;
|
||||
# a CDN cache proxy forwards upstream URLs and response bodies by design.
|
||||
gosec:
|
||||
excludes:
|
||||
- G704
|
||||
- G705
|
||||
# v1 staticcheck checks: ["all"] meant SA* only. v2 merged stylecheck (ST*)
|
||||
# and quickfix (QF*) into staticcheck; keep the previous SA*+gosimple set.
|
||||
staticcheck:
|
||||
checks:
|
||||
- all
|
||||
- "-ST*"
|
||||
- "-QF*"
|
||||
govet:
|
||||
enable-all: true
|
||||
disable:
|
||||
- fieldalignment # performance tuning not a priority for this proxy appliance
|
||||
- shadow # common idiomatic "err" redeclarations in error-handling chains (large ServeHTTP, root, parse funcs); enabling adds noise with no real bugs; would require scope refactor for little gain
|
||||
exclusions:
|
||||
generated: lax
|
||||
paths:
|
||||
- dist
|
||||
- bin
|
||||
rules:
|
||||
- path: _test\.go
|
||||
linters:
|
||||
- errcheck
|
||||
- gosec # tests often use weak patterns intentionally (e.g. error injection, temp files)
|
||||
# NOTE: narrow SA9003 exclude retained only for the one remaining intentional empty branch in test (best-effort status check; main assert is metrics side-effect).
|
||||
- path: steamcache/steamcache_test.go
|
||||
linters:
|
||||
- staticcheck
|
||||
text: "SA9003: empty branch"
|
||||
# Narrow gosec excludes for unavoidable classes after re-audit (LAN proxy threat model):
|
||||
# - G115: int64<->uint casts in eviction/GC math (all sizes positive, guarded by capacity checks; API uses uint for bytesNeeded)
|
||||
# - G304: path vars for Read/Open/Remove under trusted disk.root or user config file (sanitized keys, no traversal, no arbitrary inclusion from untrusted URLs)
|
||||
# G306 for config WriteFile kept as source //nosec (one site).
|
||||
# G301 fixed at source (0700 dirs). G104 addressed via errcheck fixes.
|
||||
- path: vfs/memory/memory.go
|
||||
linters:
|
||||
- gosec
|
||||
text: "G115"
|
||||
- path: vfs/disk/disk.go
|
||||
linters:
|
||||
- gosec
|
||||
text: "G115"
|
||||
- path: vfs/gc/gc.go
|
||||
linters:
|
||||
- gosec
|
||||
text: "G115"
|
||||
- path: config/config.go
|
||||
linters:
|
||||
- gosec
|
||||
text: "G304"
|
||||
- path: vfs/disk/disk.go
|
||||
linters:
|
||||
- gosec
|
||||
text: "G304"
|
||||
|
||||
linters-settings:
|
||||
errcheck:
|
||||
check-type-assertions: false
|
||||
check-blank: false
|
||||
gosec:
|
||||
# Broad global excludes removed (G104/G115/G301/G304/G306).
|
||||
# - G301 addressed by switching cache MkdirAll to 0700 (least privilege for CDN content).
|
||||
# - Remaining justified cases documented with precise //nosec (or #nosec) + comments at the call sites.
|
||||
# - G104 largely eliminated by errcheck + explicit _ = handling (or defer wrappers).
|
||||
staticcheck:
|
||||
checks: ["all"] # SA1019 exclusion removed (no deprecated API usages in tree)
|
||||
govet:
|
||||
enable-all: true
|
||||
disable:
|
||||
- fieldalignment # performance tuning not a priority for this proxy appliance
|
||||
- shadow # common idiomatic "err" redeclarations in error-handling chains (large ServeHTTP, root, parse funcs); enabling adds noise with no real bugs; would require scope refactor for little gain
|
||||
|
||||
# Old global errcheck disable + aspirational "re-enable after refactors" comments deleted.
|
||||
# errcheck is now on via defaults. Unavoidable cases handled at source with _ = or (rarely) narrow rules.
|
||||
formatters:
|
||||
enable:
|
||||
- goimports
|
||||
exclusions:
|
||||
generated: lax
|
||||
paths:
|
||||
- dist
|
||||
- bin
|
||||
|
||||
issues:
|
||||
max-issues-per-linter: 0
|
||||
max-same-issues: 0
|
||||
exclude-use-default: false
|
||||
exclude-dirs:
|
||||
- dist
|
||||
- bin
|
||||
exclude-rules:
|
||||
- path: _test\.go
|
||||
linters:
|
||||
- errcheck
|
||||
- gosec # tests often use weak patterns intentionally (e.g. error injection, temp files)
|
||||
# NOTE: narrow SA9003 exclude retained only for the one remaining intentional empty branch in test (best-effort status check; main assert is metrics side-effect).
|
||||
# The config one was a truly redundant check (already errored above); deleted surgically in Fix Round 1 (Issue 1), eliminating its exclude-rule.
|
||||
- path: steamcache/steamcache_test.go
|
||||
linters:
|
||||
- staticcheck
|
||||
text: "SA9003: empty branch"
|
||||
# Narrow gosec excludes for unavoidable classes after re-audit (LAN proxy threat model):
|
||||
# - G115: int64<->uint casts in eviction/GC math (all sizes positive, guarded by capacity checks; API uses uint for bytesNeeded)
|
||||
# - G304: path vars for Read/Open/Remove under trusted disk.root or user config file (sanitized keys, no traversal, no arbitrary inclusion from untrusted URLs)
|
||||
# G306 for config WriteFile kept as source //nosec (one site).
|
||||
# G301 fixed at source (0700 dirs). G104 addressed via errcheck fixes.
|
||||
- path: vfs/memory/memory.go
|
||||
linters:
|
||||
- gosec
|
||||
text: "G115"
|
||||
- path: vfs/disk/disk.go
|
||||
linters:
|
||||
- gosec
|
||||
text: "G115"
|
||||
- path: vfs/gc/gc.go
|
||||
linters:
|
||||
- gosec
|
||||
text: "G115"
|
||||
- path: config/config.go
|
||||
linters:
|
||||
- gosec
|
||||
text: "G304"
|
||||
- path: vfs/disk/disk.go
|
||||
linters:
|
||||
- gosec
|
||||
text: "G304"
|
||||
# Predictive/* rules deleted: vfs/predictive/ removed in commit 0dbb2e0; rules were stale/dead.
|
||||
# All other suppressions use source-level //nosec (gosec) or _= (errcheck) for precision and visibility.
|
||||
|
||||
@@ -0,0 +1,40 @@
|
||||
# Contributing
|
||||
|
||||
## Propose changes
|
||||
|
||||
Open a pull request against `develop`. Keep the default branch for releases and
|
||||
stable tips; land work on `develop` first.
|
||||
|
||||
Point at an existing issue when one fits. Prefer a short issue that states the
|
||||
symptom or request before a large PR.
|
||||
|
||||
## Commits
|
||||
|
||||
Subject form:
|
||||
|
||||
```
|
||||
area: Imperative summary
|
||||
```
|
||||
|
||||
- **Area** is a real package, directory, or subsystem token (`ci:`, `docs:`,
|
||||
Go package name). Not a lone filename.
|
||||
- **Imperative** mood: Fix, Add, Remove — not "Fixed" or "This patch…".
|
||||
- No trailing period. Aim ≤ ~70–75 characters for the whole subject.
|
||||
- Not conventional-commits (`feat:` / `fix:` / `chore:` as types).
|
||||
|
||||
Body explains **why**. Establish the problem, then say what you are doing.
|
||||
One logical change per commit; split fix and cleanup.
|
||||
|
||||
## Pull requests
|
||||
|
||||
Title matches the primary commit subject.
|
||||
|
||||
- **What** changed
|
||||
- **Why** (problem and impact)
|
||||
- **Test** (concrete steps; "CI green" alone is weak)
|
||||
|
||||
## Issues and closing
|
||||
|
||||
Cite leftover issues by **full URL**. Gitea closes issues when `#N` appears in
|
||||
merge text, so do not put `#N` in the merge message unless that issue is actually
|
||||
done. Use `Fixes #N` / `Closes #N` only when the leftover work is finished.
|
||||
@@ -4,21 +4,21 @@ run: ## Run the application (cross-platform; uses go run for dev on Linux/macOS/
|
||||
run-debug: ## Run the application with debug logging (cross-platform)
|
||||
@go run . --log-level debug
|
||||
|
||||
build: deps ## Build a snapshot of the application for the current platform (uses -short for fast feedback)
|
||||
build: deps lint ## Build a snapshot of the application for the current platform (uses -short for fast feedback)
|
||||
@go test -short -v ./...
|
||||
@goreleaser build --single-target --snapshot --clean
|
||||
|
||||
test: deps ## Run all tests
|
||||
test: deps lint ## Run all tests
|
||||
@go test -shuffle=on -timeout=5m -v ./...
|
||||
|
||||
test-race: deps ## Run all tests with the race detector
|
||||
test-race: deps lint ## Run all tests with the race detector
|
||||
@go test -race -shuffle=on -timeout=5m -v ./...
|
||||
|
||||
lint: deps check-review-labels ## Run golangci-lint + review label hygiene check
|
||||
@golangci-lint run ./...
|
||||
|
||||
check-review-labels: ## Fail if temporary review labels (P0-01, T1, I3, R2, etc.) are found in source
|
||||
@! grep -rnE '\b[A-Z][0-9][^a-zA-Z]' --include='*.go' . 2>/dev/null | grep -v 'G[0-9]\{3\}' || (echo "Error: Found temporary review labels (P*, T*, I*, etc.) in source. See AGENTS.md for the rule." && exit 1)
|
||||
@! grep -rnE '\b[A-Z][0-9][^a-zA-Z]' --include='*.go' --include='*.md' --include='*.yaml' --include='*.sh' --exclude='AGENTS.md' . 2>/dev/null | grep -v 'G[0-9]\{3\}' || (echo "Error: Found temporary review labels (P*, T*, I*, etc.) in source. See AGENTS.md for the rule." && exit 1)
|
||||
|
||||
deps: ## Download dependencies
|
||||
@go mod tidy
|
||||
@@ -26,6 +26,9 @@ deps: ## Download dependencies
|
||||
clean: ## Remove build artifacts and test cache
|
||||
@rm -rf bin/ dist/ *.test coverage.out steamcache2
|
||||
|
||||
clean-disk: ## Remove disk cache
|
||||
@rm -rf validate-disk/
|
||||
|
||||
bench: deps ## Run all benchmarks (MemoryFS + DiskFS variants, including all eviction strategies)
|
||||
@echo "Running MemoryFS benchmarks..."
|
||||
@go test -bench=. -benchmem -run=^$ -benchtime=1s ./vfs/memory
|
||||
@@ -38,7 +41,7 @@ setcap: build ## Explicitly set cap_net_bind_service on the (just-built) binary
|
||||
@sudo setcap 'cap_net_bind_service=+ep' dist/default_linux_amd64_v1/steamcache2
|
||||
@echo "Done. You should now be able to run 'make run-validation' as your normal user (no root)."
|
||||
|
||||
validate run-validation: build ## Start steamcache2 on :80 with small test caches (foreground)
|
||||
validate run-validation: build clean-disk ## Start steamcache2 on :80 with small test caches (foreground)
|
||||
@echo "=== Starting steamcache2 in validation mode ==="
|
||||
@echo "Port 80 + small memory/disk caches (for exercising disk tier, GC, etc.)"
|
||||
@echo "Press Ctrl-C to stop the server."
|
||||
@@ -59,17 +62,6 @@ validate run-validation: build ## Start steamcache2 on :80 with small test cache
|
||||
fi; \
|
||||
exec "$$BINARY" --config docs/examples/validate-config.yaml --log-level info
|
||||
|
||||
validate-check: ## Quick post-benchmark sanity check against a running steamcache2 (default port 80, override with PORT=xxxx)
|
||||
@echo "=== steamcache2 Full Function Validation Report ==="
|
||||
@PORT="$${PORT:-80}"; \
|
||||
echo "Server: http://localhost:$$PORT"; \
|
||||
curl -s --max-time 5 "http://localhost:$$PORT/metrics" || echo "(could not reach /metrics on port $$PORT - is the server running?)"; \
|
||||
echo ""; \
|
||||
echo "Tip: also inspect recent server logs for errors, coalesced hits, and disk activity."
|
||||
|
||||
prefill: ## Download latest SteamPrefill into bin/steam-prefill/SteamPrefill (gitignored)
|
||||
@./scripts/download-prefill.sh
|
||||
|
||||
validate-kill: ## Kill leftover steamcache2 processes (safer, checks process name)
|
||||
@echo "Looking for steamcache2 processes on common validation ports (80 is primary)..."
|
||||
@for port in 80 8040 8080; do \
|
||||
@@ -95,6 +87,11 @@ validate-kill: ## Kill leftover steamcache2 processes (safer, checks process nam
|
||||
done
|
||||
@echo "Validation server cleanup complete."
|
||||
|
||||
prefill: ## Download latest SteamPrefill into bin/steam-prefill/SteamPrefill (gitignored)
|
||||
@./scripts/download-prefill.sh
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
help: ## Show this help message
|
||||
@@ -109,9 +106,9 @@ help: ## Show this help message
|
||||
@echo " check-review-labels Fail on temporary review labels (P*, T*, I*, R*, etc.)"
|
||||
@echo " deps Download dependencies"
|
||||
@echo " clean Remove build/test artifacts"
|
||||
@echo " clean-disk Remove disk cache"
|
||||
@echo " bench Run low-level VFS microbenchmarks"
|
||||
@echo " validate / run-validation Start server on :80 (builds, auto-setcaps fresh binary, then runs as normal user)"
|
||||
@echo " validate / run-validation Start server on :80 (builds, auto-setcaps fresh binary, then runs as normal user, cleans disk cache first)"
|
||||
@echo " setcap Explicitly set cap on current build (for port 80 use outside validate)"
|
||||
@echo " validate-check Quick /metrics report after running a workload"
|
||||
@echo " validate-kill Kill leftover steamcache2 processes (safer)"
|
||||
@echo " prefill Download latest SteamPrefill into bin/steam-prefill/ (for use with run-validation)"
|
||||
@echo " prefill Download latest SteamPrefill into bin/steam-prefill/SteamPrefill (gitignored)"
|
||||
@@ -70,26 +70,7 @@ This gives you:
|
||||
- Excellent `benchmark setup` / `benchmark run` workflow with warmup, randomization, and mixed chunk sizes
|
||||
- The ability to validate a **just-built binary** end-to-end (caching, coalescing, Range support, memory+disk tiers, GC/eviction, metrics, special endpoints, startup validation, etc.)
|
||||
|
||||
#### Quick Start
|
||||
|
||||
```bash
|
||||
# 1. Build the binary (this also runs short tests)
|
||||
make build
|
||||
|
||||
# 2. Start a validation-oriented instance (small caches so disk tier + GC get exercised)
|
||||
# Uses port 80 by default; the script will automatically set the needed
|
||||
# capability on the binary via sudo setcap if it is missing.
|
||||
./scripts/validate-with-prefill.sh
|
||||
|
||||
# 3. In another terminal (or on another machine), create a workload once if you haven't already,
|
||||
# then drive it through your local steamcache2.
|
||||
# Note: when using a non-80 port you may need to give SteamPrefill the full address.
|
||||
./scripts/validate-with-prefill.sh # (shows the exact commands with the correct port)
|
||||
```
|
||||
|
||||
When the benchmark finishes, press Ctrl-C in the first terminal to cleanly stop the server.
|
||||
|
||||
#### Simple validation server (recommended for manual testing)
|
||||
#### Validation server (recommended)
|
||||
|
||||
For easy validation with external tools (SteamPrefill, etc.), use:
|
||||
|
||||
@@ -140,7 +121,7 @@ Look for:
|
||||
|
||||
#### The Validation Config
|
||||
|
||||
The script uses [docs/examples/validate-config.yaml](docs/examples/validate-config.yaml). It enables both memory and disk tiers at modest sizes (128 MB / 512 MB) with conservative concurrency. Edit or copy it if you need larger caches for bigger workloads.
|
||||
The recommended validation config is at [docs/examples/validate-config.yaml](docs/examples/validate-config.yaml). It enables both memory and disk tiers at modest sizes (128 MB / 512 MB) with conservative concurrency. Edit or copy it if you need larger caches for bigger workloads.
|
||||
|
||||
#### What Gets Validated
|
||||
|
||||
@@ -179,8 +160,9 @@ While most configuration is done via the YAML file, some runtime options are sti
|
||||
# Set logging level
|
||||
./steamcache2 --log-level debug --log-format json
|
||||
|
||||
# Set number of worker threads
|
||||
./steamcache2 --threads 8
|
||||
# Override concurrency from the CLI (0 = use config.yaml)
|
||||
./steamcache2 --max-concurrent-requests 8
|
||||
./steamcache2 --max-requests-per-client 4
|
||||
|
||||
# Show help
|
||||
./steamcache2 --help
|
||||
@@ -194,7 +176,7 @@ SteamCache2 uses a YAML configuration file (`config.yaml`) for all settings. Her
|
||||
# Server configuration
|
||||
listen_address: :80
|
||||
|
||||
# P1 hardening (see Security Hardening section)
|
||||
# Hardening (see Security Hardening section)
|
||||
max_object_size: "0" # 0=unlimited; set e.g. "256MB" for response size DoS protection
|
||||
trusted_proxies: [] # empty = safe (ignore XFF for rate limit); set CIDRs for trusted proxies
|
||||
|
||||
@@ -222,7 +204,7 @@ upstream: "https://steam.cdn.com"
|
||||
```
|
||||
|
||||
#### Startup Validation
|
||||
As of P0, `steamcache2` performs strict validation on startup (after loading config + CLI overrides, before creating the cache). Invalid configs cause immediate clean failure (no default written, no panic):
|
||||
`steamcache2` performs strict validation on startup (after loading config + CLI overrides, before creating the cache). Invalid configs cause immediate clean failure (no default written, no panic):
|
||||
|
||||
- Negative `max_concurrent_requests` / `max_requests_per_client`: "negative concurrency not allowed"
|
||||
- Invalid `gc_algorithm` (memory): "invalid memory gc algorithm: badvalue"
|
||||
@@ -236,12 +218,12 @@ Error: Invalid configuration: invalid memory gc algorithm: foo. Please fix the c
|
||||
|
||||
See `config.Validate()` and `steamcache.New` error paths. This ensures the LAN appliance fails fast on misconfig.
|
||||
|
||||
#### Security Hardening (P1)
|
||||
- `max_object_size` (default "0" = unlimited): set e.g. "256MB" or "512MB" to reject oversized upstream responses with HTTP 413 before buffering/ReadAll. Prevents OOM DoS from large or malicious responses (P1-01). Large legitimate Steam files still served if under limit.
|
||||
- `trusted_proxies`: CIDR list (default empty). When empty (safe default), X-Forwarded-For and client IP spoofing are ignored for rate limiting — always uses `r.RemoteAddr` only. When set (e.g. your reverse proxy CIDR), uses correct "rightmost untrusted" extraction. Prevents bypass of `max_requests_per_client` (P1-02). Documented for LAN proxy setups only.
|
||||
- These + P0 validation make steamcache2 safe-by-default for LAN exposure.
|
||||
#### Security Hardening
|
||||
- `max_object_size` (default "0" = unlimited): set e.g. "256MB" or "512MB" to reject oversized upstream responses with HTTP 413 before buffering/ReadAll. Prevents OOM DoS from large or malicious responses. Large legitimate Steam files still served if under limit.
|
||||
- `trusted_proxies`: CIDR list (default empty). When empty (safe default), X-Forwarded-For and client IP spoofing are ignored for rate limiting — always uses `r.RemoteAddr` only. When set (e.g. your reverse proxy CIDR), uses correct "rightmost untrusted" extraction. Prevents bypass of `max_requests_per_client`. Documented for LAN proxy setups only.
|
||||
- These + the startup validation make steamcache2 safe-by-default for LAN exposure.
|
||||
|
||||
#### Migration / Breaking Changes (P1)
|
||||
#### Migration / Breaking Changes
|
||||
- `New()` public signature gained 2 required trailing params (`maxObjectSize`, `trustedProxies`). Direct callers (rare; most use config or NewWithOptions) must update.
|
||||
- Recommended: migrate to `NewWithOptions(Options{...})` (non-breaking) or rely on YAML config + cmd/root.go.
|
||||
- No behavior change for existing configs (defaults preserve prior semantics).
|
||||
@@ -330,7 +312,7 @@ This will direct any requests to `lancache.steamcontent.com` to your SteamCache2
|
||||
|
||||
### Prerequisites
|
||||
|
||||
- Go 1.19 or later
|
||||
- Go 1.27.0 or later
|
||||
- Make (optional, but recommended)
|
||||
|
||||
### Build Commands
|
||||
@@ -338,7 +320,7 @@ This will direct any requests to `lancache.steamcontent.com` to your SteamCache2
|
||||
```bash
|
||||
# Clone the repository
|
||||
git clone <repository-url>
|
||||
cd SteamCache2
|
||||
cd steamcache2
|
||||
|
||||
# Download dependencies
|
||||
make deps
|
||||
|
||||
@@ -22,7 +22,7 @@
|
||||
#
|
||||
# Usage (typical dev workflow):
|
||||
# make build
|
||||
# ./scripts/validate-with-prefill.sh
|
||||
# make validate
|
||||
# # In another terminal:
|
||||
# SteamPrefill benchmark run -c 20 ...
|
||||
#
|
||||
@@ -43,12 +43,15 @@ trusted_proxies: ["127.0.0.0/8"]
|
||||
cache:
|
||||
memory:
|
||||
size: 1GB
|
||||
gc_algorithm: largest
|
||||
gc_algorithm: hybrid
|
||||
disk:
|
||||
size: 2GB
|
||||
path: ./validate-disk # ephemeral; clean between runs if you want a fresh test
|
||||
path: ./validate-disk # cleaned between runs by make validate or make clean-disk
|
||||
gc_algorithm: hybrid # recommended for disk in the project README
|
||||
|
||||
# Empty upstream = use Host header from the client (SteamPrefill / real Steam clients).
|
||||
# This matches the common "DNS points lancache.steamcontent.com at the cache" setup.
|
||||
# Allows for chaining steamcache2 instances if needed.
|
||||
# For example, for a lan party you could have a small fast ram only cache at each table pointing to a larger slower disk cache in the back somewhere
|
||||
# It would reduce the amount of bandwidth needed to the internet and the amount needed to each table
|
||||
# just as a little reminder there is no authentication so this is not a good idea for a public cache just out on the internet.
|
||||
upstream: ""
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
module s1d3sw1ped/steamcache2
|
||||
|
||||
go 1.23.0
|
||||
go 1.27.0
|
||||
|
||||
require (
|
||||
github.com/docker/go-units v0.5.0
|
||||
|
||||
@@ -1,182 +0,0 @@
|
||||
#!/usr/bin/env bash
|
||||
#
|
||||
# validate-with-prefill.sh
|
||||
#
|
||||
# Thin glue script to make it trivial for developers to validate complete
|
||||
# steamcache2 functionality using the external SteamPrefill (lancacheprefill)
|
||||
# tool as the realistic client simulator.
|
||||
#
|
||||
# Usage:
|
||||
# 1. make build
|
||||
# 2. ./scripts/validate-with-prefill.sh
|
||||
# (Automatically kills any leftover steamcache2 on the target port first.)
|
||||
# 3. In another terminal (or on another machine), run the printed
|
||||
# SteamPrefill benchmark commands (the script tells you the exact address/port).
|
||||
# 4. After the benchmark finishes, run the suggested metrics check.
|
||||
# 5. Ctrl-C here to cleanly stop the steamcache2 instance.
|
||||
#
|
||||
# This script + the accompanying validate-config.yaml + README docs are the
|
||||
# entire "couple little scripts to hook steamcache2 and lancacheprefill together"
|
||||
# implementation. No Go code, no new dependencies, stays outside go test / bench.
|
||||
#
|
||||
set -euo pipefail
|
||||
|
||||
# --- Locate the built steamcache2 binary (produced by "make build") ---
|
||||
BINARY=""
|
||||
for candidate in \
|
||||
"dist/default_linux_amd64_v1/steamcache2" \
|
||||
"dist/steamcache2" \
|
||||
"./steamcache2" \
|
||||
"steamcache2"
|
||||
do
|
||||
if [[ -x "$candidate" ]]; then
|
||||
BINARY="$candidate"
|
||||
break
|
||||
fi
|
||||
done
|
||||
|
||||
if [[ -z "$BINARY" ]]; then
|
||||
echo "ERROR: Could not find a built steamcache2 binary."
|
||||
echo "Run 'make build' first (or place the binary in one of the searched locations)."
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo "Using steamcache2 binary: $BINARY"
|
||||
|
||||
# --- Validation config (small dual-tier so disk + GC get real exercise) ---
|
||||
# Source of truth lives in docs/examples/ (safe from "make clean").
|
||||
# The script will also accept an explicit path via STEAMCACHE2_VALIDATE_CONFIG.
|
||||
VALIDATE_CONFIG="${STEAMCACHE2_VALIDATE_CONFIG:-docs/examples/validate-config.yaml}"
|
||||
if [[ ! -f "$VALIDATE_CONFIG" ]]; then
|
||||
echo "ERROR: Validation config not found at: $VALIDATE_CONFIG"
|
||||
echo "Set STEAMCACHE2_VALIDATE_CONFIG=/path/to/your-config.yaml or place a copy at docs/examples/validate-config.yaml"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Extract the listen port from the config (supports ":80", "127.0.0.1:80", etc.)
|
||||
# Falls back to 80 if we can't parse it.
|
||||
PORT=$(grep -E '^\s*listen_address:' "$VALIDATE_CONFIG" | head -1 | sed -E 's/.*:([0-9]+).*/\1/' || true)
|
||||
if [[ -z "$PORT" || ! "$PORT" =~ ^[0-9]+$ ]]; then
|
||||
PORT=80
|
||||
fi
|
||||
|
||||
SERVER_URL="http://localhost:${PORT}"
|
||||
|
||||
# For privileged ports (<1024, i.e. the default :80) we require the
|
||||
# cap_net_bind_service capability. We apply it automatically here (via sudo
|
||||
# setcap) on the binary we are about to run. This happens after any build
|
||||
# so the cap is never "lost" when the binary is rebuilt.
|
||||
if [ "$PORT" -lt 1024 ] && [ "$(id -u)" -ne 0 ]; then
|
||||
if ! command -v getcap >/dev/null 2>&1 || ! getcap "$BINARY" 2>/dev/null | grep -q "cap_net_bind_service"; then
|
||||
echo "Setting cap_net_bind_service on the binary (sudo may prompt)..."
|
||||
if ! sudo setcap 'cap_net_bind_service=+ep' "$BINARY"; then
|
||||
echo "ERROR: Failed to set capability."
|
||||
echo "Run 'make setcap' manually, then retry."
|
||||
exit 1
|
||||
fi
|
||||
fi
|
||||
fi
|
||||
|
||||
# Safely kill only steamcache2 processes listening on this specific port.
|
||||
# We look up PIDs on the port, check their actual process name/command,
|
||||
# and only kill via PID if it looks like steamcache2.
|
||||
echo "Checking for leftover steamcache2 processes on port ${PORT}..."
|
||||
|
||||
kill_steamcache_on_port() {
|
||||
local port=$1
|
||||
local pids=""
|
||||
|
||||
# Try ss first (modern, usually available)
|
||||
if command -v ss >/dev/null 2>&1; then
|
||||
pids=$(ss -tlnp 2>/dev/null | grep ":${port} " | sed -n 's/.*pid=\([0-9]*\).*/\1/p' | sort -u)
|
||||
fi
|
||||
|
||||
# Fallback to lsof
|
||||
if [[ -z "$pids" ]] && command -v lsof >/dev/null 2>&1; then
|
||||
pids=$(lsof -ti :${port} 2>/dev/null | sort -u)
|
||||
fi
|
||||
|
||||
if [[ -z "$pids" ]]; then
|
||||
return 0
|
||||
fi
|
||||
|
||||
for pid in $pids; do
|
||||
# Get process name and command line
|
||||
local proc_name
|
||||
local cmdline
|
||||
proc_name=$(ps -p "$pid" -o comm= 2>/dev/null || true)
|
||||
cmdline=$(ps -p "$pid" -o cmd= 2>/dev/null || true)
|
||||
|
||||
# Check if this looks like a steamcache2 process
|
||||
if echo "$proc_name $cmdline" | grep -qi "steamcache"; then
|
||||
echo " → Found steamcache2 on port ${port} (PID $pid, name: ${proc_name:-unknown})"
|
||||
kill -TERM "$pid" 2>/dev/null || true
|
||||
sleep 0.3
|
||||
# If still alive, force kill
|
||||
if kill -0 "$pid" 2>/dev/null; then
|
||||
kill -9 "$pid" 2>/dev/null || true
|
||||
fi
|
||||
echo " Killed PID $pid"
|
||||
else
|
||||
echo " → Skipping PID $pid on port ${port} (not steamcache2: ${proc_name:-$cmdline})"
|
||||
fi
|
||||
done
|
||||
}
|
||||
|
||||
kill_steamcache_on_port "$PORT"
|
||||
sleep 0.5
|
||||
|
||||
# --- Launch the server in the background ---
|
||||
echo "Starting steamcache2 with validation config (small caches for disk/GC testing)..."
|
||||
"$BINARY" --config "$VALIDATE_CONFIG" --log-level info &
|
||||
SERVER_PID=$!
|
||||
|
||||
# Ensure we always clean up the child on exit / Ctrl-C / error
|
||||
cleanup() {
|
||||
echo ""
|
||||
echo "Stopping steamcache2 (pid $SERVER_PID)..."
|
||||
if kill "$SERVER_PID" 2>/dev/null; then
|
||||
wait "$SERVER_PID" 2>/dev/null || true
|
||||
fi
|
||||
echo "Server stopped."
|
||||
}
|
||||
trap cleanup EXIT INT TERM
|
||||
|
||||
# Give the server a moment to bind and pass its own startup checks
|
||||
sleep 2
|
||||
|
||||
# Basic readiness probe using the actual configured port
|
||||
if ! curl -s --max-time 3 "${SERVER_URL}/" >/dev/null 2>&1; then
|
||||
echo "WARNING: Server did not respond quickly on ${SERVER_URL}/"
|
||||
echo " It may still be starting or bound to a different address."
|
||||
echo " Check the server logs above. You can still try the SteamPrefill commands."
|
||||
fi
|
||||
|
||||
if [[ "${VALIDATE_QUIET:-}" != "1" ]]; then
|
||||
echo ""
|
||||
echo "======================================================================"
|
||||
echo "steamcache2 is running (validation mode) on ${SERVER_URL}"
|
||||
echo ""
|
||||
echo "In another terminal (or on a machine that can reach this one), run:"
|
||||
echo ""
|
||||
echo " # One-time workload creation (run on a machine with SteamPrefill + Steam):"
|
||||
echo " SteamPrefill benchmark setup --preset LargeChunks"
|
||||
echo " # (or --use-selected, --all, --appid ..., or your own preset)"
|
||||
echo ""
|
||||
echo " # Copy the generated workload file to this machine if needed."
|
||||
echo ""
|
||||
echo " # Then run the actual benchmark (this is the realistic client simulator):"
|
||||
echo " SteamPrefill benchmark run -c 20 -i 3"
|
||||
echo ""
|
||||
echo "After the benchmark completes, you can inspect the cache with:"
|
||||
echo " curl -s ${SERVER_URL}/metrics | cat"
|
||||
echo ""
|
||||
echo "Or run: make validate-check (if the Makefile target exists)"
|
||||
echo ""
|
||||
echo "When you are finished, press Ctrl-C in this window to stop the server cleanly."
|
||||
echo "======================================================================"
|
||||
echo ""
|
||||
fi
|
||||
|
||||
# Wait for the background server (or for the user to Ctrl-C)
|
||||
wait $SERVER_PID || true
|
||||
@@ -345,6 +345,18 @@ func (sc *SteamCache) ServeHTTP(w http.ResponseWriter, r *http.Request) {
|
||||
req.Host = r.Host
|
||||
} else { // if no upstream server is configured, proxy the request to the host specified in the request
|
||||
host := r.Host
|
||||
if !hostAllowedForDirectFetch(host) {
|
||||
logger.Logger.Warn().
|
||||
Str("host", host).
|
||||
Str("client_ip", clientIP).
|
||||
Msg("Rejecting direct-fetch Host (not a Steam CDN name)")
|
||||
sc.metrics.IncrementErrors()
|
||||
if isNew {
|
||||
coalescedReq.complete(nil, fmt.Errorf("host not allowed for direct fetch"))
|
||||
}
|
||||
http.Error(w, "Invalid URL", http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
if r.Header.Get("X-Sls-Https") == "enable" {
|
||||
host = "https://" + host
|
||||
} else {
|
||||
|
||||
@@ -24,8 +24,6 @@ type Metrics struct {
|
||||
TotalBytesServed int64
|
||||
TotalBytesSaved int64 // bytes served from cache instead of being re-downloaded from upstream
|
||||
|
||||
|
||||
|
||||
// Cache metrics
|
||||
MemoryCacheSize int64
|
||||
DiskCacheSize int64
|
||||
@@ -194,8 +192,8 @@ func (m *Metrics) GetStats() *Stats {
|
||||
RateLimited: atomic.LoadInt64(&m.RateLimited),
|
||||
HitRate: hitRate,
|
||||
AvgResponseTime: avgResponseTime,
|
||||
TotalBytesServed: atomic.LoadInt64(&m.TotalBytesServed),
|
||||
TotalBytesSaved: atomic.LoadInt64(&m.TotalBytesSaved),
|
||||
TotalBytesServed: atomic.LoadInt64(&m.TotalBytesServed),
|
||||
TotalBytesSaved: atomic.LoadInt64(&m.TotalBytesSaved),
|
||||
MemoryCacheSize: atomic.LoadInt64(&m.MemoryCacheSize),
|
||||
DiskCacheSize: atomic.LoadInt64(&m.DiskCacheSize),
|
||||
MemoryCacheHits: atomic.LoadInt64(&m.MemoryCacheHits),
|
||||
@@ -242,18 +240,17 @@ func (m *Metrics) Reset() {
|
||||
|
||||
// Stats represents a snapshot of metrics
|
||||
type Stats struct {
|
||||
TotalRequests int64
|
||||
CacheHits int64
|
||||
CacheMisses int64
|
||||
CacheCoalesced int64
|
||||
Errors int64
|
||||
RateLimited int64
|
||||
HitRate float64
|
||||
AvgResponseTime time.Duration
|
||||
TotalBytesServed int64
|
||||
TotalBytesSaved int64
|
||||
MemoryCacheSize int64
|
||||
|
||||
TotalRequests int64
|
||||
CacheHits int64
|
||||
CacheMisses int64
|
||||
CacheCoalesced int64
|
||||
Errors int64
|
||||
RateLimited int64
|
||||
HitRate float64
|
||||
AvgResponseTime time.Duration
|
||||
TotalBytesServed int64
|
||||
TotalBytesSaved int64
|
||||
MemoryCacheSize int64
|
||||
|
||||
DiskCacheSize int64
|
||||
MemoryCacheHits int64
|
||||
|
||||
@@ -5,6 +5,7 @@ import (
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"fmt"
|
||||
"net"
|
||||
"net/http"
|
||||
"regexp"
|
||||
"strings"
|
||||
@@ -163,3 +164,44 @@ func generateServiceCacheKey(urlPath string, servicePrefix string) (string, erro
|
||||
}
|
||||
return servicePrefix + "/" + hash, nil
|
||||
}
|
||||
|
||||
// requestHostName strips a port and brackets from an HTTP Host header.
|
||||
func requestHostName(host string) string {
|
||||
host = strings.TrimSpace(host)
|
||||
if host == "" {
|
||||
return ""
|
||||
}
|
||||
if h, _, err := net.SplitHostPort(host); err == nil {
|
||||
host = h
|
||||
}
|
||||
return strings.Trim(host, "[]")
|
||||
}
|
||||
|
||||
func hostIsLiteralIP(host string) bool {
|
||||
return net.ParseIP(requestHostName(host)) != nil
|
||||
}
|
||||
|
||||
// defaultDirectFetchSuffixes are CDN names Steam actually uses. Applied only when
|
||||
// no configured upstream is set and the request Host is used as the fetch target.
|
||||
var defaultDirectFetchSuffixes = []string{
|
||||
"steamcontent.com",
|
||||
"steampowered.com",
|
||||
"steamstatic.com",
|
||||
}
|
||||
|
||||
// hostAllowedForDirectFetch reports whether Host may be used as an origin when
|
||||
// upstream is empty. Literal IPs are rejected (LAN/metadata SSRF). Names must
|
||||
// be Steam CDN suffixes so a spoofed User-Agent cannot turn the cache into an
|
||||
// open reverse proxy.
|
||||
func hostAllowedForDirectFetch(host string) bool {
|
||||
name := strings.ToLower(requestHostName(host))
|
||||
if name == "" || hostIsLiteralIP(host) {
|
||||
return false
|
||||
}
|
||||
for _, suf := range defaultDirectFetchSuffixes {
|
||||
if name == suf || strings.HasSuffix(name, "."+suf) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
@@ -357,7 +357,7 @@ func newHTTPTransport() *http.Transport {
|
||||
DialContext: (&net.Dialer{
|
||||
Timeout: 10 * time.Second, // Faster connection timeout
|
||||
KeepAlive: 60 * time.Second, // Longer keep-alive
|
||||
DualStack: true, // Enable dual-stack (IPv4/IPv6)
|
||||
// Dual-stack Happy Eyeballs is the default since Go 1.12 (DualStack is deprecated).
|
||||
}).DialContext,
|
||||
|
||||
// Timeout optimizations
|
||||
@@ -387,11 +387,10 @@ func newHTTPClient(transport *http.Transport) *http.Client {
|
||||
Timeout: 60 * time.Second, // Optimized timeout for better responsiveness
|
||||
// Add redirect policy for better performance
|
||||
CheckRedirect: func(req *http.Request, via []*http.Request) error {
|
||||
// Limit redirects to prevent infinite loops
|
||||
if len(via) >= 10 {
|
||||
return http.ErrUseLastResponse
|
||||
}
|
||||
return nil
|
||||
// Do not follow redirects. Steam CDN chunk/manifest fetches are
|
||||
// expected to be 200; following Location would let an origin send
|
||||
// the cache at an arbitrary internal URL.
|
||||
return http.ErrUseLastResponse
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1165,3 +1165,61 @@ func TestClientRateLimiter_BlackBox(t *testing.T) {
|
||||
t.Error("different clients must have distinct limiters")
|
||||
}
|
||||
}
|
||||
|
||||
func TestHostAllowedForDirectFetch(t *testing.T) {
|
||||
allowed := []string{
|
||||
"lancache.steamcontent.com",
|
||||
"cache1-iad1.steamcontent.com:443",
|
||||
"steamcontent.com",
|
||||
"content.steampowered.com",
|
||||
"cdn.steamstatic.com",
|
||||
}
|
||||
denied := []string{
|
||||
"",
|
||||
"127.0.0.1",
|
||||
"127.0.0.1:80",
|
||||
"[::1]:80",
|
||||
"192.168.1.1",
|
||||
"169.254.169.254",
|
||||
"evil.example",
|
||||
"example.com",
|
||||
"notsteamcontent.com",
|
||||
}
|
||||
for _, h := range allowed {
|
||||
if !hostAllowedForDirectFetch(h) {
|
||||
t.Errorf("expected allowed: %q", h)
|
||||
}
|
||||
}
|
||||
for _, h := range denied {
|
||||
if hostAllowedForDirectFetch(h) {
|
||||
t.Errorf("expected denied: %q", h)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestDirectFetchRejectsNonSteamHost(t *testing.T) {
|
||||
td := t.TempDir()
|
||||
sc, err := New("127.0.0.1:0", "1MB", "0", td, "", "lru", "lru", 200, 5, "0", nil)
|
||||
if err != nil {
|
||||
t.Fatalf("New: %v", err)
|
||||
}
|
||||
t.Cleanup(func() { sc.Shutdown() })
|
||||
|
||||
req := httptest.NewRequest("GET", "/depot/ssrf/chunk", nil)
|
||||
req.Host = "127.0.0.1"
|
||||
req.Header.Set("User-Agent", "Valve/Steam HTTP Client 1.0")
|
||||
rec := httptest.NewRecorder()
|
||||
sc.ServeHTTP(rec, req)
|
||||
if rec.Code != http.StatusBadRequest {
|
||||
t.Errorf("IP Host: expected 400, got %d", rec.Code)
|
||||
}
|
||||
|
||||
req2 := httptest.NewRequest("GET", "/depot/ssrf/chunk2", nil)
|
||||
req2.Host = "evil.example"
|
||||
req2.Header.Set("User-Agent", "Valve/Steam HTTP Client 1.0")
|
||||
rec2 := httptest.NewRecorder()
|
||||
sc.ServeHTTP(rec2, req2)
|
||||
if rec2.Code != http.StatusBadRequest {
|
||||
t.Errorf("non-CDN Host: expected 400, got %d", rec2.Code)
|
||||
}
|
||||
}
|
||||
|
||||
Vendored
+4
-2
@@ -186,7 +186,7 @@ func (tc *TieredCache) Capacity() int64 {
|
||||
func (tc *TieredCache) promoteToFast(key string, reader io.ReadCloser) {
|
||||
defer func() { _ = reader.Close() }() // best-effort close; error secondary to promotion attempt (async best-effort path)
|
||||
|
||||
// Get file info from slow tier to determine size
|
||||
// Size for the space/ReadAll guards comes from a Stat snapshot, not the live in-map FileInfo.
|
||||
var size int64
|
||||
if slow := tc.slow.Load(); slow != nil {
|
||||
if vfs, ok := slow.(vfs.VFS); ok {
|
||||
@@ -210,7 +210,7 @@ func (tc *TieredCache) promoteToFast(key string, reader io.ReadCloser) {
|
||||
}
|
||||
|
||||
// Guard promotion ReadAll using already-fetched size (in addition to space check above)
|
||||
if size > 0 && size > (1<<30) { // conservative 1GB hard limit on promotion reads (aligns with typical max_object_size)
|
||||
if size > (1 << 30) { // conservative 1GB hard limit on promotion reads (aligns with typical max_object_size)
|
||||
return
|
||||
}
|
||||
// Read the entire file content
|
||||
@@ -218,6 +218,8 @@ func (tc *TieredCache) promoteToFast(key string, reader io.ReadCloser) {
|
||||
if err != nil {
|
||||
return // Skip promotion if read fails
|
||||
}
|
||||
// Create with the bytes we actually hold so we never reuse a live FileInfo.Size.
|
||||
size = int64(len(content))
|
||||
|
||||
// Create the file in fast tier
|
||||
if fast := tc.fast.Load(); fast != nil {
|
||||
|
||||
+24
-9
@@ -248,15 +248,25 @@ func (d *DiskFS) calculateSizeAndPopulateIndex() {
|
||||
|
||||
// insertBatch populates info/LRU under lock for a bounded batch (follows maxEvictBatch pattern for short critical sections).
|
||||
// Size is incremented here only for files actually added (prevents double-count vs. concurrent Create during window).
|
||||
// Fail-closed: re-stat each path under d.mu and skip if the file is gone. Create does not wait on
|
||||
// initDone, so a file the scanner observed can be Evict/Delete'd (info + os.Remove) before this
|
||||
// insert runs. Inserting without a live-file check would resurrect the key in d.info and make
|
||||
// Stat succeed while os.Stat fails.
|
||||
func (d *DiskFS) insertBatch(batch []discoveredFile) {
|
||||
d.mu.Lock()
|
||||
for _, df := range batch {
|
||||
if _, exists := d.info[df.key]; !exists {
|
||||
fi := vfs.NewFileInfoFromOS(df.osInfo, df.key)
|
||||
d.info[df.key] = fi
|
||||
d.LRU.Add(df.key, fi)
|
||||
d.size += df.size
|
||||
if _, exists := d.info[df.key]; exists {
|
||||
continue
|
||||
}
|
||||
path := d.pathForKey(df.key)
|
||||
st, err := os.Stat(path)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
fi := vfs.NewFileInfoFromOS(st, df.key)
|
||||
d.info[df.key] = fi
|
||||
d.LRU.Add(df.key, fi)
|
||||
d.size += st.Size()
|
||||
}
|
||||
d.mu.Unlock()
|
||||
}
|
||||
@@ -602,7 +612,9 @@ func (d *DiskFS) Delete(key string) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
// Stat returns file information with lazy discovery
|
||||
// Stat returns a snapshot of file information with lazy discovery.
|
||||
// The returned *FileInfo is not the live cache entry; Close may update Size
|
||||
// on the in-map object under d.mu.
|
||||
func (d *DiskFS) Stat(key string) (*vfs.FileInfo, error) {
|
||||
if key == "" {
|
||||
return nil, vfserror.ErrInvalidKey
|
||||
@@ -617,9 +629,10 @@ func (d *DiskFS) Stat(key string) (*vfs.FileInfo, error) {
|
||||
keyMu.RLock()
|
||||
d.mu.RLock()
|
||||
if fi, ok := d.info[key]; ok {
|
||||
snap := fi.Clone()
|
||||
d.mu.RUnlock()
|
||||
keyMu.RUnlock()
|
||||
return fi, nil
|
||||
return snap, nil
|
||||
}
|
||||
d.mu.RUnlock()
|
||||
keyMu.RUnlock()
|
||||
@@ -639,8 +652,9 @@ func (d *DiskFS) Stat(key string) (*vfs.FileInfo, error) {
|
||||
// Double-check after acquiring write lock
|
||||
d.mu.Lock()
|
||||
if fi, ok := d.info[key]; ok {
|
||||
snap := fi.Clone()
|
||||
d.mu.Unlock()
|
||||
return fi, nil
|
||||
return snap, nil
|
||||
}
|
||||
|
||||
// Re-verify the file still exists on disk under the lock before inserting.
|
||||
@@ -659,9 +673,10 @@ func (d *DiskFS) Stat(key string) (*vfs.FileInfo, error) {
|
||||
fi.UpdateAccessBatched(d.timeUpdater)
|
||||
// Note: size not updated on lazy discovery (preserves prior behavior; initial on-disk accounted via bg populate at New time,
|
||||
// subsequent files come via Create which accounts size).
|
||||
snap := fi.Clone()
|
||||
d.mu.Unlock()
|
||||
|
||||
return fi, nil
|
||||
return snap, nil
|
||||
}
|
||||
|
||||
// EvictLRU evicts the least recently used files to free up space
|
||||
|
||||
+74
-41
@@ -371,7 +371,8 @@ func testKey(i int) string {
|
||||
// artifacts for victims are immediately gone (no resurrection via lazy discovery in Stat/Open),
|
||||
// and that recreating the same key produces independent content that is not subject to any
|
||||
// stale eviction unlinks. This exercises the coordinated WLock remove path for DiskFS.
|
||||
// Uses tolerant checks suitable for raw DiskFS lazy discovery + bg size.
|
||||
// Create does not wait on initDone, so this also covers insertBatch racing with eviction:
|
||||
// gone files must not be re-indexed (Stat present / disk missing).
|
||||
func TestDiskFS_EvictDiskVisibilityAndRecreateSafety(t *testing.T) {
|
||||
t.Parallel()
|
||||
td := t.TempDir()
|
||||
@@ -400,47 +401,21 @@ func TestDiskFS_EvictDiskVisibilityAndRecreateSafety(t *testing.T) {
|
||||
_ = d.EvictBySize(1024*1024, true)
|
||||
}
|
||||
|
||||
// Consistency check: never have a key absent from Stat but with a file on disk (would indicate
|
||||
// either resurrection risk or orphan). If Stat succeeds, file should exist.
|
||||
// A few retries tolerate the documented lazy discovery + eviction coordination windows under
|
||||
// artificial "force massive eviction then immediate audit" load (especially visible under -race).
|
||||
for attempt := 0; attempt < 3; attempt++ {
|
||||
bad := false
|
||||
for _, k := range created {
|
||||
p := d.pathForKey(k)
|
||||
_, statErr := d.Stat(k)
|
||||
_, diskErr := os.Stat(p)
|
||||
if statErr != nil {
|
||||
if !os.IsNotExist(diskErr) {
|
||||
bad = true
|
||||
}
|
||||
} else {
|
||||
if diskErr != nil {
|
||||
bad = true
|
||||
}
|
||||
}
|
||||
}
|
||||
if !bad {
|
||||
break
|
||||
}
|
||||
if attempt < 2 {
|
||||
time.Sleep(10 * time.Millisecond)
|
||||
} else {
|
||||
// On final attempt, report the last observed state for the keys
|
||||
for _, k := range created {
|
||||
p := d.pathForKey(k)
|
||||
_, statErr := d.Stat(k)
|
||||
_, diskErr := os.Stat(p)
|
||||
if statErr != nil {
|
||||
if !os.IsNotExist(diskErr) {
|
||||
t.Errorf("key %s absent via Stat but file lingers on disk at %s (resurrection risk)", k, p)
|
||||
}
|
||||
} else {
|
||||
if diskErr != nil {
|
||||
t.Errorf("key %s present via Stat but missing on disk: %v", k, diskErr)
|
||||
}
|
||||
}
|
||||
// Drain bg population so insertBatch cannot still be in flight when we audit.
|
||||
_ = d.Size()
|
||||
|
||||
// Consistency: Stat success iff the file exists on disk. insertBatch must not resurrect
|
||||
// keys whose backing files were already evicted.
|
||||
for _, k := range created {
|
||||
p := d.pathForKey(k)
|
||||
_, statErr := d.Stat(k)
|
||||
_, diskErr := os.Stat(p)
|
||||
if statErr != nil {
|
||||
if !os.IsNotExist(diskErr) {
|
||||
t.Errorf("key %s absent via Stat but file lingers on disk at %s (resurrection risk)", k, p)
|
||||
}
|
||||
} else if diskErr != nil {
|
||||
t.Errorf("key %s present via Stat but missing on disk: %v", k, diskErr)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -464,6 +439,64 @@ func TestDiskFS_EvictDiskVisibilityAndRecreateSafety(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// TestDiskFS_InsertBatchSkipsGoneFiles is the fail-closed contract for bg/lazy index
|
||||
// insert: a discoveredFile whose path was removed (evicted) must not be re-inserted
|
||||
// into d.info. That resurrection is what made Stat succeed while os.Stat failed.
|
||||
func TestDiskFS_InsertBatchSkipsGoneFiles(t *testing.T) {
|
||||
t.Parallel()
|
||||
td := t.TempDir()
|
||||
d, err := New(td, 10*1024*1024, nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
_ = d.Size() // finish constructor scan so it cannot also index these keys
|
||||
|
||||
liveKey := "live"
|
||||
goneKey := "gone"
|
||||
writeKey := func(key, body string) os.FileInfo {
|
||||
t.Helper()
|
||||
p := d.pathForKey(key)
|
||||
if err := os.MkdirAll(filepath.Dir(p), 0700); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.WriteFile(p, []byte(body), 0600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
st, err := os.Stat(p)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return st
|
||||
}
|
||||
liveInfo := writeKey(liveKey, "still-here")
|
||||
goneInfo := writeKey(goneKey, "about-to-vanish")
|
||||
if err := os.Remove(d.pathForKey(goneKey)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
d.insertBatch([]discoveredFile{
|
||||
{key: liveKey, size: liveInfo.Size(), osInfo: liveInfo},
|
||||
{key: goneKey, size: goneInfo.Size(), osInfo: goneInfo},
|
||||
})
|
||||
|
||||
d.mu.RLock()
|
||||
_, liveExists := d.info[liveKey]
|
||||
_, goneExists := d.info[goneKey]
|
||||
d.mu.RUnlock()
|
||||
if !liveExists {
|
||||
t.Errorf("insertBatch skipped live key %s", liveKey)
|
||||
}
|
||||
if goneExists {
|
||||
t.Errorf("insertBatch resurrected gone key %s", goneKey)
|
||||
}
|
||||
if _, err := d.Stat(goneKey); err == nil {
|
||||
t.Errorf("Stat succeeded for gone key %s", goneKey)
|
||||
}
|
||||
if _, err := os.Stat(d.pathForKey(liveKey)); err != nil {
|
||||
t.Errorf("live key %s missing on disk: %v", liveKey, err)
|
||||
}
|
||||
}
|
||||
|
||||
// TestDiskFS_EvictBoundedLargeN exercises the maxEvictBatch early-break logic (Idea #2)
|
||||
// under a map size >> batch limit. Forces repeated eviction rounds via GC-style pressure
|
||||
// and asserts progress + consistency (no resurrection/orphans). Covers bounded collection
|
||||
|
||||
@@ -289,7 +289,8 @@ func (m *MemoryFS) Delete(key string) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
// Stat returns file information
|
||||
// Stat returns a snapshot of file information. The returned *FileInfo is not
|
||||
// the live cache entry; Close may update Size on the in-map object under m.mu.
|
||||
func (m *MemoryFS) Stat(key string) (*types.FileInfo, error) {
|
||||
if key == "" {
|
||||
return nil, vfserror.ErrInvalidKey
|
||||
@@ -310,7 +311,7 @@ func (m *MemoryFS) Stat(key string) (*types.FileInfo, error) {
|
||||
defer m.mu.RUnlock()
|
||||
|
||||
if fi, ok := m.info[key]; ok {
|
||||
return fi, nil
|
||||
return fi.Clone(), nil
|
||||
}
|
||||
|
||||
return nil, vfserror.ErrNotFound
|
||||
|
||||
@@ -346,6 +346,45 @@ func TestMemoryFS_ConcurrentCloseAndEvict_RaceFree(t *testing.T) {
|
||||
_ = m.LRU.Len()
|
||||
}
|
||||
|
||||
func TestMemoryFS_StatReturnsSnapshot(t *testing.T) {
|
||||
t.Parallel()
|
||||
m, err := New(1024)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
w, err := m.Create("k", 10)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := w.Write([]byte("hello")); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := w.Close(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
fi, err := m.Stat("k")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if fi.Size != 5 {
|
||||
t.Fatalf("size %d want 5", fi.Size)
|
||||
}
|
||||
fi.Size = 999
|
||||
fi.AccessCount = 0
|
||||
|
||||
fi2, err := m.Stat("k")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if fi2.Size != 5 {
|
||||
t.Errorf("Stat returned live FileInfo; store size became %d", fi2.Size)
|
||||
}
|
||||
if fi2.AccessCount == 0 {
|
||||
t.Error("Stat returned live FileInfo; AccessCount mutation leaked")
|
||||
}
|
||||
}
|
||||
|
||||
func TestMemoryFS_EvictVariantsAndErrors(t *testing.T) {
|
||||
t.Parallel()
|
||||
m, err := New(800)
|
||||
|
||||
@@ -27,6 +27,17 @@ func NewFileInfo(key string, size int64) *FileInfo {
|
||||
}
|
||||
}
|
||||
|
||||
// Clone returns a snapshot copy of fi. Stat returns Clone() so callers can
|
||||
// read Size and other fields without racing Close/Open mutations of the
|
||||
// in-map FileInfo.
|
||||
func (fi *FileInfo) Clone() *FileInfo {
|
||||
if fi == nil {
|
||||
return nil
|
||||
}
|
||||
cp := *fi
|
||||
return &cp
|
||||
}
|
||||
|
||||
// NewFileInfoFromOS creates a FileInfo from os.FileInfo
|
||||
func NewFileInfoFromOS(info os.FileInfo, key string) *FileInfo {
|
||||
return &FileInfo{
|
||||
|
||||
@@ -16,6 +16,34 @@ func TestNewFileInfo(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestFileInfoClone(t *testing.T) {
|
||||
t.Parallel()
|
||||
fi := NewFileInfo("k", 42)
|
||||
fi.AccessCount = 7
|
||||
cp := fi.Clone()
|
||||
if cp == fi {
|
||||
t.Fatal("Clone returned the same pointer")
|
||||
}
|
||||
if cp.Key != fi.Key || cp.Size != fi.Size || cp.AccessCount != fi.AccessCount {
|
||||
t.Errorf("Clone mismatch: %+v vs %+v", cp, fi)
|
||||
}
|
||||
if !cp.ATime.Equal(fi.ATime) || !cp.CTime.Equal(fi.CTime) {
|
||||
t.Error("Clone timestamps mismatch")
|
||||
}
|
||||
cp.Size = 99
|
||||
cp.AccessCount = 1
|
||||
if fi.Size != 42 || fi.AccessCount != 7 {
|
||||
t.Error("mutating Clone affected original")
|
||||
}
|
||||
if NewFileInfo("x", 1).Clone() == nil {
|
||||
t.Error("Clone of non-nil was nil")
|
||||
}
|
||||
var none *FileInfo
|
||||
if none.Clone() != nil {
|
||||
t.Error("Clone of nil was non-nil")
|
||||
}
|
||||
}
|
||||
|
||||
func TestUpdateAccess(t *testing.T) {
|
||||
t.Parallel()
|
||||
fi := NewFileInfo("k", 1)
|
||||
|
||||
Reference in New Issue
Block a user