Compare commits
17 Commits
1.0.21
...
5e22f1054b
| Author | SHA1 | Date | |
|---|---|---|---|
| 5e22f1054b | |||
| 46aa59e877 | |||
| 8eb31143e5 | |||
| 337741e06e | |||
| 2c8ec276e7 | |||
| 144a68dcd7 | |||
| 8cebc1f96c | |||
| 12ea3ee4f6 | |||
| a500f51f17 | |||
| ac2d36f1ad | |||
| c43bfba568 | |||
| 3411defd51 | |||
| 71d5106777 | |||
| 8b1b229539 | |||
| ff1ab31327 | |||
| 85e14bc8af | |||
| 3f5175b482 |
@@ -7,6 +7,8 @@ on:
|
|||||||
jobs:
|
jobs:
|
||||||
release:
|
release:
|
||||||
runs-on: ubuntu-latest
|
runs-on: ubuntu-latest
|
||||||
|
permissions:
|
||||||
|
contents: write
|
||||||
steps:
|
steps:
|
||||||
- uses: actions/checkout@main
|
- uses: actions/checkout@main
|
||||||
with:
|
with:
|
||||||
@@ -21,4 +23,6 @@ jobs:
|
|||||||
version: 'latest'
|
version: 'latest'
|
||||||
args: release
|
args: release
|
||||||
env:
|
env:
|
||||||
GITEA_TOKEN: ${{secrets.RELEASE_TOKEN}}
|
GITEA_TOKEN: ${{ secrets.GITEA_TOKEN }}
|
||||||
|
GITHUB_TOKEN: ${{ secrets.GITEA_TOKEN }}
|
||||||
|
GORELEASER_FORCE_TOKEN: gitea
|
||||||
|
|||||||
@@ -62,6 +62,30 @@ validate run-validation: build clean-disk ## Start steamcache2 on :80 with small
|
|||||||
fi; \
|
fi; \
|
||||||
exec "$$BINARY" --config docs/examples/validate-config.yaml --log-level info
|
exec "$$BINARY" --config docs/examples/validate-config.yaml --log-level info
|
||||||
|
|
||||||
|
validate-check: ## Curl local /metrics (full dump + hit/miss + upstream/write/rate fields) and /lancache-heartbeat (default :80)
|
||||||
|
@echo "=== http://localhost/metrics ==="
|
||||||
|
@metrics=$$(curl -sf --max-time 5 http://localhost/metrics) || { \
|
||||||
|
echo "ERROR: could not fetch http://localhost/metrics"; \
|
||||||
|
echo "Is steamcache2 running on the default listen address :80?"; \
|
||||||
|
exit 1; \
|
||||||
|
}; \
|
||||||
|
printf '%s\n' "$$metrics"; \
|
||||||
|
echo ""; \
|
||||||
|
echo "=== hit/miss + upstream/write/rate fields ==="; \
|
||||||
|
printf '%s\n' "$$metrics" | grep -E '^(total_requests|cache_hits|cache_misses|hit_rate|memory_cache_hits|disk_cache_hits|errors|upstream_errors|cache_write_failures|rate_limited) ' || true; \
|
||||||
|
echo ""; \
|
||||||
|
echo "=== http://localhost/lancache-heartbeat (GET; expect 204 + X-LanCache-Processed-By: SteamCache2) ==="; \
|
||||||
|
hb=$$(curl -sD - -o /dev/null --max-time 5 http://localhost/lancache-heartbeat) || { \
|
||||||
|
echo "ERROR: could not fetch http://localhost/lancache-heartbeat"; \
|
||||||
|
echo "Is steamcache2 running on the default listen address :80?"; \
|
||||||
|
exit 1; \
|
||||||
|
}; \
|
||||||
|
printf '%s\n' "$$hb"; \
|
||||||
|
echo "$$hb" | grep -q '204' && echo "$$hb" | grep -qi 'X-LanCache-Processed-By' || { \
|
||||||
|
echo "ERROR: expected HTTP 204 and X-LanCache-Processed-By on /lancache-heartbeat"; \
|
||||||
|
exit 1; \
|
||||||
|
}
|
||||||
|
|
||||||
validate-kill: ## Kill leftover steamcache2 processes (safer, checks process name)
|
validate-kill: ## Kill leftover steamcache2 processes (safer, checks process name)
|
||||||
@echo "Looking for steamcache2 processes on common validation ports (80 is primary)..."
|
@echo "Looking for steamcache2 processes on common validation ports (80 is primary)..."
|
||||||
@for port in 80 8040 8080; do \
|
@for port in 80 8040 8080; do \
|
||||||
@@ -109,6 +133,7 @@ help: ## Show this help message
|
|||||||
@echo " clean-disk Remove disk cache"
|
@echo " clean-disk Remove disk cache"
|
||||||
@echo " bench Run low-level VFS microbenchmarks"
|
@echo " bench Run low-level VFS microbenchmarks"
|
||||||
@echo " validate / run-validation Start server on :80 (builds, auto-setcaps fresh binary, then runs as normal user, cleans disk cache first)"
|
@echo " validate / run-validation Start server on :80 (builds, auto-setcaps fresh binary, then runs as normal user, cleans disk cache first)"
|
||||||
|
@echo " validate-check Curl local /metrics (full dump + hit/miss + upstream/write/rate fields) and /lancache-heartbeat (default :80)"
|
||||||
@echo " setcap Explicitly set cap on current build (for port 80 use outside validate)"
|
@echo " setcap Explicitly set cap on current build (for port 80 use outside validate)"
|
||||||
@echo " validate-kill Kill leftover steamcache2 processes (safer)"
|
@echo " validate-kill Kill leftover steamcache2 processes (safer)"
|
||||||
@echo " prefill Download latest SteamPrefill into bin/steam-prefill/SteamPrefill (gitignored)"
|
@echo " prefill Download latest SteamPrefill into bin/steam-prefill/SteamPrefill (gitignored)"
|
||||||
@@ -10,8 +10,8 @@ SteamCache2 is a blazing fast download cache for Steam, designed to reduce bandw
|
|||||||
- Reduces bandwidth usage
|
- Reduces bandwidth usage
|
||||||
- Easy to set up and configure aside from dns stuff to trick Steam into using it
|
- Easy to set up and configure aside from dns stuff to trick Steam into using it
|
||||||
- Supports multiple clients
|
- Supports multiple clients
|
||||||
- **NEW:** YAML configuration system with automatic config generation
|
- YAML configuration with automatic config generation on first run
|
||||||
- **NEW:** Simple Makefile for development workflow
|
- Makefile for development and validation workflows
|
||||||
- Cross-platform builds (Linux, macOS, Windows)
|
- Cross-platform builds (Linux, macOS, Windows)
|
||||||
|
|
||||||
## Quick Start
|
## Quick Start
|
||||||
@@ -34,7 +34,10 @@ SteamCache2 is a blazing fast download cache for Steam, designed to reduce bandw
|
|||||||
|
|
||||||
The application will automatically create a `config.yaml` file with default settings and exit, allowing you to customize it.
|
The application will automatically create a `config.yaml` file with default settings and exit, allowing you to customize it.
|
||||||
|
|
||||||
3. **Edit the configuration** (`config.yaml`):
|
3. **Edit the configuration** (`config.yaml`) for a real Steam front-door:
|
||||||
|
|
||||||
|
Leave `upstream` empty. With an empty upstream, steamcache2 fetches from the request `Host` (the same pattern SteamPrefill / real Steam clients use when DNS points them at your cache). Do **not** paste a fake host like `https://steam.cdn.com` — that is not a Steam CDN and will not put you in front of Steam.
|
||||||
|
|
||||||
```yaml
|
```yaml
|
||||||
listen_address: :80
|
listen_address: :80
|
||||||
cache:
|
cache:
|
||||||
@@ -45,14 +48,66 @@ SteamCache2 is a blazing fast download cache for Steam, designed to reduce bandw
|
|||||||
size: 10GB
|
size: 10GB
|
||||||
path: ./disk
|
path: ./disk
|
||||||
gc_algorithm: hybrid
|
gc_algorithm: hybrid
|
||||||
upstream: "https://steam.cdn.com" # Set your upstream server
|
# Empty upstream = use the client Host as the origin (Steam CDN names only).
|
||||||
|
upstream: ""
|
||||||
```
|
```
|
||||||
|
|
||||||
4. **Run the application again:**
|
4. **Point Steam (or SteamPrefill) at this cache** before you expect hits:
|
||||||
|
|
||||||
|
- **LAN DNS:** resolve `lancache.steamcontent.com` (and other Steam content names your clients use) to this server's LAN IP.
|
||||||
|
- **Single Windows PC:** add a hosts override — see [Windows Hosts File Override](#windows-hosts-file-override) (`<cache-ip> lancache.steamcontent.com`).
|
||||||
|
- Restart Steam (or your prefill tool) after DNS/hosts changes.
|
||||||
|
|
||||||
|
Empty-upstream direct fetch only allows Steam CDN host suffixes (`steamcontent.com`, `steampowered.com`, `steamstatic.com`). Literal IPs and unrelated hosts are rejected.
|
||||||
|
|
||||||
|
5. **Run the application again:**
|
||||||
```bash
|
```bash
|
||||||
make run # or ./steamcache2
|
make run # or ./steamcache2
|
||||||
```
|
```
|
||||||
|
|
||||||
|
### Quick check: is it caching?
|
||||||
|
|
||||||
|
After steamcache2 is running (default `listen_address: :80`) and has seen a little Steam traffic — a game download, a short SteamPrefill pass, or any cacheable request — confirm hits vs misses from the existing endpoints. You do not need a full benchmark or log diving.
|
||||||
|
|
||||||
|
```bash
|
||||||
|
make validate-check
|
||||||
|
# or, manually:
|
||||||
|
curl -s http://localhost/metrics
|
||||||
|
curl -s -i http://localhost/lancache-heartbeat
|
||||||
|
```
|
||||||
|
|
||||||
|
`make validate-check` prints the full `/metrics` dump, highlights hit/miss plus `upstream_errors` / `cache_write_failures` / `rate_limited`, and curls `/lancache-heartbeat`. Read these fields:
|
||||||
|
|
||||||
|
| Field | Meaning |
|
||||||
|
| --- | --- |
|
||||||
|
| `cache_hits` / `cache_misses` / `hit_rate` | Whether later requests were served from cache |
|
||||||
|
| `range_cache` / `range_upstream` | Range GETs served as 206 from a cached object vs after a full upstream fetch |
|
||||||
|
| `memory_cache_hits` / `disk_cache_hits` | Which tier served the hits |
|
||||||
|
| `total_requests` / `errors` | Volume and failures |
|
||||||
|
| `upstream_errors` / `cache_write_failures` / `rate_limited` | Upstream pipe, cache write, and rate-limit pressure (Quick check highlights these next to hit/miss) |
|
||||||
|
| `disk_tier_ready` | `0` while disk slow-tier attach pending; `1` when attached, or when no disk configured (N/A — not waiting) |
|
||||||
|
| `capacity_pressure_events` | Soft eviction under the memory or disk cap, and/or disk Create/Write/Mkdir hitting ENOSPC (volume full). Distinct from cold-cache misses and from the existing `evictions` counter. Logs `tier` (memory or disk) and `reason` (eviction or enospc). |
|
||||||
|
|
||||||
|
A first pass through new content is mostly misses (`hit_rate` near 0). Repeat the same content and `cache_hits` / `hit_rate` should rise.
|
||||||
|
|
||||||
|
Cache entries are keyed by depot object path (not the CDN `Host` header), so when Steam rotates CDN hostnames for the same depot path, hits still climb across the aliases.
|
||||||
|
|
||||||
|
Steam clients lean on Range requests. When an object is already cached, a Range GET is served locally as 206 from that full object (`range_cache`). On a Range miss the cache still fetches and stores the full upstream body, then returns the requested byte range as 206 (`range_upstream`).
|
||||||
|
|
||||||
|
To confirm the process is up (HTTP 204 and `X-LanCache-Processed-By: SteamCache2`):
|
||||||
|
|
||||||
|
```bash
|
||||||
|
curl -s -i http://localhost/lancache-heartbeat
|
||||||
|
```
|
||||||
|
|
||||||
|
Use GET (`curl -i`), not HEAD (`curl -I`): the server only accepts GET.
|
||||||
|
|
||||||
|
Heartbeat also returns `X-SteamCache-Disk-Tier: pending|ready|disabled` (`disabled` = memory-only / no disk; `pending`/`ready` = disk configured attach state).
|
||||||
|
|
||||||
|
These are the cache process's own `/metrics` and `/lancache-heartbeat` endpoints. There is no separate metrics daemon.
|
||||||
|
|
||||||
|
If you changed `listen_address`, point curl at that host:port instead. For a full SteamPrefill validation workflow (small caches, coalescing, GC), see [Validating Full Functionality](#validating-full-functionality-with-external-tools).
|
||||||
|
|
||||||
### Development Workflow
|
### Development Workflow
|
||||||
|
|
||||||
Use `make` for the majority of common development tasks. The Makefile handles running tests, linting, hygiene checks, building, running the application, and other routine boilerplate work.
|
Use `make` for the majority of common development tasks. The Makefile handles running tests, linting, hygiene checks, building, running the application, and other routine boilerplate work.
|
||||||
@@ -96,13 +151,16 @@ When the server is running, point your external SteamPrefill (or other load gene
|
|||||||
./SteamPrefill benchmark run ...
|
./SteamPrefill benchmark run ...
|
||||||
```
|
```
|
||||||
|
|
||||||
When finished, you can get a quick metrics summary with:
|
When finished, you can get a quick metrics + heartbeat report with:
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
make validate-check
|
make validate-check
|
||||||
|
# or, manually:
|
||||||
|
curl -s http://localhost/metrics
|
||||||
|
curl -s -i http://localhost/lancache-heartbeat
|
||||||
```
|
```
|
||||||
|
|
||||||
This is the recommended simple workflow. No automatic downloading or running of external tools.
|
See [Quick check: is it caching?](#quick-check-is-it-caching) for which fields to read. This is the recommended simple workflow. No automatic downloading or running of external tools.
|
||||||
|
|
||||||
#### Inspecting the Result
|
#### Inspecting the Result
|
||||||
|
|
||||||
@@ -110,14 +168,17 @@ After a benchmark run you can ask for a quick report:
|
|||||||
|
|
||||||
```bash
|
```bash
|
||||||
make validate-check
|
make validate-check
|
||||||
# or manually:
|
# or, manually:
|
||||||
curl -s http://localhost/metrics
|
curl -s http://localhost/metrics
|
||||||
|
curl -s -i http://localhost/lancache-heartbeat
|
||||||
```
|
```
|
||||||
|
|
||||||
Look for:
|
`make validate-check` prints the full `/metrics` dump, highlights hit/miss fields, and curls `/lancache-heartbeat`. Look for:
|
||||||
- High cache hit rate after the warmup pass
|
- High cache hit rate after the warmup pass (`cache_hits`, `hit_rate`, plus `memory_cache_hits` / `disk_cache_hits`)
|
||||||
- Non-zero `coalesced` and `disk` activity
|
- Non-zero `coalesced` and `disk` activity
|
||||||
- Zero unexpected errors
|
- Zero unexpected `errors`, and quiet `upstream_errors` / `cache_write_failures` / `rate_limited`
|
||||||
|
|
||||||
|
Heartbeat should be HTTP 204 with `X-LanCache-Processed-By: SteamCache2`. Use GET (`curl -i`), not HEAD (`curl -I`).
|
||||||
|
|
||||||
#### The Validation Config
|
#### The Validation Config
|
||||||
|
|
||||||
@@ -128,7 +189,7 @@ The recommended validation config is at [docs/examples/validate-config.yaml](doc
|
|||||||
Running a realistic SteamPrefill benchmark workload through a built steamcache2 exercises the complete public surface that matters for production use:
|
Running a realistic SteamPrefill benchmark workload through a built steamcache2 exercises the complete public surface that matters for production use:
|
||||||
- Steam User-Agent detection and depot/manifest/chunk URL patterns
|
- Steam User-Agent detection and depot/manifest/chunk URL patterns
|
||||||
- Full MISS → cache write → HIT (and HIT-COALESCED) paths
|
- Full MISS → cache write → HIT (and HIT-COALESCED) paths
|
||||||
- Range request handling from cached full responses
|
- Range request handling from cached full responses (local 206 on HIT via `range_cache`; MISS fetches the full object then serves the requested slice as 206 via `range_upstream`)
|
||||||
- Request coalescing under concurrent load
|
- Request coalescing under concurrent load
|
||||||
- Memory tier + disk tier interaction (including async disk attach)
|
- Memory tier + disk tier interaction (including async disk attach)
|
||||||
- Garbage collection and eviction under pressure
|
- Garbage collection and eviction under pressure
|
||||||
@@ -160,8 +221,9 @@ While most configuration is done via the YAML file, some runtime options are sti
|
|||||||
# Set logging level
|
# Set logging level
|
||||||
./steamcache2 --log-level debug --log-format json
|
./steamcache2 --log-level debug --log-format json
|
||||||
|
|
||||||
# Set number of worker threads
|
# Override concurrency from the CLI (0 = use config.yaml)
|
||||||
./steamcache2 --threads 8
|
./steamcache2 --max-concurrent-requests 8
|
||||||
|
./steamcache2 --max-requests-per-client 4
|
||||||
|
|
||||||
# Show help
|
# Show help
|
||||||
./steamcache2 --help
|
./steamcache2 --help
|
||||||
@@ -198,8 +260,9 @@ cache:
|
|||||||
gc_algorithm: hybrid
|
gc_algorithm: hybrid
|
||||||
|
|
||||||
# Upstream server configuration
|
# Upstream server configuration
|
||||||
# The upstream server to proxy requests to
|
# Leave empty to fetch from the request Host (Steam CDN names only).
|
||||||
upstream: "https://steam.cdn.com"
|
# Set only when chaining caches (table RAM cache -> room disk cache).
|
||||||
|
upstream: ""
|
||||||
```
|
```
|
||||||
|
|
||||||
#### Startup Validation
|
#### Startup Validation
|
||||||
@@ -234,6 +297,10 @@ See `config.Validate()` and `steamcache.New` error paths. This ensures the LAN a
|
|||||||
- The explicit startup guard (reduce size if pre-existing on-disk > cap) runs as the literal last step of bg init, before the barrier opens.
|
- The explicit startup guard (reduce size if pre-existing on-disk > cap) runs as the literal last step of bg init, before the barrier opens.
|
||||||
- Add a note for operators: very large disk caches (tens/hundreds GB with millions files) may show extended "memory-only or no-cache" behavior at startup (seconds to minutes depending on storage speed); this is by design for responsiveness.
|
- Add a note for operators: very large disk caches (tens/hundreds GB with millions files) may show extended "memory-only or no-cache" behavior at startup (seconds to minutes depending on storage speed); this is by design for responsiveness.
|
||||||
- Godoc on `disk.New` and `DiskFS.Size` expanded with the barrier/attach behavior.
|
- Godoc on `disk.New` and `DiskFS.Size` expanded with the barrier/attach behavior.
|
||||||
|
- Startup logs: Info "Disk slow tier attach pending..." then later "Disk slow tier attached (...)" for disk-only and mixed modes.
|
||||||
|
- `/metrics` exposes `disk_tier_ready` 0/1 and stays responsive during attach (GetMetrics does not block on Size while pending).
|
||||||
|
- `/lancache-heartbeat` header `X-SteamCache-Disk-Tier` mirrors that state.
|
||||||
|
- `/metrics` `capacity_pressure_events` counts times the cache dropped data under capacity pressure (soft eviction at the memory or disk cap, or disk Create/Write/Mkdir returning ENOSPC). Logs include `tier=memory|disk` and `reason=eviction|enospc` so operators can grep and tell this apart from a cold cache. The existing `evictions` counter is unchanged.
|
||||||
|
|
||||||
#### Garbage Collection Algorithms
|
#### Garbage Collection Algorithms
|
||||||
|
|
||||||
@@ -311,7 +378,7 @@ This will direct any requests to `lancache.steamcontent.com` to your SteamCache2
|
|||||||
|
|
||||||
### Prerequisites
|
### Prerequisites
|
||||||
|
|
||||||
- Go 1.19 or later
|
- Go 1.27.0 or later
|
||||||
- Make (optional, but recommended)
|
- Make (optional, but recommended)
|
||||||
|
|
||||||
### Build Commands
|
### Build Commands
|
||||||
@@ -319,7 +386,7 @@ This will direct any requests to `lancache.steamcontent.com` to your SteamCache2
|
|||||||
```bash
|
```bash
|
||||||
# Clone the repository
|
# Clone the repository
|
||||||
git clone <repository-url>
|
git clone <repository-url>
|
||||||
cd SteamCache2
|
cd steamcache2
|
||||||
|
|
||||||
# Download dependencies
|
# Download dependencies
|
||||||
make deps
|
make deps
|
||||||
@@ -371,6 +438,11 @@ make
|
|||||||
- Consider using a different GC algorithm like `hybrid`
|
- Consider using a different GC algorithm like `hybrid`
|
||||||
- Adjust the disk cache size to match available storage
|
- Adjust the disk cache size to match available storage
|
||||||
|
|
||||||
|
6. **Not sure if it is caching**
|
||||||
|
- Do not start with the full SteamPrefill chapter. Use [Quick check: is it caching?](#quick-check-is-it-caching): `make validate-check` (full `/metrics`, hit/miss fields, and `/lancache-heartbeat`)
|
||||||
|
- A first pass is mostly `cache_misses`; repeating the same content should raise `cache_hits` / `hit_rate`
|
||||||
|
- Confirm the process is up with `curl -s -i http://localhost/lancache-heartbeat` (GET, not HEAD)
|
||||||
|
|
||||||
### Getting Help
|
### Getting Help
|
||||||
|
|
||||||
- Check the logs for detailed error messages
|
- Check the logs for detailed error messages
|
||||||
|
|||||||
@@ -27,7 +27,10 @@
|
|||||||
# SteamPrefill benchmark run -c 20 ...
|
# SteamPrefill benchmark run -c 20 ...
|
||||||
#
|
#
|
||||||
# After the benchmark run, inspect with:
|
# After the benchmark run, inspect with:
|
||||||
|
# make validate-check # full /metrics + hit/miss fields + /lancache-heartbeat
|
||||||
|
# # or, manually:
|
||||||
# curl -s http://localhost/metrics
|
# curl -s http://localhost/metrics
|
||||||
|
# curl -s -i http://localhost/lancache-heartbeat # GET, not HEAD
|
||||||
#
|
#
|
||||||
# Tweak sizes upward if you want to run very large workloads while still
|
# Tweak sizes upward if you want to run very large workloads while still
|
||||||
# exercising the disk tier (workload >> RAM is ideal for real disk testing).
|
# exercising the disk tier (workload >> RAM is ideal for real disk testing).
|
||||||
|
|||||||
@@ -262,6 +262,13 @@ func (sc *SteamCache) streamCachedResponse(w http.ResponseWriter, r *http.Reques
|
|||||||
// Send the range data
|
// Send the range data
|
||||||
_, _ = w.Write(rangeData) // client write error ignored (disconnect during range body send is not actionable)
|
_, _ = w.Write(rangeData) // client write error ignored (disconnect during range body send is not actionable)
|
||||||
|
|
||||||
|
// Range served from cache: count the range-specific metric and the range
|
||||||
|
// bytes actually written (handleCacheHit skips full-blob byte counting for
|
||||||
|
// Range requests so these are not double-counted).
|
||||||
|
sc.metrics.IncrementRangeCache()
|
||||||
|
sc.metrics.AddBytesServed(int64(len(rangeData)))
|
||||||
|
sc.metrics.AddBytesSaved(int64(len(rangeData)))
|
||||||
|
|
||||||
logger.Logger.Info().
|
logger.Logger.Info().
|
||||||
Str("cache_key", cacheKey).
|
Str("cache_key", cacheKey).
|
||||||
Str("url", r.URL.String()).
|
Str("url", r.URL.String()).
|
||||||
|
|||||||
+53
-9
@@ -56,6 +56,15 @@ func (sc *SteamCache) handleSpecialEndpoints(w http.ResponseWriter, r *http.Requ
|
|||||||
logger.Logger.Debug().
|
logger.Logger.Debug().
|
||||||
Str("client_ip", clientIP).
|
Str("client_ip", clientIP).
|
||||||
Msg("LanCache heartbeat request")
|
Msg("LanCache heartbeat request")
|
||||||
|
diskTier := "disabled"
|
||||||
|
if sc.disk != nil {
|
||||||
|
if sc.metrics.GetDiskTierReady() == 1 {
|
||||||
|
diskTier = "ready"
|
||||||
|
} else {
|
||||||
|
diskTier = "pending"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
w.Header().Add("X-SteamCache-Disk-Tier", diskTier)
|
||||||
w.Header().Add("X-LanCache-Processed-By", "SteamCache2")
|
w.Header().Add("X-LanCache-Processed-By", "SteamCache2")
|
||||||
w.WriteHeader(http.StatusNoContent)
|
w.WriteHeader(http.StatusNoContent)
|
||||||
_, _ = w.Write(nil) // client write error ignored (heartbeat path; nil write is no-op)
|
_, _ = w.Write(nil) // client write error ignored (heartbeat path; nil write is no-op)
|
||||||
@@ -109,8 +118,14 @@ func (sc *SteamCache) handleCacheHit(w http.ResponseWriter, r *http.Request, cac
|
|||||||
// Track cache hit metrics
|
// Track cache hit metrics
|
||||||
sc.metrics.IncrementCacheHits()
|
sc.metrics.IncrementCacheHits()
|
||||||
sc.metrics.AddResponseTime(time.Since(tstart))
|
sc.metrics.AddResponseTime(time.Since(tstart))
|
||||||
sc.metrics.AddBytesServed(int64(len(cachedData)))
|
if r.Header.Get("Range") == "" {
|
||||||
sc.metrics.AddBytesSaved(int64(len(cachedData)))
|
// Full-object HIT: count the cached blob served. Range HITs skip
|
||||||
|
// this here — streamCachedResponse counts the range bytes actually
|
||||||
|
// written to the client instead, so BytesServed/Saved reflect the
|
||||||
|
// partial body and are not double-counted.
|
||||||
|
sc.metrics.AddBytesServed(int64(len(cachedData)))
|
||||||
|
sc.metrics.AddBytesSaved(int64(len(cachedData)))
|
||||||
|
}
|
||||||
sc.metrics.IncrementServiceRequests(service.Name)
|
sc.metrics.IncrementServiceRequests(service.Name)
|
||||||
|
|
||||||
logger.Logger.Debug().
|
logger.Logger.Debug().
|
||||||
@@ -264,9 +279,12 @@ func (sc *SteamCache) ServeHTTP(w http.ResponseWriter, r *http.Request) {
|
|||||||
|
|
||||||
// Check if this is a request from a supported service
|
// Check if this is a request from a supported service
|
||||||
if service, isSupported := sc.detectService(r); isSupported {
|
if service, isSupported := sc.detectService(r); isSupported {
|
||||||
// trim the query parameters from the URL path
|
// Cache key is the path only, never the Host: Steam rotates CDN hostnames
|
||||||
// this is necessary because the cache key should not include query parameters
|
// for the same depot object, so different Host headers (or absolute-form
|
||||||
urlPath := strings.SplitN(r.URL.String(), "?", 2)[0] // trim query for cache key (SplitN makes intent explicit vs Cut + ignored bool)
|
// request targets) for the same path must share one cache entry. r.URL.Path
|
||||||
|
// is the decoded path (query is never part of it); validateURLPath checks
|
||||||
|
// this decoded form and url.JoinPath re-escapes it for the upstream join.
|
||||||
|
urlPath := r.URL.Path
|
||||||
|
|
||||||
// Validate URL path for security
|
// Validate URL path for security
|
||||||
if err := validateURLPath(urlPath); err != nil {
|
if err := validateURLPath(urlPath); err != nil {
|
||||||
@@ -549,14 +567,40 @@ func (sc *SteamCache) ServeHTTP(w http.ResponseWriter, r *http.Request) {
|
|||||||
w.Header().Set("X-LanCache-Status", "MISS")
|
w.Header().Set("X-LanCache-Status", "MISS")
|
||||||
w.Header().Set("X-LanCache-Processed-By", "SteamCache2")
|
w.Header().Set("X-LanCache-Processed-By", "SteamCache2")
|
||||||
|
|
||||||
// Stream the response body to client
|
// Stream the response body to client.
|
||||||
w.WriteHeader(resp.StatusCode)
|
// Range miss: the Range header was stripped for the upstream fetch (so the
|
||||||
_, _ = w.Write(bodyData) // client write error ignored (disconnect during MISS body send is not actionable)
|
// FULL object is cached below); serve the client's requested slice from the
|
||||||
|
// full body as 206, matching the HIT Range path.
|
||||||
|
if rangeHeader := r.Header.Get("Range"); rangeHeader != "" {
|
||||||
|
start, end, totalSize, rangeValid := parseRangeHeader(rangeHeader, int64(len(bodyData)))
|
||||||
|
if !rangeValid {
|
||||||
|
// Invalid range — 416 (consistent with the HIT Range path). Drop the
|
||||||
|
// upstream Content-Length: it describes the full body, which 416 does
|
||||||
|
// not send (a stale CL would hang clients waiting for a body).
|
||||||
|
w.Header().Del("Content-Length")
|
||||||
|
w.Header().Set("Content-Range", fmt.Sprintf("bytes */%d", len(bodyData)))
|
||||||
|
w.WriteHeader(http.StatusRequestedRangeNotSatisfiable)
|
||||||
|
} else {
|
||||||
|
rangeData := bodyData[start : end+1]
|
||||||
|
w.Header().Set("Content-Range", fmt.Sprintf("bytes %d-%d/%d", start, end, totalSize))
|
||||||
|
w.Header().Set("Content-Length", fmt.Sprintf("%d", len(rangeData)))
|
||||||
|
w.Header().Set("Accept-Ranges", "bytes")
|
||||||
|
w.WriteHeader(http.StatusPartialContent)
|
||||||
|
_, _ = w.Write(rangeData) // client write error ignored (disconnect during MISS range body send is not actionable)
|
||||||
|
|
||||||
|
// Range required an upstream fetch (full object) then served as 206
|
||||||
|
sc.metrics.IncrementRangeUpstream()
|
||||||
|
sc.metrics.AddBytesServed(int64(len(rangeData))) // range bytes only, not the full cached object
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
w.WriteHeader(resp.StatusCode)
|
||||||
|
_, _ = w.Write(bodyData) // client write error ignored (disconnect during MISS body send is not actionable)
|
||||||
|
sc.metrics.AddBytesServed(int64(len(bodyData)))
|
||||||
|
}
|
||||||
|
|
||||||
// Track cache miss metrics
|
// Track cache miss metrics
|
||||||
sc.metrics.IncrementCacheMisses()
|
sc.metrics.IncrementCacheMisses()
|
||||||
sc.metrics.AddResponseTime(time.Since(tstart))
|
sc.metrics.AddResponseTime(time.Since(tstart))
|
||||||
sc.metrics.AddBytesServed(int64(len(bodyData)))
|
|
||||||
sc.metrics.IncrementServiceRequests(service.Name)
|
sc.metrics.IncrementServiceRequests(service.Name)
|
||||||
|
|
||||||
// Verify we received the complete file by checking Content-Length
|
// Verify we received the complete file by checking Content-Length
|
||||||
|
|||||||
+115
-41
@@ -7,6 +7,8 @@ import (
|
|||||||
"sync"
|
"sync"
|
||||||
"sync/atomic"
|
"sync/atomic"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
|
"s1d3sw1ped/steamcache2/steamcache/logger"
|
||||||
)
|
)
|
||||||
|
|
||||||
// Metrics tracks various performance and operational metrics
|
// Metrics tracks various performance and operational metrics
|
||||||
@@ -16,6 +18,8 @@ type Metrics struct {
|
|||||||
CacheHits int64
|
CacheHits int64
|
||||||
CacheMisses int64
|
CacheMisses int64
|
||||||
CacheCoalesced int64
|
CacheCoalesced int64
|
||||||
|
RangeCache int64 // Range requests served as 206 from an already-cached object (HIT)
|
||||||
|
RangeUpstream int64 // Range requests that required an upstream fetch (full object), served as 206
|
||||||
Errors int64
|
Errors int64
|
||||||
RateLimited int64
|
RateLimited int64
|
||||||
|
|
||||||
@@ -25,12 +29,14 @@ type Metrics struct {
|
|||||||
TotalBytesSaved int64 // bytes served from cache instead of being re-downloaded from upstream
|
TotalBytesSaved int64 // bytes served from cache instead of being re-downloaded from upstream
|
||||||
|
|
||||||
// Cache metrics
|
// Cache metrics
|
||||||
MemoryCacheSize int64
|
MemoryCacheSize int64
|
||||||
DiskCacheSize int64
|
DiskCacheSize int64
|
||||||
MemoryCacheHits int64
|
MemoryCacheHits int64
|
||||||
DiskCacheHits int64
|
DiskCacheHits int64
|
||||||
Promotions int64
|
Promotions int64
|
||||||
Evictions int64
|
Evictions int64
|
||||||
|
CapacityPressureEvents int64 // soft eviction under cap and/or disk ENOSPC
|
||||||
|
DiskTierReady int64 // 0=pending (or unset), 1=ready or no-disk (N/A)
|
||||||
|
|
||||||
// Expanded observability (upstream breakdowns, cache write failures, per-service errors)
|
// Expanded observability (upstream breakdowns, cache write failures, per-service errors)
|
||||||
UpstreamErrors int64
|
UpstreamErrors int64
|
||||||
@@ -78,6 +84,17 @@ func (m *Metrics) IncrementCacheCoalesced() {
|
|||||||
atomic.AddInt64(&m.CacheCoalesced, 1)
|
atomic.AddInt64(&m.CacheCoalesced, 1)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// IncrementRangeCache increments the Range-from-cache counter (HIT served as 206)
|
||||||
|
func (m *Metrics) IncrementRangeCache() {
|
||||||
|
atomic.AddInt64(&m.RangeCache, 1)
|
||||||
|
}
|
||||||
|
|
||||||
|
// IncrementRangeUpstream increments the Range-from-upstream counter (full object
|
||||||
|
// fetched upstream, requested slice served as 206)
|
||||||
|
func (m *Metrics) IncrementRangeUpstream() {
|
||||||
|
atomic.AddInt64(&m.RangeUpstream, 1)
|
||||||
|
}
|
||||||
|
|
||||||
// IncrementErrors increments the error counter
|
// IncrementErrors increments the error counter
|
||||||
func (m *Metrics) IncrementErrors() {
|
func (m *Metrics) IncrementErrors() {
|
||||||
atomic.AddInt64(&m.Errors, 1)
|
atomic.AddInt64(&m.Errors, 1)
|
||||||
@@ -114,6 +131,17 @@ func (m *Metrics) SetDiskCacheSize(size int64) {
|
|||||||
atomic.StoreInt64(&m.DiskCacheSize, size)
|
atomic.StoreInt64(&m.DiskCacheSize, size)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// SetDiskTierReady sets whether the disk slow tier is attached (1) or still pending (0).
|
||||||
|
// Memory-only (no disk) also uses 1 — meaning "not waiting on disk attach". Reset does not clear this.
|
||||||
|
func (m *Metrics) SetDiskTierReady(ready int64) {
|
||||||
|
atomic.StoreInt64(&m.DiskTierReady, ready)
|
||||||
|
}
|
||||||
|
|
||||||
|
// GetDiskTierReady returns 1 if disk tier is ready (or no disk configured), else 0 while attach pending.
|
||||||
|
func (m *Metrics) GetDiskTierReady() int64 {
|
||||||
|
return atomic.LoadInt64(&m.DiskTierReady)
|
||||||
|
}
|
||||||
|
|
||||||
// IncrementMemoryCacheHits increments memory cache hits
|
// IncrementMemoryCacheHits increments memory cache hits
|
||||||
func (m *Metrics) IncrementMemoryCacheHits() {
|
func (m *Metrics) IncrementMemoryCacheHits() {
|
||||||
atomic.AddInt64(&m.MemoryCacheHits, 1)
|
atomic.AddInt64(&m.MemoryCacheHits, 1)
|
||||||
@@ -138,8 +166,39 @@ func (m *Metrics) GetServiceRequests(service string) int64 {
|
|||||||
return m.ServiceRequests[service]
|
return m.ServiceRequests[service]
|
||||||
}
|
}
|
||||||
|
|
||||||
func (m *Metrics) IncrementPromotions() { atomic.AddInt64(&m.Promotions, 1) }
|
func (m *Metrics) IncrementPromotions() { atomic.AddInt64(&m.Promotions, 1) }
|
||||||
func (m *Metrics) IncrementEvictions() { atomic.AddInt64(&m.Evictions, 1) }
|
func (m *Metrics) IncrementEvictions() { atomic.AddInt64(&m.Evictions, 1) }
|
||||||
|
func (m *Metrics) IncrementCapacityPressureEvents() { atomic.AddInt64(&m.CapacityPressureEvents, 1) }
|
||||||
|
|
||||||
|
// NoteSoftEviction records one cap-pressure eviction batch that freed bytes.
|
||||||
|
// Keeps the existing evictions counter and also increments capacity_pressure_events.
|
||||||
|
// Nil m is safe: the log still fires so ops can grep without metrics wired.
|
||||||
|
func NoteSoftEviction(m *Metrics, tier string, evicted uint) {
|
||||||
|
if evicted == 0 {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if m != nil {
|
||||||
|
m.IncrementEvictions()
|
||||||
|
m.IncrementCapacityPressureEvents()
|
||||||
|
}
|
||||||
|
logger.Logger.Info().
|
||||||
|
Str("tier", tier).
|
||||||
|
Str("reason", "eviction").
|
||||||
|
Uint("bytes_evicted", evicted).
|
||||||
|
Msg("cache capacity pressure")
|
||||||
|
}
|
||||||
|
|
||||||
|
// NoteNoSpace records a disk Create/Write/Mkdir ENOSPC (or equivalent) event.
|
||||||
|
func NoteNoSpace(m *Metrics, err error) {
|
||||||
|
if m != nil {
|
||||||
|
m.IncrementCapacityPressureEvents()
|
||||||
|
}
|
||||||
|
logger.Logger.Warn().
|
||||||
|
Str("tier", "disk").
|
||||||
|
Str("reason", "enospc").
|
||||||
|
Err(err).
|
||||||
|
Msg("cache capacity pressure")
|
||||||
|
}
|
||||||
|
|
||||||
// Additional observability counters
|
// Additional observability counters
|
||||||
func (m *Metrics) IncrementUpstreamErrors() { atomic.AddInt64(&m.UpstreamErrors, 1) }
|
func (m *Metrics) IncrementUpstreamErrors() { atomic.AddInt64(&m.UpstreamErrors, 1) }
|
||||||
@@ -184,28 +243,32 @@ func (m *Metrics) GetStats() *Stats {
|
|||||||
}
|
}
|
||||||
|
|
||||||
return &Stats{
|
return &Stats{
|
||||||
TotalRequests: totalRequests,
|
TotalRequests: totalRequests,
|
||||||
CacheHits: cacheHits,
|
CacheHits: cacheHits,
|
||||||
CacheMisses: cacheMisses,
|
CacheMisses: cacheMisses,
|
||||||
CacheCoalesced: atomic.LoadInt64(&m.CacheCoalesced),
|
CacheCoalesced: atomic.LoadInt64(&m.CacheCoalesced),
|
||||||
Errors: atomic.LoadInt64(&m.Errors),
|
RangeCache: atomic.LoadInt64(&m.RangeCache),
|
||||||
RateLimited: atomic.LoadInt64(&m.RateLimited),
|
RangeUpstream: atomic.LoadInt64(&m.RangeUpstream),
|
||||||
HitRate: hitRate,
|
Errors: atomic.LoadInt64(&m.Errors),
|
||||||
AvgResponseTime: avgResponseTime,
|
RateLimited: atomic.LoadInt64(&m.RateLimited),
|
||||||
TotalBytesServed: atomic.LoadInt64(&m.TotalBytesServed),
|
HitRate: hitRate,
|
||||||
TotalBytesSaved: atomic.LoadInt64(&m.TotalBytesSaved),
|
AvgResponseTime: avgResponseTime,
|
||||||
MemoryCacheSize: atomic.LoadInt64(&m.MemoryCacheSize),
|
TotalBytesServed: atomic.LoadInt64(&m.TotalBytesServed),
|
||||||
DiskCacheSize: atomic.LoadInt64(&m.DiskCacheSize),
|
TotalBytesSaved: atomic.LoadInt64(&m.TotalBytesSaved),
|
||||||
MemoryCacheHits: atomic.LoadInt64(&m.MemoryCacheHits),
|
MemoryCacheSize: atomic.LoadInt64(&m.MemoryCacheSize),
|
||||||
DiskCacheHits: atomic.LoadInt64(&m.DiskCacheHits),
|
DiskCacheSize: atomic.LoadInt64(&m.DiskCacheSize),
|
||||||
Promotions: atomic.LoadInt64(&m.Promotions),
|
DiskTierReady: atomic.LoadInt64(&m.DiskTierReady),
|
||||||
Evictions: atomic.LoadInt64(&m.Evictions),
|
MemoryCacheHits: atomic.LoadInt64(&m.MemoryCacheHits),
|
||||||
ServiceRequests: serviceRequests,
|
DiskCacheHits: atomic.LoadInt64(&m.DiskCacheHits),
|
||||||
UpstreamErrors: atomic.LoadInt64(&m.UpstreamErrors),
|
Promotions: atomic.LoadInt64(&m.Promotions),
|
||||||
CacheWriteFailures: atomic.LoadInt64(&m.CacheWriteFailures),
|
Evictions: atomic.LoadInt64(&m.Evictions),
|
||||||
ServiceErrors: serviceErrors,
|
CapacityPressureEvents: atomic.LoadInt64(&m.CapacityPressureEvents),
|
||||||
Uptime: time.Since(m.StartTime),
|
ServiceRequests: serviceRequests,
|
||||||
LastResetTime: m.LastResetTime,
|
UpstreamErrors: atomic.LoadInt64(&m.UpstreamErrors),
|
||||||
|
CacheWriteFailures: atomic.LoadInt64(&m.CacheWriteFailures),
|
||||||
|
ServiceErrors: serviceErrors,
|
||||||
|
Uptime: time.Since(m.StartTime),
|
||||||
|
LastResetTime: m.LastResetTime,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -215,6 +278,8 @@ func (m *Metrics) Reset() {
|
|||||||
atomic.StoreInt64(&m.CacheHits, 0)
|
atomic.StoreInt64(&m.CacheHits, 0)
|
||||||
atomic.StoreInt64(&m.CacheMisses, 0)
|
atomic.StoreInt64(&m.CacheMisses, 0)
|
||||||
atomic.StoreInt64(&m.CacheCoalesced, 0)
|
atomic.StoreInt64(&m.CacheCoalesced, 0)
|
||||||
|
atomic.StoreInt64(&m.RangeCache, 0)
|
||||||
|
atomic.StoreInt64(&m.RangeUpstream, 0)
|
||||||
atomic.StoreInt64(&m.Errors, 0)
|
atomic.StoreInt64(&m.Errors, 0)
|
||||||
atomic.StoreInt64(&m.RateLimited, 0)
|
atomic.StoreInt64(&m.RateLimited, 0)
|
||||||
atomic.StoreInt64(&m.TotalResponseTime, 0)
|
atomic.StoreInt64(&m.TotalResponseTime, 0)
|
||||||
@@ -224,6 +289,7 @@ func (m *Metrics) Reset() {
|
|||||||
atomic.StoreInt64(&m.DiskCacheHits, 0)
|
atomic.StoreInt64(&m.DiskCacheHits, 0)
|
||||||
atomic.StoreInt64(&m.Promotions, 0)
|
atomic.StoreInt64(&m.Promotions, 0)
|
||||||
atomic.StoreInt64(&m.Evictions, 0)
|
atomic.StoreInt64(&m.Evictions, 0)
|
||||||
|
atomic.StoreInt64(&m.CapacityPressureEvents, 0)
|
||||||
atomic.StoreInt64(&m.UpstreamErrors, 0)
|
atomic.StoreInt64(&m.UpstreamErrors, 0)
|
||||||
atomic.StoreInt64(&m.CacheWriteFailures, 0)
|
atomic.StoreInt64(&m.CacheWriteFailures, 0)
|
||||||
|
|
||||||
@@ -244,6 +310,8 @@ type Stats struct {
|
|||||||
CacheHits int64
|
CacheHits int64
|
||||||
CacheMisses int64
|
CacheMisses int64
|
||||||
CacheCoalesced int64
|
CacheCoalesced int64
|
||||||
|
RangeCache int64
|
||||||
|
RangeUpstream int64
|
||||||
Errors int64
|
Errors int64
|
||||||
RateLimited int64
|
RateLimited int64
|
||||||
HitRate float64
|
HitRate float64
|
||||||
@@ -252,17 +320,19 @@ type Stats struct {
|
|||||||
TotalBytesSaved int64
|
TotalBytesSaved int64
|
||||||
MemoryCacheSize int64
|
MemoryCacheSize int64
|
||||||
|
|
||||||
DiskCacheSize int64
|
DiskCacheSize int64
|
||||||
MemoryCacheHits int64
|
DiskTierReady int64
|
||||||
DiskCacheHits int64
|
MemoryCacheHits int64
|
||||||
Promotions int64
|
DiskCacheHits int64
|
||||||
Evictions int64
|
Promotions int64
|
||||||
UpstreamErrors int64
|
Evictions int64
|
||||||
CacheWriteFailures int64
|
CapacityPressureEvents int64
|
||||||
ServiceErrors map[string]int64
|
UpstreamErrors int64
|
||||||
ServiceRequests map[string]int64
|
CacheWriteFailures int64
|
||||||
Uptime time.Duration
|
ServiceErrors map[string]int64
|
||||||
LastResetTime time.Time
|
ServiceRequests map[string]int64
|
||||||
|
Uptime time.Duration
|
||||||
|
LastResetTime time.Time
|
||||||
}
|
}
|
||||||
|
|
||||||
// WriteText emits the Prometheus-style text metrics to the ResponseWriter.
|
// WriteText emits the Prometheus-style text metrics to the ResponseWriter.
|
||||||
@@ -276,6 +346,8 @@ func WriteText(w http.ResponseWriter, stats *Stats) {
|
|||||||
_, _ = fmt.Fprintf(w, "cache_hits %d\n", stats.CacheHits)
|
_, _ = fmt.Fprintf(w, "cache_hits %d\n", stats.CacheHits)
|
||||||
_, _ = fmt.Fprintf(w, "cache_misses %d\n", stats.CacheMisses)
|
_, _ = fmt.Fprintf(w, "cache_misses %d\n", stats.CacheMisses)
|
||||||
_, _ = fmt.Fprintf(w, "cache_coalesced %d\n", stats.CacheCoalesced)
|
_, _ = fmt.Fprintf(w, "cache_coalesced %d\n", stats.CacheCoalesced)
|
||||||
|
_, _ = fmt.Fprintf(w, "range_cache %d\n", stats.RangeCache)
|
||||||
|
_, _ = fmt.Fprintf(w, "range_upstream %d\n", stats.RangeUpstream)
|
||||||
_, _ = fmt.Fprintf(w, "errors %d\n", stats.Errors)
|
_, _ = fmt.Fprintf(w, "errors %d\n", stats.Errors)
|
||||||
_, _ = fmt.Fprintf(w, "rate_limited %d\n", stats.RateLimited)
|
_, _ = fmt.Fprintf(w, "rate_limited %d\n", stats.RateLimited)
|
||||||
_, _ = fmt.Fprintf(w, "upstream_errors %d\n", stats.UpstreamErrors)
|
_, _ = fmt.Fprintf(w, "upstream_errors %d\n", stats.UpstreamErrors)
|
||||||
@@ -284,6 +356,7 @@ func WriteText(w http.ResponseWriter, stats *Stats) {
|
|||||||
_, _ = fmt.Fprintf(w, "disk_cache_hits %d\n", stats.DiskCacheHits)
|
_, _ = fmt.Fprintf(w, "disk_cache_hits %d\n", stats.DiskCacheHits)
|
||||||
_, _ = fmt.Fprintf(w, "promotions %d\n", stats.Promotions)
|
_, _ = fmt.Fprintf(w, "promotions %d\n", stats.Promotions)
|
||||||
_, _ = fmt.Fprintf(w, "evictions %d\n", stats.Evictions)
|
_, _ = fmt.Fprintf(w, "evictions %d\n", stats.Evictions)
|
||||||
|
_, _ = fmt.Fprintf(w, "capacity_pressure_events %d\n", stats.CapacityPressureEvents)
|
||||||
for svc, cnt := range stats.ServiceErrors {
|
for svc, cnt := range stats.ServiceErrors {
|
||||||
_, _ = fmt.Fprintf(w, "service_errors{service=%q} %d\n", svc, cnt)
|
_, _ = fmt.Fprintf(w, "service_errors{service=%q} %d\n", svc, cnt)
|
||||||
}
|
}
|
||||||
@@ -297,5 +370,6 @@ func WriteText(w http.ResponseWriter, stats *Stats) {
|
|||||||
|
|
||||||
_, _ = fmt.Fprintf(w, "memory_cache_size %d\n", stats.MemoryCacheSize)
|
_, _ = fmt.Fprintf(w, "memory_cache_size %d\n", stats.MemoryCacheSize)
|
||||||
_, _ = fmt.Fprintf(w, "disk_cache_size %d\n", stats.DiskCacheSize)
|
_, _ = fmt.Fprintf(w, "disk_cache_size %d\n", stats.DiskCacheSize)
|
||||||
|
_, _ = fmt.Fprintf(w, "disk_tier_ready %d\n", stats.DiskTierReady)
|
||||||
_, _ = fmt.Fprintf(w, "uptime_seconds %.2f\n", stats.Uptime.Seconds())
|
_, _ = fmt.Fprintf(w, "uptime_seconds %.2f\n", stats.Uptime.Seconds())
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -0,0 +1,63 @@
|
|||||||
|
package metrics
|
||||||
|
|
||||||
|
import (
|
||||||
|
"bytes"
|
||||||
|
"errors"
|
||||||
|
"net/http/httptest"
|
||||||
|
"testing"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestCapacityPressureEventsWriteTextAndReset(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
m := NewMetrics()
|
||||||
|
if got := m.GetStats().CapacityPressureEvents; got != 0 {
|
||||||
|
t.Fatalf("initial CapacityPressureEvents=%d, want 0", got)
|
||||||
|
}
|
||||||
|
|
||||||
|
NoteSoftEviction(m, "memory", 0)
|
||||||
|
if got := m.GetStats().CapacityPressureEvents; got != 0 {
|
||||||
|
t.Fatalf("zero-byte eviction counted: %d", got)
|
||||||
|
}
|
||||||
|
|
||||||
|
NoteSoftEviction(m, "memory", 128)
|
||||||
|
st := m.GetStats()
|
||||||
|
if st.CapacityPressureEvents != 1 {
|
||||||
|
t.Fatalf("after memory eviction, CapacityPressureEvents=%d, want 1", st.CapacityPressureEvents)
|
||||||
|
}
|
||||||
|
if st.Evictions != 1 {
|
||||||
|
t.Fatalf("after memory eviction, Evictions=%d, want 1 (existing counter kept)", st.Evictions)
|
||||||
|
}
|
||||||
|
|
||||||
|
NoteSoftEviction(m, "disk", 64)
|
||||||
|
NoteNoSpace(m, errors.New("no space left on device"))
|
||||||
|
st = m.GetStats()
|
||||||
|
if st.CapacityPressureEvents != 3 {
|
||||||
|
t.Fatalf("after disk eviction + ENOSPC, CapacityPressureEvents=%d, want 3", st.CapacityPressureEvents)
|
||||||
|
}
|
||||||
|
if st.Evictions != 2 {
|
||||||
|
t.Fatalf("ENOSPC must not increment evictions; Evictions=%d, want 2", st.Evictions)
|
||||||
|
}
|
||||||
|
|
||||||
|
rec := httptest.NewRecorder()
|
||||||
|
WriteText(rec, st)
|
||||||
|
body := rec.Body.Bytes()
|
||||||
|
if !bytes.Contains(body, []byte("capacity_pressure_events 3")) {
|
||||||
|
t.Errorf("WriteText missing capacity_pressure_events 3: %q", rec.Body.String())
|
||||||
|
}
|
||||||
|
if !bytes.Contains(body, []byte("evictions 2")) {
|
||||||
|
t.Errorf("WriteText missing evictions 2: %q", rec.Body.String())
|
||||||
|
}
|
||||||
|
|
||||||
|
m.Reset()
|
||||||
|
st = m.GetStats()
|
||||||
|
if st.CapacityPressureEvents != 0 || st.Evictions != 0 {
|
||||||
|
t.Errorf("after Reset, CapacityPressureEvents=%d Evictions=%d, want 0", st.CapacityPressureEvents, st.Evictions)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestNoteSoftEvictionNilMetrics(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
// Must not panic when metrics are not wired (unit tests / early init).
|
||||||
|
NoteSoftEviction(nil, "memory", 10)
|
||||||
|
NoteNoSpace(nil, errors.New("ENOSPC"))
|
||||||
|
}
|
||||||
@@ -0,0 +1,386 @@
|
|||||||
|
// steamcache/range_test.go
|
||||||
|
package steamcache
|
||||||
|
|
||||||
|
import (
|
||||||
|
"io"
|
||||||
|
"net/http"
|
||||||
|
"net/http/httptest"
|
||||||
|
"strings"
|
||||||
|
"sync/atomic"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
)
|
||||||
|
|
||||||
|
// TestRangeHitServedLocally verifies that a Range GET against an already-cached
|
||||||
|
// object is served locally as 206 (no upstream re-fetch) and increments range_cache.
|
||||||
|
func TestRangeHitServedLocally(t *testing.T) {
|
||||||
|
body := []byte("0123456789abcdef") // 16 bytes
|
||||||
|
var upstreamCalls atomic.Int64
|
||||||
|
f := func(w http.ResponseWriter, r *http.Request) {
|
||||||
|
upstreamCalls.Add(1)
|
||||||
|
w.Header().Set("Content-Type", "application/octet-stream")
|
||||||
|
_, _ = w.Write(body)
|
||||||
|
}
|
||||||
|
sc, _ := newTestCacheWithFakeUpstream(t, f, "1MB", "0")
|
||||||
|
srv := newCacheServer(t, sc)
|
||||||
|
c := &http.Client{Timeout: 5 * time.Second}
|
||||||
|
|
||||||
|
// 1) Populate the cache with a full (non-Range) MISS.
|
||||||
|
req, err := http.NewRequest("GET", srv.URL+"/depot/rangetest/chunk", nil)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("NewRequest: %v", err)
|
||||||
|
}
|
||||||
|
req.Header.Set("User-Agent", "Valve/Steam HTTP Client 1.0")
|
||||||
|
resp, err := c.Do(req)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("miss GET: %v", err)
|
||||||
|
}
|
||||||
|
if resp.StatusCode != http.StatusOK {
|
||||||
|
t.Fatalf("miss GET: expected 200, got %d", resp.StatusCode)
|
||||||
|
}
|
||||||
|
_, _ = io.Copy(io.Discard, resp.Body)
|
||||||
|
_ = resp.Body.Close()
|
||||||
|
if got := sc.GetMetrics().CacheMisses; got < 1 {
|
||||||
|
t.Fatalf("expected CacheMisses >= 1 after first GET, got %d", got)
|
||||||
|
}
|
||||||
|
|
||||||
|
// 2) Range GET against the same URL — must be a local 206 HIT.
|
||||||
|
req2, err := http.NewRequest("GET", srv.URL+"/depot/rangetest/chunk", nil)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("NewRequest: %v", err)
|
||||||
|
}
|
||||||
|
req2.Header.Set("User-Agent", "Valve/Steam HTTP Client 1.0")
|
||||||
|
req2.Header.Set("Range", "bytes=4-7")
|
||||||
|
resp2, err := c.Do(req2)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("range GET: %v", err)
|
||||||
|
}
|
||||||
|
data, err := io.ReadAll(resp2.Body)
|
||||||
|
_ = resp2.Body.Close()
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("read range body: %v", err)
|
||||||
|
}
|
||||||
|
if resp2.StatusCode != http.StatusPartialContent {
|
||||||
|
t.Fatalf("range GET: expected 206, got %d", resp2.StatusCode)
|
||||||
|
}
|
||||||
|
if string(data) != "4567" {
|
||||||
|
t.Errorf("range GET: expected body %q, got %q", "4567", data)
|
||||||
|
}
|
||||||
|
if got := resp2.Header.Get("X-LanCache-Status"); got != "HIT" {
|
||||||
|
t.Errorf("range GET: expected X-LanCache-Status HIT, got %q", got)
|
||||||
|
}
|
||||||
|
if got := resp2.Header.Get("Content-Range"); got != "bytes 4-7/16" {
|
||||||
|
t.Errorf("range GET: expected Content-Range bytes 4-7/16, got %q", got)
|
||||||
|
}
|
||||||
|
if got := resp2.Header.Get("Accept-Ranges"); got != "bytes" {
|
||||||
|
t.Errorf("range GET: expected Accept-Ranges bytes, got %q", got)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Upstream must NOT have been hit again.
|
||||||
|
if got := upstreamCalls.Load(); got != 1 {
|
||||||
|
t.Errorf("upstream hit %d times, want exactly 1 (range HIT must be local)", got)
|
||||||
|
}
|
||||||
|
|
||||||
|
// range_cache incremented, range_upstream untouched.
|
||||||
|
stats := sc.GetMetrics()
|
||||||
|
if stats.RangeCache != 1 {
|
||||||
|
t.Errorf("expected RangeCache == 1 after range HIT, got %d", stats.RangeCache)
|
||||||
|
}
|
||||||
|
if stats.RangeUpstream != 0 {
|
||||||
|
t.Errorf("expected RangeUpstream == 0 (no range miss yet), got %d", stats.RangeUpstream)
|
||||||
|
}
|
||||||
|
if stats.CacheHits < 1 {
|
||||||
|
t.Errorf("expected CacheHits >= 1 after range HIT, got %d", stats.CacheHits)
|
||||||
|
}
|
||||||
|
// BytesServed: 16 (full miss body) + 4 (range bytes) = 20.
|
||||||
|
if stats.TotalBytesServed != 20 {
|
||||||
|
t.Errorf("expected TotalBytesServed == 20 (16 + range 4), got %d", stats.TotalBytesServed)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestRangeMissServes206FromFullFetch verifies that a Range GET on a cold key fetches
|
||||||
|
// the FULL object from upstream (Range stripped), caches it, and serves the requested
|
||||||
|
// slice as 206 with range_upstream incremented.
|
||||||
|
func TestRangeMissServes206FromFullFetch(t *testing.T) {
|
||||||
|
body := []byte("0123456789abcdef") // 16 bytes
|
||||||
|
var upstreamCalls atomic.Int64
|
||||||
|
var upstreamSawRange atomic.Bool
|
||||||
|
f := func(w http.ResponseWriter, r *http.Request) {
|
||||||
|
upstreamCalls.Add(1)
|
||||||
|
if r.Header.Get("Range") != "" {
|
||||||
|
upstreamSawRange.Store(true)
|
||||||
|
}
|
||||||
|
w.Header().Set("Content-Type", "application/octet-stream")
|
||||||
|
_, _ = w.Write(body)
|
||||||
|
}
|
||||||
|
sc, _ := newTestCacheWithFakeUpstream(t, f, "1MB", "0")
|
||||||
|
srv := newCacheServer(t, sc)
|
||||||
|
c := &http.Client{Timeout: 5 * time.Second}
|
||||||
|
|
||||||
|
// Range GET on a cold key.
|
||||||
|
req, err := http.NewRequest("GET", srv.URL+"/depot/rangetest/chunk2", nil)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("NewRequest: %v", err)
|
||||||
|
}
|
||||||
|
req.Header.Set("User-Agent", "Valve/Steam HTTP Client 1.0")
|
||||||
|
req.Header.Set("Range", "bytes=0-3")
|
||||||
|
resp, err := c.Do(req)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("range miss GET: %v", err)
|
||||||
|
}
|
||||||
|
data, err := io.ReadAll(resp.Body)
|
||||||
|
_ = resp.Body.Close()
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("read body: %v", err)
|
||||||
|
}
|
||||||
|
if resp.StatusCode != http.StatusPartialContent {
|
||||||
|
t.Fatalf("range miss GET: expected 206, got %d", resp.StatusCode)
|
||||||
|
}
|
||||||
|
if string(data) != "0123" {
|
||||||
|
t.Errorf("range miss GET: expected body %q, got %q", "0123", data)
|
||||||
|
}
|
||||||
|
if got := resp.Header.Get("X-LanCache-Status"); got != "MISS" {
|
||||||
|
t.Errorf("range miss GET: expected X-LanCache-Status MISS, got %q", got)
|
||||||
|
}
|
||||||
|
if got := resp.Header.Get("Content-Range"); got != "bytes 0-3/16" {
|
||||||
|
t.Errorf("range miss GET: expected Content-Range bytes 0-3/16, got %q", got)
|
||||||
|
}
|
||||||
|
if got := resp.Header.Get("Accept-Ranges"); got != "bytes" {
|
||||||
|
t.Errorf("range miss GET: expected Accept-Ranges bytes, got %q", got)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Range must have been stripped for the upstream fetch (full file cached).
|
||||||
|
if upstreamSawRange.Load() {
|
||||||
|
t.Error("upstream received a Range header; Range must be stripped so the full object is cached")
|
||||||
|
}
|
||||||
|
if got := upstreamCalls.Load(); got != 1 {
|
||||||
|
t.Errorf("upstream hit %d times, want exactly 1", got)
|
||||||
|
}
|
||||||
|
|
||||||
|
// range_upstream incremented, range_cache untouched.
|
||||||
|
stats := sc.GetMetrics()
|
||||||
|
if stats.RangeUpstream != 1 {
|
||||||
|
t.Errorf("expected RangeUpstream == 1 after range MISS, got %d", stats.RangeUpstream)
|
||||||
|
}
|
||||||
|
if stats.RangeCache != 0 {
|
||||||
|
t.Errorf("expected RangeCache == 0 (no range hit yet), got %d", stats.RangeCache)
|
||||||
|
}
|
||||||
|
if stats.CacheMisses < 1 {
|
||||||
|
t.Errorf("expected CacheMisses >= 1, got %d", stats.CacheMisses)
|
||||||
|
}
|
||||||
|
// BytesServed for the range miss: only the 4 range bytes, not the 16-byte fetch.
|
||||||
|
if stats.TotalBytesServed != 4 {
|
||||||
|
t.Errorf("expected TotalBytesServed == 4 (range bytes only), got %d", stats.TotalBytesServed)
|
||||||
|
}
|
||||||
|
|
||||||
|
// The FULL object must have been cached: a subsequent full GET is a HIT with the
|
||||||
|
// complete 16-byte body.
|
||||||
|
req3, err := http.NewRequest("GET", srv.URL+"/depot/rangetest/chunk2", nil)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("NewRequest: %v", err)
|
||||||
|
}
|
||||||
|
req3.Header.Set("User-Agent", "Valve/Steam HTTP Client 1.0")
|
||||||
|
resp3, err := c.Do(req3)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("full GET after range miss: %v", err)
|
||||||
|
}
|
||||||
|
fullData, err := io.ReadAll(resp3.Body)
|
||||||
|
_ = resp3.Body.Close()
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("read full body: %v", err)
|
||||||
|
}
|
||||||
|
if resp3.StatusCode != http.StatusOK {
|
||||||
|
t.Fatalf("full GET after range miss: expected 200, got %d", resp3.StatusCode)
|
||||||
|
}
|
||||||
|
if got := resp3.Header.Get("X-LanCache-Status"); got != "HIT" {
|
||||||
|
t.Errorf("full GET after range miss: expected X-LanCache-Status HIT, got %q", got)
|
||||||
|
}
|
||||||
|
if len(fullData) != len(body) || string(fullData) != string(body) {
|
||||||
|
t.Errorf("full GET after range miss: expected full %d-byte body, got %d bytes", len(body), len(fullData))
|
||||||
|
}
|
||||||
|
if got := upstreamCalls.Load(); got != 1 {
|
||||||
|
t.Errorf("upstream hit %d times after HIT, want still 1", got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestRangeMissInvalidRange416 verifies that an unsatisfiable Range on a cold key
|
||||||
|
// fetches upstream, returns 416 (as on the HIT path), and does not count range_upstream.
|
||||||
|
func TestRangeMissInvalidRange416(t *testing.T) {
|
||||||
|
body := []byte("0123456789abcdef") // 16 bytes
|
||||||
|
var upstreamCalls atomic.Int64
|
||||||
|
f := func(w http.ResponseWriter, r *http.Request) {
|
||||||
|
upstreamCalls.Add(1)
|
||||||
|
_, _ = w.Write(body)
|
||||||
|
}
|
||||||
|
sc, _ := newTestCacheWithFakeUpstream(t, f, "1MB", "0")
|
||||||
|
srv := newCacheServer(t, sc)
|
||||||
|
c := &http.Client{Timeout: 5 * time.Second}
|
||||||
|
|
||||||
|
req, err := http.NewRequest("GET", srv.URL+"/depot/rangetest/chunk3", nil)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("NewRequest: %v", err)
|
||||||
|
}
|
||||||
|
req.Header.Set("User-Agent", "Valve/Steam HTTP Client 1.0")
|
||||||
|
req.Header.Set("Range", "bytes=100-200")
|
||||||
|
resp, err := c.Do(req)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("invalid range GET: %v", err)
|
||||||
|
}
|
||||||
|
data, err := io.ReadAll(resp.Body)
|
||||||
|
_ = resp.Body.Close()
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("read body: %v", err)
|
||||||
|
}
|
||||||
|
if resp.StatusCode != http.StatusRequestedRangeNotSatisfiable {
|
||||||
|
t.Fatalf("invalid range GET: expected 416, got %d", resp.StatusCode)
|
||||||
|
}
|
||||||
|
if len(data) != 0 {
|
||||||
|
t.Errorf("invalid range GET: expected empty body, got %d bytes", len(data))
|
||||||
|
}
|
||||||
|
if got := resp.Header.Get("Content-Range"); got != "bytes */16" {
|
||||||
|
t.Errorf("invalid range GET: expected Content-Range bytes */16, got %q", got)
|
||||||
|
}
|
||||||
|
if got := upstreamCalls.Load(); got != 1 {
|
||||||
|
t.Errorf("upstream hit %d times, want exactly 1 (fetch happens, then 416 to client)", got)
|
||||||
|
}
|
||||||
|
stats := sc.GetMetrics()
|
||||||
|
if stats.RangeUpstream != 0 {
|
||||||
|
t.Errorf("expected RangeUpstream == 0 for unsatisfiable range, got %d", stats.RangeUpstream)
|
||||||
|
}
|
||||||
|
if stats.RangeCache != 0 {
|
||||||
|
t.Errorf("expected RangeCache == 0, got %d", stats.RangeCache)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestRangeMetricsWriteText verifies /metrics emits the range_cache and range_upstream
|
||||||
|
// lines with the expected values after a range HIT and a range MISS.
|
||||||
|
func TestRangeMetricsWriteText(t *testing.T) {
|
||||||
|
body := []byte("0123456789abcdef")
|
||||||
|
f := func(w http.ResponseWriter, r *http.Request) {
|
||||||
|
_, _ = w.Write(body)
|
||||||
|
}
|
||||||
|
sc, _ := newTestCacheWithFakeUpstream(t, f, "1MB", "0")
|
||||||
|
srv := newCacheServer(t, sc)
|
||||||
|
c := &http.Client{Timeout: 5 * time.Second}
|
||||||
|
|
||||||
|
get := func(path, rangeHeader string) int {
|
||||||
|
t.Helper()
|
||||||
|
req, err := http.NewRequest("GET", srv.URL+path, nil)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("NewRequest: %v", err)
|
||||||
|
}
|
||||||
|
req.Header.Set("User-Agent", "Valve/Steam HTTP Client 1.0")
|
||||||
|
if rangeHeader != "" {
|
||||||
|
req.Header.Set("Range", rangeHeader)
|
||||||
|
}
|
||||||
|
resp, err := c.Do(req)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("GET %s: %v", path, err)
|
||||||
|
}
|
||||||
|
_, _ = io.Copy(io.Discard, resp.Body)
|
||||||
|
_ = resp.Body.Close()
|
||||||
|
return resp.StatusCode
|
||||||
|
}
|
||||||
|
|
||||||
|
// Range MISS on cold key -> range_upstream; warm it; range HIT -> range_cache.
|
||||||
|
if code := get("/depot/rangetest/wt/1", "bytes=0-3"); code != http.StatusPartialContent {
|
||||||
|
t.Fatalf("range miss: expected 206, got %d", code)
|
||||||
|
}
|
||||||
|
if code := get("/depot/rangetest/wt/2", ""); code != http.StatusOK {
|
||||||
|
t.Fatalf("warm miss: expected 200, got %d", code)
|
||||||
|
}
|
||||||
|
if code := get("/depot/rangetest/wt/2", "bytes=8-11"); code != http.StatusPartialContent {
|
||||||
|
t.Fatalf("range hit: expected 206, got %d", code)
|
||||||
|
}
|
||||||
|
|
||||||
|
req, err := http.NewRequest("GET", srv.URL+"/metrics", nil)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("NewRequest: %v", err)
|
||||||
|
}
|
||||||
|
resp, err := c.Do(req)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("GET /metrics: %v", err)
|
||||||
|
}
|
||||||
|
out, err := io.ReadAll(resp.Body)
|
||||||
|
_ = resp.Body.Close()
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("read /metrics: %v", err)
|
||||||
|
}
|
||||||
|
text := string(out)
|
||||||
|
if !strings.Contains(text, "range_cache 1\n") {
|
||||||
|
t.Errorf("/metrics missing 'range_cache 1' line:\n%s", text)
|
||||||
|
}
|
||||||
|
if !strings.Contains(text, "range_upstream 1\n") {
|
||||||
|
t.Errorf("/metrics missing 'range_upstream 1' line:\n%s", text)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestStreamCachedResponseRange206 is a focused unit test for streamCachedResponse:
|
||||||
|
// valid Range yields 206 with the exact slice + metrics; invalid Range yields 416
|
||||||
|
// with no range metrics.
|
||||||
|
func TestStreamCachedResponseRange206(t *testing.T) {
|
||||||
|
body := []byte("0123456789abcdef") // 16 bytes
|
||||||
|
raw := append([]byte("HTTP/1.1 200 OK\r\nContent-Type: application/octet-stream\r\n\r\n"), body...)
|
||||||
|
serialized, err := serializeRawResponse(raw)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("serialize cache file: %v", err)
|
||||||
|
}
|
||||||
|
cf, err := deserializeCacheFile(serialized)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("build cache file: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
sc, _ := newTestCacheWithFakeUpstream(t, func(w http.ResponseWriter, r *http.Request) {
|
||||||
|
_, _ = w.Write([]byte("x"))
|
||||||
|
}, "1MB", "0")
|
||||||
|
sc.ResetMetrics()
|
||||||
|
|
||||||
|
// Valid range -> 206 + slice + metrics.
|
||||||
|
rec := httptest.NewRecorder()
|
||||||
|
req := httptest.NewRequest("GET", "/depot/rangetest/chunk", nil)
|
||||||
|
req.Header.Set("Range", "bytes=4-7")
|
||||||
|
sc.streamCachedResponse(rec, req, cf, "steam/testkey", "127.0.0.1", time.Now())
|
||||||
|
|
||||||
|
if rec.Code != http.StatusPartialContent {
|
||||||
|
t.Fatalf("expected 206, got %d", rec.Code)
|
||||||
|
}
|
||||||
|
if rec.Body.String() != "4567" {
|
||||||
|
t.Errorf("expected body %q, got %q", "4567", rec.Body.String())
|
||||||
|
}
|
||||||
|
if got := rec.Header().Get("Content-Range"); got != "bytes 4-7/16" {
|
||||||
|
t.Errorf("expected Content-Range bytes 4-7/16, got %q", got)
|
||||||
|
}
|
||||||
|
if got := rec.Header().Get("X-LanCache-Status"); got != "HIT" {
|
||||||
|
t.Errorf("expected X-LanCache-Status HIT, got %q", got)
|
||||||
|
}
|
||||||
|
stats := sc.GetMetrics()
|
||||||
|
if stats.RangeCache != 1 {
|
||||||
|
t.Errorf("expected RangeCache == 1, got %d", stats.RangeCache)
|
||||||
|
}
|
||||||
|
if stats.TotalBytesServed != 4 {
|
||||||
|
t.Errorf("expected TotalBytesServed == 4 (range bytes), got %d", stats.TotalBytesServed)
|
||||||
|
}
|
||||||
|
if stats.TotalBytesSaved != 4 {
|
||||||
|
t.Errorf("expected TotalBytesSaved == 4 (range bytes), got %d", stats.TotalBytesSaved)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Invalid range -> 416, no range metrics, no bytes served.
|
||||||
|
rec2 := httptest.NewRecorder()
|
||||||
|
req2 := httptest.NewRequest("GET", "/depot/rangetest/chunk", nil)
|
||||||
|
req2.Header.Set("Range", "bytes=100-200")
|
||||||
|
sc.streamCachedResponse(rec2, req2, cf, "steam/testkey", "127.0.0.1", time.Now())
|
||||||
|
|
||||||
|
if rec2.Code != http.StatusRequestedRangeNotSatisfiable {
|
||||||
|
t.Fatalf("expected 416, got %d", rec2.Code)
|
||||||
|
}
|
||||||
|
if got := rec2.Header().Get("Content-Range"); got != "bytes */16" {
|
||||||
|
t.Errorf("expected Content-Range bytes */16, got %q", got)
|
||||||
|
}
|
||||||
|
stats = sc.GetMetrics()
|
||||||
|
if stats.RangeCache != 1 {
|
||||||
|
t.Errorf("RangeCache must stay 1 after unsatisfiable range, got %d", stats.RangeCache)
|
||||||
|
}
|
||||||
|
if stats.TotalBytesServed != 4 {
|
||||||
|
t.Errorf("TotalBytesServed must stay 4 after 416, got %d", stats.TotalBytesServed)
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -200,23 +200,33 @@ func New(address string, memorySize string, diskSize string, diskPath, upstream,
|
|||||||
|
|
||||||
if disksize == 0 && memorysize != 0 {
|
if disksize == 0 && memorysize != 0 {
|
||||||
// memory only mode - no disk
|
// memory only mode - no disk
|
||||||
|
sc.metrics.SetDiskTierReady(1) // no disk — N/A / not pending
|
||||||
c.SetSlow(mgc)
|
c.SetSlow(mgc)
|
||||||
} else if disksize != 0 && memorysize == 0 {
|
} else if disksize != 0 && memorysize == 0 {
|
||||||
// disk only mode: delay attach until disk ready (pure-proxy during scan; Create returns ErrNotFound until slow tier Set)
|
// disk only mode: delay attach until disk ready (pure-proxy during scan; Create returns ErrNotFound until slow tier Set)
|
||||||
|
sc.metrics.SetDiskTierReady(0)
|
||||||
|
logger.Logger.Info().Msg("Disk slow tier attach pending; Size barrier in progress")
|
||||||
sc.wg.Add(1)
|
sc.wg.Add(1)
|
||||||
go func() {
|
go func() {
|
||||||
defer sc.wg.Done()
|
defer sc.wg.Done()
|
||||||
|
t0 := time.Now()
|
||||||
_ = d.Size() // block on barrier per design (all Size callers during window do this; documented)
|
_ = d.Size() // block on barrier per design (all Size callers during window do this; documented)
|
||||||
select {
|
select {
|
||||||
case <-sc.shutdownCh:
|
case <-sc.shutdownCh:
|
||||||
return // Shutdown raced; do not attach or SetSlow after stop
|
return // Shutdown raced; do not attach or SetSlow after stop
|
||||||
default:
|
default:
|
||||||
c.SetSlow(dgc)
|
c.SetSlow(dgc)
|
||||||
|
sc.metrics.SetDiskTierReady(1)
|
||||||
|
logger.Logger.Info().
|
||||||
|
Dur("attach_delay", time.Since(t0)).
|
||||||
|
Msg("Disk slow tier attached (disk-only mode); prior traffic had no disk tier")
|
||||||
}
|
}
|
||||||
}()
|
}()
|
||||||
} else if disksize != 0 && memorysize != 0 {
|
} else if disksize != 0 && memorysize != 0 {
|
||||||
// memory and disk mode: fast mem immediate, disk delayed (mem-only during scan)
|
// memory and disk mode: fast mem immediate, disk delayed (mem-only during scan)
|
||||||
c.SetFast(mgc)
|
c.SetFast(mgc)
|
||||||
|
sc.metrics.SetDiskTierReady(0)
|
||||||
|
logger.Logger.Info().Msg("Disk slow tier attach pending; Size barrier in progress")
|
||||||
sc.wg.Add(1)
|
sc.wg.Add(1)
|
||||||
go func() {
|
go func() {
|
||||||
defer sc.wg.Done()
|
defer sc.wg.Done()
|
||||||
@@ -227,6 +237,7 @@ func New(address string, memorySize string, diskSize string, diskPath, upstream,
|
|||||||
return
|
return
|
||||||
default:
|
default:
|
||||||
c.SetSlow(dgc)
|
c.SetSlow(dgc)
|
||||||
|
sc.metrics.SetDiskTierReady(1)
|
||||||
logger.Logger.Info().
|
logger.Logger.Info().
|
||||||
Dur("attach_delay", time.Since(t0)).
|
Dur("attach_delay", time.Since(t0)).
|
||||||
Msg("Disk slow tier attached (mixed mode); prior traffic was memory-only")
|
Msg("Disk slow tier attached (mixed mode); prior traffic was memory-only")
|
||||||
@@ -326,15 +337,13 @@ func (sc *SteamCache) Shutdown() {
|
|||||||
|
|
||||||
// GetMetrics returns current metrics
|
// GetMetrics returns current metrics
|
||||||
func (sc *SteamCache) GetMetrics() *metrics.Stats {
|
func (sc *SteamCache) GetMetrics() *metrics.Stats {
|
||||||
// Update cache sizes
|
|
||||||
if sc.memory != nil {
|
if sc.memory != nil {
|
||||||
sc.metrics.SetMemoryCacheSize(sc.memory.Size())
|
sc.metrics.SetMemoryCacheSize(sc.memory.Size())
|
||||||
}
|
}
|
||||||
if sc.disk != nil {
|
// Skip disk.Size() while attach pending — Size() blocks on initDone and would hang /metrics.
|
||||||
// Note: blocks on initDone (post-eviction state) for accurate post-attach size during long disk init window.
|
if sc.disk != nil && sc.metrics.GetDiskTierReady() == 1 {
|
||||||
sc.metrics.SetDiskCacheSize(sc.disk.Size())
|
sc.metrics.SetDiskCacheSize(sc.disk.Size())
|
||||||
}
|
}
|
||||||
|
|
||||||
return sc.metrics.GetStats()
|
return sc.metrics.GetStats()
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -2,21 +2,25 @@
|
|||||||
package steamcache
|
package steamcache
|
||||||
|
|
||||||
import (
|
import (
|
||||||
|
"bufio"
|
||||||
"bytes"
|
"bytes"
|
||||||
"context"
|
"context"
|
||||||
"fmt"
|
"fmt"
|
||||||
"io"
|
"io"
|
||||||
|
"net"
|
||||||
"net/http"
|
"net/http"
|
||||||
"net/http/httptest"
|
"net/http/httptest"
|
||||||
"os"
|
"os"
|
||||||
"path/filepath"
|
"path/filepath"
|
||||||
"runtime"
|
"runtime"
|
||||||
"s1d3sw1ped/steamcache2/steamcache/metrics"
|
"s1d3sw1ped/steamcache2/steamcache/metrics"
|
||||||
|
"s1d3sw1ped/steamcache2/vfs/disk"
|
||||||
"s1d3sw1ped/steamcache2/vfs/eviction"
|
"s1d3sw1ped/steamcache2/vfs/eviction"
|
||||||
"s1d3sw1ped/steamcache2/vfs/memory"
|
"s1d3sw1ped/steamcache2/vfs/memory"
|
||||||
"s1d3sw1ped/steamcache2/vfs/vfserror"
|
"s1d3sw1ped/steamcache2/vfs/vfserror"
|
||||||
"strings"
|
"strings"
|
||||||
"sync"
|
"sync"
|
||||||
|
"sync/atomic"
|
||||||
"testing"
|
"testing"
|
||||||
"time"
|
"time"
|
||||||
)
|
)
|
||||||
@@ -1057,6 +1061,12 @@ func TestDiskOnlyDelayedAttach(t *testing.T) {
|
|||||||
t.Errorf("during init window, expected ErrNotFound from disk-only tiered Create (no slow), got %v", err)
|
t.Errorf("during init window, expected ErrNotFound from disk-only tiered Create (no slow), got %v", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Disk tier is pending while the attach goroutine is in the Size barrier.
|
||||||
|
// GetMetrics must return quickly (it skips disk.Size() while pending) and report 0.
|
||||||
|
if got := sc.GetMetrics().DiskTierReady; got != 0 {
|
||||||
|
t.Errorf("during pending attach, DiskTierReady=%d, want 0", got)
|
||||||
|
}
|
||||||
|
|
||||||
// Wait the barrier (exercises the attach go's Size wait)
|
// Wait the barrier (exercises the attach go's Size wait)
|
||||||
_ = sc.disk.Size()
|
_ = sc.disk.Size()
|
||||||
|
|
||||||
@@ -1083,6 +1093,106 @@ func TestDiskOnlyDelayedAttach(t *testing.T) {
|
|||||||
} else {
|
} else {
|
||||||
rc.Close()
|
rc.Close()
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// After attach, the disk tier must be marked ready (1)
|
||||||
|
if got := sc.GetMetrics().DiskTierReady; got != 1 {
|
||||||
|
t.Errorf("post-attach DiskTierReady=%d, want 1 (ready)", got)
|
||||||
|
}
|
||||||
|
|
||||||
|
// /metrics text output includes the disk_tier_ready line
|
||||||
|
rec := httptest.NewRecorder()
|
||||||
|
metrics.WriteText(rec, sc.GetMetrics())
|
||||||
|
if !bytes.Contains(rec.Body.Bytes(), []byte("disk_tier_ready 1")) {
|
||||||
|
t.Errorf("WriteText output missing \"disk_tier_ready 1\": %q", rec.Body.String())
|
||||||
|
}
|
||||||
|
if !bytes.Contains(rec.Body.Bytes(), []byte("capacity_pressure_events")) {
|
||||||
|
t.Errorf("WriteText output missing capacity_pressure_events: %q", rec.Body.String())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestDiskTierSignalMemoryOnly covers memory-only mode: DiskTierReady=1 (N/A, not
|
||||||
|
// waiting on disk attach) and heartbeat header X-SteamCache-Disk-Tier: disabled.
|
||||||
|
func TestDiskTierSignalMemoryOnly(t *testing.T) {
|
||||||
|
sc, err := New("127.0.0.1:0", "1MB", "0", t.TempDir(), "", "lru", "lru", 10, 5, "0", nil)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("New memory-only: %v", err)
|
||||||
|
}
|
||||||
|
t.Cleanup(func() { sc.Shutdown() })
|
||||||
|
|
||||||
|
if got := sc.GetMetrics().DiskTierReady; got != 1 {
|
||||||
|
t.Errorf("DiskTierReady=%d, want 1 (memory-only = N/A/not pending)", got)
|
||||||
|
}
|
||||||
|
|
||||||
|
req := httptest.NewRequest("GET", "/lancache-heartbeat", nil)
|
||||||
|
rec := httptest.NewRecorder()
|
||||||
|
sc.ServeHTTP(rec, req)
|
||||||
|
if rec.Code != http.StatusNoContent {
|
||||||
|
t.Errorf("heartbeat status=%d, want 204", rec.Code)
|
||||||
|
}
|
||||||
|
if got := rec.Header().Get("X-SteamCache-Disk-Tier"); got != "disabled" {
|
||||||
|
t.Errorf("X-SteamCache-Disk-Tier=%q, want disabled", got)
|
||||||
|
}
|
||||||
|
if got := rec.Header().Get("X-LanCache-Processed-By"); got != "SteamCache2" {
|
||||||
|
t.Errorf("X-LanCache-Processed-By=%q, want SteamCache2", got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestDiskTierSignalMixedPendingReady covers mixed mode: DiskTierReady=0 (header
|
||||||
|
// pending) while the disk attach is in the Size barrier, then DiskTierReady=1
|
||||||
|
// (header ready) after the barrier opens and the attach goroutine sets SetSlow.
|
||||||
|
// An init hold keeps the empty-dir attach from finishing between the pending
|
||||||
|
// metric check and the heartbeat, which otherwise races under CI load.
|
||||||
|
func TestDiskTierSignalMixedPendingReady(t *testing.T) {
|
||||||
|
td := t.TempDir()
|
||||||
|
diskPath := filepath.Join(td, "disk")
|
||||||
|
if err := os.MkdirAll(diskPath, 0755); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
hold := make(chan struct{})
|
||||||
|
var holdOnce sync.Once
|
||||||
|
closeHold := func() { holdOnce.Do(func() { close(hold) }) }
|
||||||
|
disk.RegisterInitHold(diskPath, hold)
|
||||||
|
t.Cleanup(func() {
|
||||||
|
closeHold()
|
||||||
|
disk.ClearInitHold(diskPath)
|
||||||
|
})
|
||||||
|
|
||||||
|
sc, err := New("127.0.0.1:0", "1MB", "10MB", diskPath, "", "lru", "lru", 10, 1, "0", nil)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("New mixed: %v", err)
|
||||||
|
}
|
||||||
|
t.Cleanup(func() { sc.Shutdown() })
|
||||||
|
t.Cleanup(closeHold) // before Shutdown: attach is blocked in Size() until the hold closes
|
||||||
|
|
||||||
|
// Pending window is held open until closeHold; attach cannot finish.
|
||||||
|
if got := sc.GetMetrics().DiskTierReady; got != 0 {
|
||||||
|
t.Errorf("immediate DiskTierReady=%d, want 0 (pending)", got)
|
||||||
|
}
|
||||||
|
req := httptest.NewRequest("GET", "/lancache-heartbeat", nil)
|
||||||
|
rec := httptest.NewRecorder()
|
||||||
|
sc.ServeHTTP(rec, req)
|
||||||
|
if got := rec.Header().Get("X-SteamCache-Disk-Tier"); got != "pending" {
|
||||||
|
t.Errorf("heartbeat header=%q, want pending", got)
|
||||||
|
}
|
||||||
|
|
||||||
|
closeHold()
|
||||||
|
_ = sc.disk.Size()
|
||||||
|
deadline := time.Now().Add(2 * time.Second)
|
||||||
|
for time.Now().Before(deadline) {
|
||||||
|
if sc.GetMetrics().DiskTierReady == 1 {
|
||||||
|
break
|
||||||
|
}
|
||||||
|
time.Sleep(1 * time.Millisecond)
|
||||||
|
}
|
||||||
|
if got := sc.GetMetrics().DiskTierReady; got != 1 {
|
||||||
|
t.Fatalf("DiskTierReady=%d after barrier, want 1 (ready)", got)
|
||||||
|
}
|
||||||
|
|
||||||
|
rec2 := httptest.NewRecorder()
|
||||||
|
sc.ServeHTTP(rec2, httptest.NewRequest("GET", "/lancache-heartbeat", nil))
|
||||||
|
if got := rec2.Header().Get("X-SteamCache-Disk-Tier"); got != "ready" {
|
||||||
|
t.Errorf("heartbeat header=%q, want ready", got)
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// --- Phase 2: narrow black-box tests for the new wrapper types ---
|
// --- Phase 2: narrow black-box tests for the new wrapper types ---
|
||||||
@@ -1223,3 +1333,135 @@ func TestDirectFetchRejectsNonSteamHost(t *testing.T) {
|
|||||||
t.Errorf("non-CDN Host: expected 400, got %d", rec2.Code)
|
t.Errorf("non-CDN Host: expected 400, got %d", rec2.Code)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// TestCacheKeySharedAcrossCDNHostAliases verifies that one cache entry serves
|
||||||
|
// the same depot object across different Steam CDN host aliases: the key uses
|
||||||
|
// only the request path (never the Host header or an absolute-form target
|
||||||
|
// host), so hits climb instead of re-stamping upstream per host rotation.
|
||||||
|
func TestCacheKeySharedAcrossCDNHostAliases(t *testing.T) {
|
||||||
|
body := []byte("depot chunk body for host-alias keying")
|
||||||
|
var upstreamCalls atomic.Int64
|
||||||
|
f := func(w http.ResponseWriter, r *http.Request) {
|
||||||
|
upstreamCalls.Add(1)
|
||||||
|
w.Header().Set("Content-Type", "application/octet-stream")
|
||||||
|
_, _ = w.Write(body)
|
||||||
|
}
|
||||||
|
sc, _ := newTestCacheWithFakeUpstream(t, f, "1MB", "0")
|
||||||
|
srv := newCacheServer(t, sc)
|
||||||
|
const depotPath = "/depot/1684171/chunk/abc123"
|
||||||
|
const ua = "Valve/Steam HTTP Client 1.0"
|
||||||
|
c := &http.Client{Timeout: 5 * time.Second}
|
||||||
|
|
||||||
|
// 1) MISS under the first CDN alias (origin-form target, Host: cdn1).
|
||||||
|
req1, err := http.NewRequest("GET", srv.URL+depotPath, nil)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
req1.Host = "cdn1.steamcontent.com"
|
||||||
|
req1.Header.Set("User-Agent", ua)
|
||||||
|
resp1, err := c.Do(req1)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("host-alias MISS request: %v", err)
|
||||||
|
}
|
||||||
|
data1, err := io.ReadAll(resp1.Body)
|
||||||
|
resp1.Body.Close()
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if resp1.StatusCode != http.StatusOK {
|
||||||
|
t.Fatalf("host-alias MISS: expected 200, got %d", resp1.StatusCode)
|
||||||
|
}
|
||||||
|
if got := resp1.Header.Get("X-LanCache-Status"); got != "MISS" {
|
||||||
|
t.Fatalf("host-alias MISS: expected X-LanCache-Status MISS, got %q", got)
|
||||||
|
}
|
||||||
|
if !bytes.Equal(data1, body) {
|
||||||
|
t.Fatalf("host-alias MISS: body mismatch: got %q", data1)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Bounded wait for the entry to be visible before hitting the next alias
|
||||||
|
// (the MISS handler streams the body to the client before the VFS write).
|
||||||
|
key, err := generateServiceCacheKey(depotPath, "steam")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
deadline := time.Now().Add(2 * time.Second)
|
||||||
|
for {
|
||||||
|
if rc, e := sc.vfs.Open(key); e == nil {
|
||||||
|
_ = rc.Close()
|
||||||
|
break
|
||||||
|
}
|
||||||
|
if time.Now().After(deadline) {
|
||||||
|
t.Fatalf("cache entry %q not visible after MISS", key)
|
||||||
|
}
|
||||||
|
time.Sleep(5 * time.Millisecond)
|
||||||
|
}
|
||||||
|
|
||||||
|
// 2) Same depot path under the second CDN alias (Host header only) -> HIT.
|
||||||
|
req2, err := http.NewRequest("GET", srv.URL+depotPath, nil)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
req2.Host = "cdn2.steamcontent.com"
|
||||||
|
req2.Header.Set("User-Agent", ua)
|
||||||
|
resp2, err := c.Do(req2)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("second host-alias request: %v", err)
|
||||||
|
}
|
||||||
|
data2, err := io.ReadAll(resp2.Body)
|
||||||
|
resp2.Body.Close()
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if resp2.StatusCode != http.StatusOK {
|
||||||
|
t.Fatalf("second host-alias: expected 200, got %d", resp2.StatusCode)
|
||||||
|
}
|
||||||
|
if got := resp2.Header.Get("X-LanCache-Status"); got != "HIT" {
|
||||||
|
t.Fatalf("second host-alias: expected X-LanCache-Status HIT, got %q", got)
|
||||||
|
}
|
||||||
|
if !bytes.Equal(data2, body) {
|
||||||
|
t.Fatalf("second host-alias: body mismatch: got %q", data2)
|
||||||
|
}
|
||||||
|
|
||||||
|
// 3) Same depot path with an absolute-form target embedding a third CDN
|
||||||
|
// hostname in the URL itself -> still a HIT on the same entry.
|
||||||
|
// (Go's http client always sends origin-form targets, so use raw HTTP.)
|
||||||
|
conn, err := net.Dial("tcp", srv.Listener.Addr().String())
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
defer conn.Close()
|
||||||
|
rawRequest := "GET http://cdn3.steamcontent.com" + depotPath + " HTTP/1.1\r\n" +
|
||||||
|
"Host: cdn3.steamcontent.com\r\n" +
|
||||||
|
"User-Agent: " + ua + "\r\n" +
|
||||||
|
"Connection: close\r\n\r\n"
|
||||||
|
if _, err := conn.Write([]byte(rawRequest)); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
rawReq, err := http.NewRequest("GET", "http://cdn3.steamcontent.com"+depotPath, nil)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
resp3, err := http.ReadResponse(bufio.NewReader(conn), rawReq)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
defer resp3.Body.Close()
|
||||||
|
data3, err := io.ReadAll(resp3.Body)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if resp3.StatusCode != http.StatusOK {
|
||||||
|
t.Fatalf("absolute-form host-alias: expected 200, got %d", resp3.StatusCode)
|
||||||
|
}
|
||||||
|
if got := resp3.Header.Get("X-LanCache-Status"); got != "HIT" {
|
||||||
|
t.Fatalf("absolute-form host-alias: expected X-LanCache-Status HIT, got %q", got)
|
||||||
|
}
|
||||||
|
if !bytes.Equal(data3, body) {
|
||||||
|
t.Fatalf("absolute-form host-alias: body mismatch: got %q", data3)
|
||||||
|
}
|
||||||
|
|
||||||
|
// All three aliases must have shared one upstream fetch.
|
||||||
|
if got := upstreamCalls.Load(); got != 1 {
|
||||||
|
t.Errorf("upstream fetched %d times across host aliases, want 1", got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|||||||
+59
-18
@@ -45,6 +45,25 @@ type DiskFS struct {
|
|||||||
initCloseOnce sync.Once
|
initCloseOnce sync.Once
|
||||||
startupEvict func(vfs.VFS, uint) uint // passed to New (via gc.GetGCAlgorithm); invoked as last step of bg init if over cap (no post-ctor race)
|
startupEvict func(vfs.VFS, uint) uint // passed to New (via gc.GetGCAlgorithm); invoked as last step of bg init if over cap (no post-ctor race)
|
||||||
metrics *metrics.Metrics
|
metrics *metrics.Metrics
|
||||||
|
// initHold, if non-nil, is received on before closing initDone (test pending-window hold).
|
||||||
|
initHold <-chan struct{}
|
||||||
|
}
|
||||||
|
|
||||||
|
// initHolds is a per-root registry of optional init holds (root path -> <-chan struct{}).
|
||||||
|
// Tests call RegisterInitHold before New so that instance copies the channel and waits
|
||||||
|
// before closing initDone; production never registers, so init is unchanged.
|
||||||
|
var initHolds sync.Map
|
||||||
|
|
||||||
|
// RegisterInitHold registers a channel that DiskFS.New for this root copies onto that
|
||||||
|
// instance. calculateSizeAndPopulateIndex receives on it before closing initDone, so
|
||||||
|
// tests can observe the pending-attach window. Other DiskFS roots are unaffected.
|
||||||
|
func RegisterInitHold(root string, ch <-chan struct{}) {
|
||||||
|
initHolds.Store(root, ch)
|
||||||
|
}
|
||||||
|
|
||||||
|
// ClearInitHold removes a previously registered hold for root.
|
||||||
|
func ClearInitHold(root string) {
|
||||||
|
initHolds.Delete(root)
|
||||||
}
|
}
|
||||||
|
|
||||||
// shardPath converts a Steam cache key to a sharded directory path to reduce inode pressure
|
// shardPath converts a Steam cache key to a sharded directory path to reduce inode pressure
|
||||||
@@ -129,6 +148,12 @@ func New(root string, capacity int64, evict func(vfs.VFS, uint) uint) (*DiskFS,
|
|||||||
startupEvict: evict,
|
startupEvict: evict,
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if v, ok := initHolds.Load(root); ok {
|
||||||
|
if ch, ok := v.(<-chan struct{}); ok {
|
||||||
|
d.initHold = ch
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
d.initDone = make(chan struct{})
|
d.initDone = make(chan struct{})
|
||||||
// Launch heavy population asynchronously so New returns fast (scans millions of files without blocking ctor or using O(N) temp RAM).
|
// Launch heavy population asynchronously so New returns fast (scans millions of files without blocking ctor or using O(N) temp RAM).
|
||||||
// The initDone barrier ensures first Size() and subsequent ops (including late tier attach) see fully populated + post-eviction state.
|
// The initDone barrier ensures first Size() and subsequent ops (including late tier attach) see fully populated + post-eviction state.
|
||||||
@@ -152,7 +177,7 @@ func (d *DiskFS) calculateSizeAndPopulateIndex() {
|
|||||||
if r := recover(); r != nil {
|
if r := recover(); r != nil {
|
||||||
logger.Logger.Error().Interface("recovered_panic", r).Msg("calculateSizeAndPopulateIndex panicked; ensuring initDone closed to unblock Size waiters and prevent hang")
|
logger.Logger.Error().Interface("recovered_panic", r).Msg("calculateSizeAndPopulateIndex panicked; ensuring initDone closed to unblock Size waiters and prevent hang")
|
||||||
}
|
}
|
||||||
d.initCloseOnce.Do(func() { close(d.initDone) })
|
d.closeInitDone()
|
||||||
}()
|
}()
|
||||||
|
|
||||||
tstart := time.Now()
|
tstart := time.Now()
|
||||||
@@ -243,7 +268,19 @@ func (d *DiskFS) calculateSizeAndPopulateIndex() {
|
|||||||
|
|
||||||
// Signal readiness: Size() and callers (late tier attach + Evict*) now see correct populated + post-eviction state.
|
// Signal readiness: Size() and callers (late tier attach + Evict*) now see correct populated + post-eviction state.
|
||||||
// Use Once (recover path also uses it) to guarantee exactly one close even under panic.
|
// Use Once (recover path also uses it) to guarantee exactly one close even under panic.
|
||||||
d.initCloseOnce.Do(func() { close(d.initDone) })
|
d.closeInitDone()
|
||||||
|
}
|
||||||
|
|
||||||
|
// closeInitDone receives on a copied test hold (if any) then closes initDone once.
|
||||||
|
// Both the normal end of calculateSizeAndPopulateIndex and the panic-recovery defer
|
||||||
|
// call this so Size() waiters unblock in either path.
|
||||||
|
func (d *DiskFS) closeInitDone() {
|
||||||
|
d.initCloseOnce.Do(func() {
|
||||||
|
if d.initHold != nil {
|
||||||
|
<-d.initHold
|
||||||
|
}
|
||||||
|
close(d.initDone)
|
||||||
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
// insertBatch populates info/LRU under lock for a bounded batch (follows maxEvictBatch pattern for short critical sections).
|
// insertBatch populates info/LRU under lock for a bounded batch (follows maxEvictBatch pattern for short critical sections).
|
||||||
@@ -403,11 +440,13 @@ func (d *DiskFS) Create(key string, size int64) (io.WriteCloser, error) {
|
|||||||
dir := filepath.Dir(path)
|
dir := filepath.Dir(path)
|
||||||
// 0700 (not 0755): per-shard cache dirs hold untrusted CDN content; restrict to owner only (G301 addressed).
|
// 0700 (not 0755): per-shard cache dirs hold untrusted CDN content; restrict to owner only (G301 addressed).
|
||||||
if err := os.MkdirAll(dir, 0700); err != nil {
|
if err := os.MkdirAll(dir, 0700); err != nil {
|
||||||
|
d.recordIfNoSpace(err)
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
|
|
||||||
file, err := os.Create(path) // #nosec G304 -- path built by pathForKey from sanitized (Clean, no ..) hash-derived key under trusted disk.root; no untrusted file inclusion
|
file, err := os.Create(path) // #nosec G304 -- path built by pathForKey from sanitized (Clean, no ..) hash-derived key under trusted disk.root; no untrusted file inclusion
|
||||||
if err != nil {
|
if err != nil {
|
||||||
|
d.recordIfNoSpace(err)
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -438,7 +477,19 @@ type diskWriteCloser struct {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func (dwc *diskWriteCloser) Write(p []byte) (n int, err error) {
|
func (dwc *diskWriteCloser) Write(p []byte) (n int, err error) {
|
||||||
return dwc.file.Write(p)
|
n, err = dwc.file.Write(p)
|
||||||
|
if err != nil {
|
||||||
|
dwc.disk.recordIfNoSpace(err)
|
||||||
|
}
|
||||||
|
return n, err
|
||||||
|
}
|
||||||
|
|
||||||
|
// recordIfNoSpace increments capacity_pressure_events and logs when err is ENOSPC (or Windows disk-full).
|
||||||
|
func (d *DiskFS) recordIfNoSpace(err error) {
|
||||||
|
if !isNoSpaceError(err) {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
metrics.NoteNoSpace(d.metrics, err)
|
||||||
}
|
}
|
||||||
|
|
||||||
func (dwc *diskWriteCloser) Close() error {
|
func (dwc *diskWriteCloser) Close() error {
|
||||||
@@ -722,9 +773,7 @@ func (d *DiskFS) EvictLRU(bytesNeeded uint) uint {
|
|||||||
}
|
}
|
||||||
d.mu.Unlock()
|
d.mu.Unlock()
|
||||||
|
|
||||||
if d.metrics != nil && evicted > 0 {
|
metrics.NoteSoftEviction(d.metrics, "disk", evicted)
|
||||||
d.metrics.IncrementEvictions()
|
|
||||||
}
|
|
||||||
return evicted
|
return evicted
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -775,9 +824,7 @@ func (d *DiskFS) EvictBySize(bytesNeeded uint, ascending bool) uint {
|
|||||||
}
|
}
|
||||||
d.mu.Unlock()
|
d.mu.Unlock()
|
||||||
|
|
||||||
if d.metrics != nil && evicted > 0 {
|
metrics.NoteSoftEviction(d.metrics, "disk", evicted)
|
||||||
d.metrics.IncrementEvictions()
|
|
||||||
}
|
|
||||||
return evicted
|
return evicted
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -826,9 +873,7 @@ func (d *DiskFS) EvictFIFO(bytesNeeded uint) uint {
|
|||||||
}
|
}
|
||||||
d.mu.Unlock()
|
d.mu.Unlock()
|
||||||
|
|
||||||
if d.metrics != nil && evicted > 0 {
|
metrics.NoteSoftEviction(d.metrics, "disk", evicted)
|
||||||
d.metrics.IncrementEvictions()
|
|
||||||
}
|
|
||||||
return evicted
|
return evicted
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -882,9 +927,7 @@ func (d *DiskFS) EvictLFU(bytesNeeded uint) uint {
|
|||||||
}
|
}
|
||||||
d.mu.Unlock()
|
d.mu.Unlock()
|
||||||
|
|
||||||
if d.metrics != nil && evicted > 0 {
|
metrics.NoteSoftEviction(d.metrics, "disk", evicted)
|
||||||
d.metrics.IncrementEvictions()
|
|
||||||
}
|
|
||||||
return evicted
|
return evicted
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -939,8 +982,6 @@ func (d *DiskFS) EvictHybrid(bytesNeeded uint) uint {
|
|||||||
}
|
}
|
||||||
d.mu.Unlock()
|
d.mu.Unlock()
|
||||||
|
|
||||||
if d.metrics != nil && evicted > 0 {
|
metrics.NoteSoftEviction(d.metrics, "disk", evicted)
|
||||||
d.metrics.IncrementEvictions()
|
|
||||||
}
|
|
||||||
return evicted
|
return evicted
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -11,6 +11,7 @@ import (
|
|||||||
"testing"
|
"testing"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
|
"s1d3sw1ped/steamcache2/steamcache/metrics"
|
||||||
"s1d3sw1ped/steamcache2/vfs"
|
"s1d3sw1ped/steamcache2/vfs"
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -122,6 +123,42 @@ func TestDiskFS_InitPopulatesIndexOnRestart(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestDiskFS_CapacityPressureOnEvict(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
td := t.TempDir()
|
||||||
|
d, err := New(td, 500, nil)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
_ = d.Size()
|
||||||
|
met := metrics.NewMetrics()
|
||||||
|
d.SetMetrics(met)
|
||||||
|
for i := 0; i < 3; i++ {
|
||||||
|
k := "f" + string(rune('0'+i))
|
||||||
|
w, cerr := d.Create(k, 200)
|
||||||
|
if cerr != nil {
|
||||||
|
t.Fatal(cerr)
|
||||||
|
}
|
||||||
|
if _, werr := w.Write(make([]byte, 200)); werr != nil {
|
||||||
|
t.Fatal(werr)
|
||||||
|
}
|
||||||
|
if cerr := w.Close(); cerr != nil {
|
||||||
|
t.Fatal(cerr)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
evicted := d.EvictLRU(100)
|
||||||
|
if evicted == 0 {
|
||||||
|
t.Fatalf("expected eviction under cap, size=%d cap=%d", d.Size(), d.Capacity())
|
||||||
|
}
|
||||||
|
st := met.GetStats()
|
||||||
|
if st.Evictions == 0 {
|
||||||
|
t.Error("evictions counter not incremented under disk cap pressure")
|
||||||
|
}
|
||||||
|
if st.CapacityPressureEvents == 0 {
|
||||||
|
t.Error("capacity_pressure_events not incremented under disk cap pressure")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestDiskFS_EvictAndLazyStat(t *testing.T) {
|
func TestDiskFS_EvictAndLazyStat(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
td := t.TempDir()
|
td := t.TempDir()
|
||||||
@@ -138,10 +175,21 @@ func TestDiskFS_EvictAndLazyStat(t *testing.T) {
|
|||||||
w.Write(make([]byte, 120))
|
w.Write(make([]byte, 120))
|
||||||
w.Close()
|
w.Close()
|
||||||
}
|
}
|
||||||
|
met := metrics.NewMetrics()
|
||||||
|
d.SetMetrics(met)
|
||||||
ev := d.EvictLRU(200)
|
ev := d.EvictLRU(200)
|
||||||
if ev == 0 {
|
if ev == 0 {
|
||||||
t.Log("no evict (size calc async or snapshot tolerance?)")
|
t.Log("no evict (size calc async or snapshot tolerance?)")
|
||||||
}
|
}
|
||||||
|
if ev > 0 {
|
||||||
|
st := met.GetStats()
|
||||||
|
if st.Evictions == 0 {
|
||||||
|
t.Error("evictions counter not incremented after disk EvictLRU freed bytes")
|
||||||
|
}
|
||||||
|
if st.CapacityPressureEvents == 0 {
|
||||||
|
t.Error("capacity_pressure_events not incremented after disk EvictLRU freed bytes")
|
||||||
|
}
|
||||||
|
}
|
||||||
// Explicit post-evict consistency checks: for any key no longer visible via Stat, its on-disk
|
// Explicit post-evict consistency checks: for any key no longer visible via Stat, its on-disk
|
||||||
// file must be absent (verifies coordinated unlink + no resurrection via lazy discovery).
|
// file must be absent (verifies coordinated unlink + no resurrection via lazy discovery).
|
||||||
// Keys still present after this small evict are allowed (accounting tolerance in raw DiskFS).
|
// Keys still present after this small evict are allowed (accounting tolerance in raw DiskFS).
|
||||||
@@ -614,3 +662,57 @@ func TestDiskFS_NewMkdirError(t *testing.T) {
|
|||||||
t.Errorf("expected mkdir failure error for file-as-dir, got: %v", err)
|
t.Errorf("expected mkdir failure error for file-as-dir, got: %v", err)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// TestDiskFS_InitHoldBlocksOnlyRegisteredRoot covers the per-root init hold:
|
||||||
|
// Size() stays blocked while the hold is open, and a DiskFS on a different root
|
||||||
|
// does not wait on that hold.
|
||||||
|
func TestDiskFS_InitHoldBlocksOnlyRegisteredRoot(t *testing.T) {
|
||||||
|
td := t.TempDir()
|
||||||
|
hold := make(chan struct{})
|
||||||
|
var holdOnce sync.Once
|
||||||
|
closeHold := func() { holdOnce.Do(func() { close(hold) }) }
|
||||||
|
RegisterInitHold(td, hold)
|
||||||
|
t.Cleanup(func() {
|
||||||
|
closeHold()
|
||||||
|
ClearInitHold(td)
|
||||||
|
})
|
||||||
|
|
||||||
|
d, err := New(td, 10*1024*1024, nil)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
|
||||||
|
blocked := make(chan struct{})
|
||||||
|
go func() {
|
||||||
|
_ = d.Size()
|
||||||
|
close(blocked)
|
||||||
|
}()
|
||||||
|
select {
|
||||||
|
case <-blocked:
|
||||||
|
t.Fatal("Size returned while init hold still open")
|
||||||
|
case <-time.After(50 * time.Millisecond):
|
||||||
|
}
|
||||||
|
|
||||||
|
td2 := t.TempDir()
|
||||||
|
d2, err := New(td2, 10*1024*1024, nil)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
other := make(chan struct{})
|
||||||
|
go func() {
|
||||||
|
_ = d2.Size()
|
||||||
|
close(other)
|
||||||
|
}()
|
||||||
|
select {
|
||||||
|
case <-other:
|
||||||
|
case <-time.After(2 * time.Second):
|
||||||
|
t.Fatal("unrelated DiskFS Size hung; init hold leaked across roots")
|
||||||
|
}
|
||||||
|
|
||||||
|
closeHold()
|
||||||
|
select {
|
||||||
|
case <-blocked:
|
||||||
|
case <-time.After(2 * time.Second):
|
||||||
|
t.Fatal("Size did not return after init hold released")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|||||||
@@ -0,0 +1,14 @@
|
|||||||
|
//go:build !windows
|
||||||
|
|
||||||
|
package disk
|
||||||
|
|
||||||
|
import (
|
||||||
|
"errors"
|
||||||
|
|
||||||
|
"golang.org/x/sys/unix"
|
||||||
|
)
|
||||||
|
|
||||||
|
// isNoSpaceError reports whether err is ENOSPC (or wraps it).
|
||||||
|
func isNoSpaceError(err error) bool {
|
||||||
|
return err != nil && errors.Is(err, unix.ENOSPC)
|
||||||
|
}
|
||||||
@@ -0,0 +1,58 @@
|
|||||||
|
//go:build !windows
|
||||||
|
|
||||||
|
package disk
|
||||||
|
|
||||||
|
import (
|
||||||
|
"io"
|
||||||
|
"os"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"golang.org/x/sys/unix"
|
||||||
|
|
||||||
|
"s1d3sw1ped/steamcache2/steamcache/metrics"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestIsNoSpaceError(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
if isNoSpaceError(nil) {
|
||||||
|
t.Error("nil must not be ENOSPC")
|
||||||
|
}
|
||||||
|
if isNoSpaceError(io.EOF) {
|
||||||
|
t.Error("EOF must not be ENOSPC")
|
||||||
|
}
|
||||||
|
if !isNoSpaceError(unix.ENOSPC) {
|
||||||
|
t.Error("unix.ENOSPC should match")
|
||||||
|
}
|
||||||
|
wrapped := &os.PathError{Op: "write", Path: "x", Err: unix.ENOSPC}
|
||||||
|
if !isNoSpaceError(wrapped) {
|
||||||
|
t.Error("PathError wrapping ENOSPC should match")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestDiskFS_ENOSPCCapacityPressure(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
d, err := New(t.TempDir(), 1024, nil)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
met := metrics.NewMetrics()
|
||||||
|
d.SetMetrics(met)
|
||||||
|
|
||||||
|
d.recordIfNoSpace(io.EOF)
|
||||||
|
if got := met.GetStats().CapacityPressureEvents; got != 0 {
|
||||||
|
t.Fatalf("non-ENOSPC counted: %d", got)
|
||||||
|
}
|
||||||
|
|
||||||
|
d.recordIfNoSpace(unix.ENOSPC)
|
||||||
|
if got := met.GetStats().CapacityPressureEvents; got != 1 {
|
||||||
|
t.Fatalf("unix.ENOSPC: CapacityPressureEvents=%d, want 1", got)
|
||||||
|
}
|
||||||
|
if got := met.GetStats().Evictions; got != 0 {
|
||||||
|
t.Fatalf("ENOSPC must not increment evictions, got %d", got)
|
||||||
|
}
|
||||||
|
|
||||||
|
d.recordIfNoSpace(&os.PathError{Op: "write", Path: "p", Err: unix.ENOSPC})
|
||||||
|
if got := met.GetStats().CapacityPressureEvents; got != 2 {
|
||||||
|
t.Fatalf("wrapped ENOSPC: CapacityPressureEvents=%d, want 2", got)
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,17 @@
|
|||||||
|
//go:build windows
|
||||||
|
|
||||||
|
package disk
|
||||||
|
|
||||||
|
import (
|
||||||
|
"errors"
|
||||||
|
|
||||||
|
"golang.org/x/sys/windows"
|
||||||
|
)
|
||||||
|
|
||||||
|
// isNoSpaceError reports whether err is a Windows disk-full equivalent of ENOSPC.
|
||||||
|
func isNoSpaceError(err error) bool {
|
||||||
|
if err == nil {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
return errors.Is(err, windows.ERROR_DISK_FULL) || errors.Is(err, windows.ERROR_HANDLE_DISK_FULL)
|
||||||
|
}
|
||||||
@@ -0,0 +1,56 @@
|
|||||||
|
//go:build windows
|
||||||
|
|
||||||
|
package disk
|
||||||
|
|
||||||
|
import (
|
||||||
|
"io"
|
||||||
|
"os"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"golang.org/x/sys/windows"
|
||||||
|
|
||||||
|
"s1d3sw1ped/steamcache2/steamcache/metrics"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestIsNoSpaceError(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
if isNoSpaceError(nil) {
|
||||||
|
t.Error("nil must not be disk-full")
|
||||||
|
}
|
||||||
|
if isNoSpaceError(io.EOF) {
|
||||||
|
t.Error("EOF must not be disk-full")
|
||||||
|
}
|
||||||
|
if !isNoSpaceError(windows.ERROR_DISK_FULL) {
|
||||||
|
t.Error("ERROR_DISK_FULL should match")
|
||||||
|
}
|
||||||
|
if !isNoSpaceError(windows.ERROR_HANDLE_DISK_FULL) {
|
||||||
|
t.Error("ERROR_HANDLE_DISK_FULL should match")
|
||||||
|
}
|
||||||
|
wrapped := &os.PathError{Op: "write", Path: "x", Err: windows.ERROR_DISK_FULL}
|
||||||
|
if !isNoSpaceError(wrapped) {
|
||||||
|
t.Error("PathError wrapping ERROR_DISK_FULL should match")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestDiskFS_ENOSPCCapacityPressure(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
d, err := New(t.TempDir(), 1024, nil)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
met := metrics.NewMetrics()
|
||||||
|
d.SetMetrics(met)
|
||||||
|
|
||||||
|
d.recordIfNoSpace(io.EOF)
|
||||||
|
if got := met.GetStats().CapacityPressureEvents; got != 0 {
|
||||||
|
t.Fatalf("non-ENOSPC counted: %d", got)
|
||||||
|
}
|
||||||
|
|
||||||
|
d.recordIfNoSpace(windows.ERROR_DISK_FULL)
|
||||||
|
if got := met.GetStats().CapacityPressureEvents; got != 1 {
|
||||||
|
t.Fatalf("ERROR_DISK_FULL: CapacityPressureEvents=%d, want 1", got)
|
||||||
|
}
|
||||||
|
if got := met.GetStats().Evictions; got != 0 {
|
||||||
|
t.Fatalf("disk-full must not increment evictions, got %d", got)
|
||||||
|
}
|
||||||
|
}
|
||||||
+5
-15
@@ -358,9 +358,7 @@ func (m *MemoryFS) EvictLRU(bytesNeeded uint) uint {
|
|||||||
}
|
}
|
||||||
m.mu.Unlock()
|
m.mu.Unlock()
|
||||||
|
|
||||||
if m.metrics != nil && evicted > 0 {
|
metrics.NoteSoftEviction(m.metrics, "memory", evicted)
|
||||||
m.metrics.IncrementEvictions()
|
|
||||||
}
|
|
||||||
return evicted
|
return evicted
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -413,9 +411,7 @@ func (m *MemoryFS) EvictBySize(bytesNeeded uint, ascending bool) uint {
|
|||||||
}
|
}
|
||||||
m.mu.Unlock()
|
m.mu.Unlock()
|
||||||
|
|
||||||
if m.metrics != nil && evicted > 0 {
|
metrics.NoteSoftEviction(m.metrics, "memory", evicted)
|
||||||
m.metrics.IncrementEvictions()
|
|
||||||
}
|
|
||||||
return evicted
|
return evicted
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -464,9 +460,7 @@ func (m *MemoryFS) EvictFIFO(bytesNeeded uint) uint {
|
|||||||
}
|
}
|
||||||
m.mu.Unlock()
|
m.mu.Unlock()
|
||||||
|
|
||||||
if m.metrics != nil && evicted > 0 {
|
metrics.NoteSoftEviction(m.metrics, "memory", evicted)
|
||||||
m.metrics.IncrementEvictions()
|
|
||||||
}
|
|
||||||
return evicted
|
return evicted
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -520,9 +514,7 @@ func (m *MemoryFS) EvictLFU(bytesNeeded uint) uint {
|
|||||||
}
|
}
|
||||||
m.mu.Unlock()
|
m.mu.Unlock()
|
||||||
|
|
||||||
if m.metrics != nil && evicted > 0 {
|
metrics.NoteSoftEviction(m.metrics, "memory", evicted)
|
||||||
m.metrics.IncrementEvictions()
|
|
||||||
}
|
|
||||||
return evicted
|
return evicted
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -578,8 +570,6 @@ func (m *MemoryFS) EvictHybrid(bytesNeeded uint) uint {
|
|||||||
}
|
}
|
||||||
m.mu.Unlock()
|
m.mu.Unlock()
|
||||||
|
|
||||||
if m.metrics != nil && evicted > 0 {
|
metrics.NoteSoftEviction(m.metrics, "memory", evicted)
|
||||||
m.metrics.IncrementEvictions()
|
|
||||||
}
|
|
||||||
return evicted
|
return evicted
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -8,6 +8,8 @@ import (
|
|||||||
"sync/atomic"
|
"sync/atomic"
|
||||||
"testing"
|
"testing"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
|
"s1d3sw1ped/steamcache2/steamcache/metrics"
|
||||||
)
|
)
|
||||||
|
|
||||||
func TestMemoryFS_Basic(t *testing.T) {
|
func TestMemoryFS_Basic(t *testing.T) {
|
||||||
@@ -60,6 +62,8 @@ func TestMemoryFS_EvictUnderPressure(t *testing.T) {
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
|
met := metrics.NewMetrics()
|
||||||
|
m.SetMetrics(met)
|
||||||
// create 3x200 = 600 >500, should trigger internal? but direct evict call
|
// create 3x200 = 600 >500, should trigger internal? but direct evict call
|
||||||
for i := 0; i < 3; i++ {
|
for i := 0; i < 3; i++ {
|
||||||
w, _ := m.Create("f"+string(rune('0'+i)), 200)
|
w, _ := m.Create("f"+string(rune('0'+i)), 200)
|
||||||
@@ -71,6 +75,13 @@ func TestMemoryFS_EvictUnderPressure(t *testing.T) {
|
|||||||
if evicted == 0 || m.Size() > 500 {
|
if evicted == 0 || m.Size() > 500 {
|
||||||
t.Errorf("evict failed: evicted=%d size=%d", evicted, m.Size())
|
t.Errorf("evict failed: evicted=%d size=%d", evicted, m.Size())
|
||||||
}
|
}
|
||||||
|
st := met.GetStats()
|
||||||
|
if st.Evictions == 0 {
|
||||||
|
t.Error("evictions counter not incremented under memory cap pressure")
|
||||||
|
}
|
||||||
|
if st.CapacityPressureEvents == 0 {
|
||||||
|
t.Error("capacity_pressure_events not incremented under memory cap pressure")
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestMemoryFS_SizeNeverExceedsAfterEvict(t *testing.T) {
|
func TestMemoryFS_SizeNeverExceedsAfterEvict(t *testing.T) {
|
||||||
|
|||||||
Reference in New Issue
Block a user