cache: Key by depot path across CDN host aliases #48

Merged
linus merged 1 commits from moss/cache-key-depot-path into develop 2026-09-07 15:18:26 -05:00
3 changed files with 143 additions and 3 deletions
+2
View File
@@ -88,6 +88,8 @@ curl -s -i http://localhost/lancache-heartbeat
A first pass through new content is mostly misses (`hit_rate` near 0). Repeat the same content and `cache_hits` / `hit_rate` should rise.
Cache entries are keyed by depot object path (not the CDN `Host` header), so when Steam rotates CDN hostnames for the same depot path, hits still climb across the aliases.
Steam clients lean on Range requests. When an object is already cached, a Range GET is served locally as 206 from that full object (`range_cache`). On a Range miss the cache still fetches and stores the full upstream body, then returns the requested byte range as 206 (`range_upstream`).
To confirm the process is up (HTTP 204 and `X-LanCache-Processed-By: SteamCache2`):
+6 -3
View File
@@ -279,9 +279,12 @@ func (sc *SteamCache) ServeHTTP(w http.ResponseWriter, r *http.Request) {
// Check if this is a request from a supported service
if service, isSupported := sc.detectService(r); isSupported {
// trim the query parameters from the URL path
// this is necessary because the cache key should not include query parameters
urlPath := strings.SplitN(r.URL.String(), "?", 2)[0] // trim query for cache key (SplitN makes intent explicit vs Cut + ignored bool)
// Cache key is the path only, never the Host: Steam rotates CDN hostnames
// for the same depot object, so different Host headers (or absolute-form
// request targets) for the same path must share one cache entry. r.URL.Path
// is the decoded path (query is never part of it); validateURLPath checks
// this decoded form and url.JoinPath re-escapes it for the upstream join.
urlPath := r.URL.Path
// Validate URL path for security
if err := validateURLPath(urlPath); err != nil {
+135
View File
@@ -2,10 +2,12 @@
package steamcache
import (
"bufio"
"bytes"
"context"
"fmt"
"io"
"net"
"net/http"
"net/http/httptest"
"os"
@@ -17,6 +19,7 @@ import (
"s1d3sw1ped/steamcache2/vfs/vfserror"
"strings"
"sync"
"sync/atomic"
"testing"
"time"
)
@@ -1314,3 +1317,135 @@ func TestDirectFetchRejectsNonSteamHost(t *testing.T) {
t.Errorf("non-CDN Host: expected 400, got %d", rec2.Code)
}
}
// TestCacheKeySharedAcrossCDNHostAliases verifies that one cache entry serves
// the same depot object across different Steam CDN host aliases: the key uses
// only the request path (never the Host header or an absolute-form target
// host), so hits climb instead of re-stamping upstream per host rotation.
func TestCacheKeySharedAcrossCDNHostAliases(t *testing.T) {
body := []byte("depot chunk body for host-alias keying")
var upstreamCalls atomic.Int64
f := func(w http.ResponseWriter, r *http.Request) {
upstreamCalls.Add(1)
w.Header().Set("Content-Type", "application/octet-stream")
_, _ = w.Write(body)
}
sc, _ := newTestCacheWithFakeUpstream(t, f, "1MB", "0")
srv := newCacheServer(t, sc)
const depotPath = "/depot/1684171/chunk/abc123"
const ua = "Valve/Steam HTTP Client 1.0"
c := &http.Client{Timeout: 5 * time.Second}
// 1) MISS under the first CDN alias (origin-form target, Host: cdn1).
req1, err := http.NewRequest("GET", srv.URL+depotPath, nil)
if err != nil {
t.Fatal(err)
}
req1.Host = "cdn1.steamcontent.com"
req1.Header.Set("User-Agent", ua)
resp1, err := c.Do(req1)
if err != nil {
t.Fatalf("host-alias MISS request: %v", err)
}
data1, err := io.ReadAll(resp1.Body)
resp1.Body.Close()
if err != nil {
t.Fatal(err)
}
if resp1.StatusCode != http.StatusOK {
t.Fatalf("host-alias MISS: expected 200, got %d", resp1.StatusCode)
}
if got := resp1.Header.Get("X-LanCache-Status"); got != "MISS" {
t.Fatalf("host-alias MISS: expected X-LanCache-Status MISS, got %q", got)
}
if !bytes.Equal(data1, body) {
t.Fatalf("host-alias MISS: body mismatch: got %q", data1)
}
// Bounded wait for the entry to be visible before hitting the next alias
// (the MISS handler streams the body to the client before the VFS write).
key, err := generateServiceCacheKey(depotPath, "steam")
if err != nil {
t.Fatal(err)
}
deadline := time.Now().Add(2 * time.Second)
for {
if rc, e := sc.vfs.Open(key); e == nil {
_ = rc.Close()
break
}
if time.Now().After(deadline) {
t.Fatalf("cache entry %q not visible after MISS", key)
}
time.Sleep(5 * time.Millisecond)
}
// 2) Same depot path under the second CDN alias (Host header only) -> HIT.
req2, err := http.NewRequest("GET", srv.URL+depotPath, nil)
if err != nil {
t.Fatal(err)
}
req2.Host = "cdn2.steamcontent.com"
req2.Header.Set("User-Agent", ua)
resp2, err := c.Do(req2)
if err != nil {
t.Fatalf("second host-alias request: %v", err)
}
data2, err := io.ReadAll(resp2.Body)
resp2.Body.Close()
if err != nil {
t.Fatal(err)
}
if resp2.StatusCode != http.StatusOK {
t.Fatalf("second host-alias: expected 200, got %d", resp2.StatusCode)
}
if got := resp2.Header.Get("X-LanCache-Status"); got != "HIT" {
t.Fatalf("second host-alias: expected X-LanCache-Status HIT, got %q", got)
}
if !bytes.Equal(data2, body) {
t.Fatalf("second host-alias: body mismatch: got %q", data2)
}
// 3) Same depot path with an absolute-form target embedding a third CDN
// hostname in the URL itself -> still a HIT on the same entry.
// (Go's http client always sends origin-form targets, so use raw HTTP.)
conn, err := net.Dial("tcp", srv.Listener.Addr().String())
if err != nil {
t.Fatal(err)
}
defer conn.Close()
rawRequest := "GET http://cdn3.steamcontent.com" + depotPath + " HTTP/1.1\r\n" +
"Host: cdn3.steamcontent.com\r\n" +
"User-Agent: " + ua + "\r\n" +
"Connection: close\r\n\r\n"
if _, err := conn.Write([]byte(rawRequest)); err != nil {
t.Fatal(err)
}
rawReq, err := http.NewRequest("GET", "http://cdn3.steamcontent.com"+depotPath, nil)
if err != nil {
t.Fatal(err)
}
resp3, err := http.ReadResponse(bufio.NewReader(conn), rawReq)
if err != nil {
t.Fatal(err)
}
defer resp3.Body.Close()
data3, err := io.ReadAll(resp3.Body)
if err != nil {
t.Fatal(err)
}
if resp3.StatusCode != http.StatusOK {
t.Fatalf("absolute-form host-alias: expected 200, got %d", resp3.StatusCode)
}
if got := resp3.Header.Get("X-LanCache-Status"); got != "HIT" {
t.Fatalf("absolute-form host-alias: expected X-LanCache-Status HIT, got %q", got)
}
if !bytes.Equal(data3, body) {
t.Fatalf("absolute-form host-alias: body mismatch: got %q", data3)
}
// All three aliases must have shared one upstream fetch.
if got := upstreamCalls.Load(); got != 1 {
t.Errorf("upstream fetched %d times across host aliases, want 1", got)
}
}