cache: Key by depot path across CDN host aliases
This commit is contained in:
@@ -88,6 +88,8 @@ curl -s -i http://localhost/lancache-heartbeat
|
|||||||
|
|
||||||
A first pass through new content is mostly misses (`hit_rate` near 0). Repeat the same content and `cache_hits` / `hit_rate` should rise.
|
A first pass through new content is mostly misses (`hit_rate` near 0). Repeat the same content and `cache_hits` / `hit_rate` should rise.
|
||||||
|
|
||||||
|
Cache entries are keyed by depot object path (not the CDN `Host` header), so when Steam rotates CDN hostnames for the same depot path, hits still climb across the aliases.
|
||||||
|
|
||||||
Steam clients lean on Range requests. When an object is already cached, a Range GET is served locally as 206 from that full object (`range_cache`). On a Range miss the cache still fetches and stores the full upstream body, then returns the requested byte range as 206 (`range_upstream`).
|
Steam clients lean on Range requests. When an object is already cached, a Range GET is served locally as 206 from that full object (`range_cache`). On a Range miss the cache still fetches and stores the full upstream body, then returns the requested byte range as 206 (`range_upstream`).
|
||||||
|
|
||||||
To confirm the process is up (HTTP 204 and `X-LanCache-Processed-By: SteamCache2`):
|
To confirm the process is up (HTTP 204 and `X-LanCache-Processed-By: SteamCache2`):
|
||||||
|
|||||||
@@ -279,9 +279,12 @@ func (sc *SteamCache) ServeHTTP(w http.ResponseWriter, r *http.Request) {
|
|||||||
|
|
||||||
// Check if this is a request from a supported service
|
// Check if this is a request from a supported service
|
||||||
if service, isSupported := sc.detectService(r); isSupported {
|
if service, isSupported := sc.detectService(r); isSupported {
|
||||||
// trim the query parameters from the URL path
|
// Cache key is the path only, never the Host: Steam rotates CDN hostnames
|
||||||
// this is necessary because the cache key should not include query parameters
|
// for the same depot object, so different Host headers (or absolute-form
|
||||||
urlPath := strings.SplitN(r.URL.String(), "?", 2)[0] // trim query for cache key (SplitN makes intent explicit vs Cut + ignored bool)
|
// request targets) for the same path must share one cache entry. r.URL.Path
|
||||||
|
// is the decoded path (query is never part of it); validateURLPath checks
|
||||||
|
// this decoded form and url.JoinPath re-escapes it for the upstream join.
|
||||||
|
urlPath := r.URL.Path
|
||||||
|
|
||||||
// Validate URL path for security
|
// Validate URL path for security
|
||||||
if err := validateURLPath(urlPath); err != nil {
|
if err := validateURLPath(urlPath); err != nil {
|
||||||
|
|||||||
@@ -2,10 +2,12 @@
|
|||||||
package steamcache
|
package steamcache
|
||||||
|
|
||||||
import (
|
import (
|
||||||
|
"bufio"
|
||||||
"bytes"
|
"bytes"
|
||||||
"context"
|
"context"
|
||||||
"fmt"
|
"fmt"
|
||||||
"io"
|
"io"
|
||||||
|
"net"
|
||||||
"net/http"
|
"net/http"
|
||||||
"net/http/httptest"
|
"net/http/httptest"
|
||||||
"os"
|
"os"
|
||||||
@@ -17,6 +19,7 @@ import (
|
|||||||
"s1d3sw1ped/steamcache2/vfs/vfserror"
|
"s1d3sw1ped/steamcache2/vfs/vfserror"
|
||||||
"strings"
|
"strings"
|
||||||
"sync"
|
"sync"
|
||||||
|
"sync/atomic"
|
||||||
"testing"
|
"testing"
|
||||||
"time"
|
"time"
|
||||||
)
|
)
|
||||||
@@ -1314,3 +1317,135 @@ func TestDirectFetchRejectsNonSteamHost(t *testing.T) {
|
|||||||
t.Errorf("non-CDN Host: expected 400, got %d", rec2.Code)
|
t.Errorf("non-CDN Host: expected 400, got %d", rec2.Code)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// TestCacheKeySharedAcrossCDNHostAliases verifies that one cache entry serves
|
||||||
|
// the same depot object across different Steam CDN host aliases: the key uses
|
||||||
|
// only the request path (never the Host header or an absolute-form target
|
||||||
|
// host), so hits climb instead of re-stamping upstream per host rotation.
|
||||||
|
func TestCacheKeySharedAcrossCDNHostAliases(t *testing.T) {
|
||||||
|
body := []byte("depot chunk body for host-alias keying")
|
||||||
|
var upstreamCalls atomic.Int64
|
||||||
|
f := func(w http.ResponseWriter, r *http.Request) {
|
||||||
|
upstreamCalls.Add(1)
|
||||||
|
w.Header().Set("Content-Type", "application/octet-stream")
|
||||||
|
_, _ = w.Write(body)
|
||||||
|
}
|
||||||
|
sc, _ := newTestCacheWithFakeUpstream(t, f, "1MB", "0")
|
||||||
|
srv := newCacheServer(t, sc)
|
||||||
|
const depotPath = "/depot/1684171/chunk/abc123"
|
||||||
|
const ua = "Valve/Steam HTTP Client 1.0"
|
||||||
|
c := &http.Client{Timeout: 5 * time.Second}
|
||||||
|
|
||||||
|
// 1) MISS under the first CDN alias (origin-form target, Host: cdn1).
|
||||||
|
req1, err := http.NewRequest("GET", srv.URL+depotPath, nil)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
req1.Host = "cdn1.steamcontent.com"
|
||||||
|
req1.Header.Set("User-Agent", ua)
|
||||||
|
resp1, err := c.Do(req1)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("host-alias MISS request: %v", err)
|
||||||
|
}
|
||||||
|
data1, err := io.ReadAll(resp1.Body)
|
||||||
|
resp1.Body.Close()
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if resp1.StatusCode != http.StatusOK {
|
||||||
|
t.Fatalf("host-alias MISS: expected 200, got %d", resp1.StatusCode)
|
||||||
|
}
|
||||||
|
if got := resp1.Header.Get("X-LanCache-Status"); got != "MISS" {
|
||||||
|
t.Fatalf("host-alias MISS: expected X-LanCache-Status MISS, got %q", got)
|
||||||
|
}
|
||||||
|
if !bytes.Equal(data1, body) {
|
||||||
|
t.Fatalf("host-alias MISS: body mismatch: got %q", data1)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Bounded wait for the entry to be visible before hitting the next alias
|
||||||
|
// (the MISS handler streams the body to the client before the VFS write).
|
||||||
|
key, err := generateServiceCacheKey(depotPath, "steam")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
deadline := time.Now().Add(2 * time.Second)
|
||||||
|
for {
|
||||||
|
if rc, e := sc.vfs.Open(key); e == nil {
|
||||||
|
_ = rc.Close()
|
||||||
|
break
|
||||||
|
}
|
||||||
|
if time.Now().After(deadline) {
|
||||||
|
t.Fatalf("cache entry %q not visible after MISS", key)
|
||||||
|
}
|
||||||
|
time.Sleep(5 * time.Millisecond)
|
||||||
|
}
|
||||||
|
|
||||||
|
// 2) Same depot path under the second CDN alias (Host header only) -> HIT.
|
||||||
|
req2, err := http.NewRequest("GET", srv.URL+depotPath, nil)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
req2.Host = "cdn2.steamcontent.com"
|
||||||
|
req2.Header.Set("User-Agent", ua)
|
||||||
|
resp2, err := c.Do(req2)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("second host-alias request: %v", err)
|
||||||
|
}
|
||||||
|
data2, err := io.ReadAll(resp2.Body)
|
||||||
|
resp2.Body.Close()
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if resp2.StatusCode != http.StatusOK {
|
||||||
|
t.Fatalf("second host-alias: expected 200, got %d", resp2.StatusCode)
|
||||||
|
}
|
||||||
|
if got := resp2.Header.Get("X-LanCache-Status"); got != "HIT" {
|
||||||
|
t.Fatalf("second host-alias: expected X-LanCache-Status HIT, got %q", got)
|
||||||
|
}
|
||||||
|
if !bytes.Equal(data2, body) {
|
||||||
|
t.Fatalf("second host-alias: body mismatch: got %q", data2)
|
||||||
|
}
|
||||||
|
|
||||||
|
// 3) Same depot path with an absolute-form target embedding a third CDN
|
||||||
|
// hostname in the URL itself -> still a HIT on the same entry.
|
||||||
|
// (Go's http client always sends origin-form targets, so use raw HTTP.)
|
||||||
|
conn, err := net.Dial("tcp", srv.Listener.Addr().String())
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
defer conn.Close()
|
||||||
|
rawRequest := "GET http://cdn3.steamcontent.com" + depotPath + " HTTP/1.1\r\n" +
|
||||||
|
"Host: cdn3.steamcontent.com\r\n" +
|
||||||
|
"User-Agent: " + ua + "\r\n" +
|
||||||
|
"Connection: close\r\n\r\n"
|
||||||
|
if _, err := conn.Write([]byte(rawRequest)); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
rawReq, err := http.NewRequest("GET", "http://cdn3.steamcontent.com"+depotPath, nil)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
resp3, err := http.ReadResponse(bufio.NewReader(conn), rawReq)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
defer resp3.Body.Close()
|
||||||
|
data3, err := io.ReadAll(resp3.Body)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if resp3.StatusCode != http.StatusOK {
|
||||||
|
t.Fatalf("absolute-form host-alias: expected 200, got %d", resp3.StatusCode)
|
||||||
|
}
|
||||||
|
if got := resp3.Header.Get("X-LanCache-Status"); got != "HIT" {
|
||||||
|
t.Fatalf("absolute-form host-alias: expected X-LanCache-Status HIT, got %q", got)
|
||||||
|
}
|
||||||
|
if !bytes.Equal(data3, body) {
|
||||||
|
t.Fatalf("absolute-form host-alias: body mismatch: got %q", data3)
|
||||||
|
}
|
||||||
|
|
||||||
|
// All three aliases must have shared one upstream fetch.
|
||||||
|
if got := upstreamCalls.Load(); got != 1 {
|
||||||
|
t.Errorf("upstream fetched %d times across host aliases, want 1", got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|||||||
Reference in New Issue
Block a user