Justin Harms
9ca8fa4a5e
- Introduced maxConcurrentRequests and maxRequestsPerClient fields in the Config struct to manage request limits. - Updated the SteamCache implementation to utilize these new configuration options for controlling concurrent requests. - Enhanced the ServeHTTP method to enforce global and per-client rate limiting using semaphores. - Modified the root command to accept new flags for configuring concurrency limits via command-line arguments. - Updated tests to reflect changes in the SteamCache initialization and request handling logic.
755 lines
21 KiB
Go
755 lines
21 KiB
Go
// steamcache/steamcache.go
|
|
package steamcache
|
|
|
|
import (
|
|
"bufio"
|
|
"context"
|
|
"crypto/sha256"
|
|
"encoding/hex"
|
|
"io"
|
|
"net"
|
|
"net/http"
|
|
"net/url"
|
|
"os"
|
|
"s1d3sw1ped/SteamCache2/steamcache/logger"
|
|
"s1d3sw1ped/SteamCache2/vfs"
|
|
"s1d3sw1ped/SteamCache2/vfs/cache"
|
|
"s1d3sw1ped/SteamCache2/vfs/disk"
|
|
"s1d3sw1ped/SteamCache2/vfs/gc"
|
|
"s1d3sw1ped/SteamCache2/vfs/memory"
|
|
"strings"
|
|
"sync"
|
|
"time"
|
|
|
|
"github.com/docker/go-units"
|
|
"golang.org/x/sync/semaphore"
|
|
)
|
|
|
|
// generateURLHash creates a SHA256 hash of the entire URL path for cache key
|
|
func generateURLHash(urlPath string) string {
|
|
hash := sha256.Sum256([]byte(urlPath))
|
|
return hex.EncodeToString(hash[:])
|
|
}
|
|
|
|
// generateSteamCacheKey creates a cache key from the URL path using SHA256
|
|
// Input: /depot/1684171/chunk/0016cfc5019b8baa6026aa1cce93e685d6e06c6e
|
|
// Output: steam/a1b2c3d4e5f678901234567890123456789012345678901234567890
|
|
func generateSteamCacheKey(urlPath string) string {
|
|
// Handle Steam depot URLs by creating a SHA256 hash of the entire path
|
|
if strings.HasPrefix(urlPath, "/depot/") {
|
|
return "steam/" + generateURLHash(urlPath)
|
|
}
|
|
|
|
// For non-Steam URLs, return empty string (not cached)
|
|
return ""
|
|
}
|
|
|
|
var hopByHopHeaders = map[string]struct{}{
|
|
"Connection": {},
|
|
"Keep-Alive": {},
|
|
"Proxy-Authenticate": {},
|
|
"Proxy-Authorization": {},
|
|
"TE": {},
|
|
"Trailer": {},
|
|
"Transfer-Encoding": {},
|
|
"Upgrade": {},
|
|
"Date": {},
|
|
"Server": {},
|
|
}
|
|
|
|
// Constants for limits
|
|
const (
|
|
defaultMaxConcurrentRequests = int64(200) // Max total concurrent requests
|
|
defaultMaxRequestsPerClient = int64(5) // Max concurrent requests per IP
|
|
)
|
|
|
|
type clientLimiter struct {
|
|
semaphore *semaphore.Weighted
|
|
lastSeen time.Time
|
|
}
|
|
|
|
type coalescedRequest struct {
|
|
responseChan chan *http.Response
|
|
errorChan chan error
|
|
waitingCount int
|
|
done bool
|
|
mu sync.Mutex
|
|
}
|
|
|
|
func newCoalescedRequest() *coalescedRequest {
|
|
return &coalescedRequest{
|
|
responseChan: make(chan *http.Response, 1),
|
|
errorChan: make(chan error, 1),
|
|
waitingCount: 1,
|
|
done: false,
|
|
}
|
|
}
|
|
|
|
func (cr *coalescedRequest) addWaiter() {
|
|
cr.mu.Lock()
|
|
defer cr.mu.Unlock()
|
|
cr.waitingCount++
|
|
}
|
|
|
|
func (cr *coalescedRequest) complete(resp *http.Response, err error) {
|
|
cr.mu.Lock()
|
|
defer cr.mu.Unlock()
|
|
if cr.done {
|
|
return
|
|
}
|
|
cr.done = true
|
|
|
|
if err != nil {
|
|
select {
|
|
case cr.errorChan <- err:
|
|
default:
|
|
}
|
|
} else {
|
|
select {
|
|
case cr.responseChan <- resp:
|
|
default:
|
|
}
|
|
}
|
|
}
|
|
|
|
// getOrCreateCoalescedRequest gets an existing coalesced request or creates a new one
|
|
func (sc *SteamCache) getOrCreateCoalescedRequest(cacheKey string) (*coalescedRequest, bool) {
|
|
sc.coalescedRequestsMu.Lock()
|
|
defer sc.coalescedRequestsMu.Unlock()
|
|
|
|
if cr, exists := sc.coalescedRequests[cacheKey]; exists {
|
|
cr.addWaiter()
|
|
return cr, false
|
|
}
|
|
|
|
cr := newCoalescedRequest()
|
|
sc.coalescedRequests[cacheKey] = cr
|
|
return cr, true
|
|
}
|
|
|
|
// removeCoalescedRequest removes a completed coalesced request
|
|
func (sc *SteamCache) removeCoalescedRequest(cacheKey string) {
|
|
sc.coalescedRequestsMu.Lock()
|
|
defer sc.coalescedRequestsMu.Unlock()
|
|
delete(sc.coalescedRequests, cacheKey)
|
|
}
|
|
|
|
// getClientIP extracts the client IP address from the request
|
|
func getClientIP(r *http.Request) string {
|
|
// Check for forwarded headers first (common in proxy setups)
|
|
if xff := r.Header.Get("X-Forwarded-For"); xff != "" {
|
|
// X-Forwarded-For can contain multiple IPs, take the first one
|
|
if idx := strings.Index(xff, ","); idx > 0 {
|
|
return strings.TrimSpace(xff[:idx])
|
|
}
|
|
return strings.TrimSpace(xff)
|
|
}
|
|
|
|
if xri := r.Header.Get("X-Real-IP"); xri != "" {
|
|
return strings.TrimSpace(xri)
|
|
}
|
|
|
|
// Fall back to RemoteAddr
|
|
if host, _, err := net.SplitHostPort(r.RemoteAddr); err == nil {
|
|
return host
|
|
}
|
|
|
|
return r.RemoteAddr
|
|
}
|
|
|
|
// getOrCreateClientLimiter gets or creates a rate limiter for a client IP
|
|
func (sc *SteamCache) getOrCreateClientLimiter(clientIP string) *clientLimiter {
|
|
sc.clientRequestsMu.Lock()
|
|
defer sc.clientRequestsMu.Unlock()
|
|
|
|
limiter, exists := sc.clientRequests[clientIP]
|
|
if !exists || time.Since(limiter.lastSeen) > 5*time.Minute {
|
|
// Create new limiter or refresh existing one
|
|
limiter = &clientLimiter{
|
|
semaphore: semaphore.NewWeighted(sc.maxRequestsPerClient),
|
|
lastSeen: time.Now(),
|
|
}
|
|
sc.clientRequests[clientIP] = limiter
|
|
} else {
|
|
limiter.lastSeen = time.Now()
|
|
}
|
|
|
|
return limiter
|
|
}
|
|
|
|
// cleanupOldClientLimiters removes old client limiters to prevent memory leaks
|
|
func (sc *SteamCache) cleanupOldClientLimiters() {
|
|
for {
|
|
time.Sleep(10 * time.Minute) // Clean up every 10 minutes
|
|
|
|
sc.clientRequestsMu.Lock()
|
|
now := time.Now()
|
|
for ip, limiter := range sc.clientRequests {
|
|
if now.Sub(limiter.lastSeen) > 30*time.Minute {
|
|
delete(sc.clientRequests, ip)
|
|
}
|
|
}
|
|
sc.clientRequestsMu.Unlock()
|
|
}
|
|
}
|
|
|
|
type SteamCache struct {
|
|
address string
|
|
upstream string
|
|
|
|
vfs vfs.VFS
|
|
|
|
memory *memory.MemoryFS
|
|
disk *disk.DiskFS
|
|
|
|
memorygc *gc.GCFS
|
|
diskgc *gc.GCFS
|
|
|
|
server *http.Server
|
|
client *http.Client
|
|
cancel context.CancelFunc
|
|
wg sync.WaitGroup
|
|
|
|
// Request coalescing structures
|
|
coalescedRequests map[string]*coalescedRequest
|
|
coalescedRequestsMu sync.RWMutex
|
|
|
|
// Concurrency control
|
|
maxConcurrentRequests int64
|
|
requestSemaphore *semaphore.Weighted
|
|
|
|
// Per-client rate limiting
|
|
clientRequests map[string]*clientLimiter
|
|
clientRequestsMu sync.RWMutex
|
|
maxRequestsPerClient int64
|
|
}
|
|
|
|
func New(address string, memorySize string, diskSize string, diskPath, upstream, memoryGC, diskGC string, maxConcurrentRequests int64, maxRequestsPerClient int64) *SteamCache {
|
|
memorysize, err := units.FromHumanSize(memorySize)
|
|
if err != nil {
|
|
panic(err)
|
|
}
|
|
|
|
disksize, err := units.FromHumanSize(diskSize)
|
|
if err != nil {
|
|
panic(err)
|
|
}
|
|
|
|
c := cache.New()
|
|
|
|
var m *memory.MemoryFS
|
|
var mgc *gc.GCFS
|
|
if memorysize > 0 {
|
|
m = memory.New(memorysize)
|
|
memoryGCAlgo := gc.GCAlgorithm(memoryGC)
|
|
if memoryGCAlgo == "" {
|
|
memoryGCAlgo = gc.LRU // default to LRU
|
|
}
|
|
mgc = gc.New(m, memoryGCAlgo)
|
|
}
|
|
|
|
var d *disk.DiskFS
|
|
var dgc *gc.GCFS
|
|
if disksize > 0 {
|
|
d = disk.New(diskPath, disksize)
|
|
diskGCAlgo := gc.GCAlgorithm(diskGC)
|
|
if diskGCAlgo == "" {
|
|
diskGCAlgo = gc.LRU // default to LRU
|
|
}
|
|
dgc = gc.New(d, diskGCAlgo)
|
|
}
|
|
|
|
// configure the cache to match the specified mode (memory only, disk only, or memory and disk) based on the provided sizes
|
|
if disksize == 0 && memorysize != 0 {
|
|
//memory only mode - no disk
|
|
|
|
c.SetSlow(mgc)
|
|
} else if disksize != 0 && memorysize == 0 {
|
|
// disk only mode
|
|
|
|
c.SetSlow(dgc)
|
|
} else if disksize != 0 && memorysize != 0 {
|
|
// memory and disk mode
|
|
|
|
c.SetFast(mgc)
|
|
c.SetSlow(dgc)
|
|
} else {
|
|
// no memory or disk isn't a valid configuration
|
|
logger.Logger.Error().Bool("memory", false).Bool("disk", false).Msg("configuration invalid :( exiting")
|
|
os.Exit(1)
|
|
}
|
|
|
|
transport := &http.Transport{
|
|
MaxIdleConns: 200, // Increased from 100
|
|
MaxIdleConnsPerHost: 50, // Increased from 10
|
|
IdleConnTimeout: 120 * time.Second, // Increased from 90s
|
|
DialContext: (&net.Dialer{
|
|
Timeout: 30 * time.Second,
|
|
KeepAlive: 30 * time.Second,
|
|
}).DialContext,
|
|
TLSHandshakeTimeout: 15 * time.Second, // Increased from 10s
|
|
ResponseHeaderTimeout: 30 * time.Second, // Increased from 10s
|
|
ExpectContinueTimeout: 5 * time.Second, // Increased from 1s
|
|
DisableCompression: true, // Steam doesn't use compression
|
|
ForceAttemptHTTP2: true, // Enable HTTP/2 if available
|
|
}
|
|
|
|
client := &http.Client{
|
|
Transport: transport,
|
|
Timeout: 120 * time.Second, // Increased from 60s
|
|
}
|
|
|
|
sc := &SteamCache{
|
|
upstream: upstream,
|
|
address: address,
|
|
vfs: c,
|
|
memory: m,
|
|
disk: d,
|
|
memorygc: mgc,
|
|
diskgc: dgc,
|
|
client: client,
|
|
server: &http.Server{
|
|
Addr: address,
|
|
ReadTimeout: 30 * time.Second, // Increased
|
|
WriteTimeout: 60 * time.Second, // Increased
|
|
IdleTimeout: 120 * time.Second, // Good for keep-alive
|
|
ReadHeaderTimeout: 10 * time.Second, // New, for header attacks
|
|
MaxHeaderBytes: 1 << 20, // 1MB, optional
|
|
},
|
|
|
|
// Initialize concurrency control fields
|
|
coalescedRequests: make(map[string]*coalescedRequest),
|
|
maxConcurrentRequests: maxConcurrentRequests,
|
|
requestSemaphore: semaphore.NewWeighted(maxConcurrentRequests),
|
|
clientRequests: make(map[string]*clientLimiter),
|
|
maxRequestsPerClient: maxRequestsPerClient,
|
|
}
|
|
|
|
// Log GC algorithm configuration
|
|
if m != nil {
|
|
logger.Logger.Info().Str("memory_gc", memoryGC).Msg("Memory cache GC algorithm configured")
|
|
}
|
|
if d != nil {
|
|
logger.Logger.Info().Str("disk_gc", diskGC).Msg("Disk cache GC algorithm configured")
|
|
}
|
|
|
|
if d != nil {
|
|
if d.Size() > d.Capacity() {
|
|
gcHandler := gc.GetGCAlgorithm(gc.GCAlgorithm(diskGC))
|
|
gcHandler(d, uint(d.Size()-d.Capacity()))
|
|
}
|
|
}
|
|
|
|
return sc
|
|
}
|
|
|
|
func (sc *SteamCache) Run() {
|
|
if sc.upstream != "" {
|
|
resp, err := sc.client.Get(sc.upstream)
|
|
if err != nil || resp.StatusCode != http.StatusOK {
|
|
logger.Logger.Error().Err(err).Int("status_code", resp.StatusCode).Str("upstream", sc.upstream).Msg("Failed to connect to upstream server")
|
|
os.Exit(1)
|
|
}
|
|
resp.Body.Close()
|
|
}
|
|
|
|
sc.server.Handler = sc
|
|
ctx, cancel := context.WithCancel(context.Background())
|
|
sc.cancel = cancel
|
|
|
|
// Start cleanup goroutine for old client limiters
|
|
sc.wg.Add(1)
|
|
go func() {
|
|
defer sc.wg.Done()
|
|
sc.cleanupOldClientLimiters()
|
|
}()
|
|
|
|
sc.wg.Add(1)
|
|
go func() {
|
|
defer sc.wg.Done()
|
|
err := sc.server.ListenAndServe()
|
|
if err != nil && err != http.ErrServerClosed {
|
|
logger.Logger.Error().Err(err).Msg("Failed to start SteamCache2")
|
|
os.Exit(1)
|
|
}
|
|
}()
|
|
|
|
<-ctx.Done()
|
|
sc.server.Shutdown(ctx)
|
|
sc.wg.Wait()
|
|
}
|
|
|
|
func (sc *SteamCache) Shutdown() {
|
|
if sc.cancel != nil {
|
|
sc.cancel()
|
|
}
|
|
sc.wg.Wait()
|
|
}
|
|
|
|
func (sc *SteamCache) ServeHTTP(w http.ResponseWriter, r *http.Request) {
|
|
// Apply global concurrency limit first
|
|
if err := sc.requestSemaphore.Acquire(context.Background(), 1); err != nil {
|
|
logger.Logger.Warn().Str("client_ip", getClientIP(r)).Msg("Server at capacity, rejecting request")
|
|
http.Error(w, "Server busy, please try again later", http.StatusServiceUnavailable)
|
|
return
|
|
}
|
|
defer sc.requestSemaphore.Release(1)
|
|
|
|
// Apply per-client rate limiting
|
|
clientIP := getClientIP(r)
|
|
clientLimiter := sc.getOrCreateClientLimiter(clientIP)
|
|
|
|
if err := clientLimiter.semaphore.Acquire(context.Background(), 1); err != nil {
|
|
logger.Logger.Warn().
|
|
Str("client_ip", clientIP).
|
|
Int("max_per_client", int(sc.maxRequestsPerClient)).
|
|
Msg("Client exceeded concurrent request limit")
|
|
http.Error(w, "Too many concurrent requests from this client", http.StatusTooManyRequests)
|
|
return
|
|
}
|
|
defer clientLimiter.semaphore.Release(1)
|
|
|
|
if r.Method != http.MethodGet {
|
|
logger.Logger.Warn().
|
|
Str("method", r.Method).
|
|
Str("client_ip", clientIP).
|
|
Msg("Only GET method is supported")
|
|
http.Error(w, "Only GET method is supported", http.StatusMethodNotAllowed)
|
|
return
|
|
}
|
|
|
|
if r.URL.Path == "/" {
|
|
logger.Logger.Debug().
|
|
Str("client_ip", clientIP).
|
|
Msg("Health check request")
|
|
w.WriteHeader(http.StatusOK) // this is used by steamcache2's upstream verification at startup
|
|
return
|
|
}
|
|
|
|
if r.URL.String() == "/lancache-heartbeat" {
|
|
logger.Logger.Debug().
|
|
Str("client_ip", clientIP).
|
|
Msg("LanCache heartbeat request")
|
|
w.Header().Add("X-LanCache-Processed-By", "SteamCache2")
|
|
w.WriteHeader(http.StatusNoContent)
|
|
w.Write(nil)
|
|
return
|
|
}
|
|
|
|
if strings.HasPrefix(r.URL.String(), "/depot/") {
|
|
// trim the query parameters from the URL path
|
|
// this is necessary because the cache key should not include query parameters
|
|
urlPath, _, _ := strings.Cut(r.URL.String(), "?")
|
|
|
|
tstart := time.Now()
|
|
|
|
// Generate simplified Steam cache key: steam/{hash}
|
|
cacheKey := generateSteamCacheKey(urlPath)
|
|
|
|
if cacheKey == "" {
|
|
logger.Logger.Warn().Str("url", urlPath).Msg("Invalid URL")
|
|
http.Error(w, "Invalid URL", http.StatusBadRequest)
|
|
return
|
|
}
|
|
|
|
w.Header().Add("X-LanCache-Processed-By", "SteamCache2") // SteamPrefill uses this header to determine if the request was processed by the cache maybe steam uses it too
|
|
|
|
cachePath := cacheKey // You may want to add a .http or .cache extension for clarity
|
|
|
|
// Try to serve from cache
|
|
file, err := sc.vfs.Open(cachePath)
|
|
if err == nil {
|
|
defer file.Close()
|
|
buf := bufio.NewReader(file)
|
|
resp, err := http.ReadResponse(buf, nil)
|
|
if err == nil {
|
|
// Remove hop-by-hop and server-specific headers
|
|
for k, vv := range resp.Header {
|
|
if _, skip := hopByHopHeaders[http.CanonicalHeaderKey(k)]; skip {
|
|
continue
|
|
}
|
|
for _, v := range vv {
|
|
w.Header().Add(k, v)
|
|
}
|
|
}
|
|
// Add our own headers
|
|
w.Header().Set("X-LanCache-Status", "HIT")
|
|
w.Header().Set("X-LanCache-Processed-By", "SteamCache2")
|
|
w.WriteHeader(resp.StatusCode)
|
|
io.Copy(w, resp.Body)
|
|
resp.Body.Close()
|
|
logger.Logger.Info().
|
|
Str("key", cacheKey).
|
|
Str("host", r.Host).
|
|
Str("client_ip", clientIP).
|
|
Str("status", "HIT").
|
|
Dur("duration", time.Since(tstart)).
|
|
Msg("cache request")
|
|
return
|
|
}
|
|
}
|
|
|
|
// Check for coalesced request (another client already downloading this)
|
|
coalescedReq, isNew := sc.getOrCreateCoalescedRequest(cacheKey)
|
|
if !isNew {
|
|
// Wait for the existing download to complete
|
|
logger.Logger.Debug().
|
|
Str("key", cacheKey).
|
|
Str("client_ip", clientIP).
|
|
Int("waiting_clients", coalescedReq.waitingCount).
|
|
Msg("Joining coalesced request")
|
|
|
|
select {
|
|
case resp := <-coalescedReq.responseChan:
|
|
// Use the downloaded response
|
|
defer resp.Body.Close()
|
|
bodyData, err := io.ReadAll(resp.Body)
|
|
if err != nil {
|
|
logger.Logger.Error().Err(err).Str("key", cacheKey).Msg("Failed to read coalesced response body")
|
|
http.Error(w, "Failed to read response body", http.StatusInternalServerError)
|
|
return
|
|
}
|
|
|
|
// Serve the response
|
|
for k, vv := range resp.Header {
|
|
if _, skip := hopByHopHeaders[http.CanonicalHeaderKey(k)]; skip {
|
|
continue
|
|
}
|
|
for _, v := range vv {
|
|
w.Header().Add(k, v)
|
|
}
|
|
}
|
|
w.Header().Set("X-LanCache-Status", "HIT-COALESCED")
|
|
w.Header().Set("X-LanCache-Processed-By", "SteamCache2")
|
|
w.WriteHeader(resp.StatusCode)
|
|
w.Write(bodyData)
|
|
|
|
logger.Logger.Info().
|
|
Str("key", cacheKey).
|
|
Str("host", r.Host).
|
|
Str("client_ip", clientIP).
|
|
Str("status", "HIT-COALESCED").
|
|
Dur("duration", time.Since(tstart)).
|
|
Msg("cache request")
|
|
|
|
return
|
|
|
|
case err := <-coalescedReq.errorChan:
|
|
logger.Logger.Error().
|
|
Err(err).
|
|
Str("key", cacheKey).
|
|
Str("client_ip", clientIP).
|
|
Msg("Coalesced request failed")
|
|
http.Error(w, "Upstream request failed", http.StatusInternalServerError)
|
|
return
|
|
}
|
|
}
|
|
|
|
// Remove coalesced request when done
|
|
defer sc.removeCoalescedRequest(cacheKey)
|
|
|
|
var req *http.Request
|
|
if sc.upstream != "" { // if an upstream server is configured, proxy the request to the upstream server
|
|
ur, err := url.JoinPath(sc.upstream, urlPath)
|
|
if err != nil {
|
|
logger.Logger.Error().Err(err).Str("upstream", sc.upstream).Msg("Failed to join URL path")
|
|
http.Error(w, "Failed to join URL path", http.StatusInternalServerError)
|
|
return
|
|
}
|
|
|
|
req, err = http.NewRequest(http.MethodGet, ur, nil)
|
|
if err != nil {
|
|
logger.Logger.Error().Err(err).Str("upstream", sc.upstream).Msg("Failed to create request")
|
|
http.Error(w, "Failed to create request", http.StatusInternalServerError)
|
|
return
|
|
}
|
|
req.Host = r.Host
|
|
} else { // if no upstream server is configured, proxy the request to the host specified in the request
|
|
host := r.Host
|
|
if r.Header.Get("X-Sls-Https") == "enable" {
|
|
host = "https://" + host
|
|
} else {
|
|
host = "http://" + host
|
|
}
|
|
|
|
ur, err := url.JoinPath(host, urlPath)
|
|
if err != nil {
|
|
logger.Logger.Error().Err(err).Str("host", host).Msg("Failed to join URL path")
|
|
http.Error(w, "Failed to join URL path", http.StatusInternalServerError)
|
|
return
|
|
}
|
|
|
|
req, err = http.NewRequest(http.MethodGet, ur, nil)
|
|
if err != nil {
|
|
logger.Logger.Error().Err(err).Str("host", host).Msg("Failed to create request")
|
|
http.Error(w, "Failed to create request", http.StatusInternalServerError)
|
|
return
|
|
}
|
|
}
|
|
|
|
// Copy headers from the original request to the new request
|
|
for key, values := range r.Header {
|
|
for _, value := range values {
|
|
req.Header.Add(key, value)
|
|
}
|
|
}
|
|
|
|
// Retry logic
|
|
backoffSchedule := []time.Duration{1 * time.Second, 3 * time.Second, 10 * time.Second}
|
|
var resp *http.Response
|
|
for i, backoff := range backoffSchedule {
|
|
resp, err = sc.client.Do(req)
|
|
if err == nil && resp.StatusCode == http.StatusOK {
|
|
break
|
|
}
|
|
if i < len(backoffSchedule)-1 {
|
|
time.Sleep(backoff)
|
|
}
|
|
}
|
|
if err != nil || resp.StatusCode != http.StatusOK {
|
|
logger.Logger.Error().Err(err).Str("url", req.URL.String()).Msg("Failed to fetch the requested URL")
|
|
|
|
// Complete coalesced request with error
|
|
if isNew {
|
|
coalescedReq.complete(nil, err)
|
|
}
|
|
|
|
http.Error(w, "Failed to fetch the requested URL", http.StatusInternalServerError)
|
|
return
|
|
}
|
|
defer resp.Body.Close()
|
|
|
|
// Fast path: Flexible lightweight validation for all files
|
|
// Multiple validation layers ensure data integrity without blocking legitimate Steam content
|
|
|
|
// Method 1: HTTP Status Validation
|
|
if resp.StatusCode != http.StatusOK {
|
|
logger.Logger.Error().
|
|
Str("url", req.URL.String()).
|
|
Int("status_code", resp.StatusCode).
|
|
Msg("Steam returned non-OK status")
|
|
http.Error(w, "Upstream server error", http.StatusBadGateway)
|
|
return
|
|
}
|
|
|
|
// Method 2: Content-Type Validation (Steam files should be application/x-steam-chunk)
|
|
contentType := resp.Header.Get("Content-Type")
|
|
if contentType != "" && !strings.Contains(contentType, "application/x-steam-chunk") {
|
|
logger.Logger.Warn().
|
|
Str("url", req.URL.String()).
|
|
Str("content_type", contentType).
|
|
Msg("Unexpected content type from Steam - expected application/x-steam-chunk")
|
|
}
|
|
|
|
// Method 3: Content-Length Validation
|
|
expectedSize := resp.ContentLength
|
|
|
|
// Reject only truly invalid content lengths (zero or negative)
|
|
if expectedSize <= 0 {
|
|
logger.Logger.Error().
|
|
Str("url", req.URL.String()).
|
|
Int64("content_length", expectedSize).
|
|
Msg("Invalid content length, rejecting file")
|
|
http.Error(w, "Invalid content length", http.StatusBadGateway)
|
|
return
|
|
}
|
|
|
|
// Content length is valid - no size restrictions to keep logs clean
|
|
|
|
// Lightweight validation passed - trust the Content-Length and HTTP status
|
|
// This provides good integrity with minimal performance overhead
|
|
validationPassed := true
|
|
|
|
// Write to response (stream the file directly)
|
|
// Remove hop-by-hop and server-specific headers
|
|
for k, vv := range resp.Header {
|
|
if _, skip := hopByHopHeaders[http.CanonicalHeaderKey(k)]; skip {
|
|
continue
|
|
}
|
|
for _, v := range vv {
|
|
w.Header().Add(k, v)
|
|
}
|
|
}
|
|
// Add our own headers
|
|
w.Header().Set("X-LanCache-Status", "MISS")
|
|
w.Header().Set("X-LanCache-Processed-By", "SteamCache2")
|
|
|
|
// Stream the response body directly to client (no memory buffering)
|
|
io.Copy(w, resp.Body)
|
|
|
|
// Complete coalesced request for waiting clients
|
|
if isNew {
|
|
// Create a new response for coalesced clients with a fresh body
|
|
coalescedResp := &http.Response{
|
|
StatusCode: resp.StatusCode,
|
|
Status: resp.Status,
|
|
Header: make(http.Header),
|
|
Body: io.NopCloser(strings.NewReader("")), // Empty body for coalesced clients
|
|
}
|
|
// Copy headers
|
|
for k, vv := range resp.Header {
|
|
coalescedResp.Header[k] = vv
|
|
}
|
|
coalescedReq.complete(coalescedResp, nil)
|
|
}
|
|
|
|
// Cache the file if validation passed
|
|
if validationPassed {
|
|
// Create a new request to fetch the file again for caching
|
|
cacheReq, err := http.NewRequest(http.MethodGet, req.URL.String(), nil)
|
|
if err == nil {
|
|
// Copy original headers
|
|
for k, vv := range req.Header {
|
|
cacheReq.Header[k] = vv
|
|
}
|
|
|
|
// Fetch fresh copy for caching
|
|
cacheResp, err := sc.client.Do(cacheReq)
|
|
if err == nil {
|
|
defer cacheResp.Body.Close()
|
|
// Use the validated size from the original response
|
|
writer, _ := sc.vfs.Create(cachePath, expectedSize)
|
|
if writer != nil {
|
|
defer writer.Close()
|
|
io.Copy(writer, cacheResp.Body)
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
logger.Logger.Info().
|
|
Str("key", cacheKey).
|
|
Str("host", r.Host).
|
|
Str("client_ip", clientIP).
|
|
Str("status", "MISS").
|
|
Dur("duration", time.Since(tstart)).
|
|
Msg("cache request")
|
|
|
|
return
|
|
}
|
|
|
|
if r.URL.Path == "/favicon.ico" {
|
|
logger.Logger.Debug().
|
|
Str("client_ip", clientIP).
|
|
Msg("Favicon request")
|
|
w.WriteHeader(http.StatusNoContent)
|
|
return
|
|
}
|
|
|
|
if r.URL.Path == "/robots.txt" {
|
|
logger.Logger.Debug().
|
|
Str("client_ip", clientIP).
|
|
Msg("Robots.txt request")
|
|
w.Header().Set("Content-Type", "text/plain")
|
|
w.WriteHeader(http.StatusOK)
|
|
w.Write([]byte("User-agent: *\nDisallow: /\n"))
|
|
return
|
|
}
|
|
|
|
logger.Logger.Warn().
|
|
Str("url", r.URL.String()).
|
|
Str("client_ip", clientIP).
|
|
Msg("Request not found")
|
|
http.Error(w, "Not found", http.StatusNotFound)
|
|
}
|