cache: Short TTL negative cache for 404/410 depot objects

Stop re-fetching gone depot objects on every miss: store 404/410 in the
existing VFS cache under the same key with a short TTL (default 5m).

steamcache/handler.go

@@ -34,10 +34,14 @@ type Options struct {
 	// New config fields for hardening (max object size + trusted proxies)
 	MaxObjectSize  string
 	TrustedProxies []string
+
+	// NegativeTTL is a Go duration string for 404/410 negative cache entries.
+	// Empty defaults to 5m. "0" / "0s" disables storing negatives.
+	NegativeTTL string
 }
 
 func NewWithOptions(o Options) (*SteamCache, error) {
-	return New(o.Address, o.MemorySize, o.DiskSize, o.DiskPath, o.Upstream, o.MemoryGC, o.DiskGC, o.MaxConcurrentRequests, o.MaxRequestsPerClient, o.MaxObjectSize, o.TrustedProxies)
+	return New(o.Address, o.MemorySize, o.DiskSize, o.DiskPath, o.Upstream, o.MemoryGC, o.DiskGC, o.MaxConcurrentRequests, o.MaxRequestsPerClient, o.MaxObjectSize, o.TrustedProxies, o.NegativeTTL)
 }
 
 // handleSpecialEndpoints handles non-content paths (health, heartbeat, metrics) and
@@ -115,6 +119,18 @@ func (sc *SteamCache) handleCacheHit(w http.ResponseWriter, r *http.Request, cac
 					Msg("Failed to deserialize cache file - removing corrupted entry")
 				_ = sc.vfs.Delete(cachePath) // best-effort cleanup of corrupt entry; failure non-fatal (logged)
 			} else {
+				if cacheFile.ExpiresUnix > 0 {
+					if time.Now().Unix() >= cacheFile.ExpiresUnix {
+						logger.Logger.Debug().
+							Str("key", cacheKey).
+							Str("url", urlPath).
+							Int64("expires_unix", cacheFile.ExpiresUnix).
+							Msg("Negative cache entry expired - treating as miss")
+						_ = sc.vfs.Delete(cachePath) // best-effort; miss path re-fetches
+						return false
+					}
+					sc.metrics.IncrementNegativeCacheHits()
+				}
 				// Track cache hit metrics
 				sc.metrics.IncrementCacheHits()
 				sc.metrics.AddResponseTime(time.Since(tstart))
@@ -144,6 +160,126 @@ func (sc *SteamCache) handleCacheHit(w http.ResponseWriter, r *http.Request, cac
 	return false
 }
 
+const maxNegativeBody = 64 * 1024
+
+// handleNegativeUpstream serves a definitive upstream 404/410 to the client,
+// stores a short-TTL negative marker in the same VFS cache (same key as a
+// positive object), and completes coalesced waiters with that status.
+func (sc *SteamCache) handleNegativeUpstream(w http.ResponseWriter, r *http.Request, resp *http.Response, coalescedReq *coalescedRequest, isNew bool, cachePath, cacheKey, urlPath, clientIP string, service *ServiceConfig, tstart time.Time) {
+	defer func() { _ = resp.Body.Close() }() // best-effort close of gone-status body
+
+	bodyData, err := io.ReadAll(io.LimitReader(resp.Body, maxNegativeBody))
+	if err != nil {
+		logger.Logger.Warn().
+			Err(err).
+			Str("key", cacheKey).
+			Str("url", urlPath).
+			Int("status_code", resp.StatusCode).
+			Msg("Failed to read upstream 404/410 body")
+		bodyData = nil
+	}
+
+	rawResponse := sc.reconstructRawResponse(resp, bodyData)
+
+	for k, vv := range filterHopByHopHeaders(resp.Header) {
+		for _, v := range vv {
+			w.Header().Add(k, v)
+		}
+	}
+	w.Header().Set("X-LanCache-Status", "MISS")
+	w.Header().Set("X-LanCache-Processed-By", "SteamCache2")
+	w.WriteHeader(resp.StatusCode)
+	if len(bodyData) > 0 {
+		_, _ = w.Write(bodyData) // client write error ignored (disconnect during 404/410 body send is not actionable)
+	}
+
+	sc.metrics.IncrementCacheMisses()
+	sc.metrics.IncrementUpstreamErrors()
+	sc.metrics.AddResponseTime(time.Since(tstart))
+	sc.metrics.AddBytesServed(int64(len(bodyData)))
+	sc.metrics.IncrementServiceRequests(service.Name)
+
+	if sc.negativeTTL > 0 {
+		expiresUnix := time.Now().Add(sc.negativeTTL).Unix()
+		cacheData, serErr := serializeCacheFile(rawResponse, expiresUnix)
+		if serErr != nil {
+			logger.Logger.Warn().
+				Str("key", cacheKey).
+				Str("url", urlPath).
+				Err(serErr).
+				Msg("Failed to serialize negative cache file")
+			sc.metrics.IncrementCacheWriteFailures()
+			sc.metrics.IncrementServiceError("serialize")
+		} else {
+			sc.writeCacheEntry(cachePath, cacheKey, urlPath, service.Name, cacheData)
+		}
+	}
+
+	if isNew {
+		coalescedResp := &http.Response{
+			StatusCode: resp.StatusCode,
+			Status:     resp.Status,
+			Header:     make(http.Header),
+			Body:       io.NopCloser(bytes.NewReader(bodyData)),
+		}
+		for k, vv := range resp.Header {
+			coalescedResp.Header[k] = vv
+		}
+		coalescedReq.setResponseData(bodyData)
+		coalescedReq.complete(coalescedResp, nil)
+	}
+
+	logger.Logger.Info().
+		Str("cache_key", cacheKey).
+		Str("url", urlPath).
+		Str("host", r.Host).
+		Str("client_ip", clientIP).
+		Str("service", service.Name).
+		Str("cache_status", "MISS").
+		Int("status_code", resp.StatusCode).
+		Int64("file_size", int64(len(bodyData))).
+		Dur("response_time", time.Since(tstart)).
+		Msg("cache request")
+}
+
+// writeCacheEntry stores serialized SC2C bytes at cachePath. Failures increment
+// cache_write_failures; partial writes are deleted.
+func (sc *SteamCache) writeCacheEntry(cachePath, cacheKey, urlPath, serviceName string, cacheData []byte) {
+	cacheWriter, err := sc.vfs.Create(cachePath, int64(len(cacheData)))
+	if err != nil {
+		logger.Logger.Warn().
+			Str("key", cacheKey).
+			Str("url", urlPath).
+			Err(err).
+			Msg("Failed to create cache file")
+		sc.metrics.IncrementCacheWriteFailures()
+		sc.metrics.IncrementServiceError("cache_create")
+		return
+	}
+	defer func() { _ = cacheWriter.Close() }() // best-effort close of cache writer; errors on close (e.g. final sync) logged via prior write checks or non-fatal
+
+	bytesWritten, cacheErr := cacheWriter.Write(cacheData)
+	if cacheErr != nil || bytesWritten != len(cacheData) {
+		logger.Logger.Warn().
+			Str("key", cacheKey).
+			Str("url", urlPath).
+			Int("expected", len(cacheData)).
+			Int("written", bytesWritten).
+			Err(cacheErr).
+			Msg("Cache write failed or incomplete - removing corrupted entry")
+		sc.metrics.IncrementCacheWriteFailures()
+		sc.metrics.IncrementServiceError("cache_write")
+		_ = sc.vfs.Delete(cachePath) // best-effort removal of partial corrupt cache entry on write failure; non-fatal
+		return
+	}
+	logger.Logger.Debug().
+		Str("key", cacheKey).
+		Str("url", urlPath).
+		Str("service", serviceName).
+		Int("size", bytesWritten).
+		Msg("Successfully cached response")
+}
+
 // waitForCoalesced handles the follower path for a coalesced in-flight request.
 // It waits on the broadcast doneCh, serves the buffered response (or error), updates
 // coalesced metrics, and returns (the caller in ServeHTTP does the outer return).
@@ -417,12 +553,12 @@ func (sc *SteamCache) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 			}
 		}
 
-		// Retry logic
+		// Retry logic. 404/410 are definitive gone: do not retry with backoff.
 		backoffSchedule := []time.Duration{1 * time.Second, 3 * time.Second, 10 * time.Second}
 		var resp *http.Response
 		for i, backoff := range backoffSchedule {
 			resp, err = sc.client.Do(req)
-			if err == nil && resp.StatusCode == http.StatusOK {
+			if err == nil && (resp.StatusCode == http.StatusOK || isDefinitiveGone(resp.StatusCode)) {
 				break
 			}
 			if i < len(backoffSchedule)-1 {
@@ -447,6 +583,10 @@ func (sc *SteamCache) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 			return
 		}
 		if resp.StatusCode != http.StatusOK {
+			if isDefinitiveGone(resp.StatusCode) {
+				sc.handleNegativeUpstream(w, r, resp, coalescedReq, isNew, cachePath, cacheKey, urlPath, clientIP, service, tstart)
+				return
+			}
 			logger.Logger.Error().Int("status_code", resp.StatusCode).Str("url", req.URL.String()).Msg("Failed to fetch the requested URL (non-OK status after retries)")
 
 			_ = resp.Body.Close() // best-effort close on non-OK upstream; primary error path
@@ -628,42 +768,7 @@ func (sc *SteamCache) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 			sc.metrics.IncrementCacheWriteFailures()
 			sc.metrics.IncrementServiceError("serialize")
 		} else {
-			// Store the serialized cache data
-			cacheWriter, err := sc.vfs.Create(cachePath, int64(len(cacheData)))
-			if err == nil {
-				defer func() { _ = cacheWriter.Close() }() // best-effort close of cache writer; errors on close (e.g. final sync) logged via prior write checks or non-fatal
-
-				// Write the serialized cache data
-				bytesWritten, cacheErr := cacheWriter.Write(cacheData)
-
-				if cacheErr != nil || bytesWritten != len(cacheData) {
-					logger.Logger.Warn().
-						Str("key", cacheKey).
-						Str("url", urlPath).
-						Int("expected", len(cacheData)).
-						Int("written", bytesWritten).
-						Err(cacheErr).
-						Msg("Cache write failed or incomplete - removing corrupted entry")
-					sc.metrics.IncrementCacheWriteFailures()
-					sc.metrics.IncrementServiceError("cache_write")
-					_ = sc.vfs.Delete(cachePath) // best-effort removal of partial corrupt cache entry on write failure; non-fatal. Deferred cacheWriter.Close() (from earlier in block) runs after this on error unwind path (harmless per DiskFS design)
-				} else {
-					logger.Logger.Debug().
-						Str("key", cacheKey).
-						Str("url", urlPath).
-						Str("service", service.Name).
-						Int("size", bytesWritten).
-						Msg("Successfully cached response")
-				}
-			} else {
-				logger.Logger.Warn().
-					Str("key", cacheKey).
-					Str("url", urlPath).
-					Err(err).
-					Msg("Failed to create cache file")
-				sc.metrics.IncrementCacheWriteFailures()
-				sc.metrics.IncrementServiceError("cache_create")
-			}
+			sc.writeCacheEntry(cachePath, cacheKey, urlPath, service.Name, cacheData)
 		}
 
 		// Complete coalesced request with the original response