Add core components for request coalescing and service management
- Introduced coalescing logic in `coalescing.go` to handle concurrent identical upstream fetches, including a state machine and response buffering for improved performance. - Implemented a new cache file format in `format.go`, supporting serialization and deserialization of HTTP responses, along with range request handling. - Developed an HTTP handler in `handler.go` to manage requests, including special endpoint handling and metrics reporting. - Added rate limiting functionality in `ratelimit.go` to control per-client and global request rates, enhancing security and performance. - Created service management capabilities in `service.go` to define and manage cacheable services, including user-agent detection. - Updated tests in `steamcache_test.go` to cover new functionalities, ensuring robustness and reliability across the codebase.
This commit is contained in:
@@ -0,0 +1,474 @@
|
||||
// steamcache/format.go
|
||||
// On-disk cache file format (SC2C magic + SHA256 content hash + raw HTTP response),
|
||||
// plus serialization, deserialization, response reconstruction for upstream fidelity,
|
||||
// streaming with HTTP Range request support, line parsing, range header parsing,
|
||||
// completeness verification, and hop-by-hop header filtering (shared across paths).
|
||||
package steamcache
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"s1d3sw1ped/steamcache2/steamcache/logger"
|
||||
)
|
||||
|
||||
// Cache file format structures
|
||||
//
|
||||
// On-disk format (documented here at top of format.go per Phase 2 plan; stable v1):
|
||||
// File = header-line + raw-response-bytes
|
||||
// header-line = "SC2C " + 64hex(bodySHA256) + " " + strconv(len(rawResp)) + "\n"
|
||||
// raw-response-bytes = the exact bytes from reconstructRawResponse (HTTP/1.1 status\r\n + headers\r\n\r\n + body)
|
||||
// deserializeCacheFile: parses header, verifies size+SHA, returns CacheFileFormat.
|
||||
// No compression or extra fields. filterHopByHopHeaders is the shared helper
|
||||
// (used in streamCachedResponse, handler MISS, coalescing.complete).
|
||||
const (
|
||||
CacheFileMagic = "SC2C" // SteamCache2 Cache
|
||||
)
|
||||
|
||||
// CacheFileFormat represents the complete cache file structure
|
||||
type CacheFileFormat struct {
|
||||
ContentHash string // SHA256 hash of the response body (internal)
|
||||
ResponseSize int64 // Size of the entire HTTP response
|
||||
Response []byte // The entire HTTP response as raw bytes
|
||||
}
|
||||
|
||||
// serializeRawResponse serializes a raw HTTP response into our text-based cache format
|
||||
// upstreamHash and upstreamAlgo are used for verification during download but not stored
|
||||
func serializeRawResponse(rawResponse []byte) ([]byte, error) {
|
||||
// Extract body from raw response for hash calculation
|
||||
bodyStart := bytes.Index(rawResponse, []byte("\r\n\r\n"))
|
||||
if bodyStart == -1 {
|
||||
return nil, fmt.Errorf("invalid HTTP response format: no body separator found")
|
||||
}
|
||||
bodyStart += 4 // Skip the \r\n\r\n
|
||||
bodyData := rawResponse[bodyStart:]
|
||||
|
||||
// Always calculate our internal SHA256 hash
|
||||
contentHash := calculateSHA256(bodyData)
|
||||
|
||||
// Create text-based cache file
|
||||
var buf bytes.Buffer
|
||||
|
||||
// First line: magic number, content hash, response size
|
||||
headerLine := fmt.Sprintf("%s %s %d\n", CacheFileMagic, contentHash, len(rawResponse))
|
||||
buf.WriteString(headerLine)
|
||||
|
||||
// Rest of the file: raw HTTP response
|
||||
buf.Write(rawResponse)
|
||||
|
||||
return buf.Bytes(), nil
|
||||
}
|
||||
|
||||
// deserializeCacheFile deserializes our text-based cache format and returns both metadata and raw response
|
||||
func deserializeCacheFile(data []byte) (*CacheFileFormat, error) {
|
||||
if len(data) < 4 {
|
||||
return nil, fmt.Errorf("cache file too short")
|
||||
}
|
||||
|
||||
// Find the first newline to separate header from content
|
||||
newlineIndex := bytes.IndexByte(data, '\n')
|
||||
if newlineIndex == -1 {
|
||||
return nil, fmt.Errorf("invalid cache file format: no header line found")
|
||||
}
|
||||
|
||||
// Parse header line: "SC2C <hash> <size>"
|
||||
headerLine := string(data[:newlineIndex])
|
||||
parts := strings.Fields(headerLine)
|
||||
if len(parts) != 3 {
|
||||
return nil, fmt.Errorf("invalid header format: expected 3 fields, got %d", len(parts))
|
||||
}
|
||||
|
||||
// Check magic number
|
||||
if parts[0] != CacheFileMagic {
|
||||
return nil, fmt.Errorf("invalid cache file magic number: %s", parts[0])
|
||||
}
|
||||
|
||||
// Parse content hash
|
||||
contentHash := parts[1]
|
||||
if len(contentHash) != 64 {
|
||||
return nil, fmt.Errorf("invalid content hash length: expected 64, got %d", len(contentHash))
|
||||
}
|
||||
|
||||
// Parse response size
|
||||
responseSize, err := strconv.ParseInt(parts[2], 10, 64)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("invalid response size: %w", err)
|
||||
}
|
||||
|
||||
// Extract raw response (everything after the header line)
|
||||
rawResponse := data[newlineIndex+1:]
|
||||
|
||||
// Verify response size
|
||||
if int64(len(rawResponse)) != responseSize {
|
||||
return nil, fmt.Errorf("response size mismatch: expected %d, got %d",
|
||||
responseSize, len(rawResponse))
|
||||
}
|
||||
|
||||
// Extract body from response for hash verification
|
||||
bodyStart := bytes.Index(rawResponse, []byte("\r\n\r\n"))
|
||||
if bodyStart == -1 {
|
||||
return nil, fmt.Errorf("invalid HTTP response format: no body separator found")
|
||||
}
|
||||
bodyStart += 4 // Skip the \r\n\r\n
|
||||
bodyData := rawResponse[bodyStart:]
|
||||
|
||||
// Verify our internal SHA256 hash
|
||||
calculatedSHA256 := calculateSHA256(bodyData)
|
||||
if calculatedSHA256 != contentHash {
|
||||
return nil, fmt.Errorf("content hash mismatch: expected %s, got %s",
|
||||
contentHash, calculatedSHA256)
|
||||
}
|
||||
|
||||
// Create cache file structure
|
||||
cacheFile := &CacheFileFormat{
|
||||
ContentHash: contentHash,
|
||||
ResponseSize: responseSize,
|
||||
Response: rawResponse,
|
||||
}
|
||||
|
||||
return cacheFile, nil
|
||||
}
|
||||
|
||||
// reconstructRawResponse reconstructs the exact HTTP response as received from upstream
|
||||
func (sc *SteamCache) reconstructRawResponse(resp *http.Response, bodyData []byte) []byte {
|
||||
var responseBuffer bytes.Buffer
|
||||
|
||||
// Write status line exactly as it would appear from upstream
|
||||
responseBuffer.WriteString(fmt.Sprintf("HTTP/1.1 %d %s\r\n", resp.StatusCode, http.StatusText(resp.StatusCode)))
|
||||
|
||||
// Write headers in the exact order and format as received
|
||||
for k, vv := range resp.Header {
|
||||
for _, v := range vv {
|
||||
responseBuffer.WriteString(fmt.Sprintf("%s: %s\r\n", k, v))
|
||||
}
|
||||
}
|
||||
responseBuffer.WriteString("\r\n") // End of headers
|
||||
|
||||
// Write body
|
||||
responseBuffer.Write(bodyData)
|
||||
|
||||
return responseBuffer.Bytes()
|
||||
}
|
||||
|
||||
// streamCachedResponse streams the raw HTTP response bytes directly to the client
|
||||
// Supports Range requests by serving partial content from the cached full file
|
||||
func (sc *SteamCache) streamCachedResponse(w http.ResponseWriter, r *http.Request, cacheFile *CacheFileFormat, cacheKey, clientIP string, tstart time.Time) {
|
||||
// Parse the HTTP response to extract headers for our own headers
|
||||
responseReader := bytes.NewReader(cacheFile.Response)
|
||||
|
||||
// Read the status line
|
||||
statusLine, err := readLine(responseReader)
|
||||
if err != nil {
|
||||
logger.Logger.Error().
|
||||
Str("key", cacheKey).
|
||||
Str("url", r.URL.String()).
|
||||
Err(err).
|
||||
Msg("Failed to read status line from cached response")
|
||||
sc.metrics.IncrementErrors()
|
||||
sc.metrics.IncrementServiceError("cache_corrupt")
|
||||
http.Error(w, "Internal server error", http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
|
||||
// Parse status code from status line
|
||||
var statusCode int
|
||||
if _, err := fmt.Sscanf(statusLine, "HTTP/1.1 %d", &statusCode); err != nil {
|
||||
logger.Logger.Error().
|
||||
Str("key", cacheKey).
|
||||
Str("url", r.URL.String()).
|
||||
Err(err).
|
||||
Msg("Failed to parse status code from cached response")
|
||||
sc.metrics.IncrementErrors()
|
||||
sc.metrics.IncrementServiceError("cache_corrupt")
|
||||
http.Error(w, "Internal server error", http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
|
||||
// Read headers
|
||||
headers := make(map[string][]string)
|
||||
for {
|
||||
line, err := readLine(responseReader)
|
||||
if err != nil {
|
||||
logger.Logger.Error().
|
||||
Str("key", cacheKey).
|
||||
Str("url", r.URL.String()).
|
||||
Err(err).
|
||||
Msg("Failed to read headers from cached response")
|
||||
sc.metrics.IncrementErrors()
|
||||
sc.metrics.IncrementServiceError("cache_corrupt")
|
||||
http.Error(w, "Internal server error", http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
|
||||
// Empty line indicates end of headers
|
||||
if line == "" {
|
||||
break
|
||||
}
|
||||
|
||||
// Parse header line
|
||||
parts := strings.SplitN(line, ":", 2)
|
||||
if len(parts) == 2 {
|
||||
key := strings.TrimSpace(parts[0])
|
||||
value := strings.TrimSpace(parts[1])
|
||||
headers[key] = append(headers[key], value)
|
||||
}
|
||||
}
|
||||
|
||||
// Get the body data (everything after headers)
|
||||
bodyStart := responseReader.Size() - int64(responseReader.Len())
|
||||
bodyData := cacheFile.Response[bodyStart:]
|
||||
|
||||
// Handle Range requests
|
||||
rangeHeader := r.Header.Get("Range")
|
||||
if rangeHeader != "" {
|
||||
// Parse the range request
|
||||
start, end, totalSize, valid := parseRangeHeader(rangeHeader, int64(len(bodyData)))
|
||||
if !valid {
|
||||
// Invalid range - return 416 Range Not Satisfiable
|
||||
w.Header().Set("Content-Range", fmt.Sprintf("bytes */%d", len(bodyData)))
|
||||
w.WriteHeader(http.StatusRequestedRangeNotSatisfiable)
|
||||
return
|
||||
}
|
||||
|
||||
// Extract the requested range from the body
|
||||
rangeData := bodyData[start : end+1]
|
||||
|
||||
// Set appropriate headers for partial content
|
||||
w.Header().Set("Content-Range", fmt.Sprintf("bytes %d-%d/%d", start, end, totalSize))
|
||||
w.Header().Set("Content-Length", fmt.Sprintf("%d", len(rangeData)))
|
||||
w.Header().Set("Accept-Ranges", "bytes")
|
||||
|
||||
// Copy other headers (excluding Content-Length which we set above)
|
||||
for k, vv := range filterHopByHopHeaders(headers) {
|
||||
if strings.ToLower(k) == "content-length" {
|
||||
continue // We set this above for the range
|
||||
}
|
||||
for _, v := range vv {
|
||||
w.Header().Add(k, v)
|
||||
}
|
||||
}
|
||||
|
||||
// Add our own headers
|
||||
w.Header().Set("X-LanCache-Status", "HIT")
|
||||
w.Header().Set("X-LanCache-Processed-By", "SteamCache2")
|
||||
|
||||
// Write 206 Partial Content status
|
||||
w.WriteHeader(http.StatusPartialContent)
|
||||
|
||||
// Send the range data
|
||||
_, _ = w.Write(rangeData) // client write error ignored (disconnect during range body send is not actionable)
|
||||
|
||||
logger.Logger.Info().
|
||||
Str("cache_key", cacheKey).
|
||||
Str("url", r.URL.String()).
|
||||
Str("host", r.Host).
|
||||
Str("client_ip", clientIP).
|
||||
Str("cache_status", "HIT").
|
||||
Str("range", fmt.Sprintf("%d-%d/%d", start, end, totalSize)).
|
||||
Int64("range_size", end-start+1).
|
||||
Dur("response_time", time.Since(tstart)).
|
||||
Msg("cache request")
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
// No range request - serve the full file
|
||||
// Set response headers (excluding hop-by-hop headers)
|
||||
for k, vv := range filterHopByHopHeaders(headers) {
|
||||
for _, v := range vv {
|
||||
w.Header().Add(k, v)
|
||||
}
|
||||
}
|
||||
|
||||
// Add our own headers
|
||||
w.Header().Set("X-LanCache-Status", "HIT")
|
||||
w.Header().Set("X-LanCache-Processed-By", "SteamCache2")
|
||||
|
||||
// Write status code
|
||||
w.WriteHeader(statusCode)
|
||||
|
||||
// Stream the full response body
|
||||
_, _ = w.Write(bodyData) // client write error ignored (disconnect during full cached body send is not actionable)
|
||||
|
||||
logger.Logger.Info().
|
||||
Str("cache_key", cacheKey).
|
||||
Str("url", r.URL.String()).
|
||||
Str("host", r.Host).
|
||||
Str("client_ip", clientIP).
|
||||
Str("cache_status", "HIT").
|
||||
Int64("file_size", int64(len(bodyData))).
|
||||
Dur("response_time", time.Since(tstart)).
|
||||
Msg("cache request")
|
||||
}
|
||||
|
||||
// readLine reads a line from the reader, removing \r\n
|
||||
func readLine(reader *bytes.Reader) (string, error) {
|
||||
var line []byte
|
||||
for {
|
||||
b, err := reader.ReadByte()
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
if b == '\n' {
|
||||
// Remove \r if present
|
||||
if len(line) > 0 && line[len(line)-1] == '\r' {
|
||||
line = line[:len(line)-1]
|
||||
}
|
||||
return string(line), nil
|
||||
}
|
||||
line = append(line, b)
|
||||
}
|
||||
}
|
||||
|
||||
// parseRangeHeader parses a Range header and returns start, end, totalSize, and validity
|
||||
// Supports formats like "bytes=0-1023", "bytes=1024-", "bytes=-500"
|
||||
func parseRangeHeader(rangeHeader string, totalSize int64) (start, end, total int64, valid bool) {
|
||||
// Remove "bytes=" prefix
|
||||
if !strings.HasPrefix(strings.ToLower(rangeHeader), "bytes=") {
|
||||
return 0, 0, totalSize, false
|
||||
}
|
||||
|
||||
rangeSpec := strings.TrimSpace(rangeHeader[6:]) // Remove "bytes="
|
||||
|
||||
// Handle single range (we don't support multiple ranges)
|
||||
if strings.Contains(rangeSpec, ",") {
|
||||
return 0, 0, totalSize, false
|
||||
}
|
||||
|
||||
// Parse the range
|
||||
if strings.Contains(rangeSpec, "-") {
|
||||
parts := strings.Split(rangeSpec, "-")
|
||||
if len(parts) != 2 {
|
||||
return 0, 0, totalSize, false
|
||||
}
|
||||
|
||||
startStr := strings.TrimSpace(parts[0])
|
||||
endStr := strings.TrimSpace(parts[1])
|
||||
|
||||
var rangeStart, rangeEnd int64
|
||||
var parseErr error
|
||||
|
||||
if startStr == "" {
|
||||
// Suffix range: "-500" means last 500 bytes
|
||||
if endStr == "" {
|
||||
return 0, 0, totalSize, false
|
||||
}
|
||||
suffix, perr := strconv.ParseInt(endStr, 10, 64)
|
||||
if perr != nil || suffix <= 0 {
|
||||
return 0, 0, totalSize, false
|
||||
}
|
||||
rangeStart = totalSize - suffix
|
||||
if rangeStart < 0 {
|
||||
rangeStart = 0
|
||||
}
|
||||
rangeEnd = totalSize - 1
|
||||
} else if endStr == "" {
|
||||
// Open range: "1024-" means from 1024 to end
|
||||
rangeStart, parseErr = strconv.ParseInt(startStr, 10, 64)
|
||||
if parseErr != nil || rangeStart < 0 {
|
||||
return 0, 0, totalSize, false
|
||||
}
|
||||
rangeEnd = totalSize - 1
|
||||
} else {
|
||||
// Closed range: "0-1023"
|
||||
rangeStart, parseErr = strconv.ParseInt(startStr, 10, 64)
|
||||
if parseErr != nil || rangeStart < 0 {
|
||||
return 0, 0, totalSize, false
|
||||
}
|
||||
rangeEnd, parseErr = strconv.ParseInt(endStr, 10, 64)
|
||||
if parseErr != nil || rangeEnd < rangeStart {
|
||||
return 0, 0, totalSize, false
|
||||
}
|
||||
}
|
||||
|
||||
// Validate bounds
|
||||
if rangeStart >= totalSize || rangeEnd >= totalSize || rangeStart > rangeEnd {
|
||||
return 0, 0, totalSize, false
|
||||
}
|
||||
|
||||
return rangeStart, rangeEnd, totalSize, true
|
||||
}
|
||||
|
||||
return 0, 0, totalSize, false
|
||||
}
|
||||
|
||||
// verifyCompleteFile verifies that we received the complete file by checking Content-Length
|
||||
// Returns true if the file is complete, false if it's incomplete (allowing retry)
|
||||
func (sc *SteamCache) verifyCompleteFile(bodyData []byte, resp *http.Response, urlPath string, cacheKey string) bool {
|
||||
// Check if we have a Content-Length header to verify against
|
||||
if resp.ContentLength > 0 {
|
||||
receivedBytes := int64(len(bodyData))
|
||||
if receivedBytes != resp.ContentLength {
|
||||
logger.Logger.Warn().
|
||||
Str("key", cacheKey).
|
||||
Str("url", urlPath).
|
||||
Int64("received_bytes", receivedBytes).
|
||||
Int64("expected_bytes", resp.ContentLength).
|
||||
Msg("File size mismatch - incomplete download detected")
|
||||
return false
|
||||
}
|
||||
|
||||
logger.Logger.Debug().
|
||||
Str("key", cacheKey).
|
||||
Str("url", urlPath).
|
||||
Int64("file_size", receivedBytes).
|
||||
Msg("File completeness verified")
|
||||
} else {
|
||||
// No Content-Length header - we can't verify completeness
|
||||
// This is common with chunked transfer encoding
|
||||
// We don't cache chunked content to avoid risk of incomplete data
|
||||
logger.Logger.Info().
|
||||
Str("key", cacheKey).
|
||||
Str("url", urlPath).
|
||||
Int("received_bytes", len(bodyData)).
|
||||
Msg("No Content-Length header - passing through without caching")
|
||||
return false // Don't cache chunked content
|
||||
}
|
||||
|
||||
// Basic check: ensure we got some content
|
||||
if len(bodyData) == 0 {
|
||||
logger.Logger.Warn().
|
||||
Str("key", cacheKey).
|
||||
Str("url", urlPath).
|
||||
Msg("Empty file received")
|
||||
return false
|
||||
}
|
||||
|
||||
return true
|
||||
}
|
||||
|
||||
// hop-by-hop headers (per RFC) + filter helper are owned here (core to response
|
||||
// header handling in cached streaming paths) but visible package-wide.
|
||||
var hopByHopHeaders = map[string]struct{}{
|
||||
"Connection": {},
|
||||
"Keep-Alive": {},
|
||||
"Proxy-Authenticate": {},
|
||||
"Proxy-Authorization": {},
|
||||
"TE": {},
|
||||
"Trailer": {},
|
||||
"Transfer-Encoding": {},
|
||||
"Upgrade": {},
|
||||
"Date": {},
|
||||
"Server": {},
|
||||
}
|
||||
|
||||
// filterHopByHopHeaders returns a copy of src containing only headers that are
|
||||
// safe to forward (excluding hop-by-hop headers per RFC 2616 / 7230 semantics).
|
||||
// Used by streaming, MISS write, and coalesced completion paths.
|
||||
func filterHopByHopHeaders(src http.Header) http.Header {
|
||||
if src == nil {
|
||||
return nil
|
||||
}
|
||||
dst := make(http.Header, len(src))
|
||||
for k, vv := range src {
|
||||
if _, skip := hopByHopHeaders[http.CanonicalHeaderKey(k)]; skip {
|
||||
continue
|
||||
}
|
||||
dst[k] = append([]string(nil), vv...)
|
||||
}
|
||||
return dst
|
||||
}
|
||||
Reference in New Issue
Block a user