// steamcache/format.go // On-disk cache file format (SC2C magic + SHA256 content hash + raw HTTP response), // plus serialization, deserialization, response reconstruction for upstream fidelity, // streaming with HTTP Range request support, line parsing, range header parsing, // completeness verification, and hop-by-hop header filtering (shared across paths). package steamcache import ( "bytes" "fmt" "net/http" "strconv" "strings" "time" "s1d3sw1ped/steamcache2/steamcache/logger" ) // Cache file format structures // // On-disk format (documented here at top of format.go per Phase 2 plan; stable v1): // File = header-line + raw-response-bytes // header-line = "SC2C " + 64hex(bodySHA256) + " " + strconv(len(rawResp)) + "\n" // raw-response-bytes = the exact bytes from reconstructRawResponse (HTTP/1.1 status\r\n + headers\r\n\r\n + body) // deserializeCacheFile: parses header, verifies size+SHA, returns CacheFileFormat. // No compression or extra fields. filterHopByHopHeaders is the shared helper // (used in streamCachedResponse, handler MISS, coalescing.complete). const ( CacheFileMagic = "SC2C" // SteamCache2 Cache ) // CacheFileFormat represents the complete cache file structure type CacheFileFormat struct { ContentHash string // SHA256 hash of the response body (internal) ResponseSize int64 // Size of the entire HTTP response Response []byte // The entire HTTP response as raw bytes } // serializeRawResponse serializes a raw HTTP response into our text-based cache format // upstreamHash and upstreamAlgo are used for verification during download but not stored func serializeRawResponse(rawResponse []byte) ([]byte, error) { // Extract body from raw response for hash calculation bodyStart := bytes.Index(rawResponse, []byte("\r\n\r\n")) if bodyStart == -1 { return nil, fmt.Errorf("invalid HTTP response format: no body separator found") } bodyStart += 4 // Skip the \r\n\r\n bodyData := rawResponse[bodyStart:] // Always calculate our internal SHA256 hash contentHash := calculateSHA256(bodyData) // Create text-based cache file var buf bytes.Buffer // First line: magic number, content hash, response size headerLine := fmt.Sprintf("%s %s %d\n", CacheFileMagic, contentHash, len(rawResponse)) buf.WriteString(headerLine) // Rest of the file: raw HTTP response buf.Write(rawResponse) return buf.Bytes(), nil } // deserializeCacheFile deserializes our text-based cache format and returns both metadata and raw response func deserializeCacheFile(data []byte) (*CacheFileFormat, error) { if len(data) < 4 { return nil, fmt.Errorf("cache file too short") } // Find the first newline to separate header from content newlineIndex := bytes.IndexByte(data, '\n') if newlineIndex == -1 { return nil, fmt.Errorf("invalid cache file format: no header line found") } // Parse header line: "SC2C " headerLine := string(data[:newlineIndex]) parts := strings.Fields(headerLine) if len(parts) != 3 { return nil, fmt.Errorf("invalid header format: expected 3 fields, got %d", len(parts)) } // Check magic number if parts[0] != CacheFileMagic { return nil, fmt.Errorf("invalid cache file magic number: %s", parts[0]) } // Parse content hash contentHash := parts[1] if len(contentHash) != 64 { return nil, fmt.Errorf("invalid content hash length: expected 64, got %d", len(contentHash)) } // Parse response size responseSize, err := strconv.ParseInt(parts[2], 10, 64) if err != nil { return nil, fmt.Errorf("invalid response size: %w", err) } // Extract raw response (everything after the header line) rawResponse := data[newlineIndex+1:] // Verify response size if int64(len(rawResponse)) != responseSize { return nil, fmt.Errorf("response size mismatch: expected %d, got %d", responseSize, len(rawResponse)) } // Extract body from response for hash verification bodyStart := bytes.Index(rawResponse, []byte("\r\n\r\n")) if bodyStart == -1 { return nil, fmt.Errorf("invalid HTTP response format: no body separator found") } bodyStart += 4 // Skip the \r\n\r\n bodyData := rawResponse[bodyStart:] // Verify our internal SHA256 hash calculatedSHA256 := calculateSHA256(bodyData) if calculatedSHA256 != contentHash { return nil, fmt.Errorf("content hash mismatch: expected %s, got %s", contentHash, calculatedSHA256) } // Create cache file structure cacheFile := &CacheFileFormat{ ContentHash: contentHash, ResponseSize: responseSize, Response: rawResponse, } return cacheFile, nil } // reconstructRawResponse reconstructs the exact HTTP response as received from upstream func (sc *SteamCache) reconstructRawResponse(resp *http.Response, bodyData []byte) []byte { var responseBuffer bytes.Buffer // Write status line exactly as it would appear from upstream responseBuffer.WriteString(fmt.Sprintf("HTTP/1.1 %d %s\r\n", resp.StatusCode, http.StatusText(resp.StatusCode))) // Write headers in the exact order and format as received for k, vv := range resp.Header { for _, v := range vv { responseBuffer.WriteString(fmt.Sprintf("%s: %s\r\n", k, v)) } } responseBuffer.WriteString("\r\n") // End of headers // Write body responseBuffer.Write(bodyData) return responseBuffer.Bytes() } // streamCachedResponse streams the raw HTTP response bytes directly to the client // Supports Range requests by serving partial content from the cached full file func (sc *SteamCache) streamCachedResponse(w http.ResponseWriter, r *http.Request, cacheFile *CacheFileFormat, cacheKey, clientIP string, tstart time.Time) { // Parse the HTTP response to extract headers for our own headers responseReader := bytes.NewReader(cacheFile.Response) // Read the status line statusLine, err := readLine(responseReader) if err != nil { logger.Logger.Error(). Str("key", cacheKey). Str("url", r.URL.String()). Err(err). Msg("Failed to read status line from cached response") sc.metrics.IncrementErrors() sc.metrics.IncrementServiceError("cache_corrupt") http.Error(w, "Internal server error", http.StatusInternalServerError) return } // Parse status code from status line var statusCode int if _, err := fmt.Sscanf(statusLine, "HTTP/1.1 %d", &statusCode); err != nil { logger.Logger.Error(). Str("key", cacheKey). Str("url", r.URL.String()). Err(err). Msg("Failed to parse status code from cached response") sc.metrics.IncrementErrors() sc.metrics.IncrementServiceError("cache_corrupt") http.Error(w, "Internal server error", http.StatusInternalServerError) return } // Read headers headers := make(map[string][]string) for { line, err := readLine(responseReader) if err != nil { logger.Logger.Error(). Str("key", cacheKey). Str("url", r.URL.String()). Err(err). Msg("Failed to read headers from cached response") sc.metrics.IncrementErrors() sc.metrics.IncrementServiceError("cache_corrupt") http.Error(w, "Internal server error", http.StatusInternalServerError) return } // Empty line indicates end of headers if line == "" { break } // Parse header line parts := strings.SplitN(line, ":", 2) if len(parts) == 2 { key := strings.TrimSpace(parts[0]) value := strings.TrimSpace(parts[1]) headers[key] = append(headers[key], value) } } // Get the body data (everything after headers) bodyStart := responseReader.Size() - int64(responseReader.Len()) bodyData := cacheFile.Response[bodyStart:] // Handle Range requests rangeHeader := r.Header.Get("Range") if rangeHeader != "" { // Parse the range request start, end, totalSize, valid := parseRangeHeader(rangeHeader, int64(len(bodyData))) if !valid { // Invalid range - return 416 Range Not Satisfiable w.Header().Set("Content-Range", fmt.Sprintf("bytes */%d", len(bodyData))) w.WriteHeader(http.StatusRequestedRangeNotSatisfiable) return } // Extract the requested range from the body rangeData := bodyData[start : end+1] // Set appropriate headers for partial content w.Header().Set("Content-Range", fmt.Sprintf("bytes %d-%d/%d", start, end, totalSize)) w.Header().Set("Content-Length", fmt.Sprintf("%d", len(rangeData))) w.Header().Set("Accept-Ranges", "bytes") // Copy other headers (excluding Content-Length which we set above) for k, vv := range filterHopByHopHeaders(headers) { if strings.ToLower(k) == "content-length" { continue // We set this above for the range } for _, v := range vv { w.Header().Add(k, v) } } // Add our own headers w.Header().Set("X-LanCache-Status", "HIT") w.Header().Set("X-LanCache-Processed-By", "SteamCache2") // Write 206 Partial Content status w.WriteHeader(http.StatusPartialContent) // Send the range data _, _ = w.Write(rangeData) // client write error ignored (disconnect during range body send is not actionable) logger.Logger.Info(). Str("cache_key", cacheKey). Str("url", r.URL.String()). Str("host", r.Host). Str("client_ip", clientIP). Str("cache_status", "HIT"). Str("range", fmt.Sprintf("%d-%d/%d", start, end, totalSize)). Int64("range_size", end-start+1). Dur("response_time", time.Since(tstart)). Msg("cache request") return } // No range request - serve the full file // Set response headers (excluding hop-by-hop headers) for k, vv := range filterHopByHopHeaders(headers) { for _, v := range vv { w.Header().Add(k, v) } } // Add our own headers w.Header().Set("X-LanCache-Status", "HIT") w.Header().Set("X-LanCache-Processed-By", "SteamCache2") // Write status code w.WriteHeader(statusCode) // Stream the full response body _, _ = w.Write(bodyData) // client write error ignored (disconnect during full cached body send is not actionable) logger.Logger.Info(). Str("cache_key", cacheKey). Str("url", r.URL.String()). Str("host", r.Host). Str("client_ip", clientIP). Str("cache_status", "HIT"). Int64("file_size", int64(len(bodyData))). Dur("response_time", time.Since(tstart)). Msg("cache request") } // readLine reads a line from the reader, removing \r\n func readLine(reader *bytes.Reader) (string, error) { var line []byte for { b, err := reader.ReadByte() if err != nil { return "", err } if b == '\n' { // Remove \r if present if len(line) > 0 && line[len(line)-1] == '\r' { line = line[:len(line)-1] } return string(line), nil } line = append(line, b) } } // parseRangeHeader parses a Range header and returns start, end, totalSize, and validity // Supports formats like "bytes=0-1023", "bytes=1024-", "bytes=-500" func parseRangeHeader(rangeHeader string, totalSize int64) (start, end, total int64, valid bool) { // Remove "bytes=" prefix if !strings.HasPrefix(strings.ToLower(rangeHeader), "bytes=") { return 0, 0, totalSize, false } rangeSpec := strings.TrimSpace(rangeHeader[6:]) // Remove "bytes=" // Handle single range (we don't support multiple ranges) if strings.Contains(rangeSpec, ",") { return 0, 0, totalSize, false } // Parse the range if strings.Contains(rangeSpec, "-") { parts := strings.Split(rangeSpec, "-") if len(parts) != 2 { return 0, 0, totalSize, false } startStr := strings.TrimSpace(parts[0]) endStr := strings.TrimSpace(parts[1]) var rangeStart, rangeEnd int64 var parseErr error if startStr == "" { // Suffix range: "-500" means last 500 bytes if endStr == "" { return 0, 0, totalSize, false } suffix, perr := strconv.ParseInt(endStr, 10, 64) if perr != nil || suffix <= 0 { return 0, 0, totalSize, false } rangeStart = totalSize - suffix if rangeStart < 0 { rangeStart = 0 } rangeEnd = totalSize - 1 } else if endStr == "" { // Open range: "1024-" means from 1024 to end rangeStart, parseErr = strconv.ParseInt(startStr, 10, 64) if parseErr != nil || rangeStart < 0 { return 0, 0, totalSize, false } rangeEnd = totalSize - 1 } else { // Closed range: "0-1023" rangeStart, parseErr = strconv.ParseInt(startStr, 10, 64) if parseErr != nil || rangeStart < 0 { return 0, 0, totalSize, false } rangeEnd, parseErr = strconv.ParseInt(endStr, 10, 64) if parseErr != nil || rangeEnd < rangeStart { return 0, 0, totalSize, false } } // Validate bounds if rangeStart >= totalSize || rangeEnd >= totalSize || rangeStart > rangeEnd { return 0, 0, totalSize, false } return rangeStart, rangeEnd, totalSize, true } return 0, 0, totalSize, false } // verifyCompleteFile verifies that we received the complete file by checking Content-Length // Returns true if the file is complete, false if it's incomplete (allowing retry) func (sc *SteamCache) verifyCompleteFile(bodyData []byte, resp *http.Response, urlPath string, cacheKey string) bool { // Check if we have a Content-Length header to verify against if resp.ContentLength > 0 { receivedBytes := int64(len(bodyData)) if receivedBytes != resp.ContentLength { logger.Logger.Warn(). Str("key", cacheKey). Str("url", urlPath). Int64("received_bytes", receivedBytes). Int64("expected_bytes", resp.ContentLength). Msg("File size mismatch - incomplete download detected") return false } logger.Logger.Debug(). Str("key", cacheKey). Str("url", urlPath). Int64("file_size", receivedBytes). Msg("File completeness verified") } else { // No Content-Length header - we can't verify completeness // This is common with chunked transfer encoding // We don't cache chunked content to avoid risk of incomplete data logger.Logger.Info(). Str("key", cacheKey). Str("url", urlPath). Int("received_bytes", len(bodyData)). Msg("No Content-Length header - passing through without caching") return false // Don't cache chunked content } // Basic check: ensure we got some content if len(bodyData) == 0 { logger.Logger.Warn(). Str("key", cacheKey). Str("url", urlPath). Msg("Empty file received") return false } return true } // hop-by-hop headers (per RFC) + filter helper are owned here (core to response // header handling in cached streaming paths) but visible package-wide. var hopByHopHeaders = map[string]struct{}{ "Connection": {}, "Keep-Alive": {}, "Proxy-Authenticate": {}, "Proxy-Authorization": {}, "TE": {}, "Trailer": {}, "Transfer-Encoding": {}, "Upgrade": {}, "Date": {}, "Server": {}, } // filterHopByHopHeaders returns a copy of src containing only headers that are // safe to forward (excluding hop-by-hop headers per RFC 2616 / 7230 semantics). // Used by streaming, MISS write, and coalesced completion paths. func filterHopByHopHeaders(src http.Header) http.Header { if src == nil { return nil } dst := make(http.Header, len(src)) for k, vv := range src { if _, skip := hopByHopHeaders[http.CanonicalHeaderKey(k)]; skip { continue } dst[k] = append([]string(nil), vv...) } return dst }