steamcache2/steamcache/format.go

// steamcache/format.go
// On-disk cache file format (SC2C magic + SHA256 content hash + raw HTTP response),
// plus serialization, deserialization, response reconstruction for upstream fidelity,
// streaming with HTTP Range request support, line parsing, range header parsing,
// completeness verification, and hop-by-hop header filtering (shared across paths).
package steamcache

import (
	"bytes"
	"fmt"
	"net/http"
	"strconv"
	"strings"
	"time"

	"s1d3sw1ped/steamcache2/steamcache/logger"
)

// Cache file format structures
//
// On-disk format (documented here at top of format.go per Phase 2 plan; stable v1):
// File = header-line + raw-response-bytes
// header-line = "SC2C " + 64hex(bodySHA256) + " " + strconv(len(rawResp)) + "\n"
// raw-response-bytes = the exact bytes from reconstructRawResponse (HTTP/1.1 status\r\n + headers\r\n\r\n + body)
// deserializeCacheFile: parses header, verifies size+SHA, returns CacheFileFormat.
// No compression or extra fields. filterHopByHopHeaders is the shared helper
// (used in streamCachedResponse, handler MISS, coalescing.complete).
const (
	CacheFileMagic = "SC2C" // SteamCache2 Cache
)

// CacheFileFormat represents the complete cache file structure
type CacheFileFormat struct {
	ContentHash  string // SHA256 hash of the response body (internal)
	ResponseSize int64  // Size of the entire HTTP response
	Response     []byte // The entire HTTP response as raw bytes
}

// serializeRawResponse serializes a raw HTTP response into our text-based cache format
// upstreamHash and upstreamAlgo are used for verification during download but not stored
func serializeRawResponse(rawResponse []byte) ([]byte, error) {
	// Extract body from raw response for hash calculation
	bodyStart := bytes.Index(rawResponse, []byte("\r\n\r\n"))
	if bodyStart == -1 {
		return nil, fmt.Errorf("invalid HTTP response format: no body separator found")
	}
	bodyStart += 4 // Skip the \r\n\r\n
	bodyData := rawResponse[bodyStart:]

	// Always calculate our internal SHA256 hash
	contentHash := calculateSHA256(bodyData)

	// Create text-based cache file
	var buf bytes.Buffer

	// First line: magic number, content hash, response size
	headerLine := fmt.Sprintf("%s %s %d\n", CacheFileMagic, contentHash, len(rawResponse))
	buf.WriteString(headerLine)

	// Rest of the file: raw HTTP response
	buf.Write(rawResponse)

	return buf.Bytes(), nil
}

// deserializeCacheFile deserializes our text-based cache format and returns both metadata and raw response
func deserializeCacheFile(data []byte) (*CacheFileFormat, error) {
	if len(data) < 4 {
		return nil, fmt.Errorf("cache file too short")
	}

	// Find the first newline to separate header from content
	newlineIndex := bytes.IndexByte(data, '\n')
	if newlineIndex == -1 {
		return nil, fmt.Errorf("invalid cache file format: no header line found")
	}

	// Parse header line: "SC2C <hash> <size>"
	headerLine := string(data[:newlineIndex])
	parts := strings.Fields(headerLine)
	if len(parts) != 3 {
		return nil, fmt.Errorf("invalid header format: expected 3 fields, got %d", len(parts))
	}

	// Check magic number
	if parts[0] != CacheFileMagic {
		return nil, fmt.Errorf("invalid cache file magic number: %s", parts[0])
	}

	// Parse content hash
	contentHash := parts[1]
	if len(contentHash) != 64 {
		return nil, fmt.Errorf("invalid content hash length: expected 64, got %d", len(contentHash))
	}

	// Parse response size
	responseSize, err := strconv.ParseInt(parts[2], 10, 64)
	if err != nil {
		return nil, fmt.Errorf("invalid response size: %w", err)
	}

	// Extract raw response (everything after the header line)
	rawResponse := data[newlineIndex+1:]

	// Verify response size
	if int64(len(rawResponse)) != responseSize {
		return nil, fmt.Errorf("response size mismatch: expected %d, got %d",
			responseSize, len(rawResponse))
	}

	// Extract body from response for hash verification
	bodyStart := bytes.Index(rawResponse, []byte("\r\n\r\n"))
	if bodyStart == -1 {
		return nil, fmt.Errorf("invalid HTTP response format: no body separator found")
	}
	bodyStart += 4 // Skip the \r\n\r\n
	bodyData := rawResponse[bodyStart:]

	// Verify our internal SHA256 hash
	calculatedSHA256 := calculateSHA256(bodyData)
	if calculatedSHA256 != contentHash {
		return nil, fmt.Errorf("content hash mismatch: expected %s, got %s",
			contentHash, calculatedSHA256)
	}

	// Create cache file structure
	cacheFile := &CacheFileFormat{
		ContentHash:  contentHash,
		ResponseSize: responseSize,
		Response:     rawResponse,
	}

	return cacheFile, nil
}

// reconstructRawResponse reconstructs the exact HTTP response as received from upstream
func (sc *SteamCache) reconstructRawResponse(resp *http.Response, bodyData []byte) []byte {
	var responseBuffer bytes.Buffer

	// Write status line exactly as it would appear from upstream
	responseBuffer.WriteString(fmt.Sprintf("HTTP/1.1 %d %s\r\n", resp.StatusCode, http.StatusText(resp.StatusCode)))

	// Write headers in the exact order and format as received
	for k, vv := range resp.Header {
		for _, v := range vv {
			responseBuffer.WriteString(fmt.Sprintf("%s: %s\r\n", k, v))
		}
	}
	responseBuffer.WriteString("\r\n") // End of headers

	// Write body
	responseBuffer.Write(bodyData)

	return responseBuffer.Bytes()
}

// streamCachedResponse streams the raw HTTP response bytes directly to the client
// Supports Range requests by serving partial content from the cached full file
func (sc *SteamCache) streamCachedResponse(w http.ResponseWriter, r *http.Request, cacheFile *CacheFileFormat, cacheKey, clientIP string, tstart time.Time) {
	// Parse the HTTP response to extract headers for our own headers
	responseReader := bytes.NewReader(cacheFile.Response)

	// Read the status line
	statusLine, err := readLine(responseReader)
	if err != nil {
		logger.Logger.Error().
			Str("key", cacheKey).
			Str("url", r.URL.String()).
			Err(err).
			Msg("Failed to read status line from cached response")
		sc.metrics.IncrementErrors()
		sc.metrics.IncrementServiceError("cache_corrupt")
		http.Error(w, "Internal server error", http.StatusInternalServerError)
		return
	}

	// Parse status code from status line
	var statusCode int
	if _, err := fmt.Sscanf(statusLine, "HTTP/1.1 %d", &statusCode); err != nil {
		logger.Logger.Error().
			Str("key", cacheKey).
			Str("url", r.URL.String()).
			Err(err).
			Msg("Failed to parse status code from cached response")
		sc.metrics.IncrementErrors()
		sc.metrics.IncrementServiceError("cache_corrupt")
		http.Error(w, "Internal server error", http.StatusInternalServerError)
		return
	}

	// Read headers
	headers := make(map[string][]string)
	for {
		line, err := readLine(responseReader)
		if err != nil {
			logger.Logger.Error().
				Str("key", cacheKey).
				Str("url", r.URL.String()).
				Err(err).
				Msg("Failed to read headers from cached response")
			sc.metrics.IncrementErrors()
			sc.metrics.IncrementServiceError("cache_corrupt")
			http.Error(w, "Internal server error", http.StatusInternalServerError)
			return
		}

		// Empty line indicates end of headers
		if line == "" {
			break
		}

		// Parse header line
		parts := strings.SplitN(line, ":", 2)
		if len(parts) == 2 {
			key := strings.TrimSpace(parts[0])
			value := strings.TrimSpace(parts[1])
			headers[key] = append(headers[key], value)
		}
	}

	// Get the body data (everything after headers)
	bodyStart := responseReader.Size() - int64(responseReader.Len())
	bodyData := cacheFile.Response[bodyStart:]

	// Handle Range requests
	rangeHeader := r.Header.Get("Range")
	if rangeHeader != "" {
		// Parse the range request
		start, end, totalSize, valid := parseRangeHeader(rangeHeader, int64(len(bodyData)))
		if !valid {
			// Invalid range - return 416 Range Not Satisfiable
			w.Header().Set("Content-Range", fmt.Sprintf("bytes */%d", len(bodyData)))
			w.WriteHeader(http.StatusRequestedRangeNotSatisfiable)
			return
		}

		// Extract the requested range from the body
		rangeData := bodyData[start : end+1]

		// Set appropriate headers for partial content
		w.Header().Set("Content-Range", fmt.Sprintf("bytes %d-%d/%d", start, end, totalSize))
		w.Header().Set("Content-Length", fmt.Sprintf("%d", len(rangeData)))
		w.Header().Set("Accept-Ranges", "bytes")

		// Copy other headers (excluding Content-Length which we set above)
		for k, vv := range filterHopByHopHeaders(headers) {
			if strings.ToLower(k) == "content-length" {
				continue // We set this above for the range
			}
			for _, v := range vv {
				w.Header().Add(k, v)
			}
		}

		// Add our own headers
		w.Header().Set("X-LanCache-Status", "HIT")
		w.Header().Set("X-LanCache-Processed-By", "SteamCache2")

		// Write 206 Partial Content status
		w.WriteHeader(http.StatusPartialContent)

		// Send the range data
		_, _ = w.Write(rangeData) // client write error ignored (disconnect during range body send is not actionable)

		logger.Logger.Info().
			Str("cache_key", cacheKey).
			Str("url", r.URL.String()).
			Str("host", r.Host).
			Str("client_ip", clientIP).
			Str("cache_status", "HIT").
			Str("range", fmt.Sprintf("%d-%d/%d", start, end, totalSize)).
			Int64("range_size", end-start+1).
			Dur("response_time", time.Since(tstart)).
			Msg("cache request")

		return
	}

	// No range request - serve the full file
	// Set response headers (excluding hop-by-hop headers)
	for k, vv := range filterHopByHopHeaders(headers) {
		for _, v := range vv {
			w.Header().Add(k, v)
		}
	}

	// Add our own headers
	w.Header().Set("X-LanCache-Status", "HIT")
	w.Header().Set("X-LanCache-Processed-By", "SteamCache2")

	// Write status code
	w.WriteHeader(statusCode)

	// Stream the full response body
	_, _ = w.Write(bodyData) // client write error ignored (disconnect during full cached body send is not actionable)

	logger.Logger.Info().
		Str("cache_key", cacheKey).
		Str("url", r.URL.String()).
		Str("host", r.Host).
		Str("client_ip", clientIP).
		Str("cache_status", "HIT").
		Int64("file_size", int64(len(bodyData))).
		Dur("response_time", time.Since(tstart)).
		Msg("cache request")
}

// readLine reads a line from the reader, removing \r\n
func readLine(reader *bytes.Reader) (string, error) {
	var line []byte
	for {
		b, err := reader.ReadByte()
		if err != nil {
			return "", err
		}
		if b == '\n' {
			// Remove \r if present
			if len(line) > 0 && line[len(line)-1] == '\r' {
				line = line[:len(line)-1]
			}
			return string(line), nil
		}
		line = append(line, b)
	}
}

// parseRangeHeader parses a Range header and returns start, end, totalSize, and validity
// Supports formats like "bytes=0-1023", "bytes=1024-", "bytes=-500"
func parseRangeHeader(rangeHeader string, totalSize int64) (start, end, total int64, valid bool) {
	// Remove "bytes=" prefix
	if !strings.HasPrefix(strings.ToLower(rangeHeader), "bytes=") {
		return 0, 0, totalSize, false
	}

	rangeSpec := strings.TrimSpace(rangeHeader[6:]) // Remove "bytes="

	// Handle single range (we don't support multiple ranges)
	if strings.Contains(rangeSpec, ",") {
		return 0, 0, totalSize, false
	}

	// Parse the range
	if strings.Contains(rangeSpec, "-") {
		parts := strings.Split(rangeSpec, "-")
		if len(parts) != 2 {
			return 0, 0, totalSize, false
		}

		startStr := strings.TrimSpace(parts[0])
		endStr := strings.TrimSpace(parts[1])

		var rangeStart, rangeEnd int64
		var parseErr error

		if startStr == "" {
			// Suffix range: "-500" means last 500 bytes
			if endStr == "" {
				return 0, 0, totalSize, false
			}
			suffix, perr := strconv.ParseInt(endStr, 10, 64)
			if perr != nil || suffix <= 0 {
				return 0, 0, totalSize, false
			}
			rangeStart = totalSize - suffix
			if rangeStart < 0 {
				rangeStart = 0
			}
			rangeEnd = totalSize - 1
		} else if endStr == "" {
			// Open range: "1024-" means from 1024 to end
			rangeStart, parseErr = strconv.ParseInt(startStr, 10, 64)
			if parseErr != nil || rangeStart < 0 {
				return 0, 0, totalSize, false
			}
			rangeEnd = totalSize - 1
		} else {
			// Closed range: "0-1023"
			rangeStart, parseErr = strconv.ParseInt(startStr, 10, 64)
			if parseErr != nil || rangeStart < 0 {
				return 0, 0, totalSize, false
			}
			rangeEnd, parseErr = strconv.ParseInt(endStr, 10, 64)
			if parseErr != nil || rangeEnd < rangeStart {
				return 0, 0, totalSize, false
			}
		}

		// Validate bounds
		if rangeStart >= totalSize || rangeEnd >= totalSize || rangeStart > rangeEnd {
			return 0, 0, totalSize, false
		}

		return rangeStart, rangeEnd, totalSize, true
	}

	return 0, 0, totalSize, false
}

// verifyCompleteFile verifies that we received the complete file by checking Content-Length
// Returns true if the file is complete, false if it's incomplete (allowing retry)
func (sc *SteamCache) verifyCompleteFile(bodyData []byte, resp *http.Response, urlPath string, cacheKey string) bool {
	// Check if we have a Content-Length header to verify against
	if resp.ContentLength > 0 {
		receivedBytes := int64(len(bodyData))
		if receivedBytes != resp.ContentLength {
			logger.Logger.Warn().
				Str("key", cacheKey).
				Str("url", urlPath).
				Int64("received_bytes", receivedBytes).
				Int64("expected_bytes", resp.ContentLength).
				Msg("File size mismatch - incomplete download detected")
			return false
		}

		logger.Logger.Debug().
			Str("key", cacheKey).
			Str("url", urlPath).
			Int64("file_size", receivedBytes).
			Msg("File completeness verified")
	} else {
		// No Content-Length header - we can't verify completeness
		// This is common with chunked transfer encoding
		// We don't cache chunked content to avoid risk of incomplete data
		logger.Logger.Info().
			Str("key", cacheKey).
			Str("url", urlPath).
			Int("received_bytes", len(bodyData)).
			Msg("No Content-Length header - passing through without caching")
		return false // Don't cache chunked content
	}

	// Basic check: ensure we got some content
	if len(bodyData) == 0 {
		logger.Logger.Warn().
			Str("key", cacheKey).
			Str("url", urlPath).
			Msg("Empty file received")
		return false
	}

	return true
}

// hop-by-hop headers (per RFC) + filter helper are owned here (core to response
// header handling in cached streaming paths) but visible package-wide.
var hopByHopHeaders = map[string]struct{}{
	"Connection":          {},
	"Keep-Alive":          {},
	"Proxy-Authenticate":  {},
	"Proxy-Authorization": {},
	"TE":                  {},
	"Trailer":             {},
	"Transfer-Encoding":   {},
	"Upgrade":             {},
	"Date":                {},
	"Server":              {},
}

// filterHopByHopHeaders returns a copy of src containing only headers that are
// safe to forward (excluding hop-by-hop headers per RFC 2616 / 7230 semantics).
// Used by streaming, MISS write, and coalesced completion paths.
func filterHopByHopHeaders(src http.Header) http.Header {
	if src == nil {
		return nil
	}
	dst := make(http.Header, len(src))
	for k, vv := range src {
		if _, skip := hopByHopHeaders[http.CanonicalHeaderKey(k)]; skip {
			continue
		}
		dst[k] = append([]string(nil), vv...)
	}
	return dst
}