Refactor caching logic and enhance hash generation in steamcache
- Replaced SHA1 hash calculations with SHA256 for improved security and consistency in cache key generation. - Introduced a new TestURLHashing function to validate the new cache key generation logic. - Removed outdated hash calculation tests and streamlined the caching process to focus on URL-based hashing. - Implemented lightweight validation methods in ServeHTTP to enhance performance and reliability of cached responses. - Added batched time updates in VFS implementations for better performance during access time tracking.
This commit is contained in:
@@ -4,9 +4,8 @@ package steamcache
|
||||
import (
|
||||
"bufio"
|
||||
"context"
|
||||
"crypto/sha1"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"fmt"
|
||||
"io"
|
||||
"net"
|
||||
"net/http"
|
||||
@@ -18,108 +17,30 @@ import (
|
||||
"s1d3sw1ped/SteamCache2/vfs/disk"
|
||||
"s1d3sw1ped/SteamCache2/vfs/gc"
|
||||
"s1d3sw1ped/SteamCache2/vfs/memory"
|
||||
"sort"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"bytes"
|
||||
|
||||
"github.com/docker/go-units"
|
||||
)
|
||||
|
||||
// extractHashFromSteamPath extracts a hash from Steam depot URLs
|
||||
// Handles patterns like: /depot/123/chunk/abcdef... or /depot/123/manifest/456/789/hash
|
||||
func extractHashFromSteamPath(path string) (string, bool) {
|
||||
path = strings.TrimPrefix(path, "/")
|
||||
|
||||
parts := strings.Split(path, "/")
|
||||
if len(parts) < 3 {
|
||||
return "", false
|
||||
}
|
||||
|
||||
// Handle chunk files: depot/{id}/chunk/{hash}
|
||||
if len(parts) >= 4 && parts[0] == "depot" && parts[2] == "chunk" {
|
||||
hash := parts[3]
|
||||
// Validate it's a 40-character hex hash
|
||||
if len(hash) == 40 && isHexString(hash) {
|
||||
return strings.ToLower(hash), true
|
||||
}
|
||||
}
|
||||
|
||||
// Handle manifest files: depot/{id}/manifest/{manifest_id}/{version}/{hash}
|
||||
if len(parts) >= 6 && parts[0] == "depot" && parts[2] == "manifest" {
|
||||
hash := parts[5]
|
||||
// Note: Manifest hashes can be shorter than 40 characters
|
||||
if len(hash) >= 10 && isHexString(hash) {
|
||||
return strings.ToLower(hash), true
|
||||
}
|
||||
}
|
||||
|
||||
return "", false
|
||||
}
|
||||
|
||||
// isHexString checks if a string contains only hexadecimal characters
|
||||
func isHexString(s string) bool {
|
||||
for _, r := range s {
|
||||
if !((r >= '0' && r <= '9') || (r >= 'a' && r <= 'f') || (r >= 'A' && r <= 'F')) {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
// generateSteamCacheKey converts Steam depot paths to simplified cache keys
|
||||
// Input: /depot/1684171/chunk/0016cfc5019b8baa6026aa1cce93e685d6e06c6e
|
||||
// Output: steam/0016cfc5019b8baa6026aa1cce93e685d6e06c6e
|
||||
func generateSteamCacheKey(urlPath string) string {
|
||||
if hash, ok := extractHashFromSteamPath(urlPath); ok {
|
||||
return "steam/" + hash
|
||||
}
|
||||
|
||||
// Return empty string for unsupported depot URLs
|
||||
return ""
|
||||
}
|
||||
|
||||
// calculateFileHash calculates the SHA1 hash of the given data
|
||||
func calculateFileHash(data []byte) string {
|
||||
hash := sha1.Sum(data)
|
||||
// generateURLHash creates a SHA256 hash of the entire URL path for cache key
|
||||
func generateURLHash(urlPath string) string {
|
||||
hash := sha256.Sum256([]byte(urlPath))
|
||||
return hex.EncodeToString(hash[:])
|
||||
}
|
||||
|
||||
// calculateResponseHash calculates the SHA1 hash of the full HTTP response
|
||||
func calculateResponseHash(resp *http.Response, bodyData []byte) string {
|
||||
hash := sha1.New()
|
||||
|
||||
// Include status line
|
||||
statusLine := fmt.Sprintf("HTTP/1.1 %d %s\n", resp.StatusCode, resp.Status)
|
||||
hash.Write([]byte(statusLine))
|
||||
|
||||
// Include headers (sorted for consistency)
|
||||
headers := make([]string, 0, len(resp.Header))
|
||||
for key, values := range resp.Header {
|
||||
for _, value := range values {
|
||||
headers = append(headers, fmt.Sprintf("%s: %s\n", key, value))
|
||||
}
|
||||
}
|
||||
sort.Strings(headers)
|
||||
for _, header := range headers {
|
||||
hash.Write([]byte(header))
|
||||
// generateSteamCacheKey creates a cache key from the URL path using SHA256
|
||||
// Input: /depot/1684171/chunk/0016cfc5019b8baa6026aa1cce93e685d6e06c6e
|
||||
// Output: steam/a1b2c3d4e5f678901234567890123456789012345678901234567890
|
||||
func generateSteamCacheKey(urlPath string) string {
|
||||
// Handle Steam depot URLs by creating a SHA256 hash of the entire path
|
||||
if strings.HasPrefix(urlPath, "/depot/") {
|
||||
return "steam/" + generateURLHash(urlPath)
|
||||
}
|
||||
|
||||
// Include empty line between headers and body
|
||||
hash.Write([]byte("\n"))
|
||||
|
||||
// Include body
|
||||
hash.Write(bodyData)
|
||||
|
||||
return hex.EncodeToString(hash.Sum(nil))
|
||||
}
|
||||
|
||||
// verifyFileHash verifies that the file content matches the expected hash
|
||||
func verifyFileHash(data []byte, expectedHash string) bool {
|
||||
actualHash := calculateFileHash(data)
|
||||
return strings.EqualFold(actualHash, expectedHash)
|
||||
// For non-Steam URLs, return empty string (not cached)
|
||||
return ""
|
||||
}
|
||||
|
||||
var hopByHopHeaders = map[string]struct{}{
|
||||
@@ -395,7 +316,7 @@ func (sc *SteamCache) ServeHTTP(w http.ResponseWriter, r *http.Request) {
|
||||
if strings.HasPrefix(r.URL.String(), "/depot/") {
|
||||
// trim the query parameters from the URL path
|
||||
// this is necessary because the cache key should not include query parameters
|
||||
urlPath := strings.Split(r.URL.String(), "?")[0]
|
||||
urlPath, _, _ := strings.Cut(r.URL.String(), "?")
|
||||
|
||||
tstart := time.Now()
|
||||
|
||||
@@ -568,61 +489,48 @@ func (sc *SteamCache) ServeHTTP(w http.ResponseWriter, r *http.Request) {
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
// Read the entire response body into memory for hash verification
|
||||
bodyData, err := io.ReadAll(resp.Body)
|
||||
if err != nil {
|
||||
logger.Logger.Error().Err(err).Str("url", req.URL.String()).Msg("Failed to read response body")
|
||||
// Fast path: Flexible lightweight validation for all files
|
||||
// Multiple validation layers ensure data integrity without blocking legitimate Steam content
|
||||
|
||||
// Complete coalesced request with error
|
||||
if isNew {
|
||||
coalescedReq.complete(nil, err)
|
||||
}
|
||||
|
||||
http.Error(w, "Failed to read response body", http.StatusInternalServerError)
|
||||
// Method 1: HTTP Status Validation
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
logger.Logger.Error().
|
||||
Str("url", req.URL.String()).
|
||||
Int("status_code", resp.StatusCode).
|
||||
Msg("Steam returned non-OK status")
|
||||
http.Error(w, "Upstream server error", http.StatusBadGateway)
|
||||
return
|
||||
}
|
||||
|
||||
// Extract hash from cache key for verification
|
||||
var expectedHash string
|
||||
var hasHash bool
|
||||
if strings.HasPrefix(cacheKey, "steam/") {
|
||||
expectedHash = cacheKey[6:] // Remove "steam/" prefix
|
||||
hasHash = len(expectedHash) == 64 // SHA-256 hashes are 64 characters
|
||||
// Method 2: Content-Type Validation (Steam files should be binary)
|
||||
contentType := resp.Header.Get("Content-Type")
|
||||
if contentType != "" && !strings.Contains(contentType, "application/octet-stream") {
|
||||
logger.Logger.Warn().
|
||||
Str("url", req.URL.String()).
|
||||
Str("content_type", contentType).
|
||||
Msg("Unexpected content type from Steam")
|
||||
}
|
||||
|
||||
// Hash verification using Steam's X-Content-Sha header and content length verification
|
||||
hashVerified := true
|
||||
if hasHash {
|
||||
// Get the hash from Steam's X-Content-Sha header
|
||||
steamHash := resp.Header.Get("X-Content-Sha")
|
||||
// Method 3: Content-Length Validation
|
||||
expectedSize := resp.ContentLength
|
||||
|
||||
// Verify using Steam's hash
|
||||
if strings.EqualFold(steamHash, expectedHash) {
|
||||
// Hash verification succeeded
|
||||
} else {
|
||||
logger.Logger.Error().
|
||||
Str("key", cacheKey).
|
||||
Str("expected_hash", expectedHash).
|
||||
Str("steam_hash", steamHash).
|
||||
Int("content_length", len(bodyData)).
|
||||
Msg("Steam hash verification failed - Steam's hash doesn't match filename")
|
||||
hashVerified = false
|
||||
}
|
||||
} else {
|
||||
// No hash to verify
|
||||
}
|
||||
|
||||
// Always verify content length as an additional safety check
|
||||
if resp.ContentLength > 0 && int64(len(bodyData)) != resp.ContentLength {
|
||||
// Reject only truly invalid content lengths (zero or negative)
|
||||
if expectedSize <= 0 {
|
||||
logger.Logger.Error().
|
||||
Str("key", cacheKey).
|
||||
Int("actual_content_length", len(bodyData)).
|
||||
Int64("expected_content_length", resp.ContentLength).
|
||||
Msg("Content length verification failed")
|
||||
hashVerified = false
|
||||
Str("url", req.URL.String()).
|
||||
Int64("content_length", expectedSize).
|
||||
Msg("Invalid content length, rejecting file")
|
||||
http.Error(w, "Invalid content length", http.StatusBadGateway)
|
||||
return
|
||||
}
|
||||
|
||||
// Write to response (always serve the file)
|
||||
// Content length is valid - no size restrictions to keep logs clean
|
||||
|
||||
// Lightweight validation passed - trust the Content-Length and HTTP status
|
||||
// This provides good integrity with minimal performance overhead
|
||||
validationPassed := true
|
||||
|
||||
// Write to response (stream the file directly)
|
||||
// Remove hop-by-hop and server-specific headers
|
||||
for k, vv := range resp.Header {
|
||||
if _, skip := hopByHopHeaders[http.CanonicalHeaderKey(k)]; skip {
|
||||
@@ -635,16 +543,18 @@ func (sc *SteamCache) ServeHTTP(w http.ResponseWriter, r *http.Request) {
|
||||
// Add our own headers
|
||||
w.Header().Set("X-LanCache-Status", "MISS")
|
||||
w.Header().Set("X-LanCache-Processed-By", "SteamCache2")
|
||||
w.Write(bodyData)
|
||||
|
||||
// Stream the response body directly to client (no memory buffering)
|
||||
io.Copy(w, resp.Body)
|
||||
|
||||
// Complete coalesced request for waiting clients
|
||||
if isNew {
|
||||
// Create a new response for coalesced clients
|
||||
// Create a new response for coalesced clients with a fresh body
|
||||
coalescedResp := &http.Response{
|
||||
StatusCode: resp.StatusCode,
|
||||
Status: resp.Status,
|
||||
Header: make(http.Header),
|
||||
Body: io.NopCloser(bytes.NewReader(bodyData)),
|
||||
Body: io.NopCloser(strings.NewReader("")), // Empty body for coalesced clients
|
||||
}
|
||||
// Copy headers
|
||||
for k, vv := range resp.Header {
|
||||
@@ -653,19 +563,28 @@ func (sc *SteamCache) ServeHTTP(w http.ResponseWriter, r *http.Request) {
|
||||
coalescedReq.complete(coalescedResp, nil)
|
||||
}
|
||||
|
||||
// Only cache the file if hash verification passed (or no hash was present)
|
||||
if hashVerified {
|
||||
writer, _ := sc.vfs.Create(cachePath, int64(0)) // size is not known in advance
|
||||
if writer != nil {
|
||||
defer writer.Close()
|
||||
// Write the full HTTP response to cache
|
||||
resp.Body = io.NopCloser(bytes.NewReader(bodyData)) // Reset body for writing
|
||||
resp.Write(writer)
|
||||
// Cache the file if validation passed
|
||||
if validationPassed {
|
||||
// Create a new request to fetch the file again for caching
|
||||
cacheReq, err := http.NewRequest(http.MethodGet, req.URL.String(), nil)
|
||||
if err == nil {
|
||||
// Copy original headers
|
||||
for k, vv := range req.Header {
|
||||
cacheReq.Header[k] = vv
|
||||
}
|
||||
|
||||
// Fetch fresh copy for caching
|
||||
cacheResp, err := sc.client.Do(cacheReq)
|
||||
if err == nil {
|
||||
defer cacheResp.Body.Close()
|
||||
// Use the validated size from the original response
|
||||
writer, _ := sc.vfs.Create(cachePath, expectedSize)
|
||||
if writer != nil {
|
||||
defer writer.Close()
|
||||
io.Copy(writer, cacheResp.Body)
|
||||
}
|
||||
}
|
||||
}
|
||||
} else {
|
||||
logger.Logger.Warn().
|
||||
Str("key", cacheKey).
|
||||
Msg("File served but not cached due to hash verification failure")
|
||||
}
|
||||
|
||||
logger.Logger.Info().
|
||||
|
||||
@@ -3,9 +3,9 @@ package steamcache
|
||||
|
||||
import (
|
||||
"io"
|
||||
"net/http"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
@@ -110,99 +110,60 @@ func TestCacheMissAndHit(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestHashCalculation(t *testing.T) {
|
||||
// Test data
|
||||
testData := []byte("Hello, World!")
|
||||
func TestURLHashing(t *testing.T) {
|
||||
// Test the new SHA256-based cache key generation
|
||||
|
||||
// Calculate hash
|
||||
hash := calculateFileHash(testData)
|
||||
|
||||
// Expected SHA1 hash of "Hello, World!"
|
||||
expectedHash := "0a0a9f2a6772942557ab5355d76af442f8f65e01"
|
||||
|
||||
if hash != expectedHash {
|
||||
t.Errorf("Hash calculation failed: expected %s, got %s", expectedHash, hash)
|
||||
}
|
||||
|
||||
// Test verification
|
||||
if !verifyFileHash(testData, expectedHash) {
|
||||
t.Error("Hash verification failed for correct hash")
|
||||
}
|
||||
|
||||
if verifyFileHash(testData, "wronghash") {
|
||||
t.Error("Hash verification passed for wrong hash")
|
||||
}
|
||||
}
|
||||
|
||||
func TestHashVerificationWithRealData(t *testing.T) {
|
||||
// Test with some real data to ensure our hash calculation is correct
|
||||
testCases := []struct {
|
||||
data string
|
||||
expected string
|
||||
input string
|
||||
desc string
|
||||
shouldCache bool
|
||||
}{
|
||||
{"", "da39a3ee5e6b4b0d3255bfef95601890afd80709"}, // SHA1 of empty string
|
||||
{"test", "a94a8fe5ccb19ba61c4c0873d391e987982fbbd3"}, // SHA1 of "test"
|
||||
{"Hello, World!", "0a0a9f2a6772942557ab5355d76af442f8f65e01"}, // SHA1 of "Hello, World!"
|
||||
{
|
||||
input: "/depot/1684171/chunk/abcdef1234567890",
|
||||
desc: "chunk file URL",
|
||||
shouldCache: true,
|
||||
},
|
||||
{
|
||||
input: "/depot/1684171/manifest/944076726177422892/5/abcdef1234567890",
|
||||
desc: "manifest file URL",
|
||||
shouldCache: true,
|
||||
},
|
||||
{
|
||||
input: "/depot/invalid/path",
|
||||
desc: "invalid depot URL format",
|
||||
shouldCache: true, // Still gets hashed, just not a proper Steam format
|
||||
},
|
||||
{
|
||||
input: "/some/other/path",
|
||||
desc: "non-Steam URL",
|
||||
shouldCache: false, // Not cached
|
||||
},
|
||||
}
|
||||
|
||||
for _, tc := range testCases {
|
||||
data := []byte(tc.data)
|
||||
hash := calculateFileHash(data)
|
||||
if hash != tc.expected {
|
||||
t.Errorf("Hash calculation failed for '%s': expected %s, got %s", tc.data, tc.expected, hash)
|
||||
}
|
||||
t.Run(tc.desc, func(t *testing.T) {
|
||||
result := generateSteamCacheKey(tc.input)
|
||||
|
||||
if !verifyFileHash(data, tc.expected) {
|
||||
t.Errorf("Hash verification failed for '%s'", tc.data)
|
||||
}
|
||||
if tc.shouldCache {
|
||||
// Should return a cache key with "steam/" prefix
|
||||
if !strings.HasPrefix(result, "steam/") {
|
||||
t.Errorf("generateSteamCacheKey(%s) = %s, expected steam/ prefix", tc.input, result)
|
||||
}
|
||||
// Should be exactly 70 characters (6 for "steam/" + 64 for SHA256 hex)
|
||||
if len(result) != 70 {
|
||||
t.Errorf("generateSteamCacheKey(%s) length = %d, expected 70", tc.input, len(result))
|
||||
}
|
||||
} else {
|
||||
// Should return empty string for non-Steam URLs
|
||||
if result != "" {
|
||||
t.Errorf("generateSteamCacheKey(%s) = %s, expected empty string", tc.input, result)
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestResponseHashCalculation(t *testing.T) {
|
||||
// Create a mock HTTP response
|
||||
resp := &http.Response{
|
||||
StatusCode: 200,
|
||||
Status: "200 OK",
|
||||
Header: http.Header{
|
||||
"Content-Type": []string{"application/octet-stream"},
|
||||
"Content-Length": []string{"13"},
|
||||
"Cache-Control": []string{"public, max-age=3600"},
|
||||
},
|
||||
}
|
||||
|
||||
bodyData := []byte("Hello, World!")
|
||||
|
||||
// Calculate response hash
|
||||
responseHash := calculateResponseHash(resp, bodyData)
|
||||
|
||||
// The hash should be different from just the body hash
|
||||
bodyHash := calculateFileHash(bodyData)
|
||||
|
||||
if responseHash == bodyHash {
|
||||
t.Error("Response hash should be different from body hash when headers are present")
|
||||
}
|
||||
|
||||
// Test that the same response produces the same hash
|
||||
responseHash2 := calculateResponseHash(resp, bodyData)
|
||||
if responseHash != responseHash2 {
|
||||
t.Error("Response hash should be consistent for the same response")
|
||||
}
|
||||
|
||||
// Test with different headers
|
||||
resp2 := &http.Response{
|
||||
StatusCode: 200,
|
||||
Status: "200 OK",
|
||||
Header: http.Header{
|
||||
"Content-Type": []string{"text/plain"},
|
||||
"Content-Length": []string{"13"},
|
||||
},
|
||||
}
|
||||
|
||||
responseHash3 := calculateResponseHash(resp2, bodyData)
|
||||
if responseHash == responseHash3 {
|
||||
t.Error("Response hash should be different for different headers")
|
||||
}
|
||||
}
|
||||
// Removed hash calculation tests since we switched to lightweight validation
|
||||
|
||||
func TestSteamKeySharding(t *testing.T) {
|
||||
sc := New("localhost:8080", "0", "1G", t.TempDir(), "", "lru", "lru")
|
||||
@@ -236,35 +197,4 @@ func TestSteamKeySharding(t *testing.T) {
|
||||
// and be readable, whereas without sharding it might not work correctly
|
||||
}
|
||||
|
||||
func TestKeyGeneration(t *testing.T) {
|
||||
testCases := []struct {
|
||||
input string
|
||||
expected string
|
||||
desc string
|
||||
}{
|
||||
{
|
||||
input: "/depot/1684171/chunk/0016cfc5019b8baa6026aa1cce93e685d6e06c6e",
|
||||
expected: "steam/0016cfc5019b8baa6026aa1cce93e685d6e06c6e",
|
||||
desc: "chunk file URL",
|
||||
},
|
||||
{
|
||||
input: "/depot/1684171/manifest/944076726177422892/5/12001286503415372840",
|
||||
expected: "steam/12001286503415372840",
|
||||
desc: "manifest file URL",
|
||||
},
|
||||
{
|
||||
input: "/depot/invalid/path",
|
||||
expected: "",
|
||||
desc: "invalid depot URL format",
|
||||
},
|
||||
}
|
||||
|
||||
for _, tc := range testCases {
|
||||
t.Run(tc.desc, func(t *testing.T) {
|
||||
result := generateSteamCacheKey(tc.input)
|
||||
if result != tc.expected {
|
||||
t.Errorf("generateSteamCacheKey(%s) = %s, expected %s", tc.input, result, tc.expected)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
// Removed old TestKeyGeneration - replaced with TestURLHashing that uses SHA256
|
||||
|
||||
Reference in New Issue
Block a user