Enhance error handling and metrics tracking in SteamCache
- Introduced a new error handling system with custom error types for better context and clarity in error reporting. - Implemented URL validation to prevent invalid requests and enhance security. - Updated cache key generation functions to return errors, improving robustness in handling invalid inputs. - Added comprehensive metrics tracking for requests, cache hits, misses, and performance metrics, allowing for better monitoring and analysis of the caching system. - Enhanced logging to include detailed metrics and error information for improved debugging and operational insights.
This commit is contained in:
120
steamcache/errors/errors.go
Normal file
120
steamcache/errors/errors.go
Normal file
@@ -0,0 +1,120 @@
|
||||
// steamcache/errors/errors.go
|
||||
package errors
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"net/http"
|
||||
)
|
||||
|
||||
// Common SteamCache errors
|
||||
var (
|
||||
ErrInvalidURL = errors.New("steamcache: invalid URL")
|
||||
ErrUnsupportedService = errors.New("steamcache: unsupported service")
|
||||
ErrUpstreamUnavailable = errors.New("steamcache: upstream server unavailable")
|
||||
ErrCacheCorrupted = errors.New("steamcache: cache file corrupted")
|
||||
ErrInvalidContentLength = errors.New("steamcache: invalid content length")
|
||||
ErrRequestTimeout = errors.New("steamcache: request timeout")
|
||||
ErrRateLimitExceeded = errors.New("steamcache: rate limit exceeded")
|
||||
ErrInvalidUserAgent = errors.New("steamcache: invalid user agent")
|
||||
)
|
||||
|
||||
// SteamCacheError represents a SteamCache-specific error with context
|
||||
type SteamCacheError struct {
|
||||
Op string // Operation that failed
|
||||
URL string // URL that caused the error
|
||||
ClientIP string // Client IP address
|
||||
StatusCode int // HTTP status code if applicable
|
||||
Err error // Underlying error
|
||||
Context interface{} // Additional context
|
||||
}
|
||||
|
||||
// Error implements the error interface
|
||||
func (e *SteamCacheError) Error() string {
|
||||
if e.URL != "" && e.ClientIP != "" {
|
||||
return fmt.Sprintf("steamcache: %s failed for URL %q from client %s: %v", e.Op, e.URL, e.ClientIP, e.Err)
|
||||
}
|
||||
if e.URL != "" {
|
||||
return fmt.Sprintf("steamcache: %s failed for URL %q: %v", e.Op, e.URL, e.Err)
|
||||
}
|
||||
return fmt.Sprintf("steamcache: %s failed: %v", e.Op, e.Err)
|
||||
}
|
||||
|
||||
// Unwrap returns the underlying error
|
||||
func (e *SteamCacheError) Unwrap() error {
|
||||
return e.Err
|
||||
}
|
||||
|
||||
// NewSteamCacheError creates a new SteamCache error with context
|
||||
func NewSteamCacheError(op, url, clientIP string, err error) *SteamCacheError {
|
||||
return &SteamCacheError{
|
||||
Op: op,
|
||||
URL: url,
|
||||
ClientIP: clientIP,
|
||||
Err: err,
|
||||
}
|
||||
}
|
||||
|
||||
// NewSteamCacheErrorWithStatus creates a new SteamCache error with HTTP status
|
||||
func NewSteamCacheErrorWithStatus(op, url, clientIP string, statusCode int, err error) *SteamCacheError {
|
||||
return &SteamCacheError{
|
||||
Op: op,
|
||||
URL: url,
|
||||
ClientIP: clientIP,
|
||||
StatusCode: statusCode,
|
||||
Err: err,
|
||||
}
|
||||
}
|
||||
|
||||
// NewSteamCacheErrorWithContext creates a new SteamCache error with additional context
|
||||
func NewSteamCacheErrorWithContext(op, url, clientIP string, context interface{}, err error) *SteamCacheError {
|
||||
return &SteamCacheError{
|
||||
Op: op,
|
||||
URL: url,
|
||||
ClientIP: clientIP,
|
||||
Context: context,
|
||||
Err: err,
|
||||
}
|
||||
}
|
||||
|
||||
// IsRetryableError determines if an error is retryable
|
||||
func IsRetryableError(err error) bool {
|
||||
if err == nil {
|
||||
return false
|
||||
}
|
||||
|
||||
// Check for specific retryable errors
|
||||
if errors.Is(err, ErrUpstreamUnavailable) ||
|
||||
errors.Is(err, ErrRequestTimeout) {
|
||||
return true
|
||||
}
|
||||
|
||||
// Check for HTTP status codes that are retryable
|
||||
if steamErr, ok := err.(*SteamCacheError); ok {
|
||||
switch steamErr.StatusCode {
|
||||
case http.StatusServiceUnavailable,
|
||||
http.StatusGatewayTimeout,
|
||||
http.StatusTooManyRequests,
|
||||
http.StatusInternalServerError:
|
||||
return true
|
||||
}
|
||||
}
|
||||
|
||||
return false
|
||||
}
|
||||
|
||||
// IsClientError determines if an error is a client error (4xx)
|
||||
func IsClientError(err error) bool {
|
||||
if steamErr, ok := err.(*SteamCacheError); ok {
|
||||
return steamErr.StatusCode >= 400 && steamErr.StatusCode < 500
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// IsServerError determines if an error is a server error (5xx)
|
||||
func IsServerError(err error) bool {
|
||||
if steamErr, ok := err.(*SteamCacheError); ok {
|
||||
return steamErr.StatusCode >= 500
|
||||
}
|
||||
return false
|
||||
}
|
||||
213
steamcache/metrics/metrics.go
Normal file
213
steamcache/metrics/metrics.go
Normal file
@@ -0,0 +1,213 @@
|
||||
// steamcache/metrics/metrics.go
|
||||
package metrics
|
||||
|
||||
import (
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"time"
|
||||
)
|
||||
|
||||
// Metrics tracks various performance and operational metrics
|
||||
type Metrics struct {
|
||||
// Request metrics
|
||||
TotalRequests int64
|
||||
CacheHits int64
|
||||
CacheMisses int64
|
||||
CacheCoalesced int64
|
||||
Errors int64
|
||||
RateLimited int64
|
||||
|
||||
// Performance metrics
|
||||
TotalResponseTime int64 // in nanoseconds
|
||||
TotalBytesServed int64
|
||||
TotalBytesCached int64
|
||||
|
||||
// Cache metrics
|
||||
MemoryCacheSize int64
|
||||
DiskCacheSize int64
|
||||
MemoryCacheHits int64
|
||||
DiskCacheHits int64
|
||||
|
||||
// Service metrics
|
||||
ServiceRequests map[string]int64
|
||||
serviceMutex sync.RWMutex
|
||||
|
||||
// Time tracking
|
||||
StartTime time.Time
|
||||
LastResetTime time.Time
|
||||
}
|
||||
|
||||
// NewMetrics creates a new metrics instance
|
||||
func NewMetrics() *Metrics {
|
||||
now := time.Now()
|
||||
return &Metrics{
|
||||
ServiceRequests: make(map[string]int64),
|
||||
StartTime: now,
|
||||
LastResetTime: now,
|
||||
}
|
||||
}
|
||||
|
||||
// IncrementTotalRequests increments the total request counter
|
||||
func (m *Metrics) IncrementTotalRequests() {
|
||||
atomic.AddInt64(&m.TotalRequests, 1)
|
||||
}
|
||||
|
||||
// IncrementCacheHits increments the cache hit counter
|
||||
func (m *Metrics) IncrementCacheHits() {
|
||||
atomic.AddInt64(&m.CacheHits, 1)
|
||||
}
|
||||
|
||||
// IncrementCacheMisses increments the cache miss counter
|
||||
func (m *Metrics) IncrementCacheMisses() {
|
||||
atomic.AddInt64(&m.CacheMisses, 1)
|
||||
}
|
||||
|
||||
// IncrementCacheCoalesced increments the coalesced request counter
|
||||
func (m *Metrics) IncrementCacheCoalesced() {
|
||||
atomic.AddInt64(&m.CacheCoalesced, 1)
|
||||
}
|
||||
|
||||
// IncrementErrors increments the error counter
|
||||
func (m *Metrics) IncrementErrors() {
|
||||
atomic.AddInt64(&m.Errors, 1)
|
||||
}
|
||||
|
||||
// IncrementRateLimited increments the rate limited counter
|
||||
func (m *Metrics) IncrementRateLimited() {
|
||||
atomic.AddInt64(&m.RateLimited, 1)
|
||||
}
|
||||
|
||||
// AddResponseTime adds response time to the total
|
||||
func (m *Metrics) AddResponseTime(duration time.Duration) {
|
||||
atomic.AddInt64(&m.TotalResponseTime, int64(duration))
|
||||
}
|
||||
|
||||
// AddBytesServed adds bytes served to the total
|
||||
func (m *Metrics) AddBytesServed(bytes int64) {
|
||||
atomic.AddInt64(&m.TotalBytesServed, bytes)
|
||||
}
|
||||
|
||||
// AddBytesCached adds bytes cached to the total
|
||||
func (m *Metrics) AddBytesCached(bytes int64) {
|
||||
atomic.AddInt64(&m.TotalBytesCached, bytes)
|
||||
}
|
||||
|
||||
// SetMemoryCacheSize sets the current memory cache size
|
||||
func (m *Metrics) SetMemoryCacheSize(size int64) {
|
||||
atomic.StoreInt64(&m.MemoryCacheSize, size)
|
||||
}
|
||||
|
||||
// SetDiskCacheSize sets the current disk cache size
|
||||
func (m *Metrics) SetDiskCacheSize(size int64) {
|
||||
atomic.StoreInt64(&m.DiskCacheSize, size)
|
||||
}
|
||||
|
||||
// IncrementMemoryCacheHits increments memory cache hits
|
||||
func (m *Metrics) IncrementMemoryCacheHits() {
|
||||
atomic.AddInt64(&m.MemoryCacheHits, 1)
|
||||
}
|
||||
|
||||
// IncrementDiskCacheHits increments disk cache hits
|
||||
func (m *Metrics) IncrementDiskCacheHits() {
|
||||
atomic.AddInt64(&m.DiskCacheHits, 1)
|
||||
}
|
||||
|
||||
// IncrementServiceRequests increments requests for a specific service
|
||||
func (m *Metrics) IncrementServiceRequests(service string) {
|
||||
m.serviceMutex.Lock()
|
||||
defer m.serviceMutex.Unlock()
|
||||
m.ServiceRequests[service]++
|
||||
}
|
||||
|
||||
// GetServiceRequests returns the number of requests for a service
|
||||
func (m *Metrics) GetServiceRequests(service string) int64 {
|
||||
m.serviceMutex.RLock()
|
||||
defer m.serviceMutex.RUnlock()
|
||||
return m.ServiceRequests[service]
|
||||
}
|
||||
|
||||
// GetStats returns a snapshot of current metrics
|
||||
func (m *Metrics) GetStats() *Stats {
|
||||
totalRequests := atomic.LoadInt64(&m.TotalRequests)
|
||||
cacheHits := atomic.LoadInt64(&m.CacheHits)
|
||||
cacheMisses := atomic.LoadInt64(&m.CacheMisses)
|
||||
|
||||
var hitRate float64
|
||||
if totalRequests > 0 {
|
||||
hitRate = float64(cacheHits) / float64(totalRequests)
|
||||
}
|
||||
|
||||
var avgResponseTime time.Duration
|
||||
if totalRequests > 0 {
|
||||
avgResponseTime = time.Duration(atomic.LoadInt64(&m.TotalResponseTime) / totalRequests)
|
||||
}
|
||||
|
||||
m.serviceMutex.RLock()
|
||||
serviceRequests := make(map[string]int64)
|
||||
for k, v := range m.ServiceRequests {
|
||||
serviceRequests[k] = v
|
||||
}
|
||||
m.serviceMutex.RUnlock()
|
||||
|
||||
return &Stats{
|
||||
TotalRequests: totalRequests,
|
||||
CacheHits: cacheHits,
|
||||
CacheMisses: cacheMisses,
|
||||
CacheCoalesced: atomic.LoadInt64(&m.CacheCoalesced),
|
||||
Errors: atomic.LoadInt64(&m.Errors),
|
||||
RateLimited: atomic.LoadInt64(&m.RateLimited),
|
||||
HitRate: hitRate,
|
||||
AvgResponseTime: avgResponseTime,
|
||||
TotalBytesServed: atomic.LoadInt64(&m.TotalBytesServed),
|
||||
TotalBytesCached: atomic.LoadInt64(&m.TotalBytesCached),
|
||||
MemoryCacheSize: atomic.LoadInt64(&m.MemoryCacheSize),
|
||||
DiskCacheSize: atomic.LoadInt64(&m.DiskCacheSize),
|
||||
MemoryCacheHits: atomic.LoadInt64(&m.MemoryCacheHits),
|
||||
DiskCacheHits: atomic.LoadInt64(&m.DiskCacheHits),
|
||||
ServiceRequests: serviceRequests,
|
||||
Uptime: time.Since(m.StartTime),
|
||||
LastResetTime: m.LastResetTime,
|
||||
}
|
||||
}
|
||||
|
||||
// Reset resets all metrics to zero
|
||||
func (m *Metrics) Reset() {
|
||||
atomic.StoreInt64(&m.TotalRequests, 0)
|
||||
atomic.StoreInt64(&m.CacheHits, 0)
|
||||
atomic.StoreInt64(&m.CacheMisses, 0)
|
||||
atomic.StoreInt64(&m.CacheCoalesced, 0)
|
||||
atomic.StoreInt64(&m.Errors, 0)
|
||||
atomic.StoreInt64(&m.RateLimited, 0)
|
||||
atomic.StoreInt64(&m.TotalResponseTime, 0)
|
||||
atomic.StoreInt64(&m.TotalBytesServed, 0)
|
||||
atomic.StoreInt64(&m.TotalBytesCached, 0)
|
||||
atomic.StoreInt64(&m.MemoryCacheHits, 0)
|
||||
atomic.StoreInt64(&m.DiskCacheHits, 0)
|
||||
|
||||
m.serviceMutex.Lock()
|
||||
m.ServiceRequests = make(map[string]int64)
|
||||
m.serviceMutex.Unlock()
|
||||
|
||||
m.LastResetTime = time.Now()
|
||||
}
|
||||
|
||||
// Stats represents a snapshot of metrics
|
||||
type Stats struct {
|
||||
TotalRequests int64
|
||||
CacheHits int64
|
||||
CacheMisses int64
|
||||
CacheCoalesced int64
|
||||
Errors int64
|
||||
RateLimited int64
|
||||
HitRate float64
|
||||
AvgResponseTime time.Duration
|
||||
TotalBytesServed int64
|
||||
TotalBytesCached int64
|
||||
MemoryCacheSize int64
|
||||
DiskCacheSize int64
|
||||
MemoryCacheHits int64
|
||||
DiskCacheHits int64
|
||||
ServiceRequests map[string]int64
|
||||
Uptime time.Duration
|
||||
LastResetTime time.Time
|
||||
}
|
||||
@@ -13,7 +13,9 @@ import (
|
||||
"net/url"
|
||||
"os"
|
||||
"regexp"
|
||||
"s1d3sw1ped/steamcache2/steamcache/errors"
|
||||
"s1d3sw1ped/steamcache2/steamcache/logger"
|
||||
"s1d3sw1ped/steamcache2/steamcache/metrics"
|
||||
"s1d3sw1ped/steamcache2/vfs"
|
||||
"s1d3sw1ped/steamcache2/vfs/adaptive"
|
||||
"s1d3sw1ped/steamcache2/vfs/cache"
|
||||
@@ -360,13 +362,14 @@ func (sc *SteamCache) streamCachedResponse(w http.ResponseWriter, r *http.Reques
|
||||
w.Write(rangeData)
|
||||
|
||||
logger.Logger.Info().
|
||||
Str("key", cacheKey).
|
||||
Str("cache_key", cacheKey).
|
||||
Str("url", r.URL.String()).
|
||||
Str("host", r.Host).
|
||||
Str("client_ip", clientIP).
|
||||
Str("status", "HIT").
|
||||
Str("cache_status", "HIT").
|
||||
Str("range", fmt.Sprintf("%d-%d/%d", start, end, totalSize)).
|
||||
Dur("zduration", time.Since(tstart)).
|
||||
Int64("range_size", end-start+1).
|
||||
Dur("response_time", time.Since(tstart)).
|
||||
Msg("cache request")
|
||||
|
||||
return
|
||||
@@ -394,12 +397,13 @@ func (sc *SteamCache) streamCachedResponse(w http.ResponseWriter, r *http.Reques
|
||||
w.Write(bodyData)
|
||||
|
||||
logger.Logger.Info().
|
||||
Str("key", cacheKey).
|
||||
Str("cache_key", cacheKey).
|
||||
Str("url", r.URL.String()).
|
||||
Str("host", r.Host).
|
||||
Str("client_ip", clientIP).
|
||||
Str("status", "HIT").
|
||||
Dur("zduration", time.Since(tstart)).
|
||||
Str("cache_status", "HIT").
|
||||
Int64("file_size", int64(len(bodyData))).
|
||||
Dur("response_time", time.Since(tstart)).
|
||||
Msg("cache request")
|
||||
}
|
||||
|
||||
@@ -495,14 +499,19 @@ func parseRangeHeader(rangeHeader string, totalSize int64) (start, end, total in
|
||||
}
|
||||
|
||||
// generateURLHash creates a SHA256 hash of the entire URL path for cache key
|
||||
func generateURLHash(urlPath string) string {
|
||||
func generateURLHash(urlPath string) (string, error) {
|
||||
// Validate input to prevent cache key pollution
|
||||
if urlPath == "" {
|
||||
return ""
|
||||
return "", errors.NewSteamCacheError("generateURLHash", urlPath, "", errors.ErrInvalidURL)
|
||||
}
|
||||
|
||||
// Additional validation for suspicious patterns
|
||||
if strings.Contains(urlPath, "..") || strings.Contains(urlPath, "//") {
|
||||
return "", errors.NewSteamCacheError("generateURLHash", urlPath, "", errors.ErrInvalidURL)
|
||||
}
|
||||
|
||||
hash := sha256.Sum256([]byte(urlPath))
|
||||
return hex.EncodeToString(hash[:])
|
||||
return hex.EncodeToString(hash[:]), nil
|
||||
}
|
||||
|
||||
// calculateSHA256 calculates SHA256 hash of the given data
|
||||
@@ -512,6 +521,35 @@ func calculateSHA256(data []byte) string {
|
||||
return hex.EncodeToString(hasher.Sum(nil))
|
||||
}
|
||||
|
||||
// validateURLPath validates URL path for security concerns
|
||||
func validateURLPath(urlPath string) error {
|
||||
if urlPath == "" {
|
||||
return errors.NewSteamCacheError("validateURLPath", urlPath, "", errors.ErrInvalidURL)
|
||||
}
|
||||
|
||||
// Check for directory traversal attempts
|
||||
if strings.Contains(urlPath, "..") {
|
||||
return errors.NewSteamCacheError("validateURLPath", urlPath, "", errors.ErrInvalidURL)
|
||||
}
|
||||
|
||||
// Check for double slashes (potential path manipulation)
|
||||
if strings.Contains(urlPath, "//") {
|
||||
return errors.NewSteamCacheError("validateURLPath", urlPath, "", errors.ErrInvalidURL)
|
||||
}
|
||||
|
||||
// Check for suspicious characters
|
||||
if strings.ContainsAny(urlPath, "<>\"'&") {
|
||||
return errors.NewSteamCacheError("validateURLPath", urlPath, "", errors.ErrInvalidURL)
|
||||
}
|
||||
|
||||
// Check for reasonable length (prevent DoS)
|
||||
if len(urlPath) > 2048 {
|
||||
return errors.NewSteamCacheError("validateURLPath", urlPath, "", errors.ErrInvalidURL)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// verifyCompleteFile verifies that we received the complete file by checking Content-Length
|
||||
// Returns true if the file is complete, false if it's incomplete (allowing retry)
|
||||
func (sc *SteamCache) verifyCompleteFile(bodyData []byte, resp *http.Response, urlPath string, cacheKey string) bool {
|
||||
@@ -571,9 +609,20 @@ func (sc *SteamCache) detectService(r *http.Request) (*ServiceConfig, bool) {
|
||||
// The prefix indicates which service the request came from (detected via User-Agent)
|
||||
// Input: /depot/1684171/chunk/0016cfc5019b8baa6026aa1cce93e685d6e06c6e, "steam"
|
||||
// Output: steam/a1b2c3d4e5f678901234567890123456789012345678901234567890
|
||||
func generateServiceCacheKey(urlPath string, servicePrefix string) string {
|
||||
func generateServiceCacheKey(urlPath string, servicePrefix string) (string, error) {
|
||||
// Validate service prefix
|
||||
if servicePrefix == "" {
|
||||
return "", errors.NewSteamCacheError("generateServiceCacheKey", urlPath, "", errors.ErrUnsupportedService)
|
||||
}
|
||||
|
||||
// Generate hash for URL path
|
||||
hash, err := generateURLHash(urlPath)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
// Create a SHA256 hash of the entire path for all service client requests
|
||||
return servicePrefix + "/" + generateURLHash(urlPath)
|
||||
return servicePrefix + "/" + hash, nil
|
||||
}
|
||||
|
||||
var hopByHopHeaders = map[string]struct{}{
|
||||
@@ -788,6 +837,9 @@ type SteamCache struct {
|
||||
// Dynamic memory management
|
||||
memoryMonitor *memory.MemoryMonitor
|
||||
dynamicCacheMgr *memory.MemoryMonitor
|
||||
|
||||
// Metrics
|
||||
metrics *metrics.Metrics
|
||||
}
|
||||
|
||||
func New(address string, memorySize string, diskSize string, diskPath, upstream, memoryGC, diskGC string, maxConcurrentRequests int64, maxRequestsPerClient int64) *SteamCache {
|
||||
@@ -930,6 +982,9 @@ func New(address string, memorySize string, diskSize string, diskPath, upstream,
|
||||
// Initialize dynamic memory management
|
||||
memoryMonitor: memory.NewMemoryMonitor(uint64(memorysize), 10*time.Second, 0.1), // 10% threshold
|
||||
dynamicCacheMgr: nil, // Will be set after cache creation
|
||||
|
||||
// Initialize metrics
|
||||
metrics: metrics.NewMetrics(),
|
||||
}
|
||||
|
||||
// Initialize dynamic cache manager if we have memory cache
|
||||
@@ -1000,21 +1055,44 @@ func (sc *SteamCache) Shutdown() {
|
||||
sc.wg.Wait()
|
||||
}
|
||||
|
||||
// GetMetrics returns current metrics
|
||||
func (sc *SteamCache) GetMetrics() *metrics.Stats {
|
||||
// Update cache sizes
|
||||
if sc.memory != nil {
|
||||
sc.metrics.SetMemoryCacheSize(sc.memory.Size())
|
||||
}
|
||||
if sc.disk != nil {
|
||||
sc.metrics.SetDiskCacheSize(sc.disk.Size())
|
||||
}
|
||||
|
||||
return sc.metrics.GetStats()
|
||||
}
|
||||
|
||||
// ResetMetrics resets all metrics to zero
|
||||
func (sc *SteamCache) ResetMetrics() {
|
||||
sc.metrics.Reset()
|
||||
}
|
||||
|
||||
func (sc *SteamCache) ServeHTTP(w http.ResponseWriter, r *http.Request) {
|
||||
clientIP := getClientIP(r)
|
||||
|
||||
// Set keep-alive headers for better performance
|
||||
w.Header().Set("Connection", "keep-alive")
|
||||
w.Header().Set("Keep-Alive", "timeout=300, max=1000")
|
||||
|
||||
// Apply global concurrency limit first
|
||||
if err := sc.requestSemaphore.Acquire(context.Background(), 1); err != nil {
|
||||
logger.Logger.Warn().Str("client_ip", getClientIP(r)).Msg("Server at capacity, rejecting request")
|
||||
sc.metrics.IncrementRateLimited()
|
||||
logger.Logger.Warn().Str("client_ip", clientIP).Msg("Server at capacity, rejecting request")
|
||||
http.Error(w, "Server busy, please try again later", http.StatusServiceUnavailable)
|
||||
return
|
||||
}
|
||||
defer sc.requestSemaphore.Release(1)
|
||||
|
||||
// Track total requests
|
||||
sc.metrics.IncrementTotalRequests()
|
||||
|
||||
// Apply per-client rate limiting
|
||||
clientIP := getClientIP(r)
|
||||
clientLimiter := sc.getOrCreateClientLimiter(clientIP)
|
||||
|
||||
if err := clientLimiter.semaphore.Acquire(context.Background(), 1); err != nil {
|
||||
@@ -1054,19 +1132,56 @@ func (sc *SteamCache) ServeHTTP(w http.ResponseWriter, r *http.Request) {
|
||||
return
|
||||
}
|
||||
|
||||
if r.URL.String() == "/metrics" {
|
||||
// Return metrics in a simple text format
|
||||
stats := sc.GetMetrics()
|
||||
w.Header().Set("Content-Type", "text/plain")
|
||||
w.WriteHeader(http.StatusOK)
|
||||
fmt.Fprintf(w, "# SteamCache2 Metrics\n")
|
||||
fmt.Fprintf(w, "total_requests %d\n", stats.TotalRequests)
|
||||
fmt.Fprintf(w, "cache_hits %d\n", stats.CacheHits)
|
||||
fmt.Fprintf(w, "cache_misses %d\n", stats.CacheMisses)
|
||||
fmt.Fprintf(w, "cache_coalesced %d\n", stats.CacheCoalesced)
|
||||
fmt.Fprintf(w, "errors %d\n", stats.Errors)
|
||||
fmt.Fprintf(w, "rate_limited %d\n", stats.RateLimited)
|
||||
fmt.Fprintf(w, "hit_rate %.4f\n", stats.HitRate)
|
||||
fmt.Fprintf(w, "avg_response_time_ms %.2f\n", float64(stats.AvgResponseTime.Nanoseconds())/1e6)
|
||||
fmt.Fprintf(w, "total_bytes_served %d\n", stats.TotalBytesServed)
|
||||
fmt.Fprintf(w, "total_bytes_cached %d\n", stats.TotalBytesCached)
|
||||
fmt.Fprintf(w, "memory_cache_size %d\n", stats.MemoryCacheSize)
|
||||
fmt.Fprintf(w, "disk_cache_size %d\n", stats.DiskCacheSize)
|
||||
fmt.Fprintf(w, "uptime_seconds %.2f\n", stats.Uptime.Seconds())
|
||||
return
|
||||
}
|
||||
|
||||
// Check if this is a request from a supported service
|
||||
if service, isSupported := sc.detectService(r); isSupported {
|
||||
// trim the query parameters from the URL path
|
||||
// this is necessary because the cache key should not include query parameters
|
||||
urlPath, _, _ := strings.Cut(r.URL.String(), "?")
|
||||
|
||||
// Validate URL path for security
|
||||
if err := validateURLPath(urlPath); err != nil {
|
||||
logger.Logger.Warn().
|
||||
Err(err).
|
||||
Str("url", urlPath).
|
||||
Str("client_ip", clientIP).
|
||||
Msg("Invalid URL path detected")
|
||||
http.Error(w, "Invalid URL", http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
|
||||
tstart := time.Now()
|
||||
|
||||
// Generate service cache key: {service}/{hash} (prefix indicates service via User-Agent)
|
||||
cacheKey := generateServiceCacheKey(urlPath, service.Prefix)
|
||||
|
||||
if cacheKey == "" {
|
||||
logger.Logger.Warn().Str("url", urlPath).Msg("Invalid URL")
|
||||
cacheKey, err := generateServiceCacheKey(urlPath, service.Prefix)
|
||||
if err != nil {
|
||||
logger.Logger.Warn().
|
||||
Err(err).
|
||||
Str("url", urlPath).
|
||||
Str("service", service.Name).
|
||||
Str("client_ip", clientIP).
|
||||
Msg("Failed to generate cache key")
|
||||
http.Error(w, "Invalid URL", http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
@@ -1110,6 +1225,12 @@ func (sc *SteamCache) ServeHTTP(w http.ResponseWriter, r *http.Request) {
|
||||
// Cache validation passed - record access for adaptive/predictive analysis
|
||||
sc.recordCacheAccess(cacheKey, int64(len(cachedData)))
|
||||
|
||||
// Track cache hit metrics
|
||||
sc.metrics.IncrementCacheHits()
|
||||
sc.metrics.AddResponseTime(time.Since(tstart))
|
||||
sc.metrics.AddBytesServed(int64(len(cachedData)))
|
||||
sc.metrics.IncrementServiceRequests(service.Name)
|
||||
|
||||
logger.Logger.Debug().
|
||||
Str("key", cacheKey).
|
||||
Str("url", urlPath).
|
||||
@@ -1175,13 +1296,21 @@ func (sc *SteamCache) ServeHTTP(w http.ResponseWriter, r *http.Request) {
|
||||
w.WriteHeader(coalescedReq.statusCode)
|
||||
w.Write(responseData)
|
||||
|
||||
// Track coalesced cache hit metrics
|
||||
sc.metrics.IncrementCacheCoalesced()
|
||||
sc.metrics.AddResponseTime(time.Since(tstart))
|
||||
sc.metrics.AddBytesServed(int64(len(responseData)))
|
||||
sc.metrics.IncrementServiceRequests(service.Name)
|
||||
|
||||
logger.Logger.Info().
|
||||
Str("key", cacheKey).
|
||||
Str("cache_key", cacheKey).
|
||||
Str("url", urlPath).
|
||||
Str("host", r.Host).
|
||||
Str("client_ip", clientIP).
|
||||
Str("status", "HIT-COALESCED").
|
||||
Dur("zduration", time.Since(tstart)).
|
||||
Str("cache_status", "HIT-COALESCED").
|
||||
Int("waiting_clients", coalescedReq.waitingCount).
|
||||
Int64("file_size", int64(len(responseData))).
|
||||
Dur("response_time", time.Since(tstart)).
|
||||
Msg("cache request")
|
||||
|
||||
return
|
||||
@@ -1359,6 +1488,12 @@ func (sc *SteamCache) ServeHTTP(w http.ResponseWriter, r *http.Request) {
|
||||
w.WriteHeader(resp.StatusCode)
|
||||
w.Write(bodyData)
|
||||
|
||||
// Track cache miss metrics
|
||||
sc.metrics.IncrementCacheMisses()
|
||||
sc.metrics.AddResponseTime(time.Since(tstart))
|
||||
sc.metrics.AddBytesServed(int64(len(bodyData)))
|
||||
sc.metrics.IncrementServiceRequests(service.Name)
|
||||
|
||||
// Cache the file if validation passed
|
||||
if validationPassed {
|
||||
// Verify we received the complete file by checking Content-Length
|
||||
@@ -1399,6 +1534,8 @@ func (sc *SteamCache) ServeHTTP(w http.ResponseWriter, r *http.Request) {
|
||||
Msg("Cache write failed or incomplete - removing corrupted entry")
|
||||
sc.vfs.Delete(cachePath)
|
||||
} else {
|
||||
// Track successful cache write
|
||||
sc.metrics.AddBytesCached(int64(len(cacheData)))
|
||||
logger.Logger.Debug().
|
||||
Str("key", cacheKey).
|
||||
Str("url", urlPath).
|
||||
@@ -1456,12 +1593,14 @@ func (sc *SteamCache) ServeHTTP(w http.ResponseWriter, r *http.Request) {
|
||||
}
|
||||
|
||||
logger.Logger.Info().
|
||||
Str("key", cacheKey).
|
||||
Str("cache_key", cacheKey).
|
||||
Str("url", urlPath).
|
||||
Str("host", r.Host).
|
||||
Str("client_ip", clientIP).
|
||||
Str("status", "MISS").
|
||||
Dur("zduration", time.Since(tstart)).
|
||||
Str("service", service.Name).
|
||||
Str("cache_status", "MISS").
|
||||
Int64("file_size", int64(len(bodyData))).
|
||||
Dur("response_time", time.Since(tstart)).
|
||||
Msg("cache request")
|
||||
|
||||
return
|
||||
|
||||
@@ -3,6 +3,8 @@ package steamcache
|
||||
|
||||
import (
|
||||
"io"
|
||||
"s1d3sw1ped/steamcache2/steamcache/errors"
|
||||
"s1d3sw1ped/steamcache2/vfs/vfserror"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
@@ -164,10 +166,13 @@ func TestURLHashing(t *testing.T) {
|
||||
|
||||
for _, tc := range testCases {
|
||||
t.Run(tc.desc, func(t *testing.T) {
|
||||
result := generateServiceCacheKey(tc.input, "steam")
|
||||
result, err := generateServiceCacheKey(tc.input, "steam")
|
||||
|
||||
if tc.shouldCache {
|
||||
// Should return a cache key with "steam/" prefix
|
||||
if err != nil {
|
||||
t.Errorf("generateServiceCacheKey(%s, \"steam\") returned error: %v", tc.input, err)
|
||||
}
|
||||
if !strings.HasPrefix(result, "steam/") {
|
||||
t.Errorf("generateServiceCacheKey(%s, \"steam\") = %s, expected steam/ prefix", tc.input, result)
|
||||
}
|
||||
@@ -176,9 +181,9 @@ func TestURLHashing(t *testing.T) {
|
||||
t.Errorf("generateServiceCacheKey(%s, \"steam\") length = %d, expected 70", tc.input, len(result))
|
||||
}
|
||||
} else {
|
||||
// Should return empty string for non-Steam URLs
|
||||
if result != "" {
|
||||
t.Errorf("generateServiceCacheKey(%s, \"steam\") = %s, expected empty string", tc.input, result)
|
||||
// Should return error for invalid URLs
|
||||
if err == nil {
|
||||
t.Errorf("generateServiceCacheKey(%s, \"steam\") should have returned error", tc.input)
|
||||
}
|
||||
}
|
||||
})
|
||||
@@ -322,8 +327,14 @@ func TestServiceManagerExpandability(t *testing.T) {
|
||||
}
|
||||
|
||||
// Test cache key generation for different services
|
||||
steamKey := generateServiceCacheKey("/depot/123/chunk/abc", "steam")
|
||||
epicKey := generateServiceCacheKey("/epic/123/chunk/abc", "epic")
|
||||
steamKey, err := generateServiceCacheKey("/depot/123/chunk/abc", "steam")
|
||||
if err != nil {
|
||||
t.Errorf("Failed to generate Steam cache key: %v", err)
|
||||
}
|
||||
epicKey, err := generateServiceCacheKey("/epic/123/chunk/abc", "epic")
|
||||
if err != nil {
|
||||
t.Errorf("Failed to generate Epic cache key: %v", err)
|
||||
}
|
||||
|
||||
if !strings.HasPrefix(steamKey, "steam/") {
|
||||
t.Errorf("Steam cache key should start with 'steam/', got: %s", steamKey)
|
||||
@@ -367,4 +378,139 @@ func TestSteamKeySharding(t *testing.T) {
|
||||
// and be readable, whereas without sharding it might not work correctly
|
||||
}
|
||||
|
||||
// TestURLValidation tests the URL validation function
|
||||
func TestURLValidation(t *testing.T) {
|
||||
testCases := []struct {
|
||||
urlPath string
|
||||
shouldPass bool
|
||||
description string
|
||||
}{
|
||||
{
|
||||
urlPath: "/depot/123/chunk/abc",
|
||||
shouldPass: true,
|
||||
description: "valid Steam URL",
|
||||
},
|
||||
{
|
||||
urlPath: "/appinfo/456",
|
||||
shouldPass: true,
|
||||
description: "valid app info URL",
|
||||
},
|
||||
{
|
||||
urlPath: "",
|
||||
shouldPass: false,
|
||||
description: "empty URL",
|
||||
},
|
||||
{
|
||||
urlPath: "/depot/../etc/passwd",
|
||||
shouldPass: false,
|
||||
description: "directory traversal attempt",
|
||||
},
|
||||
{
|
||||
urlPath: "/depot//123/chunk/abc",
|
||||
shouldPass: false,
|
||||
description: "double slash",
|
||||
},
|
||||
{
|
||||
urlPath: "/depot/123/chunk/abc<script>",
|
||||
shouldPass: false,
|
||||
description: "suspicious characters",
|
||||
},
|
||||
{
|
||||
urlPath: strings.Repeat("/depot/123/chunk/abc", 200), // This will be much longer than 2048 chars
|
||||
shouldPass: false,
|
||||
description: "URL too long",
|
||||
},
|
||||
}
|
||||
|
||||
for _, tc := range testCases {
|
||||
t.Run(tc.description, func(t *testing.T) {
|
||||
err := validateURLPath(tc.urlPath)
|
||||
if tc.shouldPass && err != nil {
|
||||
t.Errorf("validateURLPath(%q) should pass but got error: %v", tc.urlPath, err)
|
||||
}
|
||||
if !tc.shouldPass && err == nil {
|
||||
t.Errorf("validateURLPath(%q) should fail but passed", tc.urlPath)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// TestErrorTypes tests the custom error types
|
||||
func TestErrorTypes(t *testing.T) {
|
||||
// Test VFS error
|
||||
vfsErr := vfserror.NewVFSError("test", "key1", vfserror.ErrNotFound)
|
||||
if vfsErr.Error() == "" {
|
||||
t.Error("VFS error should have a message")
|
||||
}
|
||||
if vfsErr.Unwrap() != vfserror.ErrNotFound {
|
||||
t.Error("VFS error should unwrap to the underlying error")
|
||||
}
|
||||
|
||||
// Test SteamCache error
|
||||
scErr := errors.NewSteamCacheError("test", "/test/url", "127.0.0.1", errors.ErrInvalidURL)
|
||||
if scErr.Error() == "" {
|
||||
t.Error("SteamCache error should have a message")
|
||||
}
|
||||
if scErr.Unwrap() != errors.ErrInvalidURL {
|
||||
t.Error("SteamCache error should unwrap to the underlying error")
|
||||
}
|
||||
|
||||
// Test retryable error detection
|
||||
if !errors.IsRetryableError(errors.ErrUpstreamUnavailable) {
|
||||
t.Error("Upstream unavailable should be retryable")
|
||||
}
|
||||
if errors.IsRetryableError(errors.ErrInvalidURL) {
|
||||
t.Error("Invalid URL should not be retryable")
|
||||
}
|
||||
}
|
||||
|
||||
// TestMetrics tests the metrics functionality
|
||||
func TestMetrics(t *testing.T) {
|
||||
td := t.TempDir()
|
||||
sc := New("localhost:8080", "1G", "1G", td, "", "lru", "lru", 200, 5)
|
||||
|
||||
// Test initial metrics
|
||||
stats := sc.GetMetrics()
|
||||
if stats.TotalRequests != 0 {
|
||||
t.Error("Initial total requests should be 0")
|
||||
}
|
||||
if stats.CacheHits != 0 {
|
||||
t.Error("Initial cache hits should be 0")
|
||||
}
|
||||
|
||||
// Test metrics increment
|
||||
sc.metrics.IncrementTotalRequests()
|
||||
sc.metrics.IncrementCacheHits()
|
||||
sc.metrics.IncrementCacheMisses()
|
||||
sc.metrics.AddBytesServed(1024)
|
||||
sc.metrics.IncrementServiceRequests("steam")
|
||||
|
||||
stats = sc.GetMetrics()
|
||||
if stats.TotalRequests != 1 {
|
||||
t.Error("Total requests should be 1")
|
||||
}
|
||||
if stats.CacheHits != 1 {
|
||||
t.Error("Cache hits should be 1")
|
||||
}
|
||||
if stats.CacheMisses != 1 {
|
||||
t.Error("Cache misses should be 1")
|
||||
}
|
||||
if stats.TotalBytesServed != 1024 {
|
||||
t.Error("Total bytes served should be 1024")
|
||||
}
|
||||
if stats.ServiceRequests["steam"] != 1 {
|
||||
t.Error("Steam service requests should be 1")
|
||||
}
|
||||
|
||||
// Test metrics reset
|
||||
sc.ResetMetrics()
|
||||
stats = sc.GetMetrics()
|
||||
if stats.TotalRequests != 0 {
|
||||
t.Error("After reset, total requests should be 0")
|
||||
}
|
||||
if stats.CacheHits != 0 {
|
||||
t.Error("After reset, cache hits should be 0")
|
||||
}
|
||||
}
|
||||
|
||||
// Removed old TestKeyGeneration - replaced with TestURLHashing that uses SHA256
|
||||
|
||||
Reference in New Issue
Block a user