Justin Harms
45ae234694
- Updated caching logic to support size-based promotion filtering, ensuring that not all files may be promoted based on size constraints. - Implemented adaptive caching strategies with a new AdaptiveCacheManager to analyze access patterns and adjust caching strategies dynamically. - Introduced predictive caching features with a PredictiveCacheManager to prefetch content based on access patterns. - Added a CacheWarmer to preload popular content into the cache, improving access times for frequently requested files. - Refactored memory management with a DynamicCacheManager to adjust cache sizes based on system memory usage. - Enhanced VFS interface and file metadata handling to support new features and improve performance. - Updated tests to validate new caching behaviors and ensure reliability of the caching system.
357 lines
9.0 KiB
Go
357 lines
9.0 KiB
Go
// steamcache/steamcache_test.go
|
|
package steamcache
|
|
|
|
import (
|
|
"io"
|
|
"os"
|
|
"path/filepath"
|
|
"strings"
|
|
"testing"
|
|
)
|
|
|
|
func TestCaching(t *testing.T) {
|
|
td := t.TempDir()
|
|
|
|
os.WriteFile(filepath.Join(td, "key2"), []byte("value2"), 0644)
|
|
|
|
sc := New("localhost:8080", "1G", "1G", td, "", "lru", "lru", 200, 5)
|
|
|
|
w, err := sc.vfs.Create("key", 5)
|
|
if err != nil {
|
|
t.Errorf("Create failed: %v", err)
|
|
}
|
|
w.Write([]byte("value"))
|
|
w.Close()
|
|
|
|
w, err = sc.vfs.Create("key1", 6)
|
|
if err != nil {
|
|
t.Errorf("Create failed: %v", err)
|
|
}
|
|
w.Write([]byte("value1"))
|
|
w.Close()
|
|
|
|
if sc.diskgc.Size() != 17 {
|
|
t.Errorf("Size failed: got %d, want %d", sc.diskgc.Size(), 17)
|
|
}
|
|
|
|
if sc.vfs.Size() != 17 {
|
|
t.Errorf("Size failed: got %d, want %d", sc.vfs.Size(), 17)
|
|
}
|
|
|
|
rc, err := sc.vfs.Open("key")
|
|
if err != nil {
|
|
t.Errorf("Open failed: %v", err)
|
|
}
|
|
d, _ := io.ReadAll(rc)
|
|
rc.Close()
|
|
if string(d) != "value" {
|
|
t.Errorf("Get failed: got %s, want %s", d, "value")
|
|
}
|
|
|
|
rc, err = sc.vfs.Open("key1")
|
|
if err != nil {
|
|
t.Errorf("Open failed: %v", err)
|
|
}
|
|
d, _ = io.ReadAll(rc)
|
|
rc.Close()
|
|
if string(d) != "value1" {
|
|
t.Errorf("Get failed: got %s, want %s", d, "value1")
|
|
}
|
|
|
|
rc, err = sc.vfs.Open("key2")
|
|
if err != nil {
|
|
t.Errorf("Open failed: %v", err)
|
|
}
|
|
d, _ = io.ReadAll(rc)
|
|
rc.Close()
|
|
if string(d) != "value2" {
|
|
t.Errorf("Get failed: got %s, want %s", d, "value2")
|
|
}
|
|
|
|
// With size-based promotion filtering, not all files may be promoted
|
|
// The total size should be at least the disk size (17 bytes) but may be less than 34 bytes
|
|
// if some files are filtered out due to size constraints
|
|
if sc.diskgc.Size() != 17 {
|
|
t.Errorf("Disk size failed: got %d, want %d", sc.diskgc.Size(), 17)
|
|
}
|
|
|
|
if sc.vfs.Size() < 17 {
|
|
t.Errorf("Total size too small: got %d, want at least 17", sc.vfs.Size())
|
|
}
|
|
if sc.vfs.Size() > 34 {
|
|
t.Errorf("Total size too large: got %d, want at most 34", sc.vfs.Size())
|
|
}
|
|
|
|
sc.memory.Delete("key2")
|
|
sc.disk.Delete("key2") // Also delete from disk cache
|
|
os.Remove(filepath.Join(td, "key2"))
|
|
|
|
if _, err := sc.vfs.Open("key2"); err == nil {
|
|
t.Errorf("Open failed: got nil, want error")
|
|
}
|
|
}
|
|
|
|
func TestCacheMissAndHit(t *testing.T) {
|
|
sc := New("localhost:8080", "0", "1G", t.TempDir(), "", "lru", "lru", 200, 5)
|
|
|
|
key := "testkey"
|
|
value := []byte("testvalue")
|
|
|
|
// Simulate miss: but since no upstream, skip full ServeHTTP, test VFS
|
|
w, err := sc.vfs.Create(key, int64(len(value)))
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
w.Write(value)
|
|
w.Close()
|
|
|
|
rc, err := sc.vfs.Open(key)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
got, _ := io.ReadAll(rc)
|
|
rc.Close()
|
|
|
|
if string(got) != string(value) {
|
|
t.Errorf("expected %s, got %s", value, got)
|
|
}
|
|
}
|
|
|
|
func TestURLHashing(t *testing.T) {
|
|
// Test the SHA256-based cache key generation for Steam client requests
|
|
// The "steam/" prefix indicates the request came from a Steam client (User-Agent based)
|
|
|
|
testCases := []struct {
|
|
input string
|
|
desc string
|
|
shouldCache bool
|
|
}{
|
|
{
|
|
input: "/depot/1684171/chunk/abcdef1234567890",
|
|
desc: "chunk file URL",
|
|
shouldCache: true,
|
|
},
|
|
{
|
|
input: "/depot/1684171/manifest/944076726177422892/5/abcdef1234567890",
|
|
desc: "manifest file URL",
|
|
shouldCache: true,
|
|
},
|
|
{
|
|
input: "/appinfo/123456",
|
|
desc: "app info URL",
|
|
shouldCache: true,
|
|
},
|
|
{
|
|
input: "/some/other/path",
|
|
desc: "any URL from Steam client",
|
|
shouldCache: true, // All URLs from Steam clients (detected via User-Agent) are cached
|
|
},
|
|
}
|
|
|
|
for _, tc := range testCases {
|
|
t.Run(tc.desc, func(t *testing.T) {
|
|
result := generateServiceCacheKey(tc.input, "steam")
|
|
|
|
if tc.shouldCache {
|
|
// Should return a cache key with "steam/" prefix
|
|
if !strings.HasPrefix(result, "steam/") {
|
|
t.Errorf("generateServiceCacheKey(%s, \"steam\") = %s, expected steam/ prefix", tc.input, result)
|
|
}
|
|
// Should be exactly 70 characters (6 for "steam/" + 64 for SHA256 hex)
|
|
if len(result) != 70 {
|
|
t.Errorf("generateServiceCacheKey(%s, \"steam\") length = %d, expected 70", tc.input, len(result))
|
|
}
|
|
} else {
|
|
// Should return empty string for non-Steam URLs
|
|
if result != "" {
|
|
t.Errorf("generateServiceCacheKey(%s, \"steam\") = %s, expected empty string", tc.input, result)
|
|
}
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestServiceDetection(t *testing.T) {
|
|
// Create a service manager for testing
|
|
sm := NewServiceManager()
|
|
|
|
testCases := []struct {
|
|
userAgent string
|
|
expectedName string
|
|
expectedFound bool
|
|
desc string
|
|
}{
|
|
{
|
|
userAgent: "Valve/Steam HTTP Client 1.0",
|
|
expectedName: "steam",
|
|
expectedFound: true,
|
|
desc: "Valve Steam HTTP Client",
|
|
},
|
|
{
|
|
userAgent: "Steam",
|
|
expectedName: "steam",
|
|
expectedFound: true,
|
|
desc: "Simple Steam user agent",
|
|
},
|
|
{
|
|
userAgent: "SteamClient/1.0",
|
|
expectedName: "steam",
|
|
expectedFound: true,
|
|
desc: "SteamClient with version",
|
|
},
|
|
{
|
|
userAgent: "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36",
|
|
expectedName: "",
|
|
expectedFound: false,
|
|
desc: "Browser user agent",
|
|
},
|
|
{
|
|
userAgent: "",
|
|
expectedName: "",
|
|
expectedFound: false,
|
|
desc: "Empty user agent",
|
|
},
|
|
{
|
|
userAgent: "curl/7.68.0",
|
|
expectedName: "",
|
|
expectedFound: false,
|
|
desc: "curl user agent",
|
|
},
|
|
}
|
|
|
|
for _, tc := range testCases {
|
|
t.Run(tc.desc, func(t *testing.T) {
|
|
service, found := sm.DetectService(tc.userAgent)
|
|
|
|
if found != tc.expectedFound {
|
|
t.Errorf("DetectService(%s) found = %v, expected %v", tc.userAgent, found, tc.expectedFound)
|
|
}
|
|
|
|
if found && service.Name != tc.expectedName {
|
|
t.Errorf("DetectService(%s) service name = %s, expected %s", tc.userAgent, service.Name, tc.expectedName)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestServiceManagerExpandability(t *testing.T) {
|
|
// Create a service manager for testing
|
|
sm := NewServiceManager()
|
|
|
|
// Test adding a new service (Epic Games)
|
|
epicConfig := &ServiceConfig{
|
|
Name: "epic",
|
|
Prefix: "epic",
|
|
UserAgents: []string{
|
|
`EpicGamesLauncher`,
|
|
`EpicGames`,
|
|
`Epic.*Launcher`,
|
|
},
|
|
}
|
|
|
|
err := sm.AddService(epicConfig)
|
|
if err != nil {
|
|
t.Fatalf("Failed to add Epic service: %v", err)
|
|
}
|
|
|
|
// Test Epic Games detection
|
|
epicTestCases := []struct {
|
|
userAgent string
|
|
expectedName string
|
|
expectedFound bool
|
|
desc string
|
|
}{
|
|
{
|
|
userAgent: "EpicGamesLauncher/1.0",
|
|
expectedName: "epic",
|
|
expectedFound: true,
|
|
desc: "Epic Games Launcher",
|
|
},
|
|
{
|
|
userAgent: "EpicGames/2.0",
|
|
expectedName: "epic",
|
|
expectedFound: true,
|
|
desc: "Epic Games client",
|
|
},
|
|
{
|
|
userAgent: "Epic Launcher 1.5",
|
|
expectedName: "epic",
|
|
expectedFound: true,
|
|
desc: "Epic Launcher with regex match",
|
|
},
|
|
{
|
|
userAgent: "Steam",
|
|
expectedName: "steam",
|
|
expectedFound: true,
|
|
desc: "Steam should still work",
|
|
},
|
|
{
|
|
userAgent: "Mozilla/5.0",
|
|
expectedName: "",
|
|
expectedFound: false,
|
|
desc: "Browser should not match any service",
|
|
},
|
|
}
|
|
|
|
for _, tc := range epicTestCases {
|
|
t.Run(tc.desc, func(t *testing.T) {
|
|
service, found := sm.DetectService(tc.userAgent)
|
|
|
|
if found != tc.expectedFound {
|
|
t.Errorf("DetectService(%s) found = %v, expected %v", tc.userAgent, found, tc.expectedFound)
|
|
}
|
|
|
|
if found && service.Name != tc.expectedName {
|
|
t.Errorf("DetectService(%s) service name = %s, expected %s", tc.userAgent, service.Name, tc.expectedName)
|
|
}
|
|
})
|
|
}
|
|
|
|
// Test cache key generation for different services
|
|
steamKey := generateServiceCacheKey("/depot/123/chunk/abc", "steam")
|
|
epicKey := generateServiceCacheKey("/epic/123/chunk/abc", "epic")
|
|
|
|
if !strings.HasPrefix(steamKey, "steam/") {
|
|
t.Errorf("Steam cache key should start with 'steam/', got: %s", steamKey)
|
|
}
|
|
if !strings.HasPrefix(epicKey, "epic/") {
|
|
t.Errorf("Epic cache key should start with 'epic/', got: %s", epicKey)
|
|
}
|
|
}
|
|
|
|
// Removed hash calculation tests since we switched to lightweight validation
|
|
|
|
func TestSteamKeySharding(t *testing.T) {
|
|
sc := New("localhost:8080", "0", "1G", t.TempDir(), "", "lru", "lru", 200, 5)
|
|
|
|
// Test with a Steam-style key that should trigger sharding
|
|
steamKey := "steam/0016cfc5019b8baa6026aa1cce93e685d6e06c6e"
|
|
testData := []byte("test steam cache data")
|
|
|
|
// Create a file with the steam key
|
|
w, err := sc.vfs.Create(steamKey, int64(len(testData)))
|
|
if err != nil {
|
|
t.Fatalf("Failed to create file with steam key: %v", err)
|
|
}
|
|
w.Write(testData)
|
|
w.Close()
|
|
|
|
// Verify we can read it back
|
|
rc, err := sc.vfs.Open(steamKey)
|
|
if err != nil {
|
|
t.Fatalf("Failed to open file with steam key: %v", err)
|
|
}
|
|
got, _ := io.ReadAll(rc)
|
|
rc.Close()
|
|
|
|
if string(got) != string(testData) {
|
|
t.Errorf("Data mismatch: expected %s, got %s", testData, got)
|
|
}
|
|
|
|
// Verify that the file was created (sharding is working if no error occurred)
|
|
// The key difference is that with sharding, the file should be created successfully
|
|
// and be readable, whereas without sharding it might not work correctly
|
|
}
|
|
|
|
// Removed old TestKeyGeneration - replaced with TestURLHashing that uses SHA256
|