Compare commits
7 Commits
ee6fc32a1a
...
1.0.15
| Author | SHA1 | Date | |
|---|---|---|---|
| 46495dc3aa | |||
| 45ae234694 | |||
| bbe014e334 | |||
| 694c223b00 | |||
| cc3497bc3a | |||
| 9ca8fa4a5e | |||
| 7fb1fcf21f |
4
.gitignore
vendored
4
.gitignore
vendored
@@ -9,3 +9,7 @@
|
|||||||
|
|
||||||
#windows executables
|
#windows executables
|
||||||
*.exe
|
*.exe
|
||||||
|
|
||||||
|
#test cache
|
||||||
|
/steamcache/test_cache/*
|
||||||
|
!/steamcache/test_cache/.gitkeep
|
||||||
24
Makefile
24
Makefile
@@ -1,5 +1,16 @@
|
|||||||
run: deps test ## Run the application
|
run: build-snapshot-single ## Run the application
|
||||||
@go run .
|
@dist/default_windows_amd64_v1/steamcache2.exe
|
||||||
|
run-debug: build-snapshot-single ## Run the application with debug logging
|
||||||
|
@dist/default_windows_amd64_v1/steamcache2.exe --log-level debug
|
||||||
|
|
||||||
|
test: deps ## Run all tests
|
||||||
|
@go test -v ./...
|
||||||
|
|
||||||
|
deps: ## Download dependencies
|
||||||
|
@go mod tidy
|
||||||
|
|
||||||
|
build-snapshot-single: deps test ## Build a snapshot of the application for the current platform
|
||||||
|
@goreleaser build --single-target --snapshot --clean
|
||||||
|
|
||||||
help: ## Show this help message
|
help: ## Show this help message
|
||||||
@echo SteamCache2 Makefile
|
@echo SteamCache2 Makefile
|
||||||
@@ -8,12 +19,3 @@ help: ## Show this help message
|
|||||||
@echo run-debug Run the application with debug logging
|
@echo run-debug Run the application with debug logging
|
||||||
@echo test Run all tests
|
@echo test Run all tests
|
||||||
@echo deps Download dependencies
|
@echo deps Download dependencies
|
||||||
|
|
||||||
run-debug: deps test ## Run the application with debug logging
|
|
||||||
@go run . --log-level debug
|
|
||||||
|
|
||||||
test: deps ## Run all tests
|
|
||||||
@go test -v ./...
|
|
||||||
|
|
||||||
deps: ## Download dependencies
|
|
||||||
@go mod tidy
|
|
||||||
26
cmd/root.go
26
cmd/root.go
@@ -4,7 +4,6 @@ package cmd
|
|||||||
import (
|
import (
|
||||||
"fmt"
|
"fmt"
|
||||||
"os"
|
"os"
|
||||||
"runtime"
|
|
||||||
"s1d3sw1ped/SteamCache2/config"
|
"s1d3sw1ped/SteamCache2/config"
|
||||||
"s1d3sw1ped/SteamCache2/steamcache"
|
"s1d3sw1ped/SteamCache2/steamcache"
|
||||||
"s1d3sw1ped/SteamCache2/steamcache/logger"
|
"s1d3sw1ped/SteamCache2/steamcache/logger"
|
||||||
@@ -16,11 +15,13 @@ import (
|
|||||||
)
|
)
|
||||||
|
|
||||||
var (
|
var (
|
||||||
threads int
|
|
||||||
configPath string
|
configPath string
|
||||||
|
|
||||||
logLevel string
|
logLevel string
|
||||||
logFormat string
|
logFormat string
|
||||||
|
|
||||||
|
maxConcurrentRequests int64
|
||||||
|
maxRequestsPerClient int64
|
||||||
)
|
)
|
||||||
|
|
||||||
var rootCmd = &cobra.Command{
|
var rootCmd = &cobra.Command{
|
||||||
@@ -96,11 +97,15 @@ var rootCmd = &cobra.Command{
|
|||||||
Str("config_path", configPath).
|
Str("config_path", configPath).
|
||||||
Msg("Configuration loaded successfully")
|
Msg("Configuration loaded successfully")
|
||||||
|
|
||||||
if runtime.GOMAXPROCS(-1) != threads {
|
// Use command-line flags if provided, otherwise use config values
|
||||||
runtime.GOMAXPROCS(threads)
|
finalMaxConcurrentRequests := cfg.MaxConcurrentRequests
|
||||||
logger.Logger.Info().
|
if maxConcurrentRequests > 0 {
|
||||||
Int("threads", threads).
|
finalMaxConcurrentRequests = maxConcurrentRequests
|
||||||
Msg("Maximum number of threads set")
|
}
|
||||||
|
|
||||||
|
finalMaxRequestsPerClient := cfg.MaxRequestsPerClient
|
||||||
|
if maxRequestsPerClient > 0 {
|
||||||
|
finalMaxRequestsPerClient = maxRequestsPerClient
|
||||||
}
|
}
|
||||||
|
|
||||||
sc := steamcache.New(
|
sc := steamcache.New(
|
||||||
@@ -111,6 +116,8 @@ var rootCmd = &cobra.Command{
|
|||||||
cfg.Upstream,
|
cfg.Upstream,
|
||||||
cfg.Cache.Memory.GCAlgorithm,
|
cfg.Cache.Memory.GCAlgorithm,
|
||||||
cfg.Cache.Disk.GCAlgorithm,
|
cfg.Cache.Disk.GCAlgorithm,
|
||||||
|
finalMaxConcurrentRequests,
|
||||||
|
finalMaxRequestsPerClient,
|
||||||
)
|
)
|
||||||
|
|
||||||
logger.Logger.Info().
|
logger.Logger.Info().
|
||||||
@@ -135,8 +142,9 @@ func Execute() {
|
|||||||
func init() {
|
func init() {
|
||||||
rootCmd.Flags().StringVarP(&configPath, "config", "c", "config.yaml", "Path to configuration file")
|
rootCmd.Flags().StringVarP(&configPath, "config", "c", "config.yaml", "Path to configuration file")
|
||||||
|
|
||||||
rootCmd.Flags().IntVarP(&threads, "threads", "t", runtime.GOMAXPROCS(-1), "Number of worker threads to use for processing requests")
|
|
||||||
|
|
||||||
rootCmd.Flags().StringVarP(&logLevel, "log-level", "l", "info", "Logging level: debug, info, error")
|
rootCmd.Flags().StringVarP(&logLevel, "log-level", "l", "info", "Logging level: debug, info, error")
|
||||||
rootCmd.Flags().StringVarP(&logFormat, "log-format", "f", "console", "Logging format: json, console")
|
rootCmd.Flags().StringVarP(&logFormat, "log-format", "f", "console", "Logging format: json, console")
|
||||||
|
|
||||||
|
rootCmd.Flags().Int64Var(&maxConcurrentRequests, "max-concurrent-requests", 0, "Maximum concurrent requests (0 = use config file value)")
|
||||||
|
rootCmd.Flags().Int64Var(&maxRequestsPerClient, "max-requests-per-client", 0, "Maximum concurrent requests per client IP (0 = use config file value)")
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -11,6 +11,10 @@ type Config struct {
|
|||||||
// Server configuration
|
// Server configuration
|
||||||
ListenAddress string `yaml:"listen_address" default:":80"`
|
ListenAddress string `yaml:"listen_address" default:":80"`
|
||||||
|
|
||||||
|
// Concurrency limits
|
||||||
|
MaxConcurrentRequests int64 `yaml:"max_concurrent_requests" default:"200"`
|
||||||
|
MaxRequestsPerClient int64 `yaml:"max_requests_per_client" default:"5"`
|
||||||
|
|
||||||
// Cache configuration
|
// Cache configuration
|
||||||
Cache CacheConfig `yaml:"cache"`
|
Cache CacheConfig `yaml:"cache"`
|
||||||
|
|
||||||
@@ -65,6 +69,12 @@ func LoadConfig(configPath string) (*Config, error) {
|
|||||||
if config.ListenAddress == "" {
|
if config.ListenAddress == "" {
|
||||||
config.ListenAddress = ":80"
|
config.ListenAddress = ":80"
|
||||||
}
|
}
|
||||||
|
if config.MaxConcurrentRequests == 0 {
|
||||||
|
config.MaxConcurrentRequests = 50
|
||||||
|
}
|
||||||
|
if config.MaxRequestsPerClient == 0 {
|
||||||
|
config.MaxRequestsPerClient = 3
|
||||||
|
}
|
||||||
if config.Cache.Memory.Size == "" {
|
if config.Cache.Memory.Size == "" {
|
||||||
config.Cache.Memory.Size = "0"
|
config.Cache.Memory.Size = "0"
|
||||||
}
|
}
|
||||||
@@ -89,15 +99,17 @@ func SaveDefaultConfig(configPath string) error {
|
|||||||
|
|
||||||
defaultConfig := Config{
|
defaultConfig := Config{
|
||||||
ListenAddress: ":80",
|
ListenAddress: ":80",
|
||||||
|
MaxConcurrentRequests: 50, // Reduced for home user (less concurrent load)
|
||||||
|
MaxRequestsPerClient: 3, // Reduced for home user (more conservative per client)
|
||||||
Cache: CacheConfig{
|
Cache: CacheConfig{
|
||||||
Memory: MemoryConfig{
|
Memory: MemoryConfig{
|
||||||
Size: "1GB",
|
Size: "1GB", // Recommended for systems that can spare 1GB RAM for caching
|
||||||
GCAlgorithm: "lru",
|
GCAlgorithm: "lru",
|
||||||
},
|
},
|
||||||
Disk: DiskConfig{
|
Disk: DiskConfig{
|
||||||
Size: "10GB",
|
Size: "1TB", // Large HDD cache for home user
|
||||||
Path: "./disk",
|
Path: "./disk",
|
||||||
GCAlgorithm: "hybrid",
|
GCAlgorithm: "lru", // Better for gaming patterns (keeps recently played games)
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
Upstream: "",
|
Upstream: "",
|
||||||
|
|||||||
1
go.mod
1
go.mod
@@ -7,6 +7,7 @@ require (
|
|||||||
github.com/edsrzf/mmap-go v1.1.0
|
github.com/edsrzf/mmap-go v1.1.0
|
||||||
github.com/rs/zerolog v1.33.0
|
github.com/rs/zerolog v1.33.0
|
||||||
github.com/spf13/cobra v1.8.1
|
github.com/spf13/cobra v1.8.1
|
||||||
|
golang.org/x/sync v0.16.0
|
||||||
gopkg.in/yaml.v3 v3.0.1
|
gopkg.in/yaml.v3 v3.0.1
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|||||||
2
go.sum
2
go.sum
@@ -21,6 +21,8 @@ github.com/spf13/cobra v1.8.1 h1:e5/vxKd/rZsfSJMUX1agtjeTDf+qv1/JdBF8gg5k9ZM=
|
|||||||
github.com/spf13/cobra v1.8.1/go.mod h1:wHxEcudfqmLYa8iTfL+OuZPbBZkmvliBWKIezN3kD9Y=
|
github.com/spf13/cobra v1.8.1/go.mod h1:wHxEcudfqmLYa8iTfL+OuZPbBZkmvliBWKIezN3kD9Y=
|
||||||
github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA=
|
github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA=
|
||||||
github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
|
github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
|
||||||
|
golang.org/x/sync v0.16.0 h1:ycBJEhp9p4vXvUZNszeOq0kGTPghopOL8q0fq3vstxw=
|
||||||
|
golang.org/x/sync v0.16.0/go.mod h1:1dzgHSNfp02xaA81J2MS99Qcpr2w7fw1gpm99rleRqA=
|
||||||
golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||||
golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||||
golang.org/x/sys v0.12.0 h1:CM0HF96J0hcLAwsHPJZjfdNzs0gftsLfgKt57wWHJ0o=
|
golang.org/x/sys v0.12.0 h1:CM0HF96J0hcLAwsHPJZjfdNzs0gftsLfgKt57wWHJ0o=
|
||||||
|
|||||||
279
steamcache/integration_test.go
Normal file
279
steamcache/integration_test.go
Normal file
@@ -0,0 +1,279 @@
|
|||||||
|
package steamcache
|
||||||
|
|
||||||
|
import (
|
||||||
|
"bytes"
|
||||||
|
"fmt"
|
||||||
|
"io"
|
||||||
|
"net/http"
|
||||||
|
"net/http/httptest"
|
||||||
|
"os"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
)
|
||||||
|
|
||||||
|
const SteamHostname = "cache2-den-iwst.steamcontent.com"
|
||||||
|
|
||||||
|
func TestSteamIntegration(t *testing.T) {
|
||||||
|
// Skip this test if we don't have internet access or want to avoid hitting Steam servers
|
||||||
|
if testing.Short() {
|
||||||
|
t.Skip("Skipping integration test in short mode")
|
||||||
|
}
|
||||||
|
|
||||||
|
// Test URLs from real Steam usage - these should be cached when requested by Steam clients
|
||||||
|
testURLs := []string{
|
||||||
|
"/depot/516751/patch/288061881745926019/4378193572994177373",
|
||||||
|
"/depot/516751/chunk/42e7c13eb4b4e426ec5cf6d1010abfd528e5065a",
|
||||||
|
"/depot/516751/chunk/f949f71e102d77ed6e364e2054d06429d54bebb1",
|
||||||
|
"/depot/516751/chunk/6790f5105833556d37797657be72c1c8dd2e7074",
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, testURL := range testURLs {
|
||||||
|
t.Run(fmt.Sprintf("URL_%s", testURL), func(t *testing.T) {
|
||||||
|
testSteamURL(t, testURL)
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func testSteamURL(t *testing.T, urlPath string) {
|
||||||
|
// Create a unique temporary directory for this test to avoid cache persistence issues
|
||||||
|
tempDir, err := os.MkdirTemp("", "steamcache_test_*")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Failed to create temp directory: %v", err)
|
||||||
|
}
|
||||||
|
defer os.RemoveAll(tempDir) // Clean up after test
|
||||||
|
|
||||||
|
// Create SteamCache instance with unique temp directory
|
||||||
|
sc := New(":0", "100MB", "1GB", tempDir, "", "LRU", "LRU", 10, 5)
|
||||||
|
|
||||||
|
// Use real Steam server
|
||||||
|
steamURL := "https://" + SteamHostname + urlPath
|
||||||
|
|
||||||
|
// Test direct download from Steam server
|
||||||
|
directResp, directBody := downloadDirectly(t, steamURL)
|
||||||
|
|
||||||
|
// Test download through SteamCache
|
||||||
|
cacheResp, cacheBody := downloadThroughCache(t, sc, urlPath)
|
||||||
|
|
||||||
|
// Compare responses
|
||||||
|
compareResponses(t, directResp, directBody, cacheResp, cacheBody, urlPath)
|
||||||
|
}
|
||||||
|
|
||||||
|
func downloadDirectly(t *testing.T, url string) (*http.Response, []byte) {
|
||||||
|
client := &http.Client{Timeout: 30 * time.Second}
|
||||||
|
|
||||||
|
req, err := http.NewRequest("GET", url, nil)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Failed to create request: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Add Steam user agent
|
||||||
|
req.Header.Set("User-Agent", "Valve/Steam HTTP Client 1.0")
|
||||||
|
|
||||||
|
resp, err := client.Do(req)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Failed to download directly from Steam: %v", err)
|
||||||
|
}
|
||||||
|
defer resp.Body.Close()
|
||||||
|
|
||||||
|
body, err := io.ReadAll(resp.Body)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Failed to read direct response body: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
return resp, body
|
||||||
|
}
|
||||||
|
|
||||||
|
func downloadThroughCache(t *testing.T, sc *SteamCache, urlPath string) (*http.Response, []byte) {
|
||||||
|
// Create a test server for SteamCache
|
||||||
|
cacheServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||||
|
// For real Steam URLs, we need to set the upstream to the Steam hostname
|
||||||
|
// and let SteamCache handle the full URL construction
|
||||||
|
sc.upstream = "https://" + SteamHostname
|
||||||
|
sc.ServeHTTP(w, r)
|
||||||
|
}))
|
||||||
|
defer cacheServer.Close()
|
||||||
|
|
||||||
|
// First request - should be a MISS and cache the file
|
||||||
|
client := &http.Client{Timeout: 30 * time.Second}
|
||||||
|
|
||||||
|
req1, err := http.NewRequest("GET", cacheServer.URL+urlPath, nil)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Failed to create first request: %v", err)
|
||||||
|
}
|
||||||
|
req1.Header.Set("User-Agent", "Valve/Steam HTTP Client 1.0")
|
||||||
|
|
||||||
|
resp1, err := client.Do(req1)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Failed to download through cache (first request): %v", err)
|
||||||
|
}
|
||||||
|
defer resp1.Body.Close()
|
||||||
|
|
||||||
|
body1, err := io.ReadAll(resp1.Body)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Failed to read cache response body (first request): %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Verify first request was a MISS
|
||||||
|
if resp1.Header.Get("X-LanCache-Status") != "MISS" {
|
||||||
|
t.Errorf("Expected first request to be MISS, got %s", resp1.Header.Get("X-LanCache-Status"))
|
||||||
|
}
|
||||||
|
|
||||||
|
// Second request - should be a HIT from cache
|
||||||
|
req2, err := http.NewRequest("GET", cacheServer.URL+urlPath, nil)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Failed to create second request: %v", err)
|
||||||
|
}
|
||||||
|
req2.Header.Set("User-Agent", "Valve/Steam HTTP Client 1.0")
|
||||||
|
|
||||||
|
resp2, err := client.Do(req2)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Failed to download through cache (second request): %v", err)
|
||||||
|
}
|
||||||
|
defer resp2.Body.Close()
|
||||||
|
|
||||||
|
body2, err := io.ReadAll(resp2.Body)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Failed to read cache response body (second request): %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Verify second request was a HIT (unless hash verification failed)
|
||||||
|
status2 := resp2.Header.Get("X-LanCache-Status")
|
||||||
|
if status2 != "HIT" && status2 != "MISS" {
|
||||||
|
t.Errorf("Expected second request to be HIT or MISS, got %s", status2)
|
||||||
|
}
|
||||||
|
|
||||||
|
// If it's a MISS, it means hash verification failed and content wasn't cached
|
||||||
|
// This is correct behavior - we shouldn't cache content that doesn't match the expected hash
|
||||||
|
if status2 == "MISS" {
|
||||||
|
t.Logf("Second request was MISS (hash verification failed) - this is correct behavior")
|
||||||
|
}
|
||||||
|
|
||||||
|
// Verify both cache responses are identical
|
||||||
|
if !bytes.Equal(body1, body2) {
|
||||||
|
t.Error("First and second cache responses should be identical")
|
||||||
|
}
|
||||||
|
|
||||||
|
// Return the second response (from cache)
|
||||||
|
return resp2, body2
|
||||||
|
}
|
||||||
|
|
||||||
|
func compareResponses(t *testing.T, directResp *http.Response, directBody []byte, cacheResp *http.Response, cacheBody []byte, urlPath string) {
|
||||||
|
// Compare status codes
|
||||||
|
if directResp.StatusCode != cacheResp.StatusCode {
|
||||||
|
t.Errorf("Status code mismatch: direct=%d, cache=%d", directResp.StatusCode, cacheResp.StatusCode)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Compare response bodies (this is the most important test)
|
||||||
|
if !bytes.Equal(directBody, cacheBody) {
|
||||||
|
t.Errorf("Response body mismatch for URL %s", urlPath)
|
||||||
|
t.Errorf("Direct body length: %d, Cache body length: %d", len(directBody), len(cacheBody))
|
||||||
|
|
||||||
|
// Find first difference
|
||||||
|
minLen := len(directBody)
|
||||||
|
if len(cacheBody) < minLen {
|
||||||
|
minLen = len(cacheBody)
|
||||||
|
}
|
||||||
|
|
||||||
|
for i := 0; i < minLen; i++ {
|
||||||
|
if directBody[i] != cacheBody[i] {
|
||||||
|
t.Errorf("First difference at byte %d: direct=0x%02x, cache=0x%02x", i, directBody[i], cacheBody[i])
|
||||||
|
break
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Compare important headers (excluding cache-specific ones)
|
||||||
|
importantHeaders := []string{
|
||||||
|
"Content-Type",
|
||||||
|
"Content-Length",
|
||||||
|
"X-Sha1",
|
||||||
|
"Cache-Control",
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, header := range importantHeaders {
|
||||||
|
directValue := directResp.Header.Get(header)
|
||||||
|
cacheValue := cacheResp.Header.Get(header)
|
||||||
|
|
||||||
|
if directValue != cacheValue {
|
||||||
|
t.Errorf("Header %s mismatch: direct=%s, cache=%s", header, directValue, cacheValue)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Verify cache-specific headers are present
|
||||||
|
if cacheResp.Header.Get("X-LanCache-Status") == "" {
|
||||||
|
t.Error("Cache response should have X-LanCache-Status header")
|
||||||
|
}
|
||||||
|
|
||||||
|
if cacheResp.Header.Get("X-LanCache-Processed-By") != "SteamCache2" {
|
||||||
|
t.Error("Cache response should have X-LanCache-Processed-By header set to SteamCache2")
|
||||||
|
}
|
||||||
|
|
||||||
|
t.Logf("✅ URL %s: Direct and cache responses are identical", urlPath)
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestCacheFileFormat tests the cache file format directly
|
||||||
|
func TestCacheFileFormat(t *testing.T) {
|
||||||
|
// Create test data
|
||||||
|
bodyData := []byte("test steam content")
|
||||||
|
contentHash := calculateSHA256(bodyData)
|
||||||
|
|
||||||
|
// Create mock response
|
||||||
|
resp := &http.Response{
|
||||||
|
StatusCode: 200,
|
||||||
|
Status: "200 OK",
|
||||||
|
Header: make(http.Header),
|
||||||
|
Body: http.NoBody,
|
||||||
|
}
|
||||||
|
resp.Header.Set("Content-Type", "application/x-steam-chunk")
|
||||||
|
resp.Header.Set("Content-Length", "18")
|
||||||
|
resp.Header.Set("X-Sha1", contentHash)
|
||||||
|
|
||||||
|
// Create SteamCache instance
|
||||||
|
sc := &SteamCache{}
|
||||||
|
|
||||||
|
// Reconstruct raw response
|
||||||
|
rawResponse := sc.reconstructRawResponse(resp, bodyData)
|
||||||
|
|
||||||
|
// Serialize to cache format
|
||||||
|
cacheData, err := serializeRawResponse(rawResponse)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Failed to serialize cache file: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Deserialize from cache format
|
||||||
|
cacheFile, err := deserializeCacheFile(cacheData)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Failed to deserialize cache file: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Verify cache file structure
|
||||||
|
if cacheFile.ContentHash != contentHash {
|
||||||
|
t.Errorf("ContentHash mismatch: expected %s, got %s", contentHash, cacheFile.ContentHash)
|
||||||
|
}
|
||||||
|
|
||||||
|
if cacheFile.ResponseSize != int64(len(rawResponse)) {
|
||||||
|
t.Errorf("ResponseSize mismatch: expected %d, got %d", len(rawResponse), cacheFile.ResponseSize)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Verify raw response is preserved
|
||||||
|
if !bytes.Equal(cacheFile.Response, rawResponse) {
|
||||||
|
t.Error("Raw response not preserved in cache file")
|
||||||
|
}
|
||||||
|
|
||||||
|
// Test streaming the cached response
|
||||||
|
recorder := httptest.NewRecorder()
|
||||||
|
req := httptest.NewRequest("GET", "/test/format", nil)
|
||||||
|
|
||||||
|
sc.streamCachedResponse(recorder, req, cacheFile, "test-key", "127.0.0.1", time.Now())
|
||||||
|
|
||||||
|
// Verify streamed response
|
||||||
|
if recorder.Code != 200 {
|
||||||
|
t.Errorf("Expected status code 200, got %d", recorder.Code)
|
||||||
|
}
|
||||||
|
|
||||||
|
if !bytes.Equal(recorder.Body.Bytes(), bodyData) {
|
||||||
|
t.Error("Streamed response body does not match original")
|
||||||
|
}
|
||||||
|
|
||||||
|
t.Log("✅ Cache file format test passed")
|
||||||
|
}
|
||||||
File diff suppressed because it is too large
Load Diff
@@ -14,7 +14,7 @@ func TestCaching(t *testing.T) {
|
|||||||
|
|
||||||
os.WriteFile(filepath.Join(td, "key2"), []byte("value2"), 0644)
|
os.WriteFile(filepath.Join(td, "key2"), []byte("value2"), 0644)
|
||||||
|
|
||||||
sc := New("localhost:8080", "1G", "1G", td, "", "lru", "lru")
|
sc := New("localhost:8080", "1G", "1G", td, "", "lru", "lru", 200, 5)
|
||||||
|
|
||||||
w, err := sc.vfs.Create("key", 5)
|
w, err := sc.vfs.Create("key", 5)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@@ -68,15 +68,22 @@ func TestCaching(t *testing.T) {
|
|||||||
t.Errorf("Get failed: got %s, want %s", d, "value2")
|
t.Errorf("Get failed: got %s, want %s", d, "value2")
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// With size-based promotion filtering, not all files may be promoted
|
||||||
|
// The total size should be at least the disk size (17 bytes) but may be less than 34 bytes
|
||||||
|
// if some files are filtered out due to size constraints
|
||||||
if sc.diskgc.Size() != 17 {
|
if sc.diskgc.Size() != 17 {
|
||||||
t.Errorf("Size failed: got %d, want %d", sc.diskgc.Size(), 17)
|
t.Errorf("Disk size failed: got %d, want %d", sc.diskgc.Size(), 17)
|
||||||
}
|
}
|
||||||
|
|
||||||
if sc.vfs.Size() != 17 {
|
if sc.vfs.Size() < 17 {
|
||||||
t.Errorf("Size failed: got %d, want %d", sc.vfs.Size(), 17)
|
t.Errorf("Total size too small: got %d, want at least 17", sc.vfs.Size())
|
||||||
|
}
|
||||||
|
if sc.vfs.Size() > 34 {
|
||||||
|
t.Errorf("Total size too large: got %d, want at most 34", sc.vfs.Size())
|
||||||
}
|
}
|
||||||
|
|
||||||
sc.memory.Delete("key2")
|
sc.memory.Delete("key2")
|
||||||
|
sc.disk.Delete("key2") // Also delete from disk cache
|
||||||
os.Remove(filepath.Join(td, "key2"))
|
os.Remove(filepath.Join(td, "key2"))
|
||||||
|
|
||||||
if _, err := sc.vfs.Open("key2"); err == nil {
|
if _, err := sc.vfs.Open("key2"); err == nil {
|
||||||
@@ -85,7 +92,7 @@ func TestCaching(t *testing.T) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func TestCacheMissAndHit(t *testing.T) {
|
func TestCacheMissAndHit(t *testing.T) {
|
||||||
sc := New("localhost:8080", "0", "1G", t.TempDir(), "", "lru", "lru")
|
sc := New("localhost:8080", "0", "1G", t.TempDir(), "", "lru", "lru", 200, 5)
|
||||||
|
|
||||||
key := "testkey"
|
key := "testkey"
|
||||||
value := []byte("testvalue")
|
value := []byte("testvalue")
|
||||||
@@ -111,7 +118,8 @@ func TestCacheMissAndHit(t *testing.T) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func TestURLHashing(t *testing.T) {
|
func TestURLHashing(t *testing.T) {
|
||||||
// Test the new SHA256-based cache key generation
|
// Test the SHA256-based cache key generation for Steam client requests
|
||||||
|
// The "steam/" prefix indicates the request came from a Steam client (User-Agent based)
|
||||||
|
|
||||||
testCases := []struct {
|
testCases := []struct {
|
||||||
input string
|
input string
|
||||||
@@ -129,44 +137,192 @@ func TestURLHashing(t *testing.T) {
|
|||||||
shouldCache: true,
|
shouldCache: true,
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
input: "/depot/invalid/path",
|
input: "/appinfo/123456",
|
||||||
desc: "invalid depot URL format",
|
desc: "app info URL",
|
||||||
shouldCache: true, // Still gets hashed, just not a proper Steam format
|
shouldCache: true,
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
input: "/some/other/path",
|
input: "/some/other/path",
|
||||||
desc: "non-Steam URL",
|
desc: "any URL from Steam client",
|
||||||
shouldCache: false, // Not cached
|
shouldCache: true, // All URLs from Steam clients (detected via User-Agent) are cached
|
||||||
},
|
},
|
||||||
}
|
}
|
||||||
|
|
||||||
for _, tc := range testCases {
|
for _, tc := range testCases {
|
||||||
t.Run(tc.desc, func(t *testing.T) {
|
t.Run(tc.desc, func(t *testing.T) {
|
||||||
result := generateSteamCacheKey(tc.input)
|
result := generateServiceCacheKey(tc.input, "steam")
|
||||||
|
|
||||||
if tc.shouldCache {
|
if tc.shouldCache {
|
||||||
// Should return a cache key with "steam/" prefix
|
// Should return a cache key with "steam/" prefix
|
||||||
if !strings.HasPrefix(result, "steam/") {
|
if !strings.HasPrefix(result, "steam/") {
|
||||||
t.Errorf("generateSteamCacheKey(%s) = %s, expected steam/ prefix", tc.input, result)
|
t.Errorf("generateServiceCacheKey(%s, \"steam\") = %s, expected steam/ prefix", tc.input, result)
|
||||||
}
|
}
|
||||||
// Should be exactly 70 characters (6 for "steam/" + 64 for SHA256 hex)
|
// Should be exactly 70 characters (6 for "steam/" + 64 for SHA256 hex)
|
||||||
if len(result) != 70 {
|
if len(result) != 70 {
|
||||||
t.Errorf("generateSteamCacheKey(%s) length = %d, expected 70", tc.input, len(result))
|
t.Errorf("generateServiceCacheKey(%s, \"steam\") length = %d, expected 70", tc.input, len(result))
|
||||||
}
|
}
|
||||||
} else {
|
} else {
|
||||||
// Should return empty string for non-Steam URLs
|
// Should return empty string for non-Steam URLs
|
||||||
if result != "" {
|
if result != "" {
|
||||||
t.Errorf("generateSteamCacheKey(%s) = %s, expected empty string", tc.input, result)
|
t.Errorf("generateServiceCacheKey(%s, \"steam\") = %s, expected empty string", tc.input, result)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestServiceDetection(t *testing.T) {
|
||||||
|
// Create a service manager for testing
|
||||||
|
sm := NewServiceManager()
|
||||||
|
|
||||||
|
testCases := []struct {
|
||||||
|
userAgent string
|
||||||
|
expectedName string
|
||||||
|
expectedFound bool
|
||||||
|
desc string
|
||||||
|
}{
|
||||||
|
{
|
||||||
|
userAgent: "Valve/Steam HTTP Client 1.0",
|
||||||
|
expectedName: "steam",
|
||||||
|
expectedFound: true,
|
||||||
|
desc: "Valve Steam HTTP Client",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
userAgent: "Steam",
|
||||||
|
expectedName: "steam",
|
||||||
|
expectedFound: true,
|
||||||
|
desc: "Simple Steam user agent",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
userAgent: "SteamClient/1.0",
|
||||||
|
expectedName: "steam",
|
||||||
|
expectedFound: true,
|
||||||
|
desc: "SteamClient with version",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
userAgent: "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36",
|
||||||
|
expectedName: "",
|
||||||
|
expectedFound: false,
|
||||||
|
desc: "Browser user agent",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
userAgent: "",
|
||||||
|
expectedName: "",
|
||||||
|
expectedFound: false,
|
||||||
|
desc: "Empty user agent",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
userAgent: "curl/7.68.0",
|
||||||
|
expectedName: "",
|
||||||
|
expectedFound: false,
|
||||||
|
desc: "curl user agent",
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, tc := range testCases {
|
||||||
|
t.Run(tc.desc, func(t *testing.T) {
|
||||||
|
service, found := sm.DetectService(tc.userAgent)
|
||||||
|
|
||||||
|
if found != tc.expectedFound {
|
||||||
|
t.Errorf("DetectService(%s) found = %v, expected %v", tc.userAgent, found, tc.expectedFound)
|
||||||
|
}
|
||||||
|
|
||||||
|
if found && service.Name != tc.expectedName {
|
||||||
|
t.Errorf("DetectService(%s) service name = %s, expected %s", tc.userAgent, service.Name, tc.expectedName)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestServiceManagerExpandability(t *testing.T) {
|
||||||
|
// Create a service manager for testing
|
||||||
|
sm := NewServiceManager()
|
||||||
|
|
||||||
|
// Test adding a new service (Epic Games)
|
||||||
|
epicConfig := &ServiceConfig{
|
||||||
|
Name: "epic",
|
||||||
|
Prefix: "epic",
|
||||||
|
UserAgents: []string{
|
||||||
|
`EpicGamesLauncher`,
|
||||||
|
`EpicGames`,
|
||||||
|
`Epic.*Launcher`,
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
err := sm.AddService(epicConfig)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Failed to add Epic service: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Test Epic Games detection
|
||||||
|
epicTestCases := []struct {
|
||||||
|
userAgent string
|
||||||
|
expectedName string
|
||||||
|
expectedFound bool
|
||||||
|
desc string
|
||||||
|
}{
|
||||||
|
{
|
||||||
|
userAgent: "EpicGamesLauncher/1.0",
|
||||||
|
expectedName: "epic",
|
||||||
|
expectedFound: true,
|
||||||
|
desc: "Epic Games Launcher",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
userAgent: "EpicGames/2.0",
|
||||||
|
expectedName: "epic",
|
||||||
|
expectedFound: true,
|
||||||
|
desc: "Epic Games client",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
userAgent: "Epic Launcher 1.5",
|
||||||
|
expectedName: "epic",
|
||||||
|
expectedFound: true,
|
||||||
|
desc: "Epic Launcher with regex match",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
userAgent: "Steam",
|
||||||
|
expectedName: "steam",
|
||||||
|
expectedFound: true,
|
||||||
|
desc: "Steam should still work",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
userAgent: "Mozilla/5.0",
|
||||||
|
expectedName: "",
|
||||||
|
expectedFound: false,
|
||||||
|
desc: "Browser should not match any service",
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, tc := range epicTestCases {
|
||||||
|
t.Run(tc.desc, func(t *testing.T) {
|
||||||
|
service, found := sm.DetectService(tc.userAgent)
|
||||||
|
|
||||||
|
if found != tc.expectedFound {
|
||||||
|
t.Errorf("DetectService(%s) found = %v, expected %v", tc.userAgent, found, tc.expectedFound)
|
||||||
|
}
|
||||||
|
|
||||||
|
if found && service.Name != tc.expectedName {
|
||||||
|
t.Errorf("DetectService(%s) service name = %s, expected %s", tc.userAgent, service.Name, tc.expectedName)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
// Test cache key generation for different services
|
||||||
|
steamKey := generateServiceCacheKey("/depot/123/chunk/abc", "steam")
|
||||||
|
epicKey := generateServiceCacheKey("/epic/123/chunk/abc", "epic")
|
||||||
|
|
||||||
|
if !strings.HasPrefix(steamKey, "steam/") {
|
||||||
|
t.Errorf("Steam cache key should start with 'steam/', got: %s", steamKey)
|
||||||
|
}
|
||||||
|
if !strings.HasPrefix(epicKey, "epic/") {
|
||||||
|
t.Errorf("Epic cache key should start with 'epic/', got: %s", epicKey)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
// Removed hash calculation tests since we switched to lightweight validation
|
// Removed hash calculation tests since we switched to lightweight validation
|
||||||
|
|
||||||
func TestSteamKeySharding(t *testing.T) {
|
func TestSteamKeySharding(t *testing.T) {
|
||||||
sc := New("localhost:8080", "0", "1G", t.TempDir(), "", "lru", "lru")
|
sc := New("localhost:8080", "0", "1G", t.TempDir(), "", "lru", "lru", 200, 5)
|
||||||
|
|
||||||
// Test with a Steam-style key that should trigger sharding
|
// Test with a Steam-style key that should trigger sharding
|
||||||
steamKey := "steam/0016cfc5019b8baa6026aa1cce93e685d6e06c6e"
|
steamKey := "steam/0016cfc5019b8baa6026aa1cce93e685d6e06c6e"
|
||||||
|
|||||||
0
steamcache/test_cache/.gitkeep
Normal file
0
steamcache/test_cache/.gitkeep
Normal file
273
vfs/adaptive/adaptive.go
Normal file
273
vfs/adaptive/adaptive.go
Normal file
@@ -0,0 +1,273 @@
|
|||||||
|
package adaptive
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"sync"
|
||||||
|
"sync/atomic"
|
||||||
|
"time"
|
||||||
|
)
|
||||||
|
|
||||||
|
// WorkloadPattern represents different types of workload patterns
|
||||||
|
type WorkloadPattern int
|
||||||
|
|
||||||
|
const (
|
||||||
|
PatternUnknown WorkloadPattern = iota
|
||||||
|
PatternSequential // Sequential file access (e.g., game installation)
|
||||||
|
PatternRandom // Random file access (e.g., game updates)
|
||||||
|
PatternBurst // Burst access (e.g., multiple users downloading same game)
|
||||||
|
PatternSteady // Steady access (e.g., popular games being accessed regularly)
|
||||||
|
)
|
||||||
|
|
||||||
|
// CacheStrategy represents different caching strategies
|
||||||
|
type CacheStrategy int
|
||||||
|
|
||||||
|
const (
|
||||||
|
StrategyLRU CacheStrategy = iota
|
||||||
|
StrategyLFU
|
||||||
|
StrategySizeBased
|
||||||
|
StrategyHybrid
|
||||||
|
StrategyPredictive
|
||||||
|
)
|
||||||
|
|
||||||
|
// WorkloadAnalyzer analyzes access patterns to determine optimal caching strategies
|
||||||
|
type WorkloadAnalyzer struct {
|
||||||
|
accessHistory map[string]*AccessInfo
|
||||||
|
patternCounts map[WorkloadPattern]int64
|
||||||
|
mu sync.RWMutex
|
||||||
|
analysisInterval time.Duration
|
||||||
|
ctx context.Context
|
||||||
|
cancel context.CancelFunc
|
||||||
|
}
|
||||||
|
|
||||||
|
// AccessInfo tracks access patterns for individual files
|
||||||
|
type AccessInfo struct {
|
||||||
|
Key string
|
||||||
|
AccessCount int64
|
||||||
|
LastAccess time.Time
|
||||||
|
FirstAccess time.Time
|
||||||
|
AccessTimes []time.Time
|
||||||
|
Size int64
|
||||||
|
AccessPattern WorkloadPattern
|
||||||
|
mu sync.RWMutex
|
||||||
|
}
|
||||||
|
|
||||||
|
// AdaptiveCacheManager manages adaptive caching strategies
|
||||||
|
type AdaptiveCacheManager struct {
|
||||||
|
analyzer *WorkloadAnalyzer
|
||||||
|
currentStrategy CacheStrategy
|
||||||
|
adaptationCount int64
|
||||||
|
mu sync.RWMutex
|
||||||
|
}
|
||||||
|
|
||||||
|
// NewWorkloadAnalyzer creates a new workload analyzer
|
||||||
|
func NewWorkloadAnalyzer(analysisInterval time.Duration) *WorkloadAnalyzer {
|
||||||
|
ctx, cancel := context.WithCancel(context.Background())
|
||||||
|
|
||||||
|
analyzer := &WorkloadAnalyzer{
|
||||||
|
accessHistory: make(map[string]*AccessInfo),
|
||||||
|
patternCounts: make(map[WorkloadPattern]int64),
|
||||||
|
analysisInterval: analysisInterval,
|
||||||
|
ctx: ctx,
|
||||||
|
cancel: cancel,
|
||||||
|
}
|
||||||
|
|
||||||
|
// Start background analysis with much longer interval to reduce overhead
|
||||||
|
go analyzer.analyzePatterns()
|
||||||
|
|
||||||
|
return analyzer
|
||||||
|
}
|
||||||
|
|
||||||
|
// RecordAccess records a file access for pattern analysis (lightweight version)
|
||||||
|
func (wa *WorkloadAnalyzer) RecordAccess(key string, size int64) {
|
||||||
|
// Use read lock first for better performance
|
||||||
|
wa.mu.RLock()
|
||||||
|
info, exists := wa.accessHistory[key]
|
||||||
|
wa.mu.RUnlock()
|
||||||
|
|
||||||
|
if !exists {
|
||||||
|
// Only acquire write lock when creating new entry
|
||||||
|
wa.mu.Lock()
|
||||||
|
// Double-check after acquiring write lock
|
||||||
|
if _, exists = wa.accessHistory[key]; !exists {
|
||||||
|
info = &AccessInfo{
|
||||||
|
Key: key,
|
||||||
|
AccessCount: 1,
|
||||||
|
LastAccess: time.Now(),
|
||||||
|
FirstAccess: time.Now(),
|
||||||
|
AccessTimes: []time.Time{time.Now()},
|
||||||
|
Size: size,
|
||||||
|
}
|
||||||
|
wa.accessHistory[key] = info
|
||||||
|
}
|
||||||
|
wa.mu.Unlock()
|
||||||
|
} else {
|
||||||
|
// Lightweight update - just increment counter and update timestamp
|
||||||
|
info.mu.Lock()
|
||||||
|
info.AccessCount++
|
||||||
|
info.LastAccess = time.Now()
|
||||||
|
// Only keep last 10 access times to reduce memory overhead
|
||||||
|
if len(info.AccessTimes) > 10 {
|
||||||
|
info.AccessTimes = info.AccessTimes[len(info.AccessTimes)-10:]
|
||||||
|
} else {
|
||||||
|
info.AccessTimes = append(info.AccessTimes, time.Now())
|
||||||
|
}
|
||||||
|
info.mu.Unlock()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// analyzePatterns analyzes access patterns in the background
|
||||||
|
func (wa *WorkloadAnalyzer) analyzePatterns() {
|
||||||
|
ticker := time.NewTicker(wa.analysisInterval)
|
||||||
|
defer ticker.Stop()
|
||||||
|
|
||||||
|
for {
|
||||||
|
select {
|
||||||
|
case <-wa.ctx.Done():
|
||||||
|
return
|
||||||
|
case <-ticker.C:
|
||||||
|
wa.performAnalysis()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// performAnalysis analyzes current access patterns
|
||||||
|
func (wa *WorkloadAnalyzer) performAnalysis() {
|
||||||
|
wa.mu.Lock()
|
||||||
|
defer wa.mu.Unlock()
|
||||||
|
|
||||||
|
// Reset pattern counts
|
||||||
|
wa.patternCounts = make(map[WorkloadPattern]int64)
|
||||||
|
|
||||||
|
now := time.Now()
|
||||||
|
cutoff := now.Add(-wa.analysisInterval * 2) // Analyze last 2 intervals
|
||||||
|
|
||||||
|
for _, info := range wa.accessHistory {
|
||||||
|
info.mu.RLock()
|
||||||
|
if info.LastAccess.After(cutoff) {
|
||||||
|
pattern := wa.determinePattern(info)
|
||||||
|
info.AccessPattern = pattern
|
||||||
|
wa.patternCounts[pattern]++
|
||||||
|
}
|
||||||
|
info.mu.RUnlock()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// determinePattern determines the access pattern for a file
|
||||||
|
func (wa *WorkloadAnalyzer) determinePattern(info *AccessInfo) WorkloadPattern {
|
||||||
|
if len(info.AccessTimes) < 3 {
|
||||||
|
return PatternUnknown
|
||||||
|
}
|
||||||
|
|
||||||
|
// Analyze access timing patterns
|
||||||
|
intervals := make([]time.Duration, len(info.AccessTimes)-1)
|
||||||
|
for i := 1; i < len(info.AccessTimes); i++ {
|
||||||
|
intervals[i-1] = info.AccessTimes[i].Sub(info.AccessTimes[i-1])
|
||||||
|
}
|
||||||
|
|
||||||
|
// Calculate variance in access intervals
|
||||||
|
var sum, sumSquares time.Duration
|
||||||
|
for _, interval := range intervals {
|
||||||
|
sum += interval
|
||||||
|
sumSquares += interval * interval
|
||||||
|
}
|
||||||
|
|
||||||
|
avg := sum / time.Duration(len(intervals))
|
||||||
|
variance := (sumSquares / time.Duration(len(intervals))) - (avg * avg)
|
||||||
|
|
||||||
|
// Determine pattern based on variance and access count
|
||||||
|
if info.AccessCount > 10 && variance < time.Minute {
|
||||||
|
return PatternBurst
|
||||||
|
} else if info.AccessCount > 5 && variance < time.Hour {
|
||||||
|
return PatternSteady
|
||||||
|
} else if variance < time.Minute*5 {
|
||||||
|
return PatternSequential
|
||||||
|
} else {
|
||||||
|
return PatternRandom
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// GetDominantPattern returns the most common access pattern
|
||||||
|
func (wa *WorkloadAnalyzer) GetDominantPattern() WorkloadPattern {
|
||||||
|
wa.mu.RLock()
|
||||||
|
defer wa.mu.RUnlock()
|
||||||
|
|
||||||
|
var maxCount int64
|
||||||
|
var dominantPattern WorkloadPattern
|
||||||
|
|
||||||
|
for pattern, count := range wa.patternCounts {
|
||||||
|
if count > maxCount {
|
||||||
|
maxCount = count
|
||||||
|
dominantPattern = pattern
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return dominantPattern
|
||||||
|
}
|
||||||
|
|
||||||
|
// GetAccessInfo returns access information for a key
|
||||||
|
func (wa *WorkloadAnalyzer) GetAccessInfo(key string) *AccessInfo {
|
||||||
|
wa.mu.RLock()
|
||||||
|
defer wa.mu.RUnlock()
|
||||||
|
|
||||||
|
return wa.accessHistory[key]
|
||||||
|
}
|
||||||
|
|
||||||
|
// Stop stops the workload analyzer
|
||||||
|
func (wa *WorkloadAnalyzer) Stop() {
|
||||||
|
wa.cancel()
|
||||||
|
}
|
||||||
|
|
||||||
|
// NewAdaptiveCacheManager creates a new adaptive cache manager
|
||||||
|
func NewAdaptiveCacheManager(analysisInterval time.Duration) *AdaptiveCacheManager {
|
||||||
|
return &AdaptiveCacheManager{
|
||||||
|
analyzer: NewWorkloadAnalyzer(analysisInterval),
|
||||||
|
currentStrategy: StrategyLRU, // Start with LRU
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// AdaptStrategy adapts the caching strategy based on workload patterns
|
||||||
|
func (acm *AdaptiveCacheManager) AdaptStrategy() CacheStrategy {
|
||||||
|
acm.mu.Lock()
|
||||||
|
defer acm.mu.Unlock()
|
||||||
|
|
||||||
|
dominantPattern := acm.analyzer.GetDominantPattern()
|
||||||
|
|
||||||
|
// Adapt strategy based on dominant pattern
|
||||||
|
switch dominantPattern {
|
||||||
|
case PatternBurst:
|
||||||
|
acm.currentStrategy = StrategyLFU // LFU is good for burst patterns
|
||||||
|
case PatternSteady:
|
||||||
|
acm.currentStrategy = StrategyHybrid // Hybrid for steady patterns
|
||||||
|
case PatternSequential:
|
||||||
|
acm.currentStrategy = StrategySizeBased // Size-based for sequential
|
||||||
|
case PatternRandom:
|
||||||
|
acm.currentStrategy = StrategyLRU // LRU for random patterns
|
||||||
|
default:
|
||||||
|
acm.currentStrategy = StrategyLRU // Default to LRU
|
||||||
|
}
|
||||||
|
|
||||||
|
atomic.AddInt64(&acm.adaptationCount, 1)
|
||||||
|
return acm.currentStrategy
|
||||||
|
}
|
||||||
|
|
||||||
|
// GetCurrentStrategy returns the current caching strategy
|
||||||
|
func (acm *AdaptiveCacheManager) GetCurrentStrategy() CacheStrategy {
|
||||||
|
acm.mu.RLock()
|
||||||
|
defer acm.mu.RUnlock()
|
||||||
|
return acm.currentStrategy
|
||||||
|
}
|
||||||
|
|
||||||
|
// RecordAccess records a file access for analysis
|
||||||
|
func (acm *AdaptiveCacheManager) RecordAccess(key string, size int64) {
|
||||||
|
acm.analyzer.RecordAccess(key, size)
|
||||||
|
}
|
||||||
|
|
||||||
|
// GetAdaptationCount returns the number of strategy adaptations
|
||||||
|
func (acm *AdaptiveCacheManager) GetAdaptationCount() int64 {
|
||||||
|
return atomic.LoadInt64(&acm.adaptationCount)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Stop stops the adaptive cache manager
|
||||||
|
func (acm *AdaptiveCacheManager) Stop() {
|
||||||
|
acm.analyzer.Stop()
|
||||||
|
}
|
||||||
289
vfs/cache/cache.go
vendored
289
vfs/cache/cache.go
vendored
@@ -6,6 +6,7 @@ import (
|
|||||||
"s1d3sw1ped/SteamCache2/vfs"
|
"s1d3sw1ped/SteamCache2/vfs"
|
||||||
"s1d3sw1ped/SteamCache2/vfs/vfserror"
|
"s1d3sw1ped/SteamCache2/vfs/vfserror"
|
||||||
"sync"
|
"sync"
|
||||||
|
"sync/atomic"
|
||||||
)
|
)
|
||||||
|
|
||||||
// TieredCache implements a two-tier cache with fast (memory) and slow (disk) storage
|
// TieredCache implements a two-tier cache with fast (memory) and slow (disk) storage
|
||||||
@@ -16,6 +17,12 @@ type TieredCache struct {
|
|||||||
mu sync.RWMutex
|
mu sync.RWMutex
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// LockFreeTieredCache implements a lock-free two-tier cache for better concurrency
|
||||||
|
type LockFreeTieredCache struct {
|
||||||
|
fast *atomic.Value // Memory cache (fast) - atomic.Value for lock-free access
|
||||||
|
slow *atomic.Value // Disk cache (slow) - atomic.Value for lock-free access
|
||||||
|
}
|
||||||
|
|
||||||
// New creates a new tiered cache
|
// New creates a new tiered cache
|
||||||
func New() *TieredCache {
|
func New() *TieredCache {
|
||||||
return &TieredCache{}
|
return &TieredCache{}
|
||||||
@@ -53,7 +60,7 @@ func (tc *TieredCache) Create(key string, size int64) (io.WriteCloser, error) {
|
|||||||
return nil, vfserror.ErrNotFound
|
return nil, vfserror.ErrNotFound
|
||||||
}
|
}
|
||||||
|
|
||||||
// Open opens a file, checking fast tier first, then slow tier
|
// Open opens a file, checking fast tier first, then slow tier with promotion
|
||||||
func (tc *TieredCache) Open(key string) (io.ReadCloser, error) {
|
func (tc *TieredCache) Open(key string) (io.ReadCloser, error) {
|
||||||
tc.mu.RLock()
|
tc.mu.RLock()
|
||||||
defer tc.mu.RUnlock()
|
defer tc.mu.RUnlock()
|
||||||
@@ -65,9 +72,30 @@ func (tc *TieredCache) Open(key string) (io.ReadCloser, error) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// Fall back to slow tier (disk)
|
// Fall back to slow tier (disk) and promote to fast tier
|
||||||
if tc.slow != nil {
|
if tc.slow != nil {
|
||||||
return tc.slow.Open(key)
|
reader, err := tc.slow.Open(key)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
|
||||||
|
// If we have both tiers, check if we should promote the file to fast tier
|
||||||
|
if tc.fast != nil {
|
||||||
|
// Check file size before promoting - don't promote if larger than available memory cache space
|
||||||
|
if info, err := tc.slow.Stat(key); err == nil {
|
||||||
|
availableSpace := tc.fast.Capacity() - tc.fast.Size()
|
||||||
|
// Only promote if file fits in available space (with 10% buffer for safety)
|
||||||
|
if info.Size <= int64(float64(availableSpace)*0.9) {
|
||||||
|
// Create a new reader for promotion to avoid interfering with the returned reader
|
||||||
|
promotionReader, err := tc.slow.Open(key)
|
||||||
|
if err == nil {
|
||||||
|
go tc.promoteToFast(key, promotionReader)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return reader, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
return nil, vfserror.ErrNotFound
|
return nil, vfserror.ErrNotFound
|
||||||
@@ -151,3 +179,258 @@ func (tc *TieredCache) Capacity() int64 {
|
|||||||
}
|
}
|
||||||
return total
|
return total
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// promoteToFast promotes a file from slow tier to fast tier
|
||||||
|
func (tc *TieredCache) promoteToFast(key string, reader io.ReadCloser) {
|
||||||
|
defer reader.Close()
|
||||||
|
|
||||||
|
// Get file info from slow tier to determine size
|
||||||
|
tc.mu.RLock()
|
||||||
|
var size int64
|
||||||
|
if tc.slow != nil {
|
||||||
|
if info, err := tc.slow.Stat(key); err == nil {
|
||||||
|
size = info.Size
|
||||||
|
} else {
|
||||||
|
tc.mu.RUnlock()
|
||||||
|
return // Skip promotion if we can't get file info
|
||||||
|
}
|
||||||
|
}
|
||||||
|
tc.mu.RUnlock()
|
||||||
|
|
||||||
|
// Check if file fits in available memory cache space
|
||||||
|
tc.mu.RLock()
|
||||||
|
if tc.fast != nil {
|
||||||
|
availableSpace := tc.fast.Capacity() - tc.fast.Size()
|
||||||
|
// Only promote if file fits in available space (with 10% buffer for safety)
|
||||||
|
if size > int64(float64(availableSpace)*0.9) {
|
||||||
|
tc.mu.RUnlock()
|
||||||
|
return // Skip promotion if file is too large
|
||||||
|
}
|
||||||
|
}
|
||||||
|
tc.mu.RUnlock()
|
||||||
|
|
||||||
|
// Read the entire file content
|
||||||
|
content, err := io.ReadAll(reader)
|
||||||
|
if err != nil {
|
||||||
|
return // Skip promotion if read fails
|
||||||
|
}
|
||||||
|
|
||||||
|
// Create the file in fast tier
|
||||||
|
tc.mu.RLock()
|
||||||
|
if tc.fast != nil {
|
||||||
|
writer, err := tc.fast.Create(key, size)
|
||||||
|
if err == nil {
|
||||||
|
// Write content to fast tier
|
||||||
|
writer.Write(content)
|
||||||
|
writer.Close()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
tc.mu.RUnlock()
|
||||||
|
}
|
||||||
|
|
||||||
|
// NewLockFree creates a new lock-free tiered cache
|
||||||
|
func NewLockFree() *LockFreeTieredCache {
|
||||||
|
return &LockFreeTieredCache{
|
||||||
|
fast: &atomic.Value{},
|
||||||
|
slow: &atomic.Value{},
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// SetFast sets the fast (memory) tier atomically
|
||||||
|
func (lftc *LockFreeTieredCache) SetFast(vfs vfs.VFS) {
|
||||||
|
lftc.fast.Store(vfs)
|
||||||
|
}
|
||||||
|
|
||||||
|
// SetSlow sets the slow (disk) tier atomically
|
||||||
|
func (lftc *LockFreeTieredCache) SetSlow(vfs vfs.VFS) {
|
||||||
|
lftc.slow.Store(vfs)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Create creates a new file, preferring the slow tier for persistence
|
||||||
|
func (lftc *LockFreeTieredCache) Create(key string, size int64) (io.WriteCloser, error) {
|
||||||
|
// Try slow tier first (disk) for better testability
|
||||||
|
if slow := lftc.slow.Load(); slow != nil {
|
||||||
|
if vfs, ok := slow.(vfs.VFS); ok {
|
||||||
|
return vfs.Create(key, size)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Fall back to fast tier (memory)
|
||||||
|
if fast := lftc.fast.Load(); fast != nil {
|
||||||
|
if vfs, ok := fast.(vfs.VFS); ok {
|
||||||
|
return vfs.Create(key, size)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return nil, vfserror.ErrNotFound
|
||||||
|
}
|
||||||
|
|
||||||
|
// Open opens a file, checking fast tier first, then slow tier with promotion
|
||||||
|
func (lftc *LockFreeTieredCache) Open(key string) (io.ReadCloser, error) {
|
||||||
|
// Try fast tier first (memory)
|
||||||
|
if fast := lftc.fast.Load(); fast != nil {
|
||||||
|
if vfs, ok := fast.(vfs.VFS); ok {
|
||||||
|
if reader, err := vfs.Open(key); err == nil {
|
||||||
|
return reader, nil
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Fall back to slow tier (disk) and promote to fast tier
|
||||||
|
if slow := lftc.slow.Load(); slow != nil {
|
||||||
|
if vfs, ok := slow.(vfs.VFS); ok {
|
||||||
|
reader, err := vfs.Open(key)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
|
||||||
|
// If we have both tiers, promote the file to fast tier
|
||||||
|
if fast := lftc.fast.Load(); fast != nil {
|
||||||
|
// Create a new reader for promotion to avoid interfering with the returned reader
|
||||||
|
promotionReader, err := vfs.Open(key)
|
||||||
|
if err == nil {
|
||||||
|
go lftc.promoteToFast(key, promotionReader)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return reader, nil
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return nil, vfserror.ErrNotFound
|
||||||
|
}
|
||||||
|
|
||||||
|
// Delete removes a file from all tiers
|
||||||
|
func (lftc *LockFreeTieredCache) Delete(key string) error {
|
||||||
|
var lastErr error
|
||||||
|
|
||||||
|
// Delete from fast tier
|
||||||
|
if fast := lftc.fast.Load(); fast != nil {
|
||||||
|
if vfs, ok := fast.(vfs.VFS); ok {
|
||||||
|
if err := vfs.Delete(key); err != nil {
|
||||||
|
lastErr = err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Delete from slow tier
|
||||||
|
if slow := lftc.slow.Load(); slow != nil {
|
||||||
|
if vfs, ok := slow.(vfs.VFS); ok {
|
||||||
|
if err := vfs.Delete(key); err != nil {
|
||||||
|
lastErr = err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return lastErr
|
||||||
|
}
|
||||||
|
|
||||||
|
// Stat returns file information, checking fast tier first
|
||||||
|
func (lftc *LockFreeTieredCache) Stat(key string) (*vfs.FileInfo, error) {
|
||||||
|
// Try fast tier first (memory)
|
||||||
|
if fast := lftc.fast.Load(); fast != nil {
|
||||||
|
if vfs, ok := fast.(vfs.VFS); ok {
|
||||||
|
if info, err := vfs.Stat(key); err == nil {
|
||||||
|
return info, nil
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Fall back to slow tier (disk)
|
||||||
|
if slow := lftc.slow.Load(); slow != nil {
|
||||||
|
if vfs, ok := slow.(vfs.VFS); ok {
|
||||||
|
return vfs.Stat(key)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return nil, vfserror.ErrNotFound
|
||||||
|
}
|
||||||
|
|
||||||
|
// Name returns the cache name
|
||||||
|
func (lftc *LockFreeTieredCache) Name() string {
|
||||||
|
return "LockFreeTieredCache"
|
||||||
|
}
|
||||||
|
|
||||||
|
// Size returns the total size across all tiers
|
||||||
|
func (lftc *LockFreeTieredCache) Size() int64 {
|
||||||
|
var total int64
|
||||||
|
|
||||||
|
if fast := lftc.fast.Load(); fast != nil {
|
||||||
|
if vfs, ok := fast.(vfs.VFS); ok {
|
||||||
|
total += vfs.Size()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if slow := lftc.slow.Load(); slow != nil {
|
||||||
|
if vfs, ok := slow.(vfs.VFS); ok {
|
||||||
|
total += vfs.Size()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return total
|
||||||
|
}
|
||||||
|
|
||||||
|
// Capacity returns the total capacity across all tiers
|
||||||
|
func (lftc *LockFreeTieredCache) Capacity() int64 {
|
||||||
|
var total int64
|
||||||
|
|
||||||
|
if fast := lftc.fast.Load(); fast != nil {
|
||||||
|
if vfs, ok := fast.(vfs.VFS); ok {
|
||||||
|
total += vfs.Capacity()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if slow := lftc.slow.Load(); slow != nil {
|
||||||
|
if vfs, ok := slow.(vfs.VFS); ok {
|
||||||
|
total += vfs.Capacity()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return total
|
||||||
|
}
|
||||||
|
|
||||||
|
// promoteToFast promotes a file from slow tier to fast tier (lock-free version)
|
||||||
|
func (lftc *LockFreeTieredCache) promoteToFast(key string, reader io.ReadCloser) {
|
||||||
|
defer reader.Close()
|
||||||
|
|
||||||
|
// Get file info from slow tier to determine size
|
||||||
|
var size int64
|
||||||
|
if slow := lftc.slow.Load(); slow != nil {
|
||||||
|
if vfs, ok := slow.(vfs.VFS); ok {
|
||||||
|
if info, err := vfs.Stat(key); err == nil {
|
||||||
|
size = info.Size
|
||||||
|
} else {
|
||||||
|
return // Skip promotion if we can't get file info
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Check if file fits in available memory cache space
|
||||||
|
if fast := lftc.fast.Load(); fast != nil {
|
||||||
|
if vfs, ok := fast.(vfs.VFS); ok {
|
||||||
|
availableSpace := vfs.Capacity() - vfs.Size()
|
||||||
|
// Only promote if file fits in available space (with 10% buffer for safety)
|
||||||
|
if size > int64(float64(availableSpace)*0.9) {
|
||||||
|
return // Skip promotion if file is too large
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Read the entire file content
|
||||||
|
content, err := io.ReadAll(reader)
|
||||||
|
if err != nil {
|
||||||
|
return // Skip promotion if read fails
|
||||||
|
}
|
||||||
|
|
||||||
|
// Create the file in fast tier
|
||||||
|
if fast := lftc.fast.Load(); fast != nil {
|
||||||
|
if vfs, ok := fast.(vfs.VFS); ok {
|
||||||
|
writer, err := vfs.Create(key, size)
|
||||||
|
if err == nil {
|
||||||
|
// Write content to fast tier
|
||||||
|
writer.Write(content)
|
||||||
|
writer.Close()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|||||||
@@ -184,7 +184,12 @@ func (d *DiskFS) init() {
|
|||||||
|
|
||||||
d.mu.Lock()
|
d.mu.Lock()
|
||||||
// Extract key from sharded path: remove root and convert sharding back
|
// Extract key from sharded path: remove root and convert sharding back
|
||||||
relPath := strings.ReplaceAll(npath[len(d.root)+1:], "\\", "/")
|
// Handle both "./disk" and "disk" root paths
|
||||||
|
rootPath := d.root
|
||||||
|
if strings.HasPrefix(rootPath, "./") {
|
||||||
|
rootPath = rootPath[2:] // Remove "./" prefix
|
||||||
|
}
|
||||||
|
relPath := strings.ReplaceAll(npath[len(rootPath)+1:], "\\", "/")
|
||||||
|
|
||||||
// Extract the original key from the sharded path
|
// Extract the original key from the sharded path
|
||||||
k := d.extractKeyFromPath(relPath)
|
k := d.extractKeyFromPath(relPath)
|
||||||
|
|||||||
162
vfs/gc/gc.go
162
vfs/gc/gc.go
@@ -2,10 +2,14 @@
|
|||||||
package gc
|
package gc
|
||||||
|
|
||||||
import (
|
import (
|
||||||
|
"context"
|
||||||
"io"
|
"io"
|
||||||
"s1d3sw1ped/SteamCache2/vfs"
|
"s1d3sw1ped/SteamCache2/vfs"
|
||||||
"s1d3sw1ped/SteamCache2/vfs/disk"
|
"s1d3sw1ped/SteamCache2/vfs/disk"
|
||||||
"s1d3sw1ped/SteamCache2/vfs/memory"
|
"s1d3sw1ped/SteamCache2/vfs/memory"
|
||||||
|
"sync"
|
||||||
|
"sync/atomic"
|
||||||
|
"time"
|
||||||
)
|
)
|
||||||
|
|
||||||
// GCAlgorithm represents different garbage collection strategies
|
// GCAlgorithm represents different garbage collection strategies
|
||||||
@@ -238,3 +242,161 @@ func evictHybrid(v vfs.VFS, bytesNeeded uint) uint {
|
|||||||
var AdaptivePromotionDeciderFunc = func() interface{} {
|
var AdaptivePromotionDeciderFunc = func() interface{} {
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// AsyncGCFS wraps a GCFS with asynchronous garbage collection capabilities
|
||||||
|
type AsyncGCFS struct {
|
||||||
|
*GCFS
|
||||||
|
gcQueue chan gcRequest
|
||||||
|
ctx context.Context
|
||||||
|
cancel context.CancelFunc
|
||||||
|
wg sync.WaitGroup
|
||||||
|
gcRunning int32
|
||||||
|
preemptive bool
|
||||||
|
asyncThreshold float64 // Async GC threshold as percentage of capacity (e.g., 0.8 = 80%)
|
||||||
|
syncThreshold float64 // Sync GC threshold as percentage of capacity (e.g., 0.95 = 95%)
|
||||||
|
hardLimit float64 // Hard limit threshold (e.g., 1.0 = 100%)
|
||||||
|
}
|
||||||
|
|
||||||
|
type gcRequest struct {
|
||||||
|
bytesNeeded uint
|
||||||
|
priority int // Higher number = higher priority
|
||||||
|
}
|
||||||
|
|
||||||
|
// NewAsync creates a new AsyncGCFS with asynchronous garbage collection
|
||||||
|
func NewAsync(wrappedVFS vfs.VFS, algorithm GCAlgorithm, preemptive bool, asyncThreshold, syncThreshold, hardLimit float64) *AsyncGCFS {
|
||||||
|
ctx, cancel := context.WithCancel(context.Background())
|
||||||
|
|
||||||
|
asyncGC := &AsyncGCFS{
|
||||||
|
GCFS: New(wrappedVFS, algorithm),
|
||||||
|
gcQueue: make(chan gcRequest, 100), // Buffer for GC requests
|
||||||
|
ctx: ctx,
|
||||||
|
cancel: cancel,
|
||||||
|
preemptive: preemptive,
|
||||||
|
asyncThreshold: asyncThreshold,
|
||||||
|
syncThreshold: syncThreshold,
|
||||||
|
hardLimit: hardLimit,
|
||||||
|
}
|
||||||
|
|
||||||
|
// Start the background GC worker
|
||||||
|
asyncGC.wg.Add(1)
|
||||||
|
go asyncGC.gcWorker()
|
||||||
|
|
||||||
|
// Start preemptive GC if enabled
|
||||||
|
if preemptive {
|
||||||
|
asyncGC.wg.Add(1)
|
||||||
|
go asyncGC.preemptiveGC()
|
||||||
|
}
|
||||||
|
|
||||||
|
return asyncGC
|
||||||
|
}
|
||||||
|
|
||||||
|
// Create wraps the underlying Create method with hybrid GC (async + sync hard limits)
|
||||||
|
func (agc *AsyncGCFS) Create(key string, size int64) (io.WriteCloser, error) {
|
||||||
|
currentSize := agc.vfs.Size()
|
||||||
|
capacity := agc.vfs.Capacity()
|
||||||
|
projectedSize := currentSize + size
|
||||||
|
|
||||||
|
// Calculate utilization percentages
|
||||||
|
currentUtilization := float64(currentSize) / float64(capacity)
|
||||||
|
projectedUtilization := float64(projectedSize) / float64(capacity)
|
||||||
|
|
||||||
|
// Hard limit check - never exceed the hard limit
|
||||||
|
if projectedUtilization > agc.hardLimit {
|
||||||
|
needed := uint(projectedSize - capacity)
|
||||||
|
// Immediate sync GC to prevent exceeding hard limit
|
||||||
|
agc.gcFunc(agc.vfs, needed)
|
||||||
|
} else if projectedUtilization > agc.syncThreshold {
|
||||||
|
// Near hard limit - do immediate sync GC
|
||||||
|
needed := uint(projectedSize - int64(float64(capacity)*agc.syncThreshold))
|
||||||
|
agc.gcFunc(agc.vfs, needed)
|
||||||
|
} else if currentUtilization > agc.asyncThreshold {
|
||||||
|
// Above async threshold - queue for async GC
|
||||||
|
needed := uint(projectedSize - int64(float64(capacity)*agc.asyncThreshold))
|
||||||
|
select {
|
||||||
|
case agc.gcQueue <- gcRequest{bytesNeeded: needed, priority: 2}:
|
||||||
|
default:
|
||||||
|
// Queue full, do immediate GC
|
||||||
|
agc.gcFunc(agc.vfs, needed)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return agc.vfs.Create(key, size)
|
||||||
|
}
|
||||||
|
|
||||||
|
// gcWorker processes GC requests asynchronously
|
||||||
|
func (agc *AsyncGCFS) gcWorker() {
|
||||||
|
defer agc.wg.Done()
|
||||||
|
|
||||||
|
ticker := time.NewTicker(100 * time.Millisecond) // Check every 100ms
|
||||||
|
defer ticker.Stop()
|
||||||
|
|
||||||
|
for {
|
||||||
|
select {
|
||||||
|
case <-agc.ctx.Done():
|
||||||
|
return
|
||||||
|
case req := <-agc.gcQueue:
|
||||||
|
atomic.StoreInt32(&agc.gcRunning, 1)
|
||||||
|
agc.gcFunc(agc.vfs, req.bytesNeeded)
|
||||||
|
atomic.StoreInt32(&agc.gcRunning, 0)
|
||||||
|
case <-ticker.C:
|
||||||
|
// Process any pending GC requests
|
||||||
|
select {
|
||||||
|
case req := <-agc.gcQueue:
|
||||||
|
atomic.StoreInt32(&agc.gcRunning, 1)
|
||||||
|
agc.gcFunc(agc.vfs, req.bytesNeeded)
|
||||||
|
atomic.StoreInt32(&agc.gcRunning, 0)
|
||||||
|
default:
|
||||||
|
// No pending requests
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// preemptiveGC runs background GC to keep cache utilization below threshold
|
||||||
|
func (agc *AsyncGCFS) preemptiveGC() {
|
||||||
|
defer agc.wg.Done()
|
||||||
|
|
||||||
|
ticker := time.NewTicker(5 * time.Second) // Check every 5 seconds
|
||||||
|
defer ticker.Stop()
|
||||||
|
|
||||||
|
for {
|
||||||
|
select {
|
||||||
|
case <-agc.ctx.Done():
|
||||||
|
return
|
||||||
|
case <-ticker.C:
|
||||||
|
currentSize := agc.vfs.Size()
|
||||||
|
capacity := agc.vfs.Capacity()
|
||||||
|
currentUtilization := float64(currentSize) / float64(capacity)
|
||||||
|
|
||||||
|
// Check if we're above the async threshold
|
||||||
|
if currentUtilization > agc.asyncThreshold {
|
||||||
|
// Calculate how much to free to get back to async threshold
|
||||||
|
targetSize := int64(float64(capacity) * agc.asyncThreshold)
|
||||||
|
if currentSize > targetSize {
|
||||||
|
overage := currentSize - targetSize
|
||||||
|
select {
|
||||||
|
case agc.gcQueue <- gcRequest{bytesNeeded: uint(overage), priority: 0}:
|
||||||
|
default:
|
||||||
|
// Queue full, skip this round
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Stop stops the async GC workers
|
||||||
|
func (agc *AsyncGCFS) Stop() {
|
||||||
|
agc.cancel()
|
||||||
|
agc.wg.Wait()
|
||||||
|
}
|
||||||
|
|
||||||
|
// IsGCRunning returns true if GC is currently running
|
||||||
|
func (agc *AsyncGCFS) IsGCRunning() bool {
|
||||||
|
return atomic.LoadInt32(&agc.gcRunning) == 1
|
||||||
|
}
|
||||||
|
|
||||||
|
// ForceGC forces immediate garbage collection to free the specified number of bytes
|
||||||
|
func (agc *AsyncGCFS) ForceGC(bytesNeeded uint) {
|
||||||
|
agc.gcFunc(agc.vfs, bytesNeeded)
|
||||||
|
}
|
||||||
|
|||||||
130
vfs/memory/dynamic.go
Normal file
130
vfs/memory/dynamic.go
Normal file
@@ -0,0 +1,130 @@
|
|||||||
|
package memory
|
||||||
|
|
||||||
|
import (
|
||||||
|
"s1d3sw1ped/SteamCache2/vfs"
|
||||||
|
"sync"
|
||||||
|
"sync/atomic"
|
||||||
|
"time"
|
||||||
|
)
|
||||||
|
|
||||||
|
// DynamicCacheManager manages cache size adjustments based on system memory usage
|
||||||
|
type DynamicCacheManager struct {
|
||||||
|
originalCacheSize uint64
|
||||||
|
currentCacheSize uint64
|
||||||
|
memoryMonitor *MemoryMonitor
|
||||||
|
cache vfs.VFS
|
||||||
|
adjustmentInterval time.Duration
|
||||||
|
lastAdjustment time.Time
|
||||||
|
mu sync.RWMutex
|
||||||
|
adjustmentCount int64
|
||||||
|
isAdjusting int32
|
||||||
|
}
|
||||||
|
|
||||||
|
// NewDynamicCacheManager creates a new dynamic cache manager
|
||||||
|
func NewDynamicCacheManager(cache vfs.VFS, originalSize uint64, memoryMonitor *MemoryMonitor) *DynamicCacheManager {
|
||||||
|
return &DynamicCacheManager{
|
||||||
|
originalCacheSize: originalSize,
|
||||||
|
currentCacheSize: originalSize,
|
||||||
|
memoryMonitor: memoryMonitor,
|
||||||
|
cache: cache,
|
||||||
|
adjustmentInterval: 30 * time.Second, // Adjust every 30 seconds
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Start begins the dynamic cache size adjustment process
|
||||||
|
func (dcm *DynamicCacheManager) Start() {
|
||||||
|
go dcm.adjustmentLoop()
|
||||||
|
}
|
||||||
|
|
||||||
|
// GetCurrentCacheSize returns the current cache size
|
||||||
|
func (dcm *DynamicCacheManager) GetCurrentCacheSize() uint64 {
|
||||||
|
dcm.mu.RLock()
|
||||||
|
defer dcm.mu.RUnlock()
|
||||||
|
return atomic.LoadUint64(&dcm.currentCacheSize)
|
||||||
|
}
|
||||||
|
|
||||||
|
// GetOriginalCacheSize returns the original cache size
|
||||||
|
func (dcm *DynamicCacheManager) GetOriginalCacheSize() uint64 {
|
||||||
|
dcm.mu.RLock()
|
||||||
|
defer dcm.mu.RUnlock()
|
||||||
|
return dcm.originalCacheSize
|
||||||
|
}
|
||||||
|
|
||||||
|
// GetAdjustmentCount returns the number of adjustments made
|
||||||
|
func (dcm *DynamicCacheManager) GetAdjustmentCount() int64 {
|
||||||
|
return atomic.LoadInt64(&dcm.adjustmentCount)
|
||||||
|
}
|
||||||
|
|
||||||
|
// adjustmentLoop runs the cache size adjustment loop
|
||||||
|
func (dcm *DynamicCacheManager) adjustmentLoop() {
|
||||||
|
ticker := time.NewTicker(dcm.adjustmentInterval)
|
||||||
|
defer ticker.Stop()
|
||||||
|
|
||||||
|
for range ticker.C {
|
||||||
|
dcm.performAdjustment()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// performAdjustment performs a cache size adjustment if needed
|
||||||
|
func (dcm *DynamicCacheManager) performAdjustment() {
|
||||||
|
// Prevent concurrent adjustments
|
||||||
|
if !atomic.CompareAndSwapInt32(&dcm.isAdjusting, 0, 1) {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
defer atomic.StoreInt32(&dcm.isAdjusting, 0)
|
||||||
|
|
||||||
|
// Check if enough time has passed since last adjustment
|
||||||
|
if time.Since(dcm.lastAdjustment) < dcm.adjustmentInterval {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
// Get recommended cache size
|
||||||
|
recommendedSize := dcm.memoryMonitor.GetRecommendedCacheSize(dcm.originalCacheSize)
|
||||||
|
currentSize := atomic.LoadUint64(&dcm.currentCacheSize)
|
||||||
|
|
||||||
|
// Only adjust if there's a significant difference (more than 5%)
|
||||||
|
sizeDiff := float64(recommendedSize) / float64(currentSize)
|
||||||
|
if sizeDiff < 0.95 || sizeDiff > 1.05 {
|
||||||
|
dcm.adjustCacheSize(recommendedSize)
|
||||||
|
dcm.lastAdjustment = time.Now()
|
||||||
|
atomic.AddInt64(&dcm.adjustmentCount, 1)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// adjustCacheSize adjusts the cache size to the recommended size
|
||||||
|
func (dcm *DynamicCacheManager) adjustCacheSize(newSize uint64) {
|
||||||
|
dcm.mu.Lock()
|
||||||
|
defer dcm.mu.Unlock()
|
||||||
|
|
||||||
|
oldSize := atomic.LoadUint64(&dcm.currentCacheSize)
|
||||||
|
atomic.StoreUint64(&dcm.currentCacheSize, newSize)
|
||||||
|
|
||||||
|
// If we're reducing the cache size, trigger GC to free up memory
|
||||||
|
if newSize < oldSize {
|
||||||
|
// Calculate how much to free
|
||||||
|
bytesToFree := oldSize - newSize
|
||||||
|
|
||||||
|
// Trigger GC on the cache to free up the excess memory
|
||||||
|
// This is a simplified approach - in practice, you'd want to integrate
|
||||||
|
// with the actual GC system to free the right amount
|
||||||
|
if gcCache, ok := dcm.cache.(interface{ ForceGC(uint) }); ok {
|
||||||
|
gcCache.ForceGC(uint(bytesToFree))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// GetStats returns statistics about the dynamic cache manager
|
||||||
|
func (dcm *DynamicCacheManager) GetStats() map[string]interface{} {
|
||||||
|
dcm.mu.RLock()
|
||||||
|
defer dcm.mu.RUnlock()
|
||||||
|
|
||||||
|
return map[string]interface{}{
|
||||||
|
"original_cache_size": dcm.originalCacheSize,
|
||||||
|
"current_cache_size": atomic.LoadUint64(&dcm.currentCacheSize),
|
||||||
|
"adjustment_count": atomic.LoadInt64(&dcm.adjustmentCount),
|
||||||
|
"last_adjustment": dcm.lastAdjustment,
|
||||||
|
"memory_utilization": dcm.memoryMonitor.GetMemoryUtilization(),
|
||||||
|
"target_memory_usage": dcm.memoryMonitor.GetTargetMemoryUsage(),
|
||||||
|
"current_memory_usage": dcm.memoryMonitor.GetCurrentMemoryUsage(),
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -5,7 +5,7 @@ import (
|
|||||||
"bytes"
|
"bytes"
|
||||||
"container/list"
|
"container/list"
|
||||||
"io"
|
"io"
|
||||||
"s1d3sw1ped/SteamCache2/vfs"
|
"s1d3sw1ped/SteamCache2/vfs/types"
|
||||||
"s1d3sw1ped/SteamCache2/vfs/vfserror"
|
"s1d3sw1ped/SteamCache2/vfs/vfserror"
|
||||||
"sort"
|
"sort"
|
||||||
"strings"
|
"strings"
|
||||||
@@ -13,19 +13,43 @@ import (
|
|||||||
"time"
|
"time"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
// VFS defines the interface for virtual file systems
|
||||||
|
type VFS interface {
|
||||||
|
// Create creates a new file at the given key
|
||||||
|
Create(key string, size int64) (io.WriteCloser, error)
|
||||||
|
|
||||||
|
// Open opens the file at the given key for reading
|
||||||
|
Open(key string) (io.ReadCloser, error)
|
||||||
|
|
||||||
|
// Delete removes the file at the given key
|
||||||
|
Delete(key string) error
|
||||||
|
|
||||||
|
// Stat returns information about the file at the given key
|
||||||
|
Stat(key string) (*types.FileInfo, error)
|
||||||
|
|
||||||
|
// Name returns the name of this VFS
|
||||||
|
Name() string
|
||||||
|
|
||||||
|
// Size returns the current size of the VFS
|
||||||
|
Size() int64
|
||||||
|
|
||||||
|
// Capacity returns the maximum capacity of the VFS
|
||||||
|
Capacity() int64
|
||||||
|
}
|
||||||
|
|
||||||
// Ensure MemoryFS implements VFS.
|
// Ensure MemoryFS implements VFS.
|
||||||
var _ vfs.VFS = (*MemoryFS)(nil)
|
var _ VFS = (*MemoryFS)(nil)
|
||||||
|
|
||||||
// MemoryFS is an in-memory virtual file system
|
// MemoryFS is an in-memory virtual file system
|
||||||
type MemoryFS struct {
|
type MemoryFS struct {
|
||||||
data map[string]*bytes.Buffer
|
data map[string]*bytes.Buffer
|
||||||
info map[string]*vfs.FileInfo
|
info map[string]*types.FileInfo
|
||||||
capacity int64
|
capacity int64
|
||||||
size int64
|
size int64
|
||||||
mu sync.RWMutex
|
mu sync.RWMutex
|
||||||
keyLocks []sync.Map // Sharded lock pools for better concurrency
|
keyLocks []sync.Map // Sharded lock pools for better concurrency
|
||||||
LRU *lruList
|
LRU *lruList
|
||||||
timeUpdater *vfs.BatchedTimeUpdate // Batched time updates for better performance
|
timeUpdater *types.BatchedTimeUpdate // Batched time updates for better performance
|
||||||
}
|
}
|
||||||
|
|
||||||
// Number of lock shards for reducing contention
|
// Number of lock shards for reducing contention
|
||||||
@@ -44,25 +68,25 @@ func newLruList() *lruList {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func (l *lruList) Add(key string, fi *vfs.FileInfo) {
|
func (l *lruList) Add(key string, fi *types.FileInfo) {
|
||||||
elem := l.list.PushFront(fi)
|
elem := l.list.PushFront(fi)
|
||||||
l.elem[key] = elem
|
l.elem[key] = elem
|
||||||
}
|
}
|
||||||
|
|
||||||
func (l *lruList) MoveToFront(key string, timeUpdater *vfs.BatchedTimeUpdate) {
|
func (l *lruList) MoveToFront(key string, timeUpdater *types.BatchedTimeUpdate) {
|
||||||
if elem, exists := l.elem[key]; exists {
|
if elem, exists := l.elem[key]; exists {
|
||||||
l.list.MoveToFront(elem)
|
l.list.MoveToFront(elem)
|
||||||
// Update the FileInfo in the element with new access time
|
// Update the FileInfo in the element with new access time
|
||||||
if fi := elem.Value.(*vfs.FileInfo); fi != nil {
|
if fi := elem.Value.(*types.FileInfo); fi != nil {
|
||||||
fi.UpdateAccessBatched(timeUpdater)
|
fi.UpdateAccessBatched(timeUpdater)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func (l *lruList) Remove(key string) *vfs.FileInfo {
|
func (l *lruList) Remove(key string) *types.FileInfo {
|
||||||
if elem, exists := l.elem[key]; exists {
|
if elem, exists := l.elem[key]; exists {
|
||||||
delete(l.elem, key)
|
delete(l.elem, key)
|
||||||
if fi := l.list.Remove(elem).(*vfs.FileInfo); fi != nil {
|
if fi := l.list.Remove(elem).(*types.FileInfo); fi != nil {
|
||||||
return fi
|
return fi
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -84,12 +108,12 @@ func New(capacity int64) *MemoryFS {
|
|||||||
|
|
||||||
return &MemoryFS{
|
return &MemoryFS{
|
||||||
data: make(map[string]*bytes.Buffer),
|
data: make(map[string]*bytes.Buffer),
|
||||||
info: make(map[string]*vfs.FileInfo),
|
info: make(map[string]*types.FileInfo),
|
||||||
capacity: capacity,
|
capacity: capacity,
|
||||||
size: 0,
|
size: 0,
|
||||||
keyLocks: keyLocks,
|
keyLocks: keyLocks,
|
||||||
LRU: newLruList(),
|
LRU: newLruList(),
|
||||||
timeUpdater: vfs.NewBatchedTimeUpdate(100 * time.Millisecond), // Update time every 100ms
|
timeUpdater: types.NewBatchedTimeUpdate(100 * time.Millisecond), // Update time every 100ms
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -110,6 +134,35 @@ func (m *MemoryFS) Capacity() int64 {
|
|||||||
return m.capacity
|
return m.capacity
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// GetFragmentationStats returns memory fragmentation statistics
|
||||||
|
func (m *MemoryFS) GetFragmentationStats() map[string]interface{} {
|
||||||
|
m.mu.RLock()
|
||||||
|
defer m.mu.RUnlock()
|
||||||
|
|
||||||
|
var totalCapacity int64
|
||||||
|
var totalUsed int64
|
||||||
|
var bufferCount int
|
||||||
|
|
||||||
|
for _, buffer := range m.data {
|
||||||
|
totalCapacity += int64(buffer.Cap())
|
||||||
|
totalUsed += int64(buffer.Len())
|
||||||
|
bufferCount++
|
||||||
|
}
|
||||||
|
|
||||||
|
fragmentationRatio := float64(0)
|
||||||
|
if totalCapacity > 0 {
|
||||||
|
fragmentationRatio = float64(totalCapacity-totalUsed) / float64(totalCapacity)
|
||||||
|
}
|
||||||
|
|
||||||
|
return map[string]interface{}{
|
||||||
|
"buffer_count": bufferCount,
|
||||||
|
"total_capacity": totalCapacity,
|
||||||
|
"total_used": totalUsed,
|
||||||
|
"fragmentation_ratio": fragmentationRatio,
|
||||||
|
"average_buffer_size": float64(totalUsed) / float64(bufferCount),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
// getShardIndex returns the shard index for a given key
|
// getShardIndex returns the shard index for a given key
|
||||||
func getShardIndex(key string) int {
|
func getShardIndex(key string) int {
|
||||||
// Use FNV-1a hash for good distribution
|
// Use FNV-1a hash for good distribution
|
||||||
@@ -159,7 +212,7 @@ func (m *MemoryFS) Create(key string, size int64) (io.WriteCloser, error) {
|
|||||||
|
|
||||||
buffer := &bytes.Buffer{}
|
buffer := &bytes.Buffer{}
|
||||||
m.data[key] = buffer
|
m.data[key] = buffer
|
||||||
fi := vfs.NewFileInfo(key, size)
|
fi := types.NewFileInfo(key, size)
|
||||||
m.info[key] = fi
|
m.info[key] = fi
|
||||||
m.LRU.Add(key, fi)
|
m.LRU.Add(key, fi)
|
||||||
// Initialize access time with current time
|
// Initialize access time with current time
|
||||||
@@ -230,23 +283,39 @@ func (m *MemoryFS) Open(key string) (io.ReadCloser, error) {
|
|||||||
return nil, vfserror.ErrNotFound
|
return nil, vfserror.ErrNotFound
|
||||||
}
|
}
|
||||||
|
|
||||||
// Create a copy of the buffer for reading
|
// Use zero-copy approach - return reader that reads directly from buffer
|
||||||
data := make([]byte, buffer.Len())
|
|
||||||
copy(data, buffer.Bytes())
|
|
||||||
m.mu.Unlock()
|
m.mu.Unlock()
|
||||||
|
|
||||||
return &memoryReadCloser{
|
return &memoryReadCloser{
|
||||||
reader: bytes.NewReader(data),
|
buffer: buffer,
|
||||||
|
offset: 0,
|
||||||
}, nil
|
}, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// memoryReadCloser implements io.ReadCloser for memory files
|
// memoryReadCloser implements io.ReadCloser for memory files with zero-copy optimization
|
||||||
type memoryReadCloser struct {
|
type memoryReadCloser struct {
|
||||||
reader *bytes.Reader
|
buffer *bytes.Buffer
|
||||||
|
offset int64
|
||||||
}
|
}
|
||||||
|
|
||||||
func (mrc *memoryReadCloser) Read(p []byte) (n int, err error) {
|
func (mrc *memoryReadCloser) Read(p []byte) (n int, err error) {
|
||||||
return mrc.reader.Read(p)
|
if mrc.offset >= int64(mrc.buffer.Len()) {
|
||||||
|
return 0, io.EOF
|
||||||
|
}
|
||||||
|
|
||||||
|
// Zero-copy read directly from buffer
|
||||||
|
available := mrc.buffer.Len() - int(mrc.offset)
|
||||||
|
toRead := len(p)
|
||||||
|
if toRead > available {
|
||||||
|
toRead = available
|
||||||
|
}
|
||||||
|
|
||||||
|
// Read directly from buffer without copying
|
||||||
|
data := mrc.buffer.Bytes()
|
||||||
|
copy(p, data[mrc.offset:mrc.offset+int64(toRead)])
|
||||||
|
mrc.offset += int64(toRead)
|
||||||
|
|
||||||
|
return toRead, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func (mrc *memoryReadCloser) Close() error {
|
func (mrc *memoryReadCloser) Close() error {
|
||||||
@@ -286,7 +355,7 @@ func (m *MemoryFS) Delete(key string) error {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// Stat returns file information
|
// Stat returns file information
|
||||||
func (m *MemoryFS) Stat(key string) (*vfs.FileInfo, error) {
|
func (m *MemoryFS) Stat(key string) (*types.FileInfo, error) {
|
||||||
if key == "" {
|
if key == "" {
|
||||||
return nil, vfserror.ErrInvalidKey
|
return nil, vfserror.ErrInvalidKey
|
||||||
}
|
}
|
||||||
@@ -327,7 +396,7 @@ func (m *MemoryFS) EvictLRU(bytesNeeded uint) uint {
|
|||||||
break
|
break
|
||||||
}
|
}
|
||||||
|
|
||||||
fi := elem.Value.(*vfs.FileInfo)
|
fi := elem.Value.(*types.FileInfo)
|
||||||
key := fi.Key
|
key := fi.Key
|
||||||
|
|
||||||
// Remove from LRU
|
// Remove from LRU
|
||||||
@@ -355,7 +424,7 @@ func (m *MemoryFS) EvictBySize(bytesNeeded uint, ascending bool) uint {
|
|||||||
defer m.mu.Unlock()
|
defer m.mu.Unlock()
|
||||||
|
|
||||||
var evicted uint
|
var evicted uint
|
||||||
var candidates []*vfs.FileInfo
|
var candidates []*types.FileInfo
|
||||||
|
|
||||||
// Collect all files
|
// Collect all files
|
||||||
for _, fi := range m.info {
|
for _, fi := range m.info {
|
||||||
@@ -403,7 +472,7 @@ func (m *MemoryFS) EvictFIFO(bytesNeeded uint) uint {
|
|||||||
defer m.mu.Unlock()
|
defer m.mu.Unlock()
|
||||||
|
|
||||||
var evicted uint
|
var evicted uint
|
||||||
var candidates []*vfs.FileInfo
|
var candidates []*types.FileInfo
|
||||||
|
|
||||||
// Collect all files
|
// Collect all files
|
||||||
for _, fi := range m.info {
|
for _, fi := range m.info {
|
||||||
|
|||||||
153
vfs/memory/monitor.go
Normal file
153
vfs/memory/monitor.go
Normal file
@@ -0,0 +1,153 @@
|
|||||||
|
package memory
|
||||||
|
|
||||||
|
import (
|
||||||
|
"runtime"
|
||||||
|
"sync"
|
||||||
|
"sync/atomic"
|
||||||
|
"time"
|
||||||
|
)
|
||||||
|
|
||||||
|
// MemoryMonitor tracks system memory usage and provides dynamic sizing recommendations
|
||||||
|
type MemoryMonitor struct {
|
||||||
|
targetMemoryUsage uint64 // Target total memory usage in bytes
|
||||||
|
currentMemoryUsage uint64 // Current total memory usage in bytes
|
||||||
|
monitoringInterval time.Duration
|
||||||
|
adjustmentThreshold float64 // Threshold for cache size adjustments (e.g., 0.1 = 10%)
|
||||||
|
mu sync.RWMutex
|
||||||
|
ctx chan struct{}
|
||||||
|
stopChan chan struct{}
|
||||||
|
isMonitoring int32
|
||||||
|
}
|
||||||
|
|
||||||
|
// NewMemoryMonitor creates a new memory monitor
|
||||||
|
func NewMemoryMonitor(targetMemoryUsage uint64, monitoringInterval time.Duration, adjustmentThreshold float64) *MemoryMonitor {
|
||||||
|
return &MemoryMonitor{
|
||||||
|
targetMemoryUsage: targetMemoryUsage,
|
||||||
|
monitoringInterval: monitoringInterval,
|
||||||
|
adjustmentThreshold: adjustmentThreshold,
|
||||||
|
ctx: make(chan struct{}),
|
||||||
|
stopChan: make(chan struct{}),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Start begins monitoring memory usage
|
||||||
|
func (mm *MemoryMonitor) Start() {
|
||||||
|
if atomic.CompareAndSwapInt32(&mm.isMonitoring, 0, 1) {
|
||||||
|
go mm.monitor()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Stop stops monitoring memory usage
|
||||||
|
func (mm *MemoryMonitor) Stop() {
|
||||||
|
if atomic.CompareAndSwapInt32(&mm.isMonitoring, 1, 0) {
|
||||||
|
close(mm.stopChan)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// GetCurrentMemoryUsage returns the current total memory usage
|
||||||
|
func (mm *MemoryMonitor) GetCurrentMemoryUsage() uint64 {
|
||||||
|
mm.mu.RLock()
|
||||||
|
defer mm.mu.RUnlock()
|
||||||
|
return atomic.LoadUint64(&mm.currentMemoryUsage)
|
||||||
|
}
|
||||||
|
|
||||||
|
// GetTargetMemoryUsage returns the target memory usage
|
||||||
|
func (mm *MemoryMonitor) GetTargetMemoryUsage() uint64 {
|
||||||
|
mm.mu.RLock()
|
||||||
|
defer mm.mu.RUnlock()
|
||||||
|
return mm.targetMemoryUsage
|
||||||
|
}
|
||||||
|
|
||||||
|
// GetMemoryUtilization returns the current memory utilization as a percentage
|
||||||
|
func (mm *MemoryMonitor) GetMemoryUtilization() float64 {
|
||||||
|
mm.mu.RLock()
|
||||||
|
defer mm.mu.RUnlock()
|
||||||
|
current := atomic.LoadUint64(&mm.currentMemoryUsage)
|
||||||
|
return float64(current) / float64(mm.targetMemoryUsage)
|
||||||
|
}
|
||||||
|
|
||||||
|
// GetRecommendedCacheSize calculates the recommended cache size based on current memory usage
|
||||||
|
func (mm *MemoryMonitor) GetRecommendedCacheSize(originalCacheSize uint64) uint64 {
|
||||||
|
mm.mu.RLock()
|
||||||
|
defer mm.mu.RUnlock()
|
||||||
|
|
||||||
|
current := atomic.LoadUint64(&mm.currentMemoryUsage)
|
||||||
|
target := mm.targetMemoryUsage
|
||||||
|
|
||||||
|
// If we're under target, we can use the full cache size
|
||||||
|
if current <= target {
|
||||||
|
return originalCacheSize
|
||||||
|
}
|
||||||
|
|
||||||
|
// Calculate how much we're over target
|
||||||
|
overage := current - target
|
||||||
|
|
||||||
|
// If overage is significant, reduce cache size
|
||||||
|
if overage > uint64(float64(target)*mm.adjustmentThreshold) {
|
||||||
|
// Reduce cache size by the overage amount, but don't go below 10% of original
|
||||||
|
minCacheSize := uint64(float64(originalCacheSize) * 0.1)
|
||||||
|
recommendedSize := originalCacheSize - overage
|
||||||
|
|
||||||
|
if recommendedSize < minCacheSize {
|
||||||
|
recommendedSize = minCacheSize
|
||||||
|
}
|
||||||
|
|
||||||
|
return recommendedSize
|
||||||
|
}
|
||||||
|
|
||||||
|
return originalCacheSize
|
||||||
|
}
|
||||||
|
|
||||||
|
// monitor runs the memory monitoring loop
|
||||||
|
func (mm *MemoryMonitor) monitor() {
|
||||||
|
ticker := time.NewTicker(mm.monitoringInterval)
|
||||||
|
defer ticker.Stop()
|
||||||
|
|
||||||
|
for {
|
||||||
|
select {
|
||||||
|
case <-mm.stopChan:
|
||||||
|
return
|
||||||
|
case <-ticker.C:
|
||||||
|
mm.updateMemoryUsage()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// updateMemoryUsage updates the current memory usage
|
||||||
|
func (mm *MemoryMonitor) updateMemoryUsage() {
|
||||||
|
var m runtime.MemStats
|
||||||
|
runtime.ReadMemStats(&m)
|
||||||
|
|
||||||
|
// Use Alloc (currently allocated memory) as our metric
|
||||||
|
atomic.StoreUint64(&mm.currentMemoryUsage, m.Alloc)
|
||||||
|
}
|
||||||
|
|
||||||
|
// SetTargetMemoryUsage updates the target memory usage
|
||||||
|
func (mm *MemoryMonitor) SetTargetMemoryUsage(target uint64) {
|
||||||
|
mm.mu.Lock()
|
||||||
|
defer mm.mu.Unlock()
|
||||||
|
mm.targetMemoryUsage = target
|
||||||
|
}
|
||||||
|
|
||||||
|
// GetMemoryStats returns detailed memory statistics
|
||||||
|
func (mm *MemoryMonitor) GetMemoryStats() map[string]interface{} {
|
||||||
|
var m runtime.MemStats
|
||||||
|
runtime.ReadMemStats(&m)
|
||||||
|
|
||||||
|
mm.mu.RLock()
|
||||||
|
defer mm.mu.RUnlock()
|
||||||
|
|
||||||
|
return map[string]interface{}{
|
||||||
|
"current_usage": atomic.LoadUint64(&mm.currentMemoryUsage),
|
||||||
|
"target_usage": mm.targetMemoryUsage,
|
||||||
|
"utilization": mm.GetMemoryUtilization(),
|
||||||
|
"heap_alloc": m.HeapAlloc,
|
||||||
|
"heap_sys": m.HeapSys,
|
||||||
|
"heap_idle": m.HeapIdle,
|
||||||
|
"heap_inuse": m.HeapInuse,
|
||||||
|
"stack_inuse": m.StackInuse,
|
||||||
|
"stack_sys": m.StackSys,
|
||||||
|
"gc_cycles": m.NumGC,
|
||||||
|
"gc_pause_total": m.PauseTotalNs,
|
||||||
|
}
|
||||||
|
}
|
||||||
367
vfs/predictive/predictive.go
Normal file
367
vfs/predictive/predictive.go
Normal file
@@ -0,0 +1,367 @@
|
|||||||
|
package predictive
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"sync"
|
||||||
|
"sync/atomic"
|
||||||
|
"time"
|
||||||
|
)
|
||||||
|
|
||||||
|
// PredictiveCacheManager implements predictive caching strategies
|
||||||
|
type PredictiveCacheManager struct {
|
||||||
|
accessPredictor *AccessPredictor
|
||||||
|
cacheWarmer *CacheWarmer
|
||||||
|
prefetchQueue chan PrefetchRequest
|
||||||
|
ctx context.Context
|
||||||
|
cancel context.CancelFunc
|
||||||
|
wg sync.WaitGroup
|
||||||
|
stats *PredictiveStats
|
||||||
|
}
|
||||||
|
|
||||||
|
// PrefetchRequest represents a request to prefetch content
|
||||||
|
type PrefetchRequest struct {
|
||||||
|
Key string
|
||||||
|
Priority int
|
||||||
|
Reason string
|
||||||
|
RequestedAt time.Time
|
||||||
|
}
|
||||||
|
|
||||||
|
// PredictiveStats tracks predictive caching statistics
|
||||||
|
type PredictiveStats struct {
|
||||||
|
PrefetchHits int64
|
||||||
|
PrefetchMisses int64
|
||||||
|
PrefetchRequests int64
|
||||||
|
CacheWarmHits int64
|
||||||
|
CacheWarmMisses int64
|
||||||
|
mu sync.RWMutex
|
||||||
|
}
|
||||||
|
|
||||||
|
// AccessPredictor predicts which files are likely to be accessed next
|
||||||
|
type AccessPredictor struct {
|
||||||
|
accessHistory map[string]*AccessSequence
|
||||||
|
patterns map[string][]string // Key -> likely next keys
|
||||||
|
mu sync.RWMutex
|
||||||
|
}
|
||||||
|
|
||||||
|
// AccessSequence tracks access sequences for prediction
|
||||||
|
type AccessSequence struct {
|
||||||
|
Key string
|
||||||
|
NextKeys []string
|
||||||
|
Frequency map[string]int64
|
||||||
|
LastSeen time.Time
|
||||||
|
mu sync.RWMutex
|
||||||
|
}
|
||||||
|
|
||||||
|
// CacheWarmer preloads popular content into cache
|
||||||
|
type CacheWarmer struct {
|
||||||
|
popularContent map[string]*PopularContent
|
||||||
|
warmerQueue chan WarmRequest
|
||||||
|
mu sync.RWMutex
|
||||||
|
}
|
||||||
|
|
||||||
|
// PopularContent tracks popular content for warming
|
||||||
|
type PopularContent struct {
|
||||||
|
Key string
|
||||||
|
AccessCount int64
|
||||||
|
LastAccess time.Time
|
||||||
|
Size int64
|
||||||
|
Priority int
|
||||||
|
}
|
||||||
|
|
||||||
|
// WarmRequest represents a cache warming request
|
||||||
|
type WarmRequest struct {
|
||||||
|
Key string
|
||||||
|
Priority int
|
||||||
|
Reason string
|
||||||
|
}
|
||||||
|
|
||||||
|
// NewPredictiveCacheManager creates a new predictive cache manager
|
||||||
|
func NewPredictiveCacheManager() *PredictiveCacheManager {
|
||||||
|
ctx, cancel := context.WithCancel(context.Background())
|
||||||
|
|
||||||
|
pcm := &PredictiveCacheManager{
|
||||||
|
accessPredictor: NewAccessPredictor(),
|
||||||
|
cacheWarmer: NewCacheWarmer(),
|
||||||
|
prefetchQueue: make(chan PrefetchRequest, 1000),
|
||||||
|
ctx: ctx,
|
||||||
|
cancel: cancel,
|
||||||
|
stats: &PredictiveStats{},
|
||||||
|
}
|
||||||
|
|
||||||
|
// Start background workers
|
||||||
|
pcm.wg.Add(1)
|
||||||
|
go pcm.prefetchWorker()
|
||||||
|
|
||||||
|
pcm.wg.Add(1)
|
||||||
|
go pcm.analysisWorker()
|
||||||
|
|
||||||
|
return pcm
|
||||||
|
}
|
||||||
|
|
||||||
|
// NewAccessPredictor creates a new access predictor
|
||||||
|
func NewAccessPredictor() *AccessPredictor {
|
||||||
|
return &AccessPredictor{
|
||||||
|
accessHistory: make(map[string]*AccessSequence),
|
||||||
|
patterns: make(map[string][]string),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// NewCacheWarmer creates a new cache warmer
|
||||||
|
func NewCacheWarmer() *CacheWarmer {
|
||||||
|
return &CacheWarmer{
|
||||||
|
popularContent: make(map[string]*PopularContent),
|
||||||
|
warmerQueue: make(chan WarmRequest, 100),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// RecordAccess records a file access for prediction analysis (lightweight version)
|
||||||
|
func (pcm *PredictiveCacheManager) RecordAccess(key string, previousKey string, size int64) {
|
||||||
|
// Only record if we have a previous key to avoid overhead
|
||||||
|
if previousKey != "" {
|
||||||
|
pcm.accessPredictor.RecordSequence(previousKey, key)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Lightweight popular content tracking - only for large files
|
||||||
|
if size > 1024*1024 { // Only track files > 1MB
|
||||||
|
pcm.cacheWarmer.RecordAccess(key, size)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Skip expensive prediction checks on every access
|
||||||
|
// Only check occasionally to reduce overhead
|
||||||
|
}
|
||||||
|
|
||||||
|
// PredictNextAccess predicts the next likely file to be accessed
|
||||||
|
func (pcm *PredictiveCacheManager) PredictNextAccess(currentKey string) []string {
|
||||||
|
return pcm.accessPredictor.PredictNext(currentKey)
|
||||||
|
}
|
||||||
|
|
||||||
|
// RequestPrefetch requests prefetching of predicted content
|
||||||
|
func (pcm *PredictiveCacheManager) RequestPrefetch(key string, priority int, reason string) {
|
||||||
|
select {
|
||||||
|
case pcm.prefetchQueue <- PrefetchRequest{
|
||||||
|
Key: key,
|
||||||
|
Priority: priority,
|
||||||
|
Reason: reason,
|
||||||
|
RequestedAt: time.Now(),
|
||||||
|
}:
|
||||||
|
atomic.AddInt64(&pcm.stats.PrefetchRequests, 1)
|
||||||
|
default:
|
||||||
|
// Queue full, skip prefetch
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// RecordSequence records an access sequence for prediction
|
||||||
|
func (ap *AccessPredictor) RecordSequence(previousKey, currentKey string) {
|
||||||
|
if previousKey == "" || currentKey == "" {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
ap.mu.Lock()
|
||||||
|
defer ap.mu.Unlock()
|
||||||
|
|
||||||
|
seq, exists := ap.accessHistory[previousKey]
|
||||||
|
if !exists {
|
||||||
|
seq = &AccessSequence{
|
||||||
|
Key: previousKey,
|
||||||
|
NextKeys: []string{},
|
||||||
|
Frequency: make(map[string]int64),
|
||||||
|
LastSeen: time.Now(),
|
||||||
|
}
|
||||||
|
ap.accessHistory[previousKey] = seq
|
||||||
|
}
|
||||||
|
|
||||||
|
seq.mu.Lock()
|
||||||
|
seq.Frequency[currentKey]++
|
||||||
|
seq.LastSeen = time.Now()
|
||||||
|
|
||||||
|
// Update next keys list (keep top 5)
|
||||||
|
nextKeys := make([]string, 0, 5)
|
||||||
|
for key, _ := range seq.Frequency {
|
||||||
|
nextKeys = append(nextKeys, key)
|
||||||
|
if len(nextKeys) >= 5 {
|
||||||
|
break
|
||||||
|
}
|
||||||
|
}
|
||||||
|
seq.NextKeys = nextKeys
|
||||||
|
seq.mu.Unlock()
|
||||||
|
}
|
||||||
|
|
||||||
|
// PredictNext predicts the next likely files to be accessed
|
||||||
|
func (ap *AccessPredictor) PredictNext(currentKey string) []string {
|
||||||
|
ap.mu.RLock()
|
||||||
|
defer ap.mu.RUnlock()
|
||||||
|
|
||||||
|
seq, exists := ap.accessHistory[currentKey]
|
||||||
|
if !exists {
|
||||||
|
return []string{}
|
||||||
|
}
|
||||||
|
|
||||||
|
seq.mu.RLock()
|
||||||
|
defer seq.mu.RUnlock()
|
||||||
|
|
||||||
|
// Return top predicted keys
|
||||||
|
predictions := make([]string, len(seq.NextKeys))
|
||||||
|
copy(predictions, seq.NextKeys)
|
||||||
|
return predictions
|
||||||
|
}
|
||||||
|
|
||||||
|
// IsPredictedAccess checks if an access was predicted
|
||||||
|
func (ap *AccessPredictor) IsPredictedAccess(key string) bool {
|
||||||
|
ap.mu.RLock()
|
||||||
|
defer ap.mu.RUnlock()
|
||||||
|
|
||||||
|
// Check if this key appears in any prediction lists
|
||||||
|
for _, seq := range ap.accessHistory {
|
||||||
|
seq.mu.RLock()
|
||||||
|
for _, predictedKey := range seq.NextKeys {
|
||||||
|
if predictedKey == key {
|
||||||
|
seq.mu.RUnlock()
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
seq.mu.RUnlock()
|
||||||
|
}
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
|
||||||
|
// RecordAccess records a file access for cache warming (lightweight version)
|
||||||
|
func (cw *CacheWarmer) RecordAccess(key string, size int64) {
|
||||||
|
// Use read lock first for better performance
|
||||||
|
cw.mu.RLock()
|
||||||
|
content, exists := cw.popularContent[key]
|
||||||
|
cw.mu.RUnlock()
|
||||||
|
|
||||||
|
if !exists {
|
||||||
|
// Only acquire write lock when creating new entry
|
||||||
|
cw.mu.Lock()
|
||||||
|
// Double-check after acquiring write lock
|
||||||
|
if content, exists = cw.popularContent[key]; !exists {
|
||||||
|
content = &PopularContent{
|
||||||
|
Key: key,
|
||||||
|
AccessCount: 1,
|
||||||
|
LastAccess: time.Now(),
|
||||||
|
Size: size,
|
||||||
|
Priority: 1,
|
||||||
|
}
|
||||||
|
cw.popularContent[key] = content
|
||||||
|
}
|
||||||
|
cw.mu.Unlock()
|
||||||
|
} else {
|
||||||
|
// Lightweight update - just increment counter
|
||||||
|
content.AccessCount++
|
||||||
|
content.LastAccess = time.Now()
|
||||||
|
|
||||||
|
// Only update priority occasionally to reduce overhead
|
||||||
|
if content.AccessCount%5 == 0 {
|
||||||
|
if content.AccessCount > 10 {
|
||||||
|
content.Priority = 3
|
||||||
|
} else if content.AccessCount > 5 {
|
||||||
|
content.Priority = 2
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// GetPopularContent returns the most popular content for warming
|
||||||
|
func (cw *CacheWarmer) GetPopularContent(limit int) []*PopularContent {
|
||||||
|
cw.mu.RLock()
|
||||||
|
defer cw.mu.RUnlock()
|
||||||
|
|
||||||
|
// Sort by access count and return top items
|
||||||
|
popular := make([]*PopularContent, 0, len(cw.popularContent))
|
||||||
|
for _, content := range cw.popularContent {
|
||||||
|
popular = append(popular, content)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Simple sort by access count (in production, use proper sorting)
|
||||||
|
// For now, just return the first 'limit' items
|
||||||
|
if len(popular) > limit {
|
||||||
|
popular = popular[:limit]
|
||||||
|
}
|
||||||
|
|
||||||
|
return popular
|
||||||
|
}
|
||||||
|
|
||||||
|
// prefetchWorker processes prefetch requests
|
||||||
|
func (pcm *PredictiveCacheManager) prefetchWorker() {
|
||||||
|
defer pcm.wg.Done()
|
||||||
|
|
||||||
|
for {
|
||||||
|
select {
|
||||||
|
case <-pcm.ctx.Done():
|
||||||
|
return
|
||||||
|
case req := <-pcm.prefetchQueue:
|
||||||
|
// Process prefetch request
|
||||||
|
pcm.processPrefetchRequest(req)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// analysisWorker performs periodic analysis and cache warming
|
||||||
|
func (pcm *PredictiveCacheManager) analysisWorker() {
|
||||||
|
defer pcm.wg.Done()
|
||||||
|
|
||||||
|
ticker := time.NewTicker(30 * time.Second) // Analyze every 30 seconds
|
||||||
|
defer ticker.Stop()
|
||||||
|
|
||||||
|
for {
|
||||||
|
select {
|
||||||
|
case <-pcm.ctx.Done():
|
||||||
|
return
|
||||||
|
case <-ticker.C:
|
||||||
|
pcm.performAnalysis()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// processPrefetchRequest processes a prefetch request
|
||||||
|
func (pcm *PredictiveCacheManager) processPrefetchRequest(req PrefetchRequest) {
|
||||||
|
// In a real implementation, this would:
|
||||||
|
// 1. Check if content is already cached
|
||||||
|
// 2. If not, fetch and cache it
|
||||||
|
// 3. Update statistics
|
||||||
|
|
||||||
|
// For now, just log the prefetch request
|
||||||
|
// In production, integrate with the actual cache system
|
||||||
|
}
|
||||||
|
|
||||||
|
// performAnalysis performs periodic analysis and cache warming
|
||||||
|
func (pcm *PredictiveCacheManager) performAnalysis() {
|
||||||
|
// Get popular content for warming
|
||||||
|
popular := pcm.cacheWarmer.GetPopularContent(10)
|
||||||
|
|
||||||
|
// Request warming for popular content
|
||||||
|
for _, content := range popular {
|
||||||
|
if content.AccessCount > 5 { // Only warm frequently accessed content
|
||||||
|
select {
|
||||||
|
case pcm.cacheWarmer.warmerQueue <- WarmRequest{
|
||||||
|
Key: content.Key,
|
||||||
|
Priority: content.Priority,
|
||||||
|
Reason: "popular_content",
|
||||||
|
}:
|
||||||
|
default:
|
||||||
|
// Queue full, skip
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// GetStats returns predictive caching statistics
|
||||||
|
func (pcm *PredictiveCacheManager) GetStats() *PredictiveStats {
|
||||||
|
pcm.stats.mu.RLock()
|
||||||
|
defer pcm.stats.mu.RUnlock()
|
||||||
|
|
||||||
|
return &PredictiveStats{
|
||||||
|
PrefetchHits: atomic.LoadInt64(&pcm.stats.PrefetchHits),
|
||||||
|
PrefetchMisses: atomic.LoadInt64(&pcm.stats.PrefetchMisses),
|
||||||
|
PrefetchRequests: atomic.LoadInt64(&pcm.stats.PrefetchRequests),
|
||||||
|
CacheWarmHits: atomic.LoadInt64(&pcm.stats.CacheWarmHits),
|
||||||
|
CacheWarmMisses: atomic.LoadInt64(&pcm.stats.CacheWarmMisses),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Stop stops the predictive cache manager
|
||||||
|
func (pcm *PredictiveCacheManager) Stop() {
|
||||||
|
pcm.cancel()
|
||||||
|
pcm.wg.Wait()
|
||||||
|
}
|
||||||
87
vfs/types/types.go
Normal file
87
vfs/types/types.go
Normal file
@@ -0,0 +1,87 @@
|
|||||||
|
// vfs/types/types.go
|
||||||
|
package types
|
||||||
|
|
||||||
|
import (
|
||||||
|
"os"
|
||||||
|
"time"
|
||||||
|
)
|
||||||
|
|
||||||
|
// FileInfo contains metadata about a cached file
|
||||||
|
type FileInfo struct {
|
||||||
|
Key string `json:"key"`
|
||||||
|
Size int64 `json:"size"`
|
||||||
|
ATime time.Time `json:"atime"` // Last access time
|
||||||
|
CTime time.Time `json:"ctime"` // Creation time
|
||||||
|
AccessCount int `json:"access_count"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// NewFileInfo creates a new FileInfo with the given key and current timestamp
|
||||||
|
func NewFileInfo(key string, size int64) *FileInfo {
|
||||||
|
now := time.Now()
|
||||||
|
return &FileInfo{
|
||||||
|
Key: key,
|
||||||
|
Size: size,
|
||||||
|
ATime: now,
|
||||||
|
CTime: now,
|
||||||
|
AccessCount: 1,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// NewFileInfoFromOS creates a FileInfo from os.FileInfo
|
||||||
|
func NewFileInfoFromOS(info os.FileInfo, key string) *FileInfo {
|
||||||
|
return &FileInfo{
|
||||||
|
Key: key,
|
||||||
|
Size: info.Size(),
|
||||||
|
ATime: time.Now(), // We don't have access time from os.FileInfo
|
||||||
|
CTime: info.ModTime(),
|
||||||
|
AccessCount: 1,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// UpdateAccess updates the access time and increments the access count
|
||||||
|
func (fi *FileInfo) UpdateAccess() {
|
||||||
|
fi.ATime = time.Now()
|
||||||
|
fi.AccessCount++
|
||||||
|
}
|
||||||
|
|
||||||
|
// BatchedTimeUpdate provides a way to batch time updates for better performance
|
||||||
|
type BatchedTimeUpdate struct {
|
||||||
|
currentTime time.Time
|
||||||
|
lastUpdate time.Time
|
||||||
|
updateInterval time.Duration
|
||||||
|
}
|
||||||
|
|
||||||
|
// NewBatchedTimeUpdate creates a new batched time updater
|
||||||
|
func NewBatchedTimeUpdate(interval time.Duration) *BatchedTimeUpdate {
|
||||||
|
now := time.Now()
|
||||||
|
return &BatchedTimeUpdate{
|
||||||
|
currentTime: now,
|
||||||
|
lastUpdate: now,
|
||||||
|
updateInterval: interval,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// GetTime returns the current cached time, updating it if necessary
|
||||||
|
func (btu *BatchedTimeUpdate) GetTime() time.Time {
|
||||||
|
now := time.Now()
|
||||||
|
if now.Sub(btu.lastUpdate) >= btu.updateInterval {
|
||||||
|
btu.currentTime = now
|
||||||
|
btu.lastUpdate = now
|
||||||
|
}
|
||||||
|
return btu.currentTime
|
||||||
|
}
|
||||||
|
|
||||||
|
// UpdateAccessBatched updates the access time using batched time updates
|
||||||
|
func (fi *FileInfo) UpdateAccessBatched(btu *BatchedTimeUpdate) {
|
||||||
|
fi.ATime = btu.GetTime()
|
||||||
|
fi.AccessCount++
|
||||||
|
}
|
||||||
|
|
||||||
|
// GetTimeDecayedScore calculates a score based on access time and frequency
|
||||||
|
// More recent and frequent accesses get higher scores
|
||||||
|
func (fi *FileInfo) GetTimeDecayedScore() float64 {
|
||||||
|
timeSinceAccess := time.Since(fi.ATime).Hours()
|
||||||
|
decayFactor := 1.0 / (1.0 + timeSinceAccess/24.0) // Decay over days
|
||||||
|
frequencyBonus := float64(fi.AccessCount) * 0.1
|
||||||
|
return decayFactor + frequencyBonus
|
||||||
|
}
|
||||||
90
vfs/vfs.go
90
vfs/vfs.go
@@ -3,8 +3,7 @@ package vfs
|
|||||||
|
|
||||||
import (
|
import (
|
||||||
"io"
|
"io"
|
||||||
"os"
|
"s1d3sw1ped/SteamCache2/vfs/types"
|
||||||
"time"
|
|
||||||
)
|
)
|
||||||
|
|
||||||
// VFS defines the interface for virtual file systems
|
// VFS defines the interface for virtual file systems
|
||||||
@@ -19,7 +18,7 @@ type VFS interface {
|
|||||||
Delete(key string) error
|
Delete(key string) error
|
||||||
|
|
||||||
// Stat returns information about the file at the given key
|
// Stat returns information about the file at the given key
|
||||||
Stat(key string) (*FileInfo, error)
|
Stat(key string) (*types.FileInfo, error)
|
||||||
|
|
||||||
// Name returns the name of this VFS
|
// Name returns the name of this VFS
|
||||||
Name() string
|
Name() string
|
||||||
@@ -31,82 +30,17 @@ type VFS interface {
|
|||||||
Capacity() int64
|
Capacity() int64
|
||||||
}
|
}
|
||||||
|
|
||||||
// FileInfo contains metadata about a cached file
|
// FileInfo is an alias for types.FileInfo for backward compatibility
|
||||||
type FileInfo struct {
|
type FileInfo = types.FileInfo
|
||||||
Key string `json:"key"`
|
|
||||||
Size int64 `json:"size"`
|
|
||||||
ATime time.Time `json:"atime"` // Last access time
|
|
||||||
CTime time.Time `json:"ctime"` // Creation time
|
|
||||||
AccessCount int `json:"access_count"`
|
|
||||||
}
|
|
||||||
|
|
||||||
// NewFileInfo creates a new FileInfo with the given key and current timestamp
|
// NewFileInfo is an alias for types.NewFileInfo for backward compatibility
|
||||||
func NewFileInfo(key string, size int64) *FileInfo {
|
var NewFileInfo = types.NewFileInfo
|
||||||
now := time.Now()
|
|
||||||
return &FileInfo{
|
|
||||||
Key: key,
|
|
||||||
Size: size,
|
|
||||||
ATime: now,
|
|
||||||
CTime: now,
|
|
||||||
AccessCount: 1,
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// NewFileInfoFromOS creates a FileInfo from os.FileInfo
|
// NewFileInfoFromOS is an alias for types.NewFileInfoFromOS for backward compatibility
|
||||||
func NewFileInfoFromOS(info os.FileInfo, key string) *FileInfo {
|
var NewFileInfoFromOS = types.NewFileInfoFromOS
|
||||||
return &FileInfo{
|
|
||||||
Key: key,
|
|
||||||
Size: info.Size(),
|
|
||||||
ATime: time.Now(), // We don't have access time from os.FileInfo
|
|
||||||
CTime: info.ModTime(),
|
|
||||||
AccessCount: 1,
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// UpdateAccess updates the access time and increments the access count
|
// BatchedTimeUpdate is an alias for types.BatchedTimeUpdate for backward compatibility
|
||||||
func (fi *FileInfo) UpdateAccess() {
|
type BatchedTimeUpdate = types.BatchedTimeUpdate
|
||||||
fi.ATime = time.Now()
|
|
||||||
fi.AccessCount++
|
|
||||||
}
|
|
||||||
|
|
||||||
// BatchedTimeUpdate provides a way to batch time updates for better performance
|
// NewBatchedTimeUpdate is an alias for types.NewBatchedTimeUpdate for backward compatibility
|
||||||
type BatchedTimeUpdate struct {
|
var NewBatchedTimeUpdate = types.NewBatchedTimeUpdate
|
||||||
currentTime time.Time
|
|
||||||
lastUpdate time.Time
|
|
||||||
updateInterval time.Duration
|
|
||||||
}
|
|
||||||
|
|
||||||
// NewBatchedTimeUpdate creates a new batched time updater
|
|
||||||
func NewBatchedTimeUpdate(interval time.Duration) *BatchedTimeUpdate {
|
|
||||||
now := time.Now()
|
|
||||||
return &BatchedTimeUpdate{
|
|
||||||
currentTime: now,
|
|
||||||
lastUpdate: now,
|
|
||||||
updateInterval: interval,
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// GetTime returns the current cached time, updating it if necessary
|
|
||||||
func (btu *BatchedTimeUpdate) GetTime() time.Time {
|
|
||||||
now := time.Now()
|
|
||||||
if now.Sub(btu.lastUpdate) >= btu.updateInterval {
|
|
||||||
btu.currentTime = now
|
|
||||||
btu.lastUpdate = now
|
|
||||||
}
|
|
||||||
return btu.currentTime
|
|
||||||
}
|
|
||||||
|
|
||||||
// UpdateAccessBatched updates the access time using batched time updates
|
|
||||||
func (fi *FileInfo) UpdateAccessBatched(btu *BatchedTimeUpdate) {
|
|
||||||
fi.ATime = btu.GetTime()
|
|
||||||
fi.AccessCount++
|
|
||||||
}
|
|
||||||
|
|
||||||
// GetTimeDecayedScore calculates a score based on access time and frequency
|
|
||||||
// More recent and frequent accesses get higher scores
|
|
||||||
func (fi *FileInfo) GetTimeDecayedScore() float64 {
|
|
||||||
timeSinceAccess := time.Since(fi.ATime).Hours()
|
|
||||||
decayFactor := 1.0 / (1.0 + timeSinceAccess/24.0) // Decay over days
|
|
||||||
frequencyBonus := float64(fi.AccessCount) * 0.1
|
|
||||||
return decayFactor + frequencyBonus
|
|
||||||
}
|
|
||||||
|
|||||||
300
vfs/warming/warming.go
Normal file
300
vfs/warming/warming.go
Normal file
@@ -0,0 +1,300 @@
|
|||||||
|
package warming
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"s1d3sw1ped/SteamCache2/vfs"
|
||||||
|
"sync"
|
||||||
|
"sync/atomic"
|
||||||
|
"time"
|
||||||
|
)
|
||||||
|
|
||||||
|
// CacheWarmer implements intelligent cache warming strategies
|
||||||
|
type CacheWarmer struct {
|
||||||
|
vfs vfs.VFS
|
||||||
|
warmingQueue chan WarmRequest
|
||||||
|
activeWarmers map[string]*ActiveWarmer
|
||||||
|
stats *WarmingStats
|
||||||
|
ctx context.Context
|
||||||
|
cancel context.CancelFunc
|
||||||
|
wg sync.WaitGroup
|
||||||
|
mu sync.RWMutex
|
||||||
|
maxConcurrent int
|
||||||
|
warmingEnabled bool
|
||||||
|
}
|
||||||
|
|
||||||
|
// WarmRequest represents a cache warming request
|
||||||
|
type WarmRequest struct {
|
||||||
|
Key string
|
||||||
|
Priority int
|
||||||
|
Reason string
|
||||||
|
Size int64
|
||||||
|
RequestedAt time.Time
|
||||||
|
Source string // Where the warming request came from
|
||||||
|
}
|
||||||
|
|
||||||
|
// ActiveWarmer tracks an active warming operation
|
||||||
|
type ActiveWarmer struct {
|
||||||
|
Key string
|
||||||
|
StartTime time.Time
|
||||||
|
Priority int
|
||||||
|
Reason string
|
||||||
|
mu sync.RWMutex
|
||||||
|
}
|
||||||
|
|
||||||
|
// WarmingStats tracks cache warming statistics
|
||||||
|
type WarmingStats struct {
|
||||||
|
WarmRequests int64
|
||||||
|
WarmSuccesses int64
|
||||||
|
WarmFailures int64
|
||||||
|
WarmBytes int64
|
||||||
|
WarmDuration time.Duration
|
||||||
|
ActiveWarmers int64
|
||||||
|
mu sync.RWMutex
|
||||||
|
}
|
||||||
|
|
||||||
|
// WarmingStrategy defines different warming strategies
|
||||||
|
type WarmingStrategy int
|
||||||
|
|
||||||
|
const (
|
||||||
|
StrategyImmediate WarmingStrategy = iota
|
||||||
|
StrategyBackground
|
||||||
|
StrategyScheduled
|
||||||
|
StrategyPredictive
|
||||||
|
)
|
||||||
|
|
||||||
|
// NewCacheWarmer creates a new cache warmer
|
||||||
|
func NewCacheWarmer(vfs vfs.VFS, maxConcurrent int) *CacheWarmer {
|
||||||
|
ctx, cancel := context.WithCancel(context.Background())
|
||||||
|
|
||||||
|
cw := &CacheWarmer{
|
||||||
|
vfs: vfs,
|
||||||
|
warmingQueue: make(chan WarmRequest, 1000),
|
||||||
|
activeWarmers: make(map[string]*ActiveWarmer),
|
||||||
|
stats: &WarmingStats{},
|
||||||
|
ctx: ctx,
|
||||||
|
cancel: cancel,
|
||||||
|
maxConcurrent: maxConcurrent,
|
||||||
|
warmingEnabled: true,
|
||||||
|
}
|
||||||
|
|
||||||
|
// Start warming workers
|
||||||
|
for i := 0; i < maxConcurrent; i++ {
|
||||||
|
cw.wg.Add(1)
|
||||||
|
go cw.warmingWorker(i)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Start cleanup worker
|
||||||
|
cw.wg.Add(1)
|
||||||
|
go cw.cleanupWorker()
|
||||||
|
|
||||||
|
return cw
|
||||||
|
}
|
||||||
|
|
||||||
|
// RequestWarming requests warming of content
|
||||||
|
func (cw *CacheWarmer) RequestWarming(key string, priority int, reason string, size int64, source string) {
|
||||||
|
if !cw.warmingEnabled {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
// Check if already warming
|
||||||
|
cw.mu.RLock()
|
||||||
|
if _, exists := cw.activeWarmers[key]; exists {
|
||||||
|
cw.mu.RUnlock()
|
||||||
|
return // Already warming
|
||||||
|
}
|
||||||
|
cw.mu.RUnlock()
|
||||||
|
|
||||||
|
// Check if already cached
|
||||||
|
if _, err := cw.vfs.Stat(key); err == nil {
|
||||||
|
return // Already cached
|
||||||
|
}
|
||||||
|
|
||||||
|
select {
|
||||||
|
case cw.warmingQueue <- WarmRequest{
|
||||||
|
Key: key,
|
||||||
|
Priority: priority,
|
||||||
|
Reason: reason,
|
||||||
|
Size: size,
|
||||||
|
RequestedAt: time.Now(),
|
||||||
|
Source: source,
|
||||||
|
}:
|
||||||
|
atomic.AddInt64(&cw.stats.WarmRequests, 1)
|
||||||
|
default:
|
||||||
|
// Queue full, skip warming
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// warmingWorker processes warming requests
|
||||||
|
func (cw *CacheWarmer) warmingWorker(workerID int) {
|
||||||
|
defer cw.wg.Done()
|
||||||
|
|
||||||
|
for {
|
||||||
|
select {
|
||||||
|
case <-cw.ctx.Done():
|
||||||
|
return
|
||||||
|
case req := <-cw.warmingQueue:
|
||||||
|
cw.processWarmingRequest(req, workerID)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// processWarmingRequest processes a warming request
|
||||||
|
func (cw *CacheWarmer) processWarmingRequest(req WarmRequest, workerID int) {
|
||||||
|
// Mark as active warmer
|
||||||
|
cw.mu.Lock()
|
||||||
|
cw.activeWarmers[req.Key] = &ActiveWarmer{
|
||||||
|
Key: req.Key,
|
||||||
|
StartTime: time.Now(),
|
||||||
|
Priority: req.Priority,
|
||||||
|
Reason: req.Reason,
|
||||||
|
}
|
||||||
|
cw.mu.Unlock()
|
||||||
|
|
||||||
|
atomic.AddInt64(&cw.stats.ActiveWarmers, 1)
|
||||||
|
|
||||||
|
// Simulate warming process
|
||||||
|
// In a real implementation, this would:
|
||||||
|
// 1. Fetch content from upstream
|
||||||
|
// 2. Store in cache
|
||||||
|
// 3. Update statistics
|
||||||
|
|
||||||
|
startTime := time.Now()
|
||||||
|
|
||||||
|
// Simulate warming delay based on priority
|
||||||
|
warmingDelay := time.Duration(100-req.Priority*10) * time.Millisecond
|
||||||
|
if warmingDelay < 10*time.Millisecond {
|
||||||
|
warmingDelay = 10 * time.Millisecond
|
||||||
|
}
|
||||||
|
|
||||||
|
select {
|
||||||
|
case <-time.After(warmingDelay):
|
||||||
|
// Warming completed successfully
|
||||||
|
atomic.AddInt64(&cw.stats.WarmSuccesses, 1)
|
||||||
|
atomic.AddInt64(&cw.stats.WarmBytes, req.Size)
|
||||||
|
case <-cw.ctx.Done():
|
||||||
|
// Context cancelled
|
||||||
|
atomic.AddInt64(&cw.stats.WarmFailures, 1)
|
||||||
|
}
|
||||||
|
|
||||||
|
duration := time.Since(startTime)
|
||||||
|
cw.stats.mu.Lock()
|
||||||
|
cw.stats.WarmDuration += duration
|
||||||
|
cw.stats.mu.Unlock()
|
||||||
|
|
||||||
|
// Remove from active warmers
|
||||||
|
cw.mu.Lock()
|
||||||
|
delete(cw.activeWarmers, req.Key)
|
||||||
|
cw.mu.Unlock()
|
||||||
|
|
||||||
|
atomic.AddInt64(&cw.stats.ActiveWarmers, -1)
|
||||||
|
}
|
||||||
|
|
||||||
|
// cleanupWorker cleans up old warming requests
|
||||||
|
func (cw *CacheWarmer) cleanupWorker() {
|
||||||
|
defer cw.wg.Done()
|
||||||
|
|
||||||
|
ticker := time.NewTicker(1 * time.Minute)
|
||||||
|
defer ticker.Stop()
|
||||||
|
|
||||||
|
for {
|
||||||
|
select {
|
||||||
|
case <-cw.ctx.Done():
|
||||||
|
return
|
||||||
|
case <-ticker.C:
|
||||||
|
cw.cleanupOldWarmers()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// cleanupOldWarmers removes old warming requests
|
||||||
|
func (cw *CacheWarmer) cleanupOldWarmers() {
|
||||||
|
cw.mu.Lock()
|
||||||
|
defer cw.mu.Unlock()
|
||||||
|
|
||||||
|
now := time.Now()
|
||||||
|
cutoff := now.Add(-5 * time.Minute) // Remove warmers older than 5 minutes
|
||||||
|
|
||||||
|
for key, warmer := range cw.activeWarmers {
|
||||||
|
warmer.mu.RLock()
|
||||||
|
if warmer.StartTime.Before(cutoff) {
|
||||||
|
warmer.mu.RUnlock()
|
||||||
|
delete(cw.activeWarmers, key)
|
||||||
|
atomic.AddInt64(&cw.stats.WarmFailures, 1)
|
||||||
|
} else {
|
||||||
|
warmer.mu.RUnlock()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// GetActiveWarmers returns currently active warming operations
|
||||||
|
func (cw *CacheWarmer) GetActiveWarmers() []*ActiveWarmer {
|
||||||
|
cw.mu.RLock()
|
||||||
|
defer cw.mu.RUnlock()
|
||||||
|
|
||||||
|
warmers := make([]*ActiveWarmer, 0, len(cw.activeWarmers))
|
||||||
|
for _, warmer := range cw.activeWarmers {
|
||||||
|
warmers = append(warmers, warmer)
|
||||||
|
}
|
||||||
|
|
||||||
|
return warmers
|
||||||
|
}
|
||||||
|
|
||||||
|
// GetStats returns warming statistics
|
||||||
|
func (cw *CacheWarmer) GetStats() *WarmingStats {
|
||||||
|
cw.stats.mu.RLock()
|
||||||
|
defer cw.stats.mu.RUnlock()
|
||||||
|
|
||||||
|
return &WarmingStats{
|
||||||
|
WarmRequests: atomic.LoadInt64(&cw.stats.WarmRequests),
|
||||||
|
WarmSuccesses: atomic.LoadInt64(&cw.stats.WarmSuccesses),
|
||||||
|
WarmFailures: atomic.LoadInt64(&cw.stats.WarmFailures),
|
||||||
|
WarmBytes: atomic.LoadInt64(&cw.stats.WarmBytes),
|
||||||
|
WarmDuration: cw.stats.WarmDuration,
|
||||||
|
ActiveWarmers: atomic.LoadInt64(&cw.stats.ActiveWarmers),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// SetWarmingEnabled enables or disables cache warming
|
||||||
|
func (cw *CacheWarmer) SetWarmingEnabled(enabled bool) {
|
||||||
|
cw.mu.Lock()
|
||||||
|
defer cw.mu.Unlock()
|
||||||
|
cw.warmingEnabled = enabled
|
||||||
|
}
|
||||||
|
|
||||||
|
// IsWarmingEnabled returns whether warming is enabled
|
||||||
|
func (cw *CacheWarmer) IsWarmingEnabled() bool {
|
||||||
|
cw.mu.RLock()
|
||||||
|
defer cw.mu.RUnlock()
|
||||||
|
return cw.warmingEnabled
|
||||||
|
}
|
||||||
|
|
||||||
|
// Stop stops the cache warmer
|
||||||
|
func (cw *CacheWarmer) Stop() {
|
||||||
|
cw.cancel()
|
||||||
|
cw.wg.Wait()
|
||||||
|
}
|
||||||
|
|
||||||
|
// WarmPopularContent warms popular content based on access patterns
|
||||||
|
func (cw *CacheWarmer) WarmPopularContent(popularKeys []string, priority int) {
|
||||||
|
for _, key := range popularKeys {
|
||||||
|
cw.RequestWarming(key, priority, "popular_content", 0, "popular_analyzer")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// WarmPredictedContent warms predicted content
|
||||||
|
func (cw *CacheWarmer) WarmPredictedContent(predictedKeys []string, priority int) {
|
||||||
|
for _, key := range predictedKeys {
|
||||||
|
cw.RequestWarming(key, priority, "predicted_access", 0, "predictor")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// WarmSequentialContent warms content in sequential order
|
||||||
|
func (cw *CacheWarmer) WarmSequentialContent(sequentialKeys []string, priority int) {
|
||||||
|
for i, key := range sequentialKeys {
|
||||||
|
// Stagger warming requests to avoid overwhelming the system
|
||||||
|
go func(k string, delay time.Duration) {
|
||||||
|
time.Sleep(delay)
|
||||||
|
cw.RequestWarming(k, priority, "sequential_access", 0, "sequential_analyzer")
|
||||||
|
}(key, time.Duration(i)*100*time.Millisecond)
|
||||||
|
}
|
||||||
|
}
|
||||||
Reference in New Issue
Block a user