Compare commits
8 Commits
9ca8fa4a5e
...
1.0.18
| Author | SHA1 | Date | |
|---|---|---|---|
| bfe29dea75 | |||
| 9b2affe95a | |||
| bd123bc63a | |||
| 46495dc3aa | |||
| 45ae234694 | |||
| bbe014e334 | |||
| 694c223b00 | |||
| cc3497bc3a |
4
.gitignore
vendored
4
.gitignore
vendored
@@ -9,3 +9,7 @@
|
||||
|
||||
#windows executables
|
||||
*.exe
|
||||
|
||||
#test cache
|
||||
/steamcache/test_cache/*
|
||||
!/steamcache/test_cache/.gitkeep
|
||||
@@ -11,8 +11,8 @@ builds:
|
||||
- -s
|
||||
- -w
|
||||
- -extldflags "-static"
|
||||
- -X s1d3sw1ped/SteamCache2/version.Version={{.Version}}
|
||||
- -X s1d3sw1ped/SteamCache2/version.Date={{.Date}}
|
||||
- -X s1d3sw1ped/steamcache2/version.Version={{.Version}}
|
||||
- -X s1d3sw1ped/steamcache2/version.Date={{.Date}}
|
||||
env:
|
||||
- CGO_ENABLED=0
|
||||
goos:
|
||||
|
||||
28
Makefile
28
Makefile
@@ -1,19 +1,21 @@
|
||||
run: deps test ## Run the application
|
||||
@go run .
|
||||
|
||||
help: ## Show this help message
|
||||
@echo SteamCache2 Makefile
|
||||
@echo Available targets:
|
||||
@echo run Run the application
|
||||
@echo run-debug Run the application with debug logging
|
||||
@echo test Run all tests
|
||||
@echo deps Download dependencies
|
||||
|
||||
run-debug: deps test ## Run the application with debug logging
|
||||
@go run . --log-level debug
|
||||
run: build-snapshot-single ## Run the application
|
||||
@dist/default_windows_amd64_v1/steamcache2.exe
|
||||
run-debug: build-snapshot-single ## Run the application with debug logging
|
||||
@dist/default_windows_amd64_v1/steamcache2.exe --log-level debug
|
||||
|
||||
test: deps ## Run all tests
|
||||
@go test -v ./...
|
||||
|
||||
deps: ## Download dependencies
|
||||
@go mod tidy
|
||||
|
||||
build-snapshot-single: deps test ## Build a snapshot of the application for the current platform
|
||||
@goreleaser build --single-target --snapshot --clean
|
||||
|
||||
help: ## Show this help message
|
||||
@echo steamcache2 Makefile
|
||||
@echo Available targets:
|
||||
@echo run Run the application
|
||||
@echo run-debug Run the application with debug logging
|
||||
@echo test Run all tests
|
||||
@echo deps Download dependencies
|
||||
@@ -21,7 +21,7 @@ SteamCache2 is a blazing fast download cache for Steam, designed to reduce bandw
|
||||
1. **Clone and build:**
|
||||
```bash
|
||||
git clone <repository-url>
|
||||
cd SteamCache2
|
||||
cd steamcache2
|
||||
make # This will run tests and build the application
|
||||
```
|
||||
|
||||
|
||||
20
cmd/root.go
20
cmd/root.go
@@ -4,10 +4,10 @@ package cmd
|
||||
import (
|
||||
"fmt"
|
||||
"os"
|
||||
"s1d3sw1ped/SteamCache2/config"
|
||||
"s1d3sw1ped/SteamCache2/steamcache"
|
||||
"s1d3sw1ped/SteamCache2/steamcache/logger"
|
||||
"s1d3sw1ped/SteamCache2/version"
|
||||
"s1d3sw1ped/steamcache2/config"
|
||||
"s1d3sw1ped/steamcache2/steamcache"
|
||||
"s1d3sw1ped/steamcache2/steamcache/logger"
|
||||
"s1d3sw1ped/steamcache2/version"
|
||||
"strings"
|
||||
|
||||
"github.com/rs/zerolog"
|
||||
@@ -25,9 +25,9 @@ var (
|
||||
)
|
||||
|
||||
var rootCmd = &cobra.Command{
|
||||
Use: "SteamCache2",
|
||||
Short: "SteamCache2 is a caching solution for Steam game updates and installations",
|
||||
Long: `SteamCache2 is a caching solution designed to optimize the delivery of Steam game updates and installations.
|
||||
Use: "steamcache2",
|
||||
Short: "steamcache2 is a caching solution for Steam game updates and installations",
|
||||
Long: `steamcache2 is a caching solution designed to optimize the delivery of Steam game updates and installations.
|
||||
It reduces bandwidth usage and speeds up the download process by caching game files locally.
|
||||
This tool is particularly useful for environments with multiple Steam users, such as gaming cafes or households with multiple gamers.
|
||||
By caching game files, SteamCache2 ensures that subsequent downloads of the same files are served from the local cache,
|
||||
@@ -53,7 +53,7 @@ var rootCmd = &cobra.Command{
|
||||
logger.Logger = zerolog.New(writer).With().Timestamp().Logger()
|
||||
|
||||
logger.Logger.Info().
|
||||
Msg("SteamCache2 " + version.Version + " " + version.Date + " starting...")
|
||||
Msg("steamcache2 " + version.Version + " " + version.Date + " starting...")
|
||||
|
||||
// Load configuration
|
||||
cfg, err := config.LoadConfig(configPath)
|
||||
@@ -121,11 +121,11 @@ var rootCmd = &cobra.Command{
|
||||
)
|
||||
|
||||
logger.Logger.Info().
|
||||
Msg("SteamCache2 " + version.Version + " started on " + cfg.ListenAddress)
|
||||
Msg("steamcache2 " + version.Version + " started on " + cfg.ListenAddress)
|
||||
|
||||
sc.Run()
|
||||
|
||||
logger.Logger.Info().Msg("SteamCache2 stopped")
|
||||
logger.Logger.Info().Msg("steamcache2 stopped")
|
||||
os.Exit(0)
|
||||
},
|
||||
}
|
||||
|
||||
@@ -4,7 +4,7 @@ package cmd
|
||||
import (
|
||||
"fmt"
|
||||
"os"
|
||||
"s1d3sw1ped/SteamCache2/version"
|
||||
"s1d3sw1ped/steamcache2/version"
|
||||
|
||||
"github.com/spf13/cobra"
|
||||
)
|
||||
@@ -12,10 +12,10 @@ import (
|
||||
// versionCmd represents the version command
|
||||
var versionCmd = &cobra.Command{
|
||||
Use: "version",
|
||||
Short: "prints the version of SteamCache2",
|
||||
Long: `Prints the version of SteamCache2. This command is useful for checking the version of the application.`,
|
||||
Short: "prints the version of steamcache2",
|
||||
Long: `Prints the version of steamcache2. This command is useful for checking the version of the application.`,
|
||||
Run: func(cmd *cobra.Command, args []string) {
|
||||
fmt.Fprintln(os.Stderr, "SteamCache2", version.Version, version.Date)
|
||||
fmt.Fprintln(os.Stderr, "steamcache2", version.Version, version.Date)
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
4
go.mod
4
go.mod
@@ -1,4 +1,4 @@
|
||||
module s1d3sw1ped/SteamCache2
|
||||
module s1d3sw1ped/steamcache2
|
||||
|
||||
go 1.23.0
|
||||
|
||||
@@ -7,6 +7,7 @@ require (
|
||||
github.com/edsrzf/mmap-go v1.1.0
|
||||
github.com/rs/zerolog v1.33.0
|
||||
github.com/spf13/cobra v1.8.1
|
||||
golang.org/x/sync v0.16.0
|
||||
gopkg.in/yaml.v3 v3.0.1
|
||||
)
|
||||
|
||||
@@ -15,6 +16,5 @@ require (
|
||||
github.com/mattn/go-colorable v0.1.13 // indirect
|
||||
github.com/mattn/go-isatty v0.0.19 // indirect
|
||||
github.com/spf13/pflag v1.0.5 // indirect
|
||||
golang.org/x/sync v0.16.0 // indirect
|
||||
golang.org/x/sys v0.12.0 // indirect
|
||||
)
|
||||
|
||||
4
main.go
4
main.go
@@ -2,8 +2,8 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"s1d3sw1ped/SteamCache2/cmd"
|
||||
_ "s1d3sw1ped/SteamCache2/version" // Import the version package for global version variable
|
||||
"s1d3sw1ped/steamcache2/cmd"
|
||||
_ "s1d3sw1ped/steamcache2/version" // Import the version package for global version variable
|
||||
)
|
||||
|
||||
func main() {
|
||||
|
||||
279
steamcache/integration_test.go
Normal file
279
steamcache/integration_test.go
Normal file
@@ -0,0 +1,279 @@
|
||||
package steamcache
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"os"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
const SteamHostname = "cache2-den-iwst.steamcontent.com"
|
||||
|
||||
func TestSteamIntegration(t *testing.T) {
|
||||
// Skip this test if we don't have internet access or want to avoid hitting Steam servers
|
||||
if testing.Short() {
|
||||
t.Skip("Skipping integration test in short mode")
|
||||
}
|
||||
|
||||
// Test URLs from real Steam usage - these should be cached when requested by Steam clients
|
||||
testURLs := []string{
|
||||
"/depot/516751/patch/288061881745926019/4378193572994177373",
|
||||
"/depot/516751/chunk/42e7c13eb4b4e426ec5cf6d1010abfd528e5065a",
|
||||
"/depot/516751/chunk/f949f71e102d77ed6e364e2054d06429d54bebb1",
|
||||
"/depot/516751/chunk/6790f5105833556d37797657be72c1c8dd2e7074",
|
||||
}
|
||||
|
||||
for _, testURL := range testURLs {
|
||||
t.Run(fmt.Sprintf("URL_%s", testURL), func(t *testing.T) {
|
||||
testSteamURL(t, testURL)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func testSteamURL(t *testing.T, urlPath string) {
|
||||
// Create a unique temporary directory for this test to avoid cache persistence issues
|
||||
tempDir, err := os.MkdirTemp("", "steamcache_test_*")
|
||||
if err != nil {
|
||||
t.Fatalf("Failed to create temp directory: %v", err)
|
||||
}
|
||||
defer os.RemoveAll(tempDir) // Clean up after test
|
||||
|
||||
// Create SteamCache instance with unique temp directory
|
||||
sc := New(":0", "100MB", "1GB", tempDir, "", "LRU", "LRU", 10, 5)
|
||||
|
||||
// Use real Steam server
|
||||
steamURL := "https://" + SteamHostname + urlPath
|
||||
|
||||
// Test direct download from Steam server
|
||||
directResp, directBody := downloadDirectly(t, steamURL)
|
||||
|
||||
// Test download through SteamCache
|
||||
cacheResp, cacheBody := downloadThroughCache(t, sc, urlPath)
|
||||
|
||||
// Compare responses
|
||||
compareResponses(t, directResp, directBody, cacheResp, cacheBody, urlPath)
|
||||
}
|
||||
|
||||
func downloadDirectly(t *testing.T, url string) (*http.Response, []byte) {
|
||||
client := &http.Client{Timeout: 30 * time.Second}
|
||||
|
||||
req, err := http.NewRequest("GET", url, nil)
|
||||
if err != nil {
|
||||
t.Fatalf("Failed to create request: %v", err)
|
||||
}
|
||||
|
||||
// Add Steam user agent
|
||||
req.Header.Set("User-Agent", "Valve/Steam HTTP Client 1.0")
|
||||
|
||||
resp, err := client.Do(req)
|
||||
if err != nil {
|
||||
t.Fatalf("Failed to download directly from Steam: %v", err)
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
body, err := io.ReadAll(resp.Body)
|
||||
if err != nil {
|
||||
t.Fatalf("Failed to read direct response body: %v", err)
|
||||
}
|
||||
|
||||
return resp, body
|
||||
}
|
||||
|
||||
func downloadThroughCache(t *testing.T, sc *SteamCache, urlPath string) (*http.Response, []byte) {
|
||||
// Create a test server for SteamCache
|
||||
cacheServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
// For real Steam URLs, we need to set the upstream to the Steam hostname
|
||||
// and let SteamCache handle the full URL construction
|
||||
sc.upstream = "https://" + SteamHostname
|
||||
sc.ServeHTTP(w, r)
|
||||
}))
|
||||
defer cacheServer.Close()
|
||||
|
||||
// First request - should be a MISS and cache the file
|
||||
client := &http.Client{Timeout: 30 * time.Second}
|
||||
|
||||
req1, err := http.NewRequest("GET", cacheServer.URL+urlPath, nil)
|
||||
if err != nil {
|
||||
t.Fatalf("Failed to create first request: %v", err)
|
||||
}
|
||||
req1.Header.Set("User-Agent", "Valve/Steam HTTP Client 1.0")
|
||||
|
||||
resp1, err := client.Do(req1)
|
||||
if err != nil {
|
||||
t.Fatalf("Failed to download through cache (first request): %v", err)
|
||||
}
|
||||
defer resp1.Body.Close()
|
||||
|
||||
body1, err := io.ReadAll(resp1.Body)
|
||||
if err != nil {
|
||||
t.Fatalf("Failed to read cache response body (first request): %v", err)
|
||||
}
|
||||
|
||||
// Verify first request was a MISS
|
||||
if resp1.Header.Get("X-LanCache-Status") != "MISS" {
|
||||
t.Errorf("Expected first request to be MISS, got %s", resp1.Header.Get("X-LanCache-Status"))
|
||||
}
|
||||
|
||||
// Second request - should be a HIT from cache
|
||||
req2, err := http.NewRequest("GET", cacheServer.URL+urlPath, nil)
|
||||
if err != nil {
|
||||
t.Fatalf("Failed to create second request: %v", err)
|
||||
}
|
||||
req2.Header.Set("User-Agent", "Valve/Steam HTTP Client 1.0")
|
||||
|
||||
resp2, err := client.Do(req2)
|
||||
if err != nil {
|
||||
t.Fatalf("Failed to download through cache (second request): %v", err)
|
||||
}
|
||||
defer resp2.Body.Close()
|
||||
|
||||
body2, err := io.ReadAll(resp2.Body)
|
||||
if err != nil {
|
||||
t.Fatalf("Failed to read cache response body (second request): %v", err)
|
||||
}
|
||||
|
||||
// Verify second request was a HIT (unless hash verification failed)
|
||||
status2 := resp2.Header.Get("X-LanCache-Status")
|
||||
if status2 != "HIT" && status2 != "MISS" {
|
||||
t.Errorf("Expected second request to be HIT or MISS, got %s", status2)
|
||||
}
|
||||
|
||||
// If it's a MISS, it means hash verification failed and content wasn't cached
|
||||
// This is correct behavior - we shouldn't cache content that doesn't match the expected hash
|
||||
if status2 == "MISS" {
|
||||
t.Logf("Second request was MISS (hash verification failed) - this is correct behavior")
|
||||
}
|
||||
|
||||
// Verify both cache responses are identical
|
||||
if !bytes.Equal(body1, body2) {
|
||||
t.Error("First and second cache responses should be identical")
|
||||
}
|
||||
|
||||
// Return the second response (from cache)
|
||||
return resp2, body2
|
||||
}
|
||||
|
||||
func compareResponses(t *testing.T, directResp *http.Response, directBody []byte, cacheResp *http.Response, cacheBody []byte, urlPath string) {
|
||||
// Compare status codes
|
||||
if directResp.StatusCode != cacheResp.StatusCode {
|
||||
t.Errorf("Status code mismatch: direct=%d, cache=%d", directResp.StatusCode, cacheResp.StatusCode)
|
||||
}
|
||||
|
||||
// Compare response bodies (this is the most important test)
|
||||
if !bytes.Equal(directBody, cacheBody) {
|
||||
t.Errorf("Response body mismatch for URL %s", urlPath)
|
||||
t.Errorf("Direct body length: %d, Cache body length: %d", len(directBody), len(cacheBody))
|
||||
|
||||
// Find first difference
|
||||
minLen := len(directBody)
|
||||
if len(cacheBody) < minLen {
|
||||
minLen = len(cacheBody)
|
||||
}
|
||||
|
||||
for i := 0; i < minLen; i++ {
|
||||
if directBody[i] != cacheBody[i] {
|
||||
t.Errorf("First difference at byte %d: direct=0x%02x, cache=0x%02x", i, directBody[i], cacheBody[i])
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Compare important headers (excluding cache-specific ones)
|
||||
importantHeaders := []string{
|
||||
"Content-Type",
|
||||
"Content-Length",
|
||||
"X-Sha1",
|
||||
"Cache-Control",
|
||||
}
|
||||
|
||||
for _, header := range importantHeaders {
|
||||
directValue := directResp.Header.Get(header)
|
||||
cacheValue := cacheResp.Header.Get(header)
|
||||
|
||||
if directValue != cacheValue {
|
||||
t.Errorf("Header %s mismatch: direct=%s, cache=%s", header, directValue, cacheValue)
|
||||
}
|
||||
}
|
||||
|
||||
// Verify cache-specific headers are present
|
||||
if cacheResp.Header.Get("X-LanCache-Status") == "" {
|
||||
t.Error("Cache response should have X-LanCache-Status header")
|
||||
}
|
||||
|
||||
if cacheResp.Header.Get("X-LanCache-Processed-By") != "SteamCache2" {
|
||||
t.Error("Cache response should have X-LanCache-Processed-By header set to SteamCache2")
|
||||
}
|
||||
|
||||
t.Logf("✅ URL %s: Direct and cache responses are identical", urlPath)
|
||||
}
|
||||
|
||||
// TestCacheFileFormat tests the cache file format directly
|
||||
func TestCacheFileFormat(t *testing.T) {
|
||||
// Create test data
|
||||
bodyData := []byte("test steam content")
|
||||
contentHash := calculateSHA256(bodyData)
|
||||
|
||||
// Create mock response
|
||||
resp := &http.Response{
|
||||
StatusCode: 200,
|
||||
Status: "200 OK",
|
||||
Header: make(http.Header),
|
||||
Body: http.NoBody,
|
||||
}
|
||||
resp.Header.Set("Content-Type", "application/x-steam-chunk")
|
||||
resp.Header.Set("Content-Length", "18")
|
||||
resp.Header.Set("X-Sha1", contentHash)
|
||||
|
||||
// Create SteamCache instance
|
||||
sc := &SteamCache{}
|
||||
|
||||
// Reconstruct raw response
|
||||
rawResponse := sc.reconstructRawResponse(resp, bodyData)
|
||||
|
||||
// Serialize to cache format
|
||||
cacheData, err := serializeRawResponse(rawResponse)
|
||||
if err != nil {
|
||||
t.Fatalf("Failed to serialize cache file: %v", err)
|
||||
}
|
||||
|
||||
// Deserialize from cache format
|
||||
cacheFile, err := deserializeCacheFile(cacheData)
|
||||
if err != nil {
|
||||
t.Fatalf("Failed to deserialize cache file: %v", err)
|
||||
}
|
||||
|
||||
// Verify cache file structure
|
||||
if cacheFile.ContentHash != contentHash {
|
||||
t.Errorf("ContentHash mismatch: expected %s, got %s", contentHash, cacheFile.ContentHash)
|
||||
}
|
||||
|
||||
if cacheFile.ResponseSize != int64(len(rawResponse)) {
|
||||
t.Errorf("ResponseSize mismatch: expected %d, got %d", len(rawResponse), cacheFile.ResponseSize)
|
||||
}
|
||||
|
||||
// Verify raw response is preserved
|
||||
if !bytes.Equal(cacheFile.Response, rawResponse) {
|
||||
t.Error("Raw response not preserved in cache file")
|
||||
}
|
||||
|
||||
// Test streaming the cached response
|
||||
recorder := httptest.NewRecorder()
|
||||
req := httptest.NewRequest("GET", "/test/format", nil)
|
||||
|
||||
sc.streamCachedResponse(recorder, req, cacheFile, "test-key", "127.0.0.1", time.Now())
|
||||
|
||||
// Verify streamed response
|
||||
if recorder.Code != 200 {
|
||||
t.Errorf("Expected status code 200, got %d", recorder.Code)
|
||||
}
|
||||
|
||||
if !bytes.Equal(recorder.Body.Bytes(), bodyData) {
|
||||
t.Error("Streamed response body does not match original")
|
||||
}
|
||||
|
||||
t.Log("✅ Cache file format test passed")
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
@@ -3,20 +3,25 @@ package steamcache
|
||||
|
||||
import (
|
||||
"io"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func TestCaching(t *testing.T) {
|
||||
td := t.TempDir()
|
||||
|
||||
os.WriteFile(filepath.Join(td, "key2"), []byte("value2"), 0644)
|
||||
|
||||
sc := New("localhost:8080", "1G", "1G", td, "", "lru", "lru", 200, 5)
|
||||
|
||||
w, err := sc.vfs.Create("key", 5)
|
||||
// Create key2 through the VFS system instead of directly
|
||||
w, err := sc.vfs.Create("key2", 6)
|
||||
if err != nil {
|
||||
t.Errorf("Create key2 failed: %v", err)
|
||||
}
|
||||
w.Write([]byte("value2"))
|
||||
w.Close()
|
||||
|
||||
w, err = sc.vfs.Create("key", 5)
|
||||
if err != nil {
|
||||
t.Errorf("Create failed: %v", err)
|
||||
}
|
||||
@@ -68,16 +73,32 @@ func TestCaching(t *testing.T) {
|
||||
t.Errorf("Get failed: got %s, want %s", d, "value2")
|
||||
}
|
||||
|
||||
// With size-based promotion filtering, not all files may be promoted
|
||||
// The total size should be at least the disk size (17 bytes) but may be less than 34 bytes
|
||||
// if some files are filtered out due to size constraints
|
||||
if sc.diskgc.Size() != 17 {
|
||||
t.Errorf("Size failed: got %d, want %d", sc.diskgc.Size(), 17)
|
||||
t.Errorf("Disk size failed: got %d, want %d", sc.diskgc.Size(), 17)
|
||||
}
|
||||
|
||||
if sc.vfs.Size() != 17 {
|
||||
t.Errorf("Size failed: got %d, want %d", sc.vfs.Size(), 17)
|
||||
if sc.vfs.Size() < 17 {
|
||||
t.Errorf("Total size too small: got %d, want at least 17", sc.vfs.Size())
|
||||
}
|
||||
if sc.vfs.Size() > 34 {
|
||||
t.Errorf("Total size too large: got %d, want at most 34", sc.vfs.Size())
|
||||
}
|
||||
|
||||
// First ensure the file is indexed by opening it
|
||||
rc, err = sc.vfs.Open("key2")
|
||||
if err != nil {
|
||||
t.Errorf("Open key2 failed: %v", err)
|
||||
}
|
||||
rc.Close()
|
||||
|
||||
// Give promotion goroutine time to complete before deleting
|
||||
time.Sleep(100 * time.Millisecond)
|
||||
|
||||
sc.memory.Delete("key2")
|
||||
os.Remove(filepath.Join(td, "key2"))
|
||||
sc.disk.Delete("key2") // Also delete from disk cache
|
||||
|
||||
if _, err := sc.vfs.Open("key2"); err == nil {
|
||||
t.Errorf("Open failed: got nil, want error")
|
||||
@@ -111,7 +132,8 @@ func TestCacheMissAndHit(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestURLHashing(t *testing.T) {
|
||||
// Test the new SHA256-based cache key generation
|
||||
// Test the SHA256-based cache key generation for Steam client requests
|
||||
// The "steam/" prefix indicates the request came from a Steam client (User-Agent based)
|
||||
|
||||
testCases := []struct {
|
||||
input string
|
||||
@@ -129,40 +151,188 @@ func TestURLHashing(t *testing.T) {
|
||||
shouldCache: true,
|
||||
},
|
||||
{
|
||||
input: "/depot/invalid/path",
|
||||
desc: "invalid depot URL format",
|
||||
shouldCache: true, // Still gets hashed, just not a proper Steam format
|
||||
input: "/appinfo/123456",
|
||||
desc: "app info URL",
|
||||
shouldCache: true,
|
||||
},
|
||||
{
|
||||
input: "/some/other/path",
|
||||
desc: "non-Steam URL",
|
||||
shouldCache: false, // Not cached
|
||||
desc: "any URL from Steam client",
|
||||
shouldCache: true, // All URLs from Steam clients (detected via User-Agent) are cached
|
||||
},
|
||||
}
|
||||
|
||||
for _, tc := range testCases {
|
||||
t.Run(tc.desc, func(t *testing.T) {
|
||||
result := generateSteamCacheKey(tc.input)
|
||||
result := generateServiceCacheKey(tc.input, "steam")
|
||||
|
||||
if tc.shouldCache {
|
||||
// Should return a cache key with "steam/" prefix
|
||||
if !strings.HasPrefix(result, "steam/") {
|
||||
t.Errorf("generateSteamCacheKey(%s) = %s, expected steam/ prefix", tc.input, result)
|
||||
t.Errorf("generateServiceCacheKey(%s, \"steam\") = %s, expected steam/ prefix", tc.input, result)
|
||||
}
|
||||
// Should be exactly 70 characters (6 for "steam/" + 64 for SHA256 hex)
|
||||
if len(result) != 70 {
|
||||
t.Errorf("generateSteamCacheKey(%s) length = %d, expected 70", tc.input, len(result))
|
||||
t.Errorf("generateServiceCacheKey(%s, \"steam\") length = %d, expected 70", tc.input, len(result))
|
||||
}
|
||||
} else {
|
||||
// Should return empty string for non-Steam URLs
|
||||
if result != "" {
|
||||
t.Errorf("generateSteamCacheKey(%s) = %s, expected empty string", tc.input, result)
|
||||
t.Errorf("generateServiceCacheKey(%s, \"steam\") = %s, expected empty string", tc.input, result)
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestServiceDetection(t *testing.T) {
|
||||
// Create a service manager for testing
|
||||
sm := NewServiceManager()
|
||||
|
||||
testCases := []struct {
|
||||
userAgent string
|
||||
expectedName string
|
||||
expectedFound bool
|
||||
desc string
|
||||
}{
|
||||
{
|
||||
userAgent: "Valve/Steam HTTP Client 1.0",
|
||||
expectedName: "steam",
|
||||
expectedFound: true,
|
||||
desc: "Valve Steam HTTP Client",
|
||||
},
|
||||
{
|
||||
userAgent: "Steam",
|
||||
expectedName: "steam",
|
||||
expectedFound: true,
|
||||
desc: "Simple Steam user agent",
|
||||
},
|
||||
{
|
||||
userAgent: "SteamClient/1.0",
|
||||
expectedName: "steam",
|
||||
expectedFound: true,
|
||||
desc: "SteamClient with version",
|
||||
},
|
||||
{
|
||||
userAgent: "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36",
|
||||
expectedName: "",
|
||||
expectedFound: false,
|
||||
desc: "Browser user agent",
|
||||
},
|
||||
{
|
||||
userAgent: "",
|
||||
expectedName: "",
|
||||
expectedFound: false,
|
||||
desc: "Empty user agent",
|
||||
},
|
||||
{
|
||||
userAgent: "curl/7.68.0",
|
||||
expectedName: "",
|
||||
expectedFound: false,
|
||||
desc: "curl user agent",
|
||||
},
|
||||
}
|
||||
|
||||
for _, tc := range testCases {
|
||||
t.Run(tc.desc, func(t *testing.T) {
|
||||
service, found := sm.DetectService(tc.userAgent)
|
||||
|
||||
if found != tc.expectedFound {
|
||||
t.Errorf("DetectService(%s) found = %v, expected %v", tc.userAgent, found, tc.expectedFound)
|
||||
}
|
||||
|
||||
if found && service.Name != tc.expectedName {
|
||||
t.Errorf("DetectService(%s) service name = %s, expected %s", tc.userAgent, service.Name, tc.expectedName)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestServiceManagerExpandability(t *testing.T) {
|
||||
// Create a service manager for testing
|
||||
sm := NewServiceManager()
|
||||
|
||||
// Test adding a new service (Epic Games)
|
||||
epicConfig := &ServiceConfig{
|
||||
Name: "epic",
|
||||
Prefix: "epic",
|
||||
UserAgents: []string{
|
||||
`EpicGamesLauncher`,
|
||||
`EpicGames`,
|
||||
`Epic.*Launcher`,
|
||||
},
|
||||
}
|
||||
|
||||
err := sm.AddService(epicConfig)
|
||||
if err != nil {
|
||||
t.Fatalf("Failed to add Epic service: %v", err)
|
||||
}
|
||||
|
||||
// Test Epic Games detection
|
||||
epicTestCases := []struct {
|
||||
userAgent string
|
||||
expectedName string
|
||||
expectedFound bool
|
||||
desc string
|
||||
}{
|
||||
{
|
||||
userAgent: "EpicGamesLauncher/1.0",
|
||||
expectedName: "epic",
|
||||
expectedFound: true,
|
||||
desc: "Epic Games Launcher",
|
||||
},
|
||||
{
|
||||
userAgent: "EpicGames/2.0",
|
||||
expectedName: "epic",
|
||||
expectedFound: true,
|
||||
desc: "Epic Games client",
|
||||
},
|
||||
{
|
||||
userAgent: "Epic Launcher 1.5",
|
||||
expectedName: "epic",
|
||||
expectedFound: true,
|
||||
desc: "Epic Launcher with regex match",
|
||||
},
|
||||
{
|
||||
userAgent: "Steam",
|
||||
expectedName: "steam",
|
||||
expectedFound: true,
|
||||
desc: "Steam should still work",
|
||||
},
|
||||
{
|
||||
userAgent: "Mozilla/5.0",
|
||||
expectedName: "",
|
||||
expectedFound: false,
|
||||
desc: "Browser should not match any service",
|
||||
},
|
||||
}
|
||||
|
||||
for _, tc := range epicTestCases {
|
||||
t.Run(tc.desc, func(t *testing.T) {
|
||||
service, found := sm.DetectService(tc.userAgent)
|
||||
|
||||
if found != tc.expectedFound {
|
||||
t.Errorf("DetectService(%s) found = %v, expected %v", tc.userAgent, found, tc.expectedFound)
|
||||
}
|
||||
|
||||
if found && service.Name != tc.expectedName {
|
||||
t.Errorf("DetectService(%s) service name = %s, expected %s", tc.userAgent, service.Name, tc.expectedName)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
// Test cache key generation for different services
|
||||
steamKey := generateServiceCacheKey("/depot/123/chunk/abc", "steam")
|
||||
epicKey := generateServiceCacheKey("/epic/123/chunk/abc", "epic")
|
||||
|
||||
if !strings.HasPrefix(steamKey, "steam/") {
|
||||
t.Errorf("Steam cache key should start with 'steam/', got: %s", steamKey)
|
||||
}
|
||||
if !strings.HasPrefix(epicKey, "epic/") {
|
||||
t.Errorf("Epic cache key should start with 'epic/', got: %s", epicKey)
|
||||
}
|
||||
}
|
||||
|
||||
// Removed hash calculation tests since we switched to lightweight validation
|
||||
|
||||
func TestSteamKeySharding(t *testing.T) {
|
||||
|
||||
0
steamcache/test_cache/.gitkeep
Normal file
0
steamcache/test_cache/.gitkeep
Normal file
273
vfs/adaptive/adaptive.go
Normal file
273
vfs/adaptive/adaptive.go
Normal file
@@ -0,0 +1,273 @@
|
||||
package adaptive
|
||||
|
||||
import (
|
||||
"context"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"time"
|
||||
)
|
||||
|
||||
// WorkloadPattern represents different types of workload patterns
|
||||
type WorkloadPattern int
|
||||
|
||||
const (
|
||||
PatternUnknown WorkloadPattern = iota
|
||||
PatternSequential // Sequential file access (e.g., game installation)
|
||||
PatternRandom // Random file access (e.g., game updates)
|
||||
PatternBurst // Burst access (e.g., multiple users downloading same game)
|
||||
PatternSteady // Steady access (e.g., popular games being accessed regularly)
|
||||
)
|
||||
|
||||
// CacheStrategy represents different caching strategies
|
||||
type CacheStrategy int
|
||||
|
||||
const (
|
||||
StrategyLRU CacheStrategy = iota
|
||||
StrategyLFU
|
||||
StrategySizeBased
|
||||
StrategyHybrid
|
||||
StrategyPredictive
|
||||
)
|
||||
|
||||
// WorkloadAnalyzer analyzes access patterns to determine optimal caching strategies
|
||||
type WorkloadAnalyzer struct {
|
||||
accessHistory map[string]*AccessInfo
|
||||
patternCounts map[WorkloadPattern]int64
|
||||
mu sync.RWMutex
|
||||
analysisInterval time.Duration
|
||||
ctx context.Context
|
||||
cancel context.CancelFunc
|
||||
}
|
||||
|
||||
// AccessInfo tracks access patterns for individual files
|
||||
type AccessInfo struct {
|
||||
Key string
|
||||
AccessCount int64
|
||||
LastAccess time.Time
|
||||
FirstAccess time.Time
|
||||
AccessTimes []time.Time
|
||||
Size int64
|
||||
AccessPattern WorkloadPattern
|
||||
mu sync.RWMutex
|
||||
}
|
||||
|
||||
// AdaptiveCacheManager manages adaptive caching strategies
|
||||
type AdaptiveCacheManager struct {
|
||||
analyzer *WorkloadAnalyzer
|
||||
currentStrategy CacheStrategy
|
||||
adaptationCount int64
|
||||
mu sync.RWMutex
|
||||
}
|
||||
|
||||
// NewWorkloadAnalyzer creates a new workload analyzer
|
||||
func NewWorkloadAnalyzer(analysisInterval time.Duration) *WorkloadAnalyzer {
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
|
||||
analyzer := &WorkloadAnalyzer{
|
||||
accessHistory: make(map[string]*AccessInfo),
|
||||
patternCounts: make(map[WorkloadPattern]int64),
|
||||
analysisInterval: analysisInterval,
|
||||
ctx: ctx,
|
||||
cancel: cancel,
|
||||
}
|
||||
|
||||
// Start background analysis with much longer interval to reduce overhead
|
||||
go analyzer.analyzePatterns()
|
||||
|
||||
return analyzer
|
||||
}
|
||||
|
||||
// RecordAccess records a file access for pattern analysis (lightweight version)
|
||||
func (wa *WorkloadAnalyzer) RecordAccess(key string, size int64) {
|
||||
// Use read lock first for better performance
|
||||
wa.mu.RLock()
|
||||
info, exists := wa.accessHistory[key]
|
||||
wa.mu.RUnlock()
|
||||
|
||||
if !exists {
|
||||
// Only acquire write lock when creating new entry
|
||||
wa.mu.Lock()
|
||||
// Double-check after acquiring write lock
|
||||
if _, exists = wa.accessHistory[key]; !exists {
|
||||
info = &AccessInfo{
|
||||
Key: key,
|
||||
AccessCount: 1,
|
||||
LastAccess: time.Now(),
|
||||
FirstAccess: time.Now(),
|
||||
AccessTimes: []time.Time{time.Now()},
|
||||
Size: size,
|
||||
}
|
||||
wa.accessHistory[key] = info
|
||||
}
|
||||
wa.mu.Unlock()
|
||||
} else {
|
||||
// Lightweight update - just increment counter and update timestamp
|
||||
info.mu.Lock()
|
||||
info.AccessCount++
|
||||
info.LastAccess = time.Now()
|
||||
// Only keep last 10 access times to reduce memory overhead
|
||||
if len(info.AccessTimes) > 10 {
|
||||
info.AccessTimes = info.AccessTimes[len(info.AccessTimes)-10:]
|
||||
} else {
|
||||
info.AccessTimes = append(info.AccessTimes, time.Now())
|
||||
}
|
||||
info.mu.Unlock()
|
||||
}
|
||||
}
|
||||
|
||||
// analyzePatterns analyzes access patterns in the background
|
||||
func (wa *WorkloadAnalyzer) analyzePatterns() {
|
||||
ticker := time.NewTicker(wa.analysisInterval)
|
||||
defer ticker.Stop()
|
||||
|
||||
for {
|
||||
select {
|
||||
case <-wa.ctx.Done():
|
||||
return
|
||||
case <-ticker.C:
|
||||
wa.performAnalysis()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// performAnalysis analyzes current access patterns
|
||||
func (wa *WorkloadAnalyzer) performAnalysis() {
|
||||
wa.mu.Lock()
|
||||
defer wa.mu.Unlock()
|
||||
|
||||
// Reset pattern counts
|
||||
wa.patternCounts = make(map[WorkloadPattern]int64)
|
||||
|
||||
now := time.Now()
|
||||
cutoff := now.Add(-wa.analysisInterval * 2) // Analyze last 2 intervals
|
||||
|
||||
for _, info := range wa.accessHistory {
|
||||
info.mu.RLock()
|
||||
if info.LastAccess.After(cutoff) {
|
||||
pattern := wa.determinePattern(info)
|
||||
info.AccessPattern = pattern
|
||||
wa.patternCounts[pattern]++
|
||||
}
|
||||
info.mu.RUnlock()
|
||||
}
|
||||
}
|
||||
|
||||
// determinePattern determines the access pattern for a file
|
||||
func (wa *WorkloadAnalyzer) determinePattern(info *AccessInfo) WorkloadPattern {
|
||||
if len(info.AccessTimes) < 3 {
|
||||
return PatternUnknown
|
||||
}
|
||||
|
||||
// Analyze access timing patterns
|
||||
intervals := make([]time.Duration, len(info.AccessTimes)-1)
|
||||
for i := 1; i < len(info.AccessTimes); i++ {
|
||||
intervals[i-1] = info.AccessTimes[i].Sub(info.AccessTimes[i-1])
|
||||
}
|
||||
|
||||
// Calculate variance in access intervals
|
||||
var sum, sumSquares time.Duration
|
||||
for _, interval := range intervals {
|
||||
sum += interval
|
||||
sumSquares += interval * interval
|
||||
}
|
||||
|
||||
avg := sum / time.Duration(len(intervals))
|
||||
variance := (sumSquares / time.Duration(len(intervals))) - (avg * avg)
|
||||
|
||||
// Determine pattern based on variance and access count
|
||||
if info.AccessCount > 10 && variance < time.Minute {
|
||||
return PatternBurst
|
||||
} else if info.AccessCount > 5 && variance < time.Hour {
|
||||
return PatternSteady
|
||||
} else if variance < time.Minute*5 {
|
||||
return PatternSequential
|
||||
} else {
|
||||
return PatternRandom
|
||||
}
|
||||
}
|
||||
|
||||
// GetDominantPattern returns the most common access pattern
|
||||
func (wa *WorkloadAnalyzer) GetDominantPattern() WorkloadPattern {
|
||||
wa.mu.RLock()
|
||||
defer wa.mu.RUnlock()
|
||||
|
||||
var maxCount int64
|
||||
var dominantPattern WorkloadPattern
|
||||
|
||||
for pattern, count := range wa.patternCounts {
|
||||
if count > maxCount {
|
||||
maxCount = count
|
||||
dominantPattern = pattern
|
||||
}
|
||||
}
|
||||
|
||||
return dominantPattern
|
||||
}
|
||||
|
||||
// GetAccessInfo returns access information for a key
|
||||
func (wa *WorkloadAnalyzer) GetAccessInfo(key string) *AccessInfo {
|
||||
wa.mu.RLock()
|
||||
defer wa.mu.RUnlock()
|
||||
|
||||
return wa.accessHistory[key]
|
||||
}
|
||||
|
||||
// Stop stops the workload analyzer
|
||||
func (wa *WorkloadAnalyzer) Stop() {
|
||||
wa.cancel()
|
||||
}
|
||||
|
||||
// NewAdaptiveCacheManager creates a new adaptive cache manager
|
||||
func NewAdaptiveCacheManager(analysisInterval time.Duration) *AdaptiveCacheManager {
|
||||
return &AdaptiveCacheManager{
|
||||
analyzer: NewWorkloadAnalyzer(analysisInterval),
|
||||
currentStrategy: StrategyLRU, // Start with LRU
|
||||
}
|
||||
}
|
||||
|
||||
// AdaptStrategy adapts the caching strategy based on workload patterns
|
||||
func (acm *AdaptiveCacheManager) AdaptStrategy() CacheStrategy {
|
||||
acm.mu.Lock()
|
||||
defer acm.mu.Unlock()
|
||||
|
||||
dominantPattern := acm.analyzer.GetDominantPattern()
|
||||
|
||||
// Adapt strategy based on dominant pattern
|
||||
switch dominantPattern {
|
||||
case PatternBurst:
|
||||
acm.currentStrategy = StrategyLFU // LFU is good for burst patterns
|
||||
case PatternSteady:
|
||||
acm.currentStrategy = StrategyHybrid // Hybrid for steady patterns
|
||||
case PatternSequential:
|
||||
acm.currentStrategy = StrategySizeBased // Size-based for sequential
|
||||
case PatternRandom:
|
||||
acm.currentStrategy = StrategyLRU // LRU for random patterns
|
||||
default:
|
||||
acm.currentStrategy = StrategyLRU // Default to LRU
|
||||
}
|
||||
|
||||
atomic.AddInt64(&acm.adaptationCount, 1)
|
||||
return acm.currentStrategy
|
||||
}
|
||||
|
||||
// GetCurrentStrategy returns the current caching strategy
|
||||
func (acm *AdaptiveCacheManager) GetCurrentStrategy() CacheStrategy {
|
||||
acm.mu.RLock()
|
||||
defer acm.mu.RUnlock()
|
||||
return acm.currentStrategy
|
||||
}
|
||||
|
||||
// RecordAccess records a file access for analysis
|
||||
func (acm *AdaptiveCacheManager) RecordAccess(key string, size int64) {
|
||||
acm.analyzer.RecordAccess(key, size)
|
||||
}
|
||||
|
||||
// GetAdaptationCount returns the number of strategy adaptations
|
||||
func (acm *AdaptiveCacheManager) GetAdaptationCount() int64 {
|
||||
return atomic.LoadInt64(&acm.adaptationCount)
|
||||
}
|
||||
|
||||
// Stop stops the adaptive cache manager
|
||||
func (acm *AdaptiveCacheManager) Stop() {
|
||||
acm.analyzer.Stop()
|
||||
}
|
||||
193
vfs/cache/cache.go
vendored
193
vfs/cache/cache.go
vendored
@@ -3,71 +3,84 @@ package cache
|
||||
|
||||
import (
|
||||
"io"
|
||||
"s1d3sw1ped/SteamCache2/vfs"
|
||||
"s1d3sw1ped/SteamCache2/vfs/vfserror"
|
||||
"sync"
|
||||
"s1d3sw1ped/steamcache2/vfs"
|
||||
"s1d3sw1ped/steamcache2/vfs/vfserror"
|
||||
"sync/atomic"
|
||||
)
|
||||
|
||||
// TieredCache implements a two-tier cache with fast (memory) and slow (disk) storage
|
||||
// TieredCache implements a lock-free two-tier cache for better concurrency
|
||||
type TieredCache struct {
|
||||
fast vfs.VFS // Memory cache (fast)
|
||||
slow vfs.VFS // Disk cache (slow)
|
||||
|
||||
mu sync.RWMutex
|
||||
fast *atomic.Value // Memory cache (fast) - atomic.Value for lock-free access
|
||||
slow *atomic.Value // Disk cache (slow) - atomic.Value for lock-free access
|
||||
}
|
||||
|
||||
// New creates a new tiered cache
|
||||
func New() *TieredCache {
|
||||
return &TieredCache{}
|
||||
return &TieredCache{
|
||||
fast: &atomic.Value{},
|
||||
slow: &atomic.Value{},
|
||||
}
|
||||
}
|
||||
|
||||
// SetFast sets the fast (memory) tier
|
||||
// SetFast sets the fast (memory) tier atomically
|
||||
func (tc *TieredCache) SetFast(vfs vfs.VFS) {
|
||||
tc.mu.Lock()
|
||||
defer tc.mu.Unlock()
|
||||
tc.fast = vfs
|
||||
tc.fast.Store(vfs)
|
||||
}
|
||||
|
||||
// SetSlow sets the slow (disk) tier
|
||||
// SetSlow sets the slow (disk) tier atomically
|
||||
func (tc *TieredCache) SetSlow(vfs vfs.VFS) {
|
||||
tc.mu.Lock()
|
||||
defer tc.mu.Unlock()
|
||||
tc.slow = vfs
|
||||
tc.slow.Store(vfs)
|
||||
}
|
||||
|
||||
// Create creates a new file, preferring the slow tier for persistence testing
|
||||
// Create creates a new file, preferring the slow tier for persistence
|
||||
func (tc *TieredCache) Create(key string, size int64) (io.WriteCloser, error) {
|
||||
tc.mu.RLock()
|
||||
defer tc.mu.RUnlock()
|
||||
|
||||
// Try slow tier first (disk) for better testability
|
||||
if tc.slow != nil {
|
||||
return tc.slow.Create(key, size)
|
||||
if slow := tc.slow.Load(); slow != nil {
|
||||
if vfs, ok := slow.(vfs.VFS); ok {
|
||||
return vfs.Create(key, size)
|
||||
}
|
||||
}
|
||||
|
||||
// Fall back to fast tier (memory)
|
||||
if tc.fast != nil {
|
||||
return tc.fast.Create(key, size)
|
||||
if fast := tc.fast.Load(); fast != nil {
|
||||
if vfs, ok := fast.(vfs.VFS); ok {
|
||||
return vfs.Create(key, size)
|
||||
}
|
||||
}
|
||||
|
||||
return nil, vfserror.ErrNotFound
|
||||
}
|
||||
|
||||
// Open opens a file, checking fast tier first, then slow tier
|
||||
// Open opens a file, checking fast tier first, then slow tier with promotion
|
||||
func (tc *TieredCache) Open(key string) (io.ReadCloser, error) {
|
||||
tc.mu.RLock()
|
||||
defer tc.mu.RUnlock()
|
||||
|
||||
// Try fast tier first (memory)
|
||||
if tc.fast != nil {
|
||||
if reader, err := tc.fast.Open(key); err == nil {
|
||||
if fast := tc.fast.Load(); fast != nil {
|
||||
if vfs, ok := fast.(vfs.VFS); ok {
|
||||
if reader, err := vfs.Open(key); err == nil {
|
||||
return reader, nil
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Fall back to slow tier (disk)
|
||||
if tc.slow != nil {
|
||||
return tc.slow.Open(key)
|
||||
// Fall back to slow tier (disk) and promote to fast tier
|
||||
if slow := tc.slow.Load(); slow != nil {
|
||||
if vfs, ok := slow.(vfs.VFS); ok {
|
||||
reader, err := vfs.Open(key)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// If we have both tiers, promote the file to fast tier
|
||||
if fast := tc.fast.Load(); fast != nil {
|
||||
// Create a new reader for promotion to avoid interfering with the returned reader
|
||||
promotionReader, err := vfs.Open(key)
|
||||
if err == nil {
|
||||
go tc.promoteToFast(key, promotionReader)
|
||||
}
|
||||
}
|
||||
|
||||
return reader, nil
|
||||
}
|
||||
}
|
||||
|
||||
return nil, vfserror.ErrNotFound
|
||||
@@ -75,43 +88,45 @@ func (tc *TieredCache) Open(key string) (io.ReadCloser, error) {
|
||||
|
||||
// Delete removes a file from all tiers
|
||||
func (tc *TieredCache) Delete(key string) error {
|
||||
tc.mu.RLock()
|
||||
defer tc.mu.RUnlock()
|
||||
|
||||
var lastErr error
|
||||
|
||||
// Delete from fast tier
|
||||
if tc.fast != nil {
|
||||
if err := tc.fast.Delete(key); err != nil {
|
||||
if fast := tc.fast.Load(); fast != nil {
|
||||
if vfs, ok := fast.(vfs.VFS); ok {
|
||||
if err := vfs.Delete(key); err != nil {
|
||||
lastErr = err
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Delete from slow tier
|
||||
if tc.slow != nil {
|
||||
if err := tc.slow.Delete(key); err != nil {
|
||||
if slow := tc.slow.Load(); slow != nil {
|
||||
if vfs, ok := slow.(vfs.VFS); ok {
|
||||
if err := vfs.Delete(key); err != nil {
|
||||
lastErr = err
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return lastErr
|
||||
}
|
||||
|
||||
// Stat returns file information, checking fast tier first
|
||||
func (tc *TieredCache) Stat(key string) (*vfs.FileInfo, error) {
|
||||
tc.mu.RLock()
|
||||
defer tc.mu.RUnlock()
|
||||
|
||||
// Try fast tier first (memory)
|
||||
if tc.fast != nil {
|
||||
if info, err := tc.fast.Stat(key); err == nil {
|
||||
if fast := tc.fast.Load(); fast != nil {
|
||||
if vfs, ok := fast.(vfs.VFS); ok {
|
||||
if info, err := vfs.Stat(key); err == nil {
|
||||
return info, nil
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Fall back to slow tier (disk)
|
||||
if tc.slow != nil {
|
||||
return tc.slow.Stat(key)
|
||||
if slow := tc.slow.Load(); slow != nil {
|
||||
if vfs, ok := slow.(vfs.VFS); ok {
|
||||
return vfs.Stat(key)
|
||||
}
|
||||
}
|
||||
|
||||
return nil, vfserror.ErrNotFound
|
||||
@@ -124,30 +139,84 @@ func (tc *TieredCache) Name() string {
|
||||
|
||||
// Size returns the total size across all tiers
|
||||
func (tc *TieredCache) Size() int64 {
|
||||
tc.mu.RLock()
|
||||
defer tc.mu.RUnlock()
|
||||
|
||||
var total int64
|
||||
if tc.fast != nil {
|
||||
total += tc.fast.Size()
|
||||
|
||||
if fast := tc.fast.Load(); fast != nil {
|
||||
if vfs, ok := fast.(vfs.VFS); ok {
|
||||
total += vfs.Size()
|
||||
}
|
||||
if tc.slow != nil {
|
||||
total += tc.slow.Size()
|
||||
}
|
||||
|
||||
if slow := tc.slow.Load(); slow != nil {
|
||||
if vfs, ok := slow.(vfs.VFS); ok {
|
||||
total += vfs.Size()
|
||||
}
|
||||
}
|
||||
|
||||
return total
|
||||
}
|
||||
|
||||
// Capacity returns the total capacity across all tiers
|
||||
func (tc *TieredCache) Capacity() int64 {
|
||||
tc.mu.RLock()
|
||||
defer tc.mu.RUnlock()
|
||||
|
||||
var total int64
|
||||
if tc.fast != nil {
|
||||
total += tc.fast.Capacity()
|
||||
|
||||
if fast := tc.fast.Load(); fast != nil {
|
||||
if vfs, ok := fast.(vfs.VFS); ok {
|
||||
total += vfs.Capacity()
|
||||
}
|
||||
if tc.slow != nil {
|
||||
total += tc.slow.Capacity()
|
||||
}
|
||||
|
||||
if slow := tc.slow.Load(); slow != nil {
|
||||
if vfs, ok := slow.(vfs.VFS); ok {
|
||||
total += vfs.Capacity()
|
||||
}
|
||||
}
|
||||
|
||||
return total
|
||||
}
|
||||
|
||||
// promoteToFast promotes a file from slow tier to fast tier
|
||||
func (tc *TieredCache) promoteToFast(key string, reader io.ReadCloser) {
|
||||
defer reader.Close()
|
||||
|
||||
// Get file info from slow tier to determine size
|
||||
var size int64
|
||||
if slow := tc.slow.Load(); slow != nil {
|
||||
if vfs, ok := slow.(vfs.VFS); ok {
|
||||
if info, err := vfs.Stat(key); err == nil {
|
||||
size = info.Size
|
||||
} else {
|
||||
return // Skip promotion if we can't get file info
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Check if file fits in available memory cache space
|
||||
if fast := tc.fast.Load(); fast != nil {
|
||||
if vfs, ok := fast.(vfs.VFS); ok {
|
||||
availableSpace := vfs.Capacity() - vfs.Size()
|
||||
// Only promote if file fits in available space (with 10% buffer for safety)
|
||||
if size > int64(float64(availableSpace)*0.9) {
|
||||
return // Skip promotion if file is too large
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Read the entire file content
|
||||
content, err := io.ReadAll(reader)
|
||||
if err != nil {
|
||||
return // Skip promotion if read fails
|
||||
}
|
||||
|
||||
// Create the file in fast tier
|
||||
if fast := tc.fast.Load(); fast != nil {
|
||||
if vfs, ok := fast.(vfs.VFS); ok {
|
||||
writer, err := vfs.Create(key, size)
|
||||
if err == nil {
|
||||
// Write content to fast tier
|
||||
writer.Write(content)
|
||||
writer.Close()
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
422
vfs/disk/disk.go
422
vfs/disk/disk.go
@@ -2,17 +2,19 @@
|
||||
package disk
|
||||
|
||||
import (
|
||||
"container/list"
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"s1d3sw1ped/SteamCache2/steamcache/logger"
|
||||
"s1d3sw1ped/SteamCache2/vfs"
|
||||
"s1d3sw1ped/SteamCache2/vfs/vfserror"
|
||||
"s1d3sw1ped/steamcache2/steamcache/logger"
|
||||
"s1d3sw1ped/steamcache2/vfs"
|
||||
"s1d3sw1ped/steamcache2/vfs/locks"
|
||||
"s1d3sw1ped/steamcache2/vfs/lru"
|
||||
"s1d3sw1ped/steamcache2/vfs/vfserror"
|
||||
"sort"
|
||||
"strings"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"time"
|
||||
|
||||
"github.com/docker/go-units"
|
||||
@@ -31,55 +33,10 @@ type DiskFS struct {
|
||||
size int64
|
||||
mu sync.RWMutex
|
||||
keyLocks []sync.Map // Sharded lock pools for better concurrency
|
||||
LRU *lruList
|
||||
LRU *lru.LRUList[*vfs.FileInfo]
|
||||
timeUpdater *vfs.BatchedTimeUpdate // Batched time updates for better performance
|
||||
}
|
||||
|
||||
// Number of lock shards for reducing contention
|
||||
const numLockShards = 32
|
||||
|
||||
// lruList for time-decayed LRU eviction
|
||||
type lruList struct {
|
||||
list *list.List
|
||||
elem map[string]*list.Element
|
||||
}
|
||||
|
||||
func newLruList() *lruList {
|
||||
return &lruList{
|
||||
list: list.New(),
|
||||
elem: make(map[string]*list.Element),
|
||||
}
|
||||
}
|
||||
|
||||
func (l *lruList) Add(key string, fi *vfs.FileInfo) {
|
||||
elem := l.list.PushFront(fi)
|
||||
l.elem[key] = elem
|
||||
}
|
||||
|
||||
func (l *lruList) MoveToFront(key string, timeUpdater *vfs.BatchedTimeUpdate) {
|
||||
if elem, exists := l.elem[key]; exists {
|
||||
l.list.MoveToFront(elem)
|
||||
// Update the FileInfo in the element with new access time
|
||||
if fi := elem.Value.(*vfs.FileInfo); fi != nil {
|
||||
fi.UpdateAccessBatched(timeUpdater)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (l *lruList) Remove(key string) *vfs.FileInfo {
|
||||
if elem, exists := l.elem[key]; exists {
|
||||
delete(l.elem, key)
|
||||
if fi := l.list.Remove(elem).(*vfs.FileInfo); fi != nil {
|
||||
return fi
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (l *lruList) Len() int {
|
||||
return l.list.Len()
|
||||
}
|
||||
|
||||
// shardPath converts a Steam cache key to a sharded directory path to reduce inode pressure
|
||||
func (d *DiskFS) shardPath(key string) string {
|
||||
if !strings.HasPrefix(key, "steam/") {
|
||||
@@ -104,43 +61,6 @@ func (d *DiskFS) shardPath(key string) string {
|
||||
return filepath.Join("steam", shard1, shard2, hashPart)
|
||||
}
|
||||
|
||||
// extractKeyFromPath reverses the sharding logic to get the original key from a sharded path
|
||||
func (d *DiskFS) extractKeyFromPath(path string) string {
|
||||
// Fast path: if no slashes, it's not a sharded path
|
||||
if !strings.Contains(path, "/") {
|
||||
return path
|
||||
}
|
||||
|
||||
parts := strings.SplitN(path, "/", 5)
|
||||
numParts := len(parts)
|
||||
|
||||
if numParts >= 4 && parts[0] == "steam" {
|
||||
lastThree := parts[numParts-3:]
|
||||
shard1 := lastThree[0]
|
||||
shard2 := lastThree[1]
|
||||
filename := lastThree[2]
|
||||
|
||||
// Verify sharding is correct
|
||||
if len(filename) >= 4 && filename[:2] == shard1 && filename[2:4] == shard2 {
|
||||
return "steam/" + filename
|
||||
}
|
||||
}
|
||||
|
||||
// Handle single-level sharding for short hashes: steam/shard1/filename
|
||||
if numParts >= 3 && parts[0] == "steam" {
|
||||
lastTwo := parts[numParts-2:]
|
||||
shard1 := lastTwo[0]
|
||||
filename := lastTwo[1]
|
||||
|
||||
if len(filename) >= 2 && filename[:2] == shard1 {
|
||||
return "steam/" + filename
|
||||
}
|
||||
}
|
||||
|
||||
// Fallback: return as-is for any unrecognized format
|
||||
return path
|
||||
}
|
||||
|
||||
// New creates a new DiskFS.
|
||||
func New(root string, capacity int64) *DiskFS {
|
||||
if capacity <= 0 {
|
||||
@@ -151,7 +71,7 @@ func New(root string, capacity int64) *DiskFS {
|
||||
os.MkdirAll(root, 0755)
|
||||
|
||||
// Initialize sharded locks
|
||||
keyLocks := make([]sync.Map, numLockShards)
|
||||
keyLocks := make([]sync.Map, locks.NumLockShards)
|
||||
|
||||
d := &DiskFS{
|
||||
root: root,
|
||||
@@ -159,7 +79,7 @@ func New(root string, capacity int64) *DiskFS {
|
||||
capacity: capacity,
|
||||
size: 0,
|
||||
keyLocks: keyLocks,
|
||||
LRU: newLruList(),
|
||||
LRU: lru.NewLRUList[*vfs.FileInfo](),
|
||||
timeUpdater: vfs.NewBatchedTimeUpdate(100 * time.Millisecond), // Update time every 100ms
|
||||
}
|
||||
|
||||
@@ -167,53 +87,15 @@ func New(root string, capacity int64) *DiskFS {
|
||||
return d
|
||||
}
|
||||
|
||||
// init loads existing files from disk and migrates legacy depot files to sharded structure
|
||||
// init loads existing files from disk with ultra-fast lazy initialization
|
||||
func (d *DiskFS) init() {
|
||||
tstart := time.Now()
|
||||
|
||||
var depotFiles []string // Track depot files that need migration
|
||||
// Ultra-fast initialization: only scan directory structure, defer file stats
|
||||
d.scanDirectoriesOnly()
|
||||
|
||||
err := filepath.Walk(d.root, func(npath string, info os.FileInfo, err error) error {
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if info.IsDir() {
|
||||
return nil
|
||||
}
|
||||
|
||||
d.mu.Lock()
|
||||
// Extract key from sharded path: remove root and convert sharding back
|
||||
relPath := strings.ReplaceAll(npath[len(d.root)+1:], "\\", "/")
|
||||
|
||||
// Extract the original key from the sharded path
|
||||
k := d.extractKeyFromPath(relPath)
|
||||
|
||||
fi := vfs.NewFileInfoFromOS(info, k)
|
||||
d.info[k] = fi
|
||||
d.LRU.Add(k, fi)
|
||||
// Initialize access time with file modification time
|
||||
fi.UpdateAccessBatched(d.timeUpdater)
|
||||
d.size += info.Size()
|
||||
|
||||
// Track depot files for potential migration
|
||||
if strings.HasPrefix(relPath, "depot/") {
|
||||
depotFiles = append(depotFiles, relPath)
|
||||
}
|
||||
|
||||
d.mu.Unlock()
|
||||
|
||||
return nil
|
||||
})
|
||||
if err != nil {
|
||||
logger.Logger.Error().Err(err).Msg("Walk failed")
|
||||
}
|
||||
|
||||
// Migrate depot files to sharded structure if any exist
|
||||
if len(depotFiles) > 0 {
|
||||
logger.Logger.Info().Int("count", len(depotFiles)).Msg("Found legacy depot files, starting migration")
|
||||
d.migrateDepotFiles(depotFiles)
|
||||
}
|
||||
// Start background size calculation in a separate goroutine
|
||||
go d.calculateSizeInBackground()
|
||||
|
||||
logger.Logger.Info().
|
||||
Str("name", d.Name()).
|
||||
@@ -225,69 +107,144 @@ func (d *DiskFS) init() {
|
||||
Msg("init")
|
||||
}
|
||||
|
||||
// migrateDepotFiles moves legacy depot files to the sharded steam structure
|
||||
func (d *DiskFS) migrateDepotFiles(depotFiles []string) {
|
||||
migratedCount := 0
|
||||
errorCount := 0
|
||||
|
||||
for _, relPath := range depotFiles {
|
||||
// Extract the steam key from the depot path
|
||||
steamKey := d.extractKeyFromPath(relPath)
|
||||
if !strings.HasPrefix(steamKey, "steam/") {
|
||||
// Skip if we can't extract a proper steam key
|
||||
errorCount++
|
||||
continue
|
||||
}
|
||||
|
||||
// Get the source and destination paths
|
||||
sourcePath := filepath.Join(d.root, relPath)
|
||||
shardedPath := d.shardPath(steamKey)
|
||||
destPath := filepath.Join(d.root, shardedPath)
|
||||
|
||||
// Create destination directory
|
||||
destDir := filepath.Dir(destPath)
|
||||
if err := os.MkdirAll(destDir, 0755); err != nil {
|
||||
logger.Logger.Error().Err(err).Str("path", destDir).Msg("Failed to create migration destination directory")
|
||||
errorCount++
|
||||
continue
|
||||
}
|
||||
|
||||
// Move the file
|
||||
if err := os.Rename(sourcePath, destPath); err != nil {
|
||||
logger.Logger.Error().Err(err).Str("from", sourcePath).Str("to", destPath).Msg("Failed to migrate depot file")
|
||||
errorCount++
|
||||
continue
|
||||
}
|
||||
|
||||
migratedCount++
|
||||
|
||||
// Clean up empty depot directories (this is a simple cleanup, may not handle all cases)
|
||||
d.cleanupEmptyDepotDirs(filepath.Dir(sourcePath))
|
||||
}
|
||||
|
||||
logger.Logger.Info().
|
||||
Int("migrated", migratedCount).
|
||||
Int("errors", errorCount).
|
||||
Msg("Depot file migration completed")
|
||||
// scanDirectoriesOnly performs ultra-fast directory structure scanning without file stats
|
||||
func (d *DiskFS) scanDirectoriesOnly() {
|
||||
// Just ensure the root directory exists and is accessible
|
||||
// No file scanning during init - files will be discovered on-demand
|
||||
logger.Logger.Debug().
|
||||
Str("root", d.root).
|
||||
Msg("Directory structure scan completed (lazy file discovery enabled)")
|
||||
}
|
||||
|
||||
// cleanupEmptyDepotDirs removes empty depot directories after migration
|
||||
func (d *DiskFS) cleanupEmptyDepotDirs(dirPath string) {
|
||||
for dirPath != d.root && strings.HasPrefix(dirPath, filepath.Join(d.root, "depot")) {
|
||||
// calculateSizeInBackground calculates the total size of all files in the background
|
||||
func (d *DiskFS) calculateSizeInBackground() {
|
||||
tstart := time.Now()
|
||||
|
||||
// Channel for collecting file information
|
||||
fileChan := make(chan fileSizeInfo, 1000)
|
||||
|
||||
// Progress tracking
|
||||
var totalFiles int64
|
||||
var processedFiles int64
|
||||
progressTicker := time.NewTicker(2 * time.Second)
|
||||
defer progressTicker.Stop()
|
||||
|
||||
// Wait group for workers
|
||||
var wg sync.WaitGroup
|
||||
|
||||
// Start directory scanner
|
||||
wg.Add(1)
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
defer close(fileChan)
|
||||
d.scanFilesForSize(d.root, fileChan, &totalFiles)
|
||||
}()
|
||||
|
||||
// Collect results with progress reporting
|
||||
var totalSize int64
|
||||
|
||||
// Use a separate goroutine to collect results
|
||||
done := make(chan struct{})
|
||||
go func() {
|
||||
defer close(done)
|
||||
for {
|
||||
select {
|
||||
case fi, ok := <-fileChan:
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
totalSize += fi.size
|
||||
processedFiles++
|
||||
case <-progressTicker.C:
|
||||
if totalFiles > 0 {
|
||||
logger.Logger.Debug().
|
||||
Int64("processed", processedFiles).
|
||||
Int64("total", totalFiles).
|
||||
Int64("size", totalSize).
|
||||
Float64("progress", float64(processedFiles)/float64(totalFiles)*100).
|
||||
Msg("Background size calculation progress")
|
||||
}
|
||||
}
|
||||
}
|
||||
}()
|
||||
|
||||
// Wait for scanning to complete
|
||||
wg.Wait()
|
||||
<-done
|
||||
|
||||
// Update the total size
|
||||
d.mu.Lock()
|
||||
d.size = totalSize
|
||||
d.mu.Unlock()
|
||||
|
||||
logger.Logger.Info().
|
||||
Int64("files_scanned", processedFiles).
|
||||
Int64("total_size", totalSize).
|
||||
Str("duration", time.Since(tstart).String()).
|
||||
Msg("Background size calculation completed")
|
||||
}
|
||||
|
||||
// fileSizeInfo represents a file found during size calculation
|
||||
type fileSizeInfo struct {
|
||||
size int64
|
||||
}
|
||||
|
||||
// scanFilesForSize performs recursive file scanning for size calculation only
|
||||
func (d *DiskFS) scanFilesForSize(dirPath string, fileChan chan<- fileSizeInfo, totalFiles *int64) {
|
||||
// Use ReadDir for faster directory listing
|
||||
entries, err := os.ReadDir(dirPath)
|
||||
if err != nil || len(entries) > 0 {
|
||||
break
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
|
||||
// Directory is empty, remove it
|
||||
if err := os.Remove(dirPath); err != nil {
|
||||
logger.Logger.Error().Err(err).Str("dir", dirPath).Msg("Failed to remove empty depot directory")
|
||||
break
|
||||
// Count files first for progress tracking
|
||||
fileCount := 0
|
||||
for _, entry := range entries {
|
||||
if !entry.IsDir() {
|
||||
fileCount++
|
||||
}
|
||||
}
|
||||
atomic.AddInt64(totalFiles, int64(fileCount))
|
||||
|
||||
// Process entries concurrently with limited workers
|
||||
semaphore := make(chan struct{}, 16) // More workers for size calculation
|
||||
var wg sync.WaitGroup
|
||||
|
||||
for _, entry := range entries {
|
||||
entryPath := filepath.Join(dirPath, entry.Name())
|
||||
|
||||
if entry.IsDir() {
|
||||
// Recursively scan subdirectories
|
||||
wg.Add(1)
|
||||
go func(path string) {
|
||||
defer wg.Done()
|
||||
semaphore <- struct{}{} // Acquire semaphore
|
||||
defer func() { <-semaphore }() // Release semaphore
|
||||
d.scanFilesForSize(path, fileChan, totalFiles)
|
||||
}(entryPath)
|
||||
} else {
|
||||
// Process file for size only
|
||||
wg.Add(1)
|
||||
go func(entry os.DirEntry) {
|
||||
defer wg.Done()
|
||||
semaphore <- struct{}{} // Acquire semaphore
|
||||
defer func() { <-semaphore }() // Release semaphore
|
||||
|
||||
// Get file info for size calculation
|
||||
info, err := entry.Info()
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
|
||||
// Move up to parent directory
|
||||
dirPath = filepath.Dir(dirPath)
|
||||
// Send file size info
|
||||
fileChan <- fileSizeInfo{
|
||||
size: info.Size(),
|
||||
}
|
||||
}(entry)
|
||||
}
|
||||
}
|
||||
|
||||
wg.Wait()
|
||||
}
|
||||
|
||||
// Name returns the name of this VFS
|
||||
@@ -307,24 +264,9 @@ func (d *DiskFS) Capacity() int64 {
|
||||
return d.capacity
|
||||
}
|
||||
|
||||
// getShardIndex returns the shard index for a given key
|
||||
func getShardIndex(key string) int {
|
||||
// Use FNV-1a hash for good distribution
|
||||
var h uint32 = 2166136261 // FNV offset basis
|
||||
for i := 0; i < len(key); i++ {
|
||||
h ^= uint32(key[i])
|
||||
h *= 16777619 // FNV prime
|
||||
}
|
||||
return int(h % numLockShards)
|
||||
}
|
||||
|
||||
// getKeyLock returns a lock for the given key using sharding
|
||||
func (d *DiskFS) getKeyLock(key string) *sync.RWMutex {
|
||||
shardIndex := getShardIndex(key)
|
||||
shard := &d.keyLocks[shardIndex]
|
||||
|
||||
keyLock, _ := shard.LoadOrStore(key, &sync.RWMutex{})
|
||||
return keyLock.(*sync.RWMutex)
|
||||
return locks.GetKeyLock(d.keyLocks, key)
|
||||
}
|
||||
|
||||
// Create creates a new file
|
||||
@@ -376,6 +318,7 @@ func (d *DiskFS) Create(key string, size int64) (io.WriteCloser, error) {
|
||||
d.LRU.Add(key, fi)
|
||||
// Initialize access time with current time
|
||||
fi.UpdateAccessBatched(d.timeUpdater)
|
||||
// Add to size for new files (not discovered files)
|
||||
d.size += size
|
||||
d.mu.Unlock()
|
||||
|
||||
@@ -421,7 +364,7 @@ func (dwc *diskWriteCloser) Close() error {
|
||||
return dwc.file.Close()
|
||||
}
|
||||
|
||||
// Open opens a file for reading
|
||||
// Open opens a file for reading with lazy discovery
|
||||
func (d *DiskFS) Open(key string) (io.ReadCloser, error) {
|
||||
if key == "" {
|
||||
return nil, vfserror.ErrInvalidKey
|
||||
@@ -437,16 +380,22 @@ func (d *DiskFS) Open(key string) (io.ReadCloser, error) {
|
||||
return nil, vfserror.ErrInvalidKey
|
||||
}
|
||||
|
||||
keyMu := d.getKeyLock(key)
|
||||
keyMu.RLock()
|
||||
defer keyMu.RUnlock()
|
||||
|
||||
d.mu.Lock()
|
||||
// First, try to get the file info
|
||||
d.mu.RLock()
|
||||
fi, exists := d.info[key]
|
||||
d.mu.RUnlock()
|
||||
|
||||
if !exists {
|
||||
d.mu.Unlock()
|
||||
return nil, vfserror.ErrNotFound
|
||||
// Try lazy discovery
|
||||
var err error
|
||||
fi, err = d.Stat(key)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
|
||||
// Update access time and LRU
|
||||
d.mu.Lock()
|
||||
fi.UpdateAccessBatched(d.timeUpdater)
|
||||
d.LRU.MoveToFront(key, d.timeUpdater)
|
||||
d.mu.Unlock()
|
||||
@@ -547,7 +496,7 @@ func (d *DiskFS) Delete(key string) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
// Stat returns file information
|
||||
// Stat returns file information with lazy discovery
|
||||
func (d *DiskFS) Stat(key string) (*vfs.FileInfo, error) {
|
||||
if key == "" {
|
||||
return nil, vfserror.ErrInvalidKey
|
||||
@@ -557,30 +506,49 @@ func (d *DiskFS) Stat(key string) (*vfs.FileInfo, error) {
|
||||
}
|
||||
|
||||
keyMu := d.getKeyLock(key)
|
||||
|
||||
// First, try to get the file info with read lock
|
||||
keyMu.RLock()
|
||||
defer keyMu.RUnlock()
|
||||
|
||||
d.mu.RLock()
|
||||
defer d.mu.RUnlock()
|
||||
|
||||
if fi, ok := d.info[key]; ok {
|
||||
d.mu.RUnlock()
|
||||
keyMu.RUnlock()
|
||||
return fi, nil
|
||||
}
|
||||
d.mu.RUnlock()
|
||||
keyMu.RUnlock()
|
||||
|
||||
// Check if file exists on disk but wasn't indexed (for migration)
|
||||
// Lazy discovery: check if file exists on disk and index it
|
||||
shardedPath := d.shardPath(key)
|
||||
path := filepath.Join(d.root, shardedPath)
|
||||
path = strings.ReplaceAll(path, "\\", "/")
|
||||
|
||||
if info, err := os.Stat(path); err == nil {
|
||||
// File exists in sharded location but not indexed, re-index it
|
||||
fi := vfs.NewFileInfoFromOS(info, key)
|
||||
// We can't modify the map here because we're in a read lock
|
||||
// This is a simplified version - in production you'd need to handle this properly
|
||||
info, err := os.Stat(path)
|
||||
if err != nil {
|
||||
return nil, vfserror.ErrNotFound
|
||||
}
|
||||
|
||||
// File exists, add it to the index with write lock
|
||||
keyMu.Lock()
|
||||
defer keyMu.Unlock()
|
||||
|
||||
// Double-check after acquiring write lock
|
||||
d.mu.Lock()
|
||||
if fi, ok := d.info[key]; ok {
|
||||
d.mu.Unlock()
|
||||
return fi, nil
|
||||
}
|
||||
|
||||
return nil, vfserror.ErrNotFound
|
||||
// Create and add file info
|
||||
fi := vfs.NewFileInfoFromOS(info, key)
|
||||
d.info[key] = fi
|
||||
d.LRU.Add(key, fi)
|
||||
fi.UpdateAccessBatched(d.timeUpdater)
|
||||
// Note: Don't add to d.size here as it's being calculated in background
|
||||
// The background calculation will handle the total size
|
||||
d.mu.Unlock()
|
||||
|
||||
return fi, nil
|
||||
}
|
||||
|
||||
// EvictLRU evicts the least recently used files to free up space
|
||||
@@ -593,7 +561,7 @@ func (d *DiskFS) EvictLRU(bytesNeeded uint) uint {
|
||||
// Evict from LRU list until we free enough space
|
||||
for d.size > d.capacity-int64(bytesNeeded) && d.LRU.Len() > 0 {
|
||||
// Get the least recently used item
|
||||
elem := d.LRU.list.Back()
|
||||
elem := d.LRU.Back()
|
||||
if elem == nil {
|
||||
break
|
||||
}
|
||||
@@ -622,7 +590,7 @@ func (d *DiskFS) EvictLRU(bytesNeeded uint) uint {
|
||||
evicted += uint(fi.Size)
|
||||
|
||||
// Clean up key lock
|
||||
shardIndex := getShardIndex(key)
|
||||
shardIndex := locks.GetShardIndex(key)
|
||||
d.keyLocks[shardIndex].Delete(key)
|
||||
}
|
||||
|
||||
@@ -678,7 +646,7 @@ func (d *DiskFS) EvictBySize(bytesNeeded uint, ascending bool) uint {
|
||||
evicted += uint(fi.Size)
|
||||
|
||||
// Clean up key lock
|
||||
shardIndex := getShardIndex(key)
|
||||
shardIndex := locks.GetShardIndex(key)
|
||||
d.keyLocks[shardIndex].Delete(key)
|
||||
}
|
||||
|
||||
@@ -731,7 +699,7 @@ func (d *DiskFS) EvictFIFO(bytesNeeded uint) uint {
|
||||
evicted += uint(fi.Size)
|
||||
|
||||
// Clean up key lock
|
||||
shardIndex := getShardIndex(key)
|
||||
shardIndex := locks.GetShardIndex(key)
|
||||
d.keyLocks[shardIndex].Delete(key)
|
||||
}
|
||||
|
||||
|
||||
110
vfs/eviction/eviction.go
Normal file
110
vfs/eviction/eviction.go
Normal file
@@ -0,0 +1,110 @@
|
||||
package eviction
|
||||
|
||||
import (
|
||||
"s1d3sw1ped/steamcache2/vfs"
|
||||
"s1d3sw1ped/steamcache2/vfs/disk"
|
||||
"s1d3sw1ped/steamcache2/vfs/memory"
|
||||
)
|
||||
|
||||
// EvictionStrategy defines different eviction strategies
|
||||
type EvictionStrategy string
|
||||
|
||||
const (
|
||||
StrategyLRU EvictionStrategy = "lru"
|
||||
StrategyLFU EvictionStrategy = "lfu"
|
||||
StrategyFIFO EvictionStrategy = "fifo"
|
||||
StrategyLargest EvictionStrategy = "largest"
|
||||
StrategySmallest EvictionStrategy = "smallest"
|
||||
StrategyHybrid EvictionStrategy = "hybrid"
|
||||
)
|
||||
|
||||
// EvictLRU performs LRU eviction by removing least recently used files
|
||||
func EvictLRU(v vfs.VFS, bytesNeeded uint) uint {
|
||||
switch fs := v.(type) {
|
||||
case *memory.MemoryFS:
|
||||
return fs.EvictLRU(bytesNeeded)
|
||||
case *disk.DiskFS:
|
||||
return fs.EvictLRU(bytesNeeded)
|
||||
default:
|
||||
return 0
|
||||
}
|
||||
}
|
||||
|
||||
// EvictFIFO performs FIFO (First In First Out) eviction
|
||||
func EvictFIFO(v vfs.VFS, bytesNeeded uint) uint {
|
||||
switch fs := v.(type) {
|
||||
case *memory.MemoryFS:
|
||||
return fs.EvictFIFO(bytesNeeded)
|
||||
case *disk.DiskFS:
|
||||
return fs.EvictFIFO(bytesNeeded)
|
||||
default:
|
||||
return 0
|
||||
}
|
||||
}
|
||||
|
||||
// EvictBySizeAsc evicts smallest files first
|
||||
func EvictBySizeAsc(v vfs.VFS, bytesNeeded uint) uint {
|
||||
switch fs := v.(type) {
|
||||
case *memory.MemoryFS:
|
||||
return fs.EvictBySize(bytesNeeded, true) // true = ascending (smallest first)
|
||||
case *disk.DiskFS:
|
||||
return fs.EvictBySize(bytesNeeded, true) // true = ascending (smallest first)
|
||||
default:
|
||||
return 0
|
||||
}
|
||||
}
|
||||
|
||||
// EvictBySizeDesc evicts largest files first
|
||||
func EvictBySizeDesc(v vfs.VFS, bytesNeeded uint) uint {
|
||||
switch fs := v.(type) {
|
||||
case *memory.MemoryFS:
|
||||
return fs.EvictBySize(bytesNeeded, false) // false = descending (largest first)
|
||||
case *disk.DiskFS:
|
||||
return fs.EvictBySize(bytesNeeded, false) // false = descending (largest first)
|
||||
default:
|
||||
return 0
|
||||
}
|
||||
}
|
||||
|
||||
// EvictLargest evicts largest files first
|
||||
func EvictLargest(v vfs.VFS, bytesNeeded uint) uint {
|
||||
return EvictBySizeDesc(v, bytesNeeded)
|
||||
}
|
||||
|
||||
// EvictSmallest evicts smallest files first
|
||||
func EvictSmallest(v vfs.VFS, bytesNeeded uint) uint {
|
||||
return EvictBySizeAsc(v, bytesNeeded)
|
||||
}
|
||||
|
||||
// EvictLFU performs LFU (Least Frequently Used) eviction
|
||||
func EvictLFU(v vfs.VFS, bytesNeeded uint) uint {
|
||||
// For now, fall back to size-based eviction
|
||||
// TODO: Implement proper LFU tracking
|
||||
return EvictBySizeAsc(v, bytesNeeded)
|
||||
}
|
||||
|
||||
// EvictHybrid implements a hybrid eviction strategy
|
||||
func EvictHybrid(v vfs.VFS, bytesNeeded uint) uint {
|
||||
// Use LRU as primary strategy, but consider size as tiebreaker
|
||||
return EvictLRU(v, bytesNeeded)
|
||||
}
|
||||
|
||||
// GetEvictionFunction returns the eviction function for the given strategy
|
||||
func GetEvictionFunction(strategy EvictionStrategy) func(vfs.VFS, uint) uint {
|
||||
switch strategy {
|
||||
case StrategyLRU:
|
||||
return EvictLRU
|
||||
case StrategyLFU:
|
||||
return EvictLFU
|
||||
case StrategyFIFO:
|
||||
return EvictFIFO
|
||||
case StrategyLargest:
|
||||
return EvictLargest
|
||||
case StrategySmallest:
|
||||
return EvictSmallest
|
||||
case StrategyHybrid:
|
||||
return EvictHybrid
|
||||
default:
|
||||
return EvictLRU
|
||||
}
|
||||
}
|
||||
315
vfs/gc/gc.go
315
vfs/gc/gc.go
@@ -2,10 +2,13 @@
|
||||
package gc
|
||||
|
||||
import (
|
||||
"context"
|
||||
"io"
|
||||
"s1d3sw1ped/SteamCache2/vfs"
|
||||
"s1d3sw1ped/SteamCache2/vfs/disk"
|
||||
"s1d3sw1ped/SteamCache2/vfs/memory"
|
||||
"s1d3sw1ped/steamcache2/vfs"
|
||||
"s1d3sw1ped/steamcache2/vfs/eviction"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"time"
|
||||
)
|
||||
|
||||
// GCAlgorithm represents different garbage collection strategies
|
||||
@@ -34,45 +37,14 @@ func New(wrappedVFS vfs.VFS, algorithm GCAlgorithm) *GCFS {
|
||||
algorithm: algorithm,
|
||||
}
|
||||
|
||||
switch algorithm {
|
||||
case LRU:
|
||||
gcfs.gcFunc = gcLRU
|
||||
case LFU:
|
||||
gcfs.gcFunc = gcLFU
|
||||
case FIFO:
|
||||
gcfs.gcFunc = gcFIFO
|
||||
case Largest:
|
||||
gcfs.gcFunc = gcLargest
|
||||
case Smallest:
|
||||
gcfs.gcFunc = gcSmallest
|
||||
case Hybrid:
|
||||
gcfs.gcFunc = gcHybrid
|
||||
default:
|
||||
// Default to LRU
|
||||
gcfs.gcFunc = gcLRU
|
||||
}
|
||||
gcfs.gcFunc = eviction.GetEvictionFunction(eviction.EvictionStrategy(algorithm))
|
||||
|
||||
return gcfs
|
||||
}
|
||||
|
||||
// GetGCAlgorithm returns the GC function for the given algorithm
|
||||
func GetGCAlgorithm(algorithm GCAlgorithm) func(vfs.VFS, uint) uint {
|
||||
switch algorithm {
|
||||
case LRU:
|
||||
return gcLRU
|
||||
case LFU:
|
||||
return gcLFU
|
||||
case FIFO:
|
||||
return gcFIFO
|
||||
case Largest:
|
||||
return gcLargest
|
||||
case Smallest:
|
||||
return gcSmallest
|
||||
case Hybrid:
|
||||
return gcHybrid
|
||||
default:
|
||||
return gcLRU
|
||||
}
|
||||
return eviction.GetEvictionFunction(eviction.EvictionStrategy(algorithm))
|
||||
}
|
||||
|
||||
// Create wraps the underlying Create method
|
||||
@@ -121,120 +93,165 @@ type EvictionStrategy interface {
|
||||
Evict(vfs vfs.VFS, bytesNeeded uint) uint
|
||||
}
|
||||
|
||||
// GC functions
|
||||
|
||||
// gcLRU implements Least Recently Used eviction
|
||||
func gcLRU(v vfs.VFS, bytesNeeded uint) uint {
|
||||
return evictLRU(v, bytesNeeded)
|
||||
}
|
||||
|
||||
// gcLFU implements Least Frequently Used eviction
|
||||
func gcLFU(v vfs.VFS, bytesNeeded uint) uint {
|
||||
return evictLFU(v, bytesNeeded)
|
||||
}
|
||||
|
||||
// gcFIFO implements First In First Out eviction
|
||||
func gcFIFO(v vfs.VFS, bytesNeeded uint) uint {
|
||||
return evictFIFO(v, bytesNeeded)
|
||||
}
|
||||
|
||||
// gcLargest implements largest file first eviction
|
||||
func gcLargest(v vfs.VFS, bytesNeeded uint) uint {
|
||||
return evictLargest(v, bytesNeeded)
|
||||
}
|
||||
|
||||
// gcSmallest implements smallest file first eviction
|
||||
func gcSmallest(v vfs.VFS, bytesNeeded uint) uint {
|
||||
return evictSmallest(v, bytesNeeded)
|
||||
}
|
||||
|
||||
// gcHybrid implements a hybrid eviction strategy
|
||||
func gcHybrid(v vfs.VFS, bytesNeeded uint) uint {
|
||||
return evictHybrid(v, bytesNeeded)
|
||||
}
|
||||
|
||||
// evictLRU performs LRU eviction by removing least recently used files
|
||||
func evictLRU(v vfs.VFS, bytesNeeded uint) uint {
|
||||
// Try to use specific eviction methods if available
|
||||
switch fs := v.(type) {
|
||||
case *memory.MemoryFS:
|
||||
return fs.EvictLRU(bytesNeeded)
|
||||
case *disk.DiskFS:
|
||||
return fs.EvictLRU(bytesNeeded)
|
||||
default:
|
||||
// No fallback - return 0 (no eviction performed)
|
||||
return 0
|
||||
}
|
||||
}
|
||||
|
||||
// evictLFU performs LFU (Least Frequently Used) eviction
|
||||
func evictLFU(v vfs.VFS, bytesNeeded uint) uint {
|
||||
// For now, fall back to size-based eviction
|
||||
// TODO: Implement proper LFU tracking
|
||||
return evictBySize(v, bytesNeeded)
|
||||
}
|
||||
|
||||
// evictFIFO performs FIFO (First In First Out) eviction
|
||||
func evictFIFO(v vfs.VFS, bytesNeeded uint) uint {
|
||||
switch fs := v.(type) {
|
||||
case *memory.MemoryFS:
|
||||
return fs.EvictFIFO(bytesNeeded)
|
||||
case *disk.DiskFS:
|
||||
return fs.EvictFIFO(bytesNeeded)
|
||||
default:
|
||||
// No fallback - return 0 (no eviction performed)
|
||||
return 0
|
||||
}
|
||||
}
|
||||
|
||||
// evictLargest evicts largest files first
|
||||
func evictLargest(v vfs.VFS, bytesNeeded uint) uint {
|
||||
return evictBySizeDesc(v, bytesNeeded)
|
||||
}
|
||||
|
||||
// evictSmallest evicts smallest files first
|
||||
func evictSmallest(v vfs.VFS, bytesNeeded uint) uint {
|
||||
return evictBySizeAsc(v, bytesNeeded)
|
||||
}
|
||||
|
||||
// evictBySize evicts files based on size (smallest first)
|
||||
func evictBySize(v vfs.VFS, bytesNeeded uint) uint {
|
||||
return evictBySizeAsc(v, bytesNeeded)
|
||||
}
|
||||
|
||||
// evictBySizeAsc evicts smallest files first
|
||||
func evictBySizeAsc(v vfs.VFS, bytesNeeded uint) uint {
|
||||
switch fs := v.(type) {
|
||||
case *memory.MemoryFS:
|
||||
return fs.EvictBySize(bytesNeeded, true) // true = ascending (smallest first)
|
||||
case *disk.DiskFS:
|
||||
return fs.EvictBySize(bytesNeeded, true) // true = ascending (smallest first)
|
||||
default:
|
||||
// No fallback - return 0 (no eviction performed)
|
||||
return 0
|
||||
}
|
||||
}
|
||||
|
||||
// evictBySizeDesc evicts largest files first
|
||||
func evictBySizeDesc(v vfs.VFS, bytesNeeded uint) uint {
|
||||
switch fs := v.(type) {
|
||||
case *memory.MemoryFS:
|
||||
return fs.EvictBySize(bytesNeeded, false) // false = descending (largest first)
|
||||
case *disk.DiskFS:
|
||||
return fs.EvictBySize(bytesNeeded, false) // false = descending (largest first)
|
||||
default:
|
||||
// No fallback - return 0 (no eviction performed)
|
||||
return 0
|
||||
}
|
||||
}
|
||||
|
||||
// evictHybrid implements a hybrid eviction strategy
|
||||
func evictHybrid(v vfs.VFS, bytesNeeded uint) uint {
|
||||
// Use LRU as primary strategy, but consider size as tiebreaker
|
||||
return evictLRU(v, bytesNeeded)
|
||||
}
|
||||
|
||||
// AdaptivePromotionDeciderFunc is a placeholder for the adaptive promotion logic
|
||||
var AdaptivePromotionDeciderFunc = func() interface{} {
|
||||
return nil
|
||||
}
|
||||
|
||||
// AsyncGCFS wraps a GCFS with asynchronous garbage collection capabilities
|
||||
type AsyncGCFS struct {
|
||||
*GCFS
|
||||
gcQueue chan gcRequest
|
||||
ctx context.Context
|
||||
cancel context.CancelFunc
|
||||
wg sync.WaitGroup
|
||||
gcRunning int32
|
||||
preemptive bool
|
||||
asyncThreshold float64 // Async GC threshold as percentage of capacity (e.g., 0.8 = 80%)
|
||||
syncThreshold float64 // Sync GC threshold as percentage of capacity (e.g., 0.95 = 95%)
|
||||
hardLimit float64 // Hard limit threshold (e.g., 1.0 = 100%)
|
||||
}
|
||||
|
||||
type gcRequest struct {
|
||||
bytesNeeded uint
|
||||
priority int // Higher number = higher priority
|
||||
}
|
||||
|
||||
// NewAsync creates a new AsyncGCFS with asynchronous garbage collection
|
||||
func NewAsync(wrappedVFS vfs.VFS, algorithm GCAlgorithm, preemptive bool, asyncThreshold, syncThreshold, hardLimit float64) *AsyncGCFS {
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
|
||||
asyncGC := &AsyncGCFS{
|
||||
GCFS: New(wrappedVFS, algorithm),
|
||||
gcQueue: make(chan gcRequest, 100), // Buffer for GC requests
|
||||
ctx: ctx,
|
||||
cancel: cancel,
|
||||
preemptive: preemptive,
|
||||
asyncThreshold: asyncThreshold,
|
||||
syncThreshold: syncThreshold,
|
||||
hardLimit: hardLimit,
|
||||
}
|
||||
|
||||
// Start the background GC worker
|
||||
asyncGC.wg.Add(1)
|
||||
go asyncGC.gcWorker()
|
||||
|
||||
// Start preemptive GC if enabled
|
||||
if preemptive {
|
||||
asyncGC.wg.Add(1)
|
||||
go asyncGC.preemptiveGC()
|
||||
}
|
||||
|
||||
return asyncGC
|
||||
}
|
||||
|
||||
// Create wraps the underlying Create method with hybrid GC (async + sync hard limits)
|
||||
func (agc *AsyncGCFS) Create(key string, size int64) (io.WriteCloser, error) {
|
||||
currentSize := agc.vfs.Size()
|
||||
capacity := agc.vfs.Capacity()
|
||||
projectedSize := currentSize + size
|
||||
|
||||
// Calculate utilization percentages
|
||||
currentUtilization := float64(currentSize) / float64(capacity)
|
||||
projectedUtilization := float64(projectedSize) / float64(capacity)
|
||||
|
||||
// Hard limit check - never exceed the hard limit
|
||||
if projectedUtilization > agc.hardLimit {
|
||||
needed := uint(projectedSize - capacity)
|
||||
// Immediate sync GC to prevent exceeding hard limit
|
||||
agc.gcFunc(agc.vfs, needed)
|
||||
} else if projectedUtilization > agc.syncThreshold {
|
||||
// Near hard limit - do immediate sync GC
|
||||
needed := uint(projectedSize - int64(float64(capacity)*agc.syncThreshold))
|
||||
agc.gcFunc(agc.vfs, needed)
|
||||
} else if currentUtilization > agc.asyncThreshold {
|
||||
// Above async threshold - queue for async GC
|
||||
needed := uint(projectedSize - int64(float64(capacity)*agc.asyncThreshold))
|
||||
select {
|
||||
case agc.gcQueue <- gcRequest{bytesNeeded: needed, priority: 2}:
|
||||
default:
|
||||
// Queue full, do immediate GC
|
||||
agc.gcFunc(agc.vfs, needed)
|
||||
}
|
||||
}
|
||||
|
||||
return agc.vfs.Create(key, size)
|
||||
}
|
||||
|
||||
// gcWorker processes GC requests asynchronously
|
||||
func (agc *AsyncGCFS) gcWorker() {
|
||||
defer agc.wg.Done()
|
||||
|
||||
ticker := time.NewTicker(100 * time.Millisecond) // Check every 100ms
|
||||
defer ticker.Stop()
|
||||
|
||||
for {
|
||||
select {
|
||||
case <-agc.ctx.Done():
|
||||
return
|
||||
case req := <-agc.gcQueue:
|
||||
atomic.StoreInt32(&agc.gcRunning, 1)
|
||||
agc.gcFunc(agc.vfs, req.bytesNeeded)
|
||||
atomic.StoreInt32(&agc.gcRunning, 0)
|
||||
case <-ticker.C:
|
||||
// Process any pending GC requests
|
||||
select {
|
||||
case req := <-agc.gcQueue:
|
||||
atomic.StoreInt32(&agc.gcRunning, 1)
|
||||
agc.gcFunc(agc.vfs, req.bytesNeeded)
|
||||
atomic.StoreInt32(&agc.gcRunning, 0)
|
||||
default:
|
||||
// No pending requests
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// preemptiveGC runs background GC to keep cache utilization below threshold
|
||||
func (agc *AsyncGCFS) preemptiveGC() {
|
||||
defer agc.wg.Done()
|
||||
|
||||
ticker := time.NewTicker(5 * time.Second) // Check every 5 seconds
|
||||
defer ticker.Stop()
|
||||
|
||||
for {
|
||||
select {
|
||||
case <-agc.ctx.Done():
|
||||
return
|
||||
case <-ticker.C:
|
||||
currentSize := agc.vfs.Size()
|
||||
capacity := agc.vfs.Capacity()
|
||||
currentUtilization := float64(currentSize) / float64(capacity)
|
||||
|
||||
// Check if we're above the async threshold
|
||||
if currentUtilization > agc.asyncThreshold {
|
||||
// Calculate how much to free to get back to async threshold
|
||||
targetSize := int64(float64(capacity) * agc.asyncThreshold)
|
||||
if currentSize > targetSize {
|
||||
overage := currentSize - targetSize
|
||||
select {
|
||||
case agc.gcQueue <- gcRequest{bytesNeeded: uint(overage), priority: 0}:
|
||||
default:
|
||||
// Queue full, skip this round
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Stop stops the async GC workers
|
||||
func (agc *AsyncGCFS) Stop() {
|
||||
agc.cancel()
|
||||
agc.wg.Wait()
|
||||
}
|
||||
|
||||
// IsGCRunning returns true if GC is currently running
|
||||
func (agc *AsyncGCFS) IsGCRunning() bool {
|
||||
return atomic.LoadInt32(&agc.gcRunning) == 1
|
||||
}
|
||||
|
||||
// ForceGC forces immediate garbage collection to free the specified number of bytes
|
||||
func (agc *AsyncGCFS) ForceGC(bytesNeeded uint) {
|
||||
agc.gcFunc(agc.vfs, bytesNeeded)
|
||||
}
|
||||
|
||||
28
vfs/locks/sharding.go
Normal file
28
vfs/locks/sharding.go
Normal file
@@ -0,0 +1,28 @@
|
||||
package locks
|
||||
|
||||
import (
|
||||
"sync"
|
||||
)
|
||||
|
||||
// Number of lock shards for reducing contention
|
||||
const NumLockShards = 32
|
||||
|
||||
// GetShardIndex returns the shard index for a given key using FNV-1a hash
|
||||
func GetShardIndex(key string) int {
|
||||
// Use FNV-1a hash for good distribution
|
||||
var h uint32 = 2166136261 // FNV offset basis
|
||||
for i := 0; i < len(key); i++ {
|
||||
h ^= uint32(key[i])
|
||||
h *= 16777619 // FNV prime
|
||||
}
|
||||
return int(h % NumLockShards)
|
||||
}
|
||||
|
||||
// GetKeyLock returns a lock for the given key using sharding
|
||||
func GetKeyLock(keyLocks []sync.Map, key string) *sync.RWMutex {
|
||||
shardIndex := GetShardIndex(key)
|
||||
shard := &keyLocks[shardIndex]
|
||||
|
||||
keyLock, _ := shard.LoadOrStore(key, &sync.RWMutex{})
|
||||
return keyLock.(*sync.RWMutex)
|
||||
}
|
||||
66
vfs/lru/lru.go
Normal file
66
vfs/lru/lru.go
Normal file
@@ -0,0 +1,66 @@
|
||||
package lru
|
||||
|
||||
import (
|
||||
"container/list"
|
||||
"s1d3sw1ped/steamcache2/vfs/types"
|
||||
)
|
||||
|
||||
// LRUList represents a least recently used list for cache eviction
|
||||
type LRUList[T any] struct {
|
||||
list *list.List
|
||||
elem map[string]*list.Element
|
||||
}
|
||||
|
||||
// NewLRUList creates a new LRU list
|
||||
func NewLRUList[T any]() *LRUList[T] {
|
||||
return &LRUList[T]{
|
||||
list: list.New(),
|
||||
elem: make(map[string]*list.Element),
|
||||
}
|
||||
}
|
||||
|
||||
// Add adds an item to the front of the LRU list
|
||||
func (l *LRUList[T]) Add(key string, item T) {
|
||||
elem := l.list.PushFront(item)
|
||||
l.elem[key] = elem
|
||||
}
|
||||
|
||||
// MoveToFront moves an item to the front of the LRU list
|
||||
func (l *LRUList[T]) MoveToFront(key string, timeUpdater *types.BatchedTimeUpdate) {
|
||||
if elem, exists := l.elem[key]; exists {
|
||||
l.list.MoveToFront(elem)
|
||||
// Update the FileInfo in the element with new access time
|
||||
if fi, ok := any(elem.Value).(interface {
|
||||
UpdateAccessBatched(*types.BatchedTimeUpdate)
|
||||
}); ok {
|
||||
fi.UpdateAccessBatched(timeUpdater)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Remove removes an item from the LRU list
|
||||
func (l *LRUList[T]) Remove(key string) (T, bool) {
|
||||
if elem, exists := l.elem[key]; exists {
|
||||
delete(l.elem, key)
|
||||
if item, ok := l.list.Remove(elem).(T); ok {
|
||||
return item, true
|
||||
}
|
||||
}
|
||||
var zero T
|
||||
return zero, false
|
||||
}
|
||||
|
||||
// Len returns the number of items in the LRU list
|
||||
func (l *LRUList[T]) Len() int {
|
||||
return l.list.Len()
|
||||
}
|
||||
|
||||
// Back returns the least recently used item (at the back of the list)
|
||||
func (l *LRUList[T]) Back() *list.Element {
|
||||
return l.list.Back()
|
||||
}
|
||||
|
||||
// Front returns the most recently used item (at the front of the list)
|
||||
func (l *LRUList[T]) Front() *list.Element {
|
||||
return l.list.Front()
|
||||
}
|
||||
@@ -3,10 +3,12 @@ package memory
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"container/list"
|
||||
"io"
|
||||
"s1d3sw1ped/SteamCache2/vfs"
|
||||
"s1d3sw1ped/SteamCache2/vfs/vfserror"
|
||||
"s1d3sw1ped/steamcache2/vfs"
|
||||
"s1d3sw1ped/steamcache2/vfs/locks"
|
||||
"s1d3sw1ped/steamcache2/vfs/lru"
|
||||
"s1d3sw1ped/steamcache2/vfs/types"
|
||||
"s1d3sw1ped/steamcache2/vfs/vfserror"
|
||||
"sort"
|
||||
"strings"
|
||||
"sync"
|
||||
@@ -19,58 +21,13 @@ var _ vfs.VFS = (*MemoryFS)(nil)
|
||||
// MemoryFS is an in-memory virtual file system
|
||||
type MemoryFS struct {
|
||||
data map[string]*bytes.Buffer
|
||||
info map[string]*vfs.FileInfo
|
||||
info map[string]*types.FileInfo
|
||||
capacity int64
|
||||
size int64
|
||||
mu sync.RWMutex
|
||||
keyLocks []sync.Map // Sharded lock pools for better concurrency
|
||||
LRU *lruList
|
||||
timeUpdater *vfs.BatchedTimeUpdate // Batched time updates for better performance
|
||||
}
|
||||
|
||||
// Number of lock shards for reducing contention
|
||||
const numLockShards = 32
|
||||
|
||||
// lruList for time-decayed LRU eviction
|
||||
type lruList struct {
|
||||
list *list.List
|
||||
elem map[string]*list.Element
|
||||
}
|
||||
|
||||
func newLruList() *lruList {
|
||||
return &lruList{
|
||||
list: list.New(),
|
||||
elem: make(map[string]*list.Element),
|
||||
}
|
||||
}
|
||||
|
||||
func (l *lruList) Add(key string, fi *vfs.FileInfo) {
|
||||
elem := l.list.PushFront(fi)
|
||||
l.elem[key] = elem
|
||||
}
|
||||
|
||||
func (l *lruList) MoveToFront(key string, timeUpdater *vfs.BatchedTimeUpdate) {
|
||||
if elem, exists := l.elem[key]; exists {
|
||||
l.list.MoveToFront(elem)
|
||||
// Update the FileInfo in the element with new access time
|
||||
if fi := elem.Value.(*vfs.FileInfo); fi != nil {
|
||||
fi.UpdateAccessBatched(timeUpdater)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (l *lruList) Remove(key string) *vfs.FileInfo {
|
||||
if elem, exists := l.elem[key]; exists {
|
||||
delete(l.elem, key)
|
||||
if fi := l.list.Remove(elem).(*vfs.FileInfo); fi != nil {
|
||||
return fi
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (l *lruList) Len() int {
|
||||
return l.list.Len()
|
||||
LRU *lru.LRUList[*types.FileInfo]
|
||||
timeUpdater *types.BatchedTimeUpdate // Batched time updates for better performance
|
||||
}
|
||||
|
||||
// New creates a new MemoryFS
|
||||
@@ -80,16 +37,16 @@ func New(capacity int64) *MemoryFS {
|
||||
}
|
||||
|
||||
// Initialize sharded locks
|
||||
keyLocks := make([]sync.Map, numLockShards)
|
||||
keyLocks := make([]sync.Map, locks.NumLockShards)
|
||||
|
||||
return &MemoryFS{
|
||||
data: make(map[string]*bytes.Buffer),
|
||||
info: make(map[string]*vfs.FileInfo),
|
||||
info: make(map[string]*types.FileInfo),
|
||||
capacity: capacity,
|
||||
size: 0,
|
||||
keyLocks: keyLocks,
|
||||
LRU: newLruList(),
|
||||
timeUpdater: vfs.NewBatchedTimeUpdate(100 * time.Millisecond), // Update time every 100ms
|
||||
LRU: lru.NewLRUList[*types.FileInfo](),
|
||||
timeUpdater: types.NewBatchedTimeUpdate(100 * time.Millisecond), // Update time every 100ms
|
||||
}
|
||||
}
|
||||
|
||||
@@ -110,24 +67,38 @@ func (m *MemoryFS) Capacity() int64 {
|
||||
return m.capacity
|
||||
}
|
||||
|
||||
// getShardIndex returns the shard index for a given key
|
||||
func getShardIndex(key string) int {
|
||||
// Use FNV-1a hash for good distribution
|
||||
var h uint32 = 2166136261 // FNV offset basis
|
||||
for i := 0; i < len(key); i++ {
|
||||
h ^= uint32(key[i])
|
||||
h *= 16777619 // FNV prime
|
||||
// GetFragmentationStats returns memory fragmentation statistics
|
||||
func (m *MemoryFS) GetFragmentationStats() map[string]interface{} {
|
||||
m.mu.RLock()
|
||||
defer m.mu.RUnlock()
|
||||
|
||||
var totalCapacity int64
|
||||
var totalUsed int64
|
||||
var bufferCount int
|
||||
|
||||
for _, buffer := range m.data {
|
||||
totalCapacity += int64(buffer.Cap())
|
||||
totalUsed += int64(buffer.Len())
|
||||
bufferCount++
|
||||
}
|
||||
|
||||
fragmentationRatio := float64(0)
|
||||
if totalCapacity > 0 {
|
||||
fragmentationRatio = float64(totalCapacity-totalUsed) / float64(totalCapacity)
|
||||
}
|
||||
|
||||
return map[string]interface{}{
|
||||
"buffer_count": bufferCount,
|
||||
"total_capacity": totalCapacity,
|
||||
"total_used": totalUsed,
|
||||
"fragmentation_ratio": fragmentationRatio,
|
||||
"average_buffer_size": float64(totalUsed) / float64(bufferCount),
|
||||
}
|
||||
return int(h % numLockShards)
|
||||
}
|
||||
|
||||
// getKeyLock returns a lock for the given key using sharding
|
||||
func (m *MemoryFS) getKeyLock(key string) *sync.RWMutex {
|
||||
shardIndex := getShardIndex(key)
|
||||
shard := &m.keyLocks[shardIndex]
|
||||
|
||||
keyLock, _ := shard.LoadOrStore(key, &sync.RWMutex{})
|
||||
return keyLock.(*sync.RWMutex)
|
||||
return locks.GetKeyLock(m.keyLocks, key)
|
||||
}
|
||||
|
||||
// Create creates a new file
|
||||
@@ -159,7 +130,7 @@ func (m *MemoryFS) Create(key string, size int64) (io.WriteCloser, error) {
|
||||
|
||||
buffer := &bytes.Buffer{}
|
||||
m.data[key] = buffer
|
||||
fi := vfs.NewFileInfo(key, size)
|
||||
fi := types.NewFileInfo(key, size)
|
||||
m.info[key] = fi
|
||||
m.LRU.Add(key, fi)
|
||||
// Initialize access time with current time
|
||||
@@ -230,23 +201,39 @@ func (m *MemoryFS) Open(key string) (io.ReadCloser, error) {
|
||||
return nil, vfserror.ErrNotFound
|
||||
}
|
||||
|
||||
// Create a copy of the buffer for reading
|
||||
data := make([]byte, buffer.Len())
|
||||
copy(data, buffer.Bytes())
|
||||
// Use zero-copy approach - return reader that reads directly from buffer
|
||||
m.mu.Unlock()
|
||||
|
||||
return &memoryReadCloser{
|
||||
reader: bytes.NewReader(data),
|
||||
buffer: buffer,
|
||||
offset: 0,
|
||||
}, nil
|
||||
}
|
||||
|
||||
// memoryReadCloser implements io.ReadCloser for memory files
|
||||
// memoryReadCloser implements io.ReadCloser for memory files with zero-copy optimization
|
||||
type memoryReadCloser struct {
|
||||
reader *bytes.Reader
|
||||
buffer *bytes.Buffer
|
||||
offset int64
|
||||
}
|
||||
|
||||
func (mrc *memoryReadCloser) Read(p []byte) (n int, err error) {
|
||||
return mrc.reader.Read(p)
|
||||
if mrc.offset >= int64(mrc.buffer.Len()) {
|
||||
return 0, io.EOF
|
||||
}
|
||||
|
||||
// Zero-copy read directly from buffer
|
||||
available := mrc.buffer.Len() - int(mrc.offset)
|
||||
toRead := len(p)
|
||||
if toRead > available {
|
||||
toRead = available
|
||||
}
|
||||
|
||||
// Read directly from buffer without copying
|
||||
data := mrc.buffer.Bytes()
|
||||
copy(p, data[mrc.offset:mrc.offset+int64(toRead)])
|
||||
mrc.offset += int64(toRead)
|
||||
|
||||
return toRead, nil
|
||||
}
|
||||
|
||||
func (mrc *memoryReadCloser) Close() error {
|
||||
@@ -286,7 +273,7 @@ func (m *MemoryFS) Delete(key string) error {
|
||||
}
|
||||
|
||||
// Stat returns file information
|
||||
func (m *MemoryFS) Stat(key string) (*vfs.FileInfo, error) {
|
||||
func (m *MemoryFS) Stat(key string) (*types.FileInfo, error) {
|
||||
if key == "" {
|
||||
return nil, vfserror.ErrInvalidKey
|
||||
}
|
||||
@@ -322,12 +309,12 @@ func (m *MemoryFS) EvictLRU(bytesNeeded uint) uint {
|
||||
// Evict from LRU list until we free enough space
|
||||
for m.size > m.capacity-int64(bytesNeeded) && m.LRU.Len() > 0 {
|
||||
// Get the least recently used item
|
||||
elem := m.LRU.list.Back()
|
||||
elem := m.LRU.Back()
|
||||
if elem == nil {
|
||||
break
|
||||
}
|
||||
|
||||
fi := elem.Value.(*vfs.FileInfo)
|
||||
fi := elem.Value.(*types.FileInfo)
|
||||
key := fi.Key
|
||||
|
||||
// Remove from LRU
|
||||
@@ -342,7 +329,7 @@ func (m *MemoryFS) EvictLRU(bytesNeeded uint) uint {
|
||||
evicted += uint(fi.Size)
|
||||
|
||||
// Clean up key lock
|
||||
shardIndex := getShardIndex(key)
|
||||
shardIndex := locks.GetShardIndex(key)
|
||||
m.keyLocks[shardIndex].Delete(key)
|
||||
}
|
||||
|
||||
@@ -355,7 +342,7 @@ func (m *MemoryFS) EvictBySize(bytesNeeded uint, ascending bool) uint {
|
||||
defer m.mu.Unlock()
|
||||
|
||||
var evicted uint
|
||||
var candidates []*vfs.FileInfo
|
||||
var candidates []*types.FileInfo
|
||||
|
||||
// Collect all files
|
||||
for _, fi := range m.info {
|
||||
@@ -390,7 +377,7 @@ func (m *MemoryFS) EvictBySize(bytesNeeded uint, ascending bool) uint {
|
||||
evicted += uint(fi.Size)
|
||||
|
||||
// Clean up key lock
|
||||
shardIndex := getShardIndex(key)
|
||||
shardIndex := locks.GetShardIndex(key)
|
||||
m.keyLocks[shardIndex].Delete(key)
|
||||
}
|
||||
|
||||
@@ -403,7 +390,7 @@ func (m *MemoryFS) EvictFIFO(bytesNeeded uint) uint {
|
||||
defer m.mu.Unlock()
|
||||
|
||||
var evicted uint
|
||||
var candidates []*vfs.FileInfo
|
||||
var candidates []*types.FileInfo
|
||||
|
||||
// Collect all files
|
||||
for _, fi := range m.info {
|
||||
@@ -435,7 +422,7 @@ func (m *MemoryFS) EvictFIFO(bytesNeeded uint) uint {
|
||||
evicted += uint(fi.Size)
|
||||
|
||||
// Clean up key lock
|
||||
shardIndex := getShardIndex(key)
|
||||
shardIndex := locks.GetShardIndex(key)
|
||||
m.keyLocks[shardIndex].Delete(key)
|
||||
}
|
||||
|
||||
|
||||
274
vfs/memory/monitor.go
Normal file
274
vfs/memory/monitor.go
Normal file
@@ -0,0 +1,274 @@
|
||||
package memory
|
||||
|
||||
import (
|
||||
"runtime"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"time"
|
||||
)
|
||||
|
||||
// MemoryMonitor tracks system memory usage and provides dynamic sizing recommendations
|
||||
type MemoryMonitor struct {
|
||||
targetMemoryUsage uint64 // Target total memory usage in bytes
|
||||
currentMemoryUsage uint64 // Current total memory usage in bytes
|
||||
monitoringInterval time.Duration
|
||||
adjustmentThreshold float64 // Threshold for cache size adjustments (e.g., 0.1 = 10%)
|
||||
mu sync.RWMutex
|
||||
ctx chan struct{}
|
||||
stopChan chan struct{}
|
||||
isMonitoring int32
|
||||
|
||||
// Dynamic cache management fields
|
||||
originalCacheSize uint64
|
||||
currentCacheSize uint64
|
||||
cache interface{} // Generic cache interface
|
||||
adjustmentInterval time.Duration
|
||||
lastAdjustment time.Time
|
||||
adjustmentCount int64
|
||||
isAdjusting int32
|
||||
}
|
||||
|
||||
// NewMemoryMonitor creates a new memory monitor
|
||||
func NewMemoryMonitor(targetMemoryUsage uint64, monitoringInterval time.Duration, adjustmentThreshold float64) *MemoryMonitor {
|
||||
return &MemoryMonitor{
|
||||
targetMemoryUsage: targetMemoryUsage,
|
||||
monitoringInterval: monitoringInterval,
|
||||
adjustmentThreshold: adjustmentThreshold,
|
||||
ctx: make(chan struct{}),
|
||||
stopChan: make(chan struct{}),
|
||||
adjustmentInterval: 30 * time.Second, // Default adjustment interval
|
||||
}
|
||||
}
|
||||
|
||||
// NewMemoryMonitorWithCache creates a new memory monitor with cache management
|
||||
func NewMemoryMonitorWithCache(targetMemoryUsage uint64, monitoringInterval time.Duration, adjustmentThreshold float64, cache interface{}, originalCacheSize uint64) *MemoryMonitor {
|
||||
mm := NewMemoryMonitor(targetMemoryUsage, monitoringInterval, adjustmentThreshold)
|
||||
mm.cache = cache
|
||||
mm.originalCacheSize = originalCacheSize
|
||||
mm.currentCacheSize = originalCacheSize
|
||||
return mm
|
||||
}
|
||||
|
||||
// Start begins monitoring memory usage
|
||||
func (mm *MemoryMonitor) Start() {
|
||||
if atomic.CompareAndSwapInt32(&mm.isMonitoring, 0, 1) {
|
||||
go mm.monitor()
|
||||
}
|
||||
}
|
||||
|
||||
// Stop stops monitoring memory usage
|
||||
func (mm *MemoryMonitor) Stop() {
|
||||
if atomic.CompareAndSwapInt32(&mm.isMonitoring, 1, 0) {
|
||||
close(mm.stopChan)
|
||||
}
|
||||
}
|
||||
|
||||
// GetCurrentMemoryUsage returns the current total memory usage
|
||||
func (mm *MemoryMonitor) GetCurrentMemoryUsage() uint64 {
|
||||
mm.mu.RLock()
|
||||
defer mm.mu.RUnlock()
|
||||
return atomic.LoadUint64(&mm.currentMemoryUsage)
|
||||
}
|
||||
|
||||
// GetTargetMemoryUsage returns the target memory usage
|
||||
func (mm *MemoryMonitor) GetTargetMemoryUsage() uint64 {
|
||||
mm.mu.RLock()
|
||||
defer mm.mu.RUnlock()
|
||||
return mm.targetMemoryUsage
|
||||
}
|
||||
|
||||
// GetMemoryUtilization returns the current memory utilization as a percentage
|
||||
func (mm *MemoryMonitor) GetMemoryUtilization() float64 {
|
||||
mm.mu.RLock()
|
||||
defer mm.mu.RUnlock()
|
||||
current := atomic.LoadUint64(&mm.currentMemoryUsage)
|
||||
return float64(current) / float64(mm.targetMemoryUsage)
|
||||
}
|
||||
|
||||
// GetRecommendedCacheSize calculates the recommended cache size based on current memory usage
|
||||
func (mm *MemoryMonitor) GetRecommendedCacheSize(originalCacheSize uint64) uint64 {
|
||||
mm.mu.RLock()
|
||||
defer mm.mu.RUnlock()
|
||||
|
||||
current := atomic.LoadUint64(&mm.currentMemoryUsage)
|
||||
target := mm.targetMemoryUsage
|
||||
|
||||
// If we're under target, we can use the full cache size
|
||||
if current <= target {
|
||||
return originalCacheSize
|
||||
}
|
||||
|
||||
// Calculate how much we're over target
|
||||
overage := current - target
|
||||
|
||||
// If overage is significant, reduce cache size
|
||||
if overage > uint64(float64(target)*mm.adjustmentThreshold) {
|
||||
// Reduce cache size by the overage amount, but don't go below 10% of original
|
||||
minCacheSize := uint64(float64(originalCacheSize) * 0.1)
|
||||
recommendedSize := originalCacheSize - overage
|
||||
|
||||
if recommendedSize < minCacheSize {
|
||||
recommendedSize = minCacheSize
|
||||
}
|
||||
|
||||
return recommendedSize
|
||||
}
|
||||
|
||||
return originalCacheSize
|
||||
}
|
||||
|
||||
// monitor runs the memory monitoring loop
|
||||
func (mm *MemoryMonitor) monitor() {
|
||||
ticker := time.NewTicker(mm.monitoringInterval)
|
||||
defer ticker.Stop()
|
||||
|
||||
for {
|
||||
select {
|
||||
case <-mm.stopChan:
|
||||
return
|
||||
case <-ticker.C:
|
||||
mm.updateMemoryUsage()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// updateMemoryUsage updates the current memory usage
|
||||
func (mm *MemoryMonitor) updateMemoryUsage() {
|
||||
var m runtime.MemStats
|
||||
runtime.ReadMemStats(&m)
|
||||
|
||||
// Use Alloc (currently allocated memory) as our metric
|
||||
atomic.StoreUint64(&mm.currentMemoryUsage, m.Alloc)
|
||||
}
|
||||
|
||||
// SetTargetMemoryUsage updates the target memory usage
|
||||
func (mm *MemoryMonitor) SetTargetMemoryUsage(target uint64) {
|
||||
mm.mu.Lock()
|
||||
defer mm.mu.Unlock()
|
||||
mm.targetMemoryUsage = target
|
||||
}
|
||||
|
||||
// GetMemoryStats returns detailed memory statistics
|
||||
func (mm *MemoryMonitor) GetMemoryStats() map[string]interface{} {
|
||||
var m runtime.MemStats
|
||||
runtime.ReadMemStats(&m)
|
||||
|
||||
mm.mu.RLock()
|
||||
defer mm.mu.RUnlock()
|
||||
|
||||
return map[string]interface{}{
|
||||
"current_usage": atomic.LoadUint64(&mm.currentMemoryUsage),
|
||||
"target_usage": mm.targetMemoryUsage,
|
||||
"utilization": mm.GetMemoryUtilization(),
|
||||
"heap_alloc": m.HeapAlloc,
|
||||
"heap_sys": m.HeapSys,
|
||||
"heap_idle": m.HeapIdle,
|
||||
"heap_inuse": m.HeapInuse,
|
||||
"stack_inuse": m.StackInuse,
|
||||
"stack_sys": m.StackSys,
|
||||
"gc_cycles": m.NumGC,
|
||||
"gc_pause_total": m.PauseTotalNs,
|
||||
}
|
||||
}
|
||||
|
||||
// Dynamic Cache Management Methods
|
||||
|
||||
// StartDynamicAdjustment begins the dynamic cache size adjustment process
|
||||
func (mm *MemoryMonitor) StartDynamicAdjustment() {
|
||||
if mm.cache != nil {
|
||||
go mm.adjustmentLoop()
|
||||
}
|
||||
}
|
||||
|
||||
// GetCurrentCacheSize returns the current cache size
|
||||
func (mm *MemoryMonitor) GetCurrentCacheSize() uint64 {
|
||||
mm.mu.RLock()
|
||||
defer mm.mu.RUnlock()
|
||||
return atomic.LoadUint64(&mm.currentCacheSize)
|
||||
}
|
||||
|
||||
// GetOriginalCacheSize returns the original cache size
|
||||
func (mm *MemoryMonitor) GetOriginalCacheSize() uint64 {
|
||||
mm.mu.RLock()
|
||||
defer mm.mu.RUnlock()
|
||||
return mm.originalCacheSize
|
||||
}
|
||||
|
||||
// GetAdjustmentCount returns the number of adjustments made
|
||||
func (mm *MemoryMonitor) GetAdjustmentCount() int64 {
|
||||
return atomic.LoadInt64(&mm.adjustmentCount)
|
||||
}
|
||||
|
||||
// adjustmentLoop runs the cache size adjustment loop
|
||||
func (mm *MemoryMonitor) adjustmentLoop() {
|
||||
ticker := time.NewTicker(mm.adjustmentInterval)
|
||||
defer ticker.Stop()
|
||||
|
||||
for range ticker.C {
|
||||
mm.performAdjustment()
|
||||
}
|
||||
}
|
||||
|
||||
// performAdjustment performs a cache size adjustment if needed
|
||||
func (mm *MemoryMonitor) performAdjustment() {
|
||||
// Prevent concurrent adjustments
|
||||
if !atomic.CompareAndSwapInt32(&mm.isAdjusting, 0, 1) {
|
||||
return
|
||||
}
|
||||
defer atomic.StoreInt32(&mm.isAdjusting, 0)
|
||||
|
||||
// Check if enough time has passed since last adjustment
|
||||
if time.Since(mm.lastAdjustment) < mm.adjustmentInterval {
|
||||
return
|
||||
}
|
||||
|
||||
// Get recommended cache size
|
||||
recommendedSize := mm.GetRecommendedCacheSize(mm.originalCacheSize)
|
||||
currentSize := atomic.LoadUint64(&mm.currentCacheSize)
|
||||
|
||||
// Only adjust if there's a significant difference (more than 5%)
|
||||
sizeDiff := float64(recommendedSize) / float64(currentSize)
|
||||
if sizeDiff < 0.95 || sizeDiff > 1.05 {
|
||||
mm.adjustCacheSize(recommendedSize)
|
||||
mm.lastAdjustment = time.Now()
|
||||
atomic.AddInt64(&mm.adjustmentCount, 1)
|
||||
}
|
||||
}
|
||||
|
||||
// adjustCacheSize adjusts the cache size to the recommended size
|
||||
func (mm *MemoryMonitor) adjustCacheSize(newSize uint64) {
|
||||
mm.mu.Lock()
|
||||
defer mm.mu.Unlock()
|
||||
|
||||
oldSize := atomic.LoadUint64(&mm.currentCacheSize)
|
||||
atomic.StoreUint64(&mm.currentCacheSize, newSize)
|
||||
|
||||
// If we're reducing the cache size, trigger GC to free up memory
|
||||
if newSize < oldSize {
|
||||
// Calculate how much to free
|
||||
bytesToFree := oldSize - newSize
|
||||
|
||||
// Trigger GC on the cache to free up the excess memory
|
||||
// This is a simplified approach - in practice, you'd want to integrate
|
||||
// with the actual GC system to free the right amount
|
||||
if gcCache, ok := mm.cache.(interface{ ForceGC(uint) }); ok {
|
||||
gcCache.ForceGC(uint(bytesToFree))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// GetDynamicStats returns statistics about the dynamic cache manager
|
||||
func (mm *MemoryMonitor) GetDynamicStats() map[string]interface{} {
|
||||
mm.mu.RLock()
|
||||
defer mm.mu.RUnlock()
|
||||
|
||||
return map[string]interface{}{
|
||||
"original_cache_size": mm.originalCacheSize,
|
||||
"current_cache_size": atomic.LoadUint64(&mm.currentCacheSize),
|
||||
"adjustment_count": atomic.LoadInt64(&mm.adjustmentCount),
|
||||
"last_adjustment": mm.lastAdjustment,
|
||||
"memory_utilization": mm.GetMemoryUtilization(),
|
||||
"target_memory_usage": mm.GetTargetMemoryUsage(),
|
||||
"current_memory_usage": mm.GetCurrentMemoryUsage(),
|
||||
}
|
||||
}
|
||||
425
vfs/predictive/predictive.go
Normal file
425
vfs/predictive/predictive.go
Normal file
@@ -0,0 +1,425 @@
|
||||
package predictive
|
||||
|
||||
import (
|
||||
"context"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"time"
|
||||
)
|
||||
|
||||
// PredictiveCacheManager implements predictive caching strategies
|
||||
type PredictiveCacheManager struct {
|
||||
accessPredictor *AccessPredictor
|
||||
cacheWarmer *CacheWarmer
|
||||
prefetchQueue chan PrefetchRequest
|
||||
ctx context.Context
|
||||
cancel context.CancelFunc
|
||||
wg sync.WaitGroup
|
||||
stats *PredictiveStats
|
||||
}
|
||||
|
||||
// PrefetchRequest represents a request to prefetch content
|
||||
type PrefetchRequest struct {
|
||||
Key string
|
||||
Priority int
|
||||
Reason string
|
||||
RequestedAt time.Time
|
||||
}
|
||||
|
||||
// PredictiveStats tracks predictive caching statistics
|
||||
type PredictiveStats struct {
|
||||
PrefetchHits int64
|
||||
PrefetchMisses int64
|
||||
PrefetchRequests int64
|
||||
CacheWarmHits int64
|
||||
CacheWarmMisses int64
|
||||
mu sync.RWMutex
|
||||
}
|
||||
|
||||
// AccessPredictor predicts which files are likely to be accessed next
|
||||
type AccessPredictor struct {
|
||||
accessHistory map[string]*AccessSequence
|
||||
patterns map[string][]string // Key -> likely next keys
|
||||
mu sync.RWMutex
|
||||
}
|
||||
|
||||
// AccessSequence tracks access sequences for prediction
|
||||
type AccessSequence struct {
|
||||
Key string
|
||||
NextKeys []string
|
||||
Frequency map[string]int64
|
||||
LastSeen time.Time
|
||||
mu sync.RWMutex
|
||||
}
|
||||
|
||||
// CacheWarmer preloads popular content into cache
|
||||
type CacheWarmer struct {
|
||||
popularContent map[string]*PopularContent
|
||||
warmerQueue chan WarmRequest
|
||||
mu sync.RWMutex
|
||||
}
|
||||
|
||||
// PopularContent tracks popular content for warming
|
||||
type PopularContent struct {
|
||||
Key string
|
||||
AccessCount int64
|
||||
LastAccess time.Time
|
||||
Size int64
|
||||
Priority int
|
||||
}
|
||||
|
||||
// WarmRequest represents a cache warming request
|
||||
type WarmRequest struct {
|
||||
Key string
|
||||
Priority int
|
||||
Reason string
|
||||
Size int64
|
||||
RequestedAt time.Time
|
||||
Source string // Where the warming request came from
|
||||
}
|
||||
|
||||
// ActiveWarmer tracks an active warming operation
|
||||
type ActiveWarmer struct {
|
||||
Key string
|
||||
StartTime time.Time
|
||||
Priority int
|
||||
Reason string
|
||||
mu sync.RWMutex
|
||||
}
|
||||
|
||||
// WarmingStats tracks cache warming statistics
|
||||
type WarmingStats struct {
|
||||
WarmRequests int64
|
||||
WarmSuccesses int64
|
||||
WarmFailures int64
|
||||
WarmBytes int64
|
||||
WarmDuration time.Duration
|
||||
PrefetchRequests int64
|
||||
PrefetchSuccesses int64
|
||||
PrefetchFailures int64
|
||||
PrefetchBytes int64
|
||||
PrefetchDuration time.Duration
|
||||
}
|
||||
|
||||
// NewPredictiveCacheManager creates a new predictive cache manager
|
||||
func NewPredictiveCacheManager() *PredictiveCacheManager {
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
|
||||
pcm := &PredictiveCacheManager{
|
||||
accessPredictor: NewAccessPredictor(),
|
||||
cacheWarmer: NewCacheWarmer(),
|
||||
prefetchQueue: make(chan PrefetchRequest, 1000),
|
||||
ctx: ctx,
|
||||
cancel: cancel,
|
||||
stats: &PredictiveStats{},
|
||||
}
|
||||
|
||||
// Start background workers
|
||||
pcm.wg.Add(1)
|
||||
go pcm.prefetchWorker()
|
||||
|
||||
pcm.wg.Add(1)
|
||||
go pcm.analysisWorker()
|
||||
|
||||
return pcm
|
||||
}
|
||||
|
||||
// NewAccessPredictor creates a new access predictor
|
||||
func NewAccessPredictor() *AccessPredictor {
|
||||
return &AccessPredictor{
|
||||
accessHistory: make(map[string]*AccessSequence),
|
||||
patterns: make(map[string][]string),
|
||||
}
|
||||
}
|
||||
|
||||
// NewCacheWarmer creates a new cache warmer
|
||||
func NewCacheWarmer() *CacheWarmer {
|
||||
return &CacheWarmer{
|
||||
popularContent: make(map[string]*PopularContent),
|
||||
warmerQueue: make(chan WarmRequest, 100),
|
||||
}
|
||||
}
|
||||
|
||||
// NewWarmingStats creates a new warming stats tracker
|
||||
func NewWarmingStats() *WarmingStats {
|
||||
return &WarmingStats{}
|
||||
}
|
||||
|
||||
// NewActiveWarmer creates a new active warmer tracker
|
||||
func NewActiveWarmer(key string, priority int, reason string) *ActiveWarmer {
|
||||
return &ActiveWarmer{
|
||||
Key: key,
|
||||
StartTime: time.Now(),
|
||||
Priority: priority,
|
||||
Reason: reason,
|
||||
}
|
||||
}
|
||||
|
||||
// RecordAccess records a file access for prediction analysis (lightweight version)
|
||||
func (pcm *PredictiveCacheManager) RecordAccess(key string, previousKey string, size int64) {
|
||||
// Only record if we have a previous key to avoid overhead
|
||||
if previousKey != "" {
|
||||
pcm.accessPredictor.RecordSequence(previousKey, key)
|
||||
}
|
||||
|
||||
// Lightweight popular content tracking - only for large files
|
||||
if size > 1024*1024 { // Only track files > 1MB
|
||||
pcm.cacheWarmer.RecordAccess(key, size)
|
||||
}
|
||||
|
||||
// Skip expensive prediction checks on every access
|
||||
// Only check occasionally to reduce overhead
|
||||
}
|
||||
|
||||
// PredictNextAccess predicts the next likely file to be accessed
|
||||
func (pcm *PredictiveCacheManager) PredictNextAccess(currentKey string) []string {
|
||||
return pcm.accessPredictor.PredictNext(currentKey)
|
||||
}
|
||||
|
||||
// RequestPrefetch requests prefetching of predicted content
|
||||
func (pcm *PredictiveCacheManager) RequestPrefetch(key string, priority int, reason string) {
|
||||
select {
|
||||
case pcm.prefetchQueue <- PrefetchRequest{
|
||||
Key: key,
|
||||
Priority: priority,
|
||||
Reason: reason,
|
||||
RequestedAt: time.Now(),
|
||||
}:
|
||||
atomic.AddInt64(&pcm.stats.PrefetchRequests, 1)
|
||||
default:
|
||||
// Queue full, skip prefetch
|
||||
}
|
||||
}
|
||||
|
||||
// RecordSequence records an access sequence for prediction
|
||||
func (ap *AccessPredictor) RecordSequence(previousKey, currentKey string) {
|
||||
if previousKey == "" || currentKey == "" {
|
||||
return
|
||||
}
|
||||
|
||||
ap.mu.Lock()
|
||||
defer ap.mu.Unlock()
|
||||
|
||||
seq, exists := ap.accessHistory[previousKey]
|
||||
if !exists {
|
||||
seq = &AccessSequence{
|
||||
Key: previousKey,
|
||||
NextKeys: []string{},
|
||||
Frequency: make(map[string]int64),
|
||||
LastSeen: time.Now(),
|
||||
}
|
||||
ap.accessHistory[previousKey] = seq
|
||||
}
|
||||
|
||||
seq.mu.Lock()
|
||||
seq.Frequency[currentKey]++
|
||||
seq.LastSeen = time.Now()
|
||||
|
||||
// Update next keys list (keep top 5)
|
||||
nextKeys := make([]string, 0, 5)
|
||||
for key, _ := range seq.Frequency {
|
||||
nextKeys = append(nextKeys, key)
|
||||
if len(nextKeys) >= 5 {
|
||||
break
|
||||
}
|
||||
}
|
||||
seq.NextKeys = nextKeys
|
||||
seq.mu.Unlock()
|
||||
}
|
||||
|
||||
// PredictNext predicts the next likely files to be accessed
|
||||
func (ap *AccessPredictor) PredictNext(currentKey string) []string {
|
||||
ap.mu.RLock()
|
||||
defer ap.mu.RUnlock()
|
||||
|
||||
seq, exists := ap.accessHistory[currentKey]
|
||||
if !exists {
|
||||
return []string{}
|
||||
}
|
||||
|
||||
seq.mu.RLock()
|
||||
defer seq.mu.RUnlock()
|
||||
|
||||
// Return top predicted keys
|
||||
predictions := make([]string, len(seq.NextKeys))
|
||||
copy(predictions, seq.NextKeys)
|
||||
return predictions
|
||||
}
|
||||
|
||||
// IsPredictedAccess checks if an access was predicted
|
||||
func (ap *AccessPredictor) IsPredictedAccess(key string) bool {
|
||||
ap.mu.RLock()
|
||||
defer ap.mu.RUnlock()
|
||||
|
||||
// Check if this key appears in any prediction lists
|
||||
for _, seq := range ap.accessHistory {
|
||||
seq.mu.RLock()
|
||||
for _, predictedKey := range seq.NextKeys {
|
||||
if predictedKey == key {
|
||||
seq.mu.RUnlock()
|
||||
return true
|
||||
}
|
||||
}
|
||||
seq.mu.RUnlock()
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// RecordAccess records a file access for cache warming (lightweight version)
|
||||
func (cw *CacheWarmer) RecordAccess(key string, size int64) {
|
||||
// Use read lock first for better performance
|
||||
cw.mu.RLock()
|
||||
content, exists := cw.popularContent[key]
|
||||
cw.mu.RUnlock()
|
||||
|
||||
if !exists {
|
||||
// Only acquire write lock when creating new entry
|
||||
cw.mu.Lock()
|
||||
// Double-check after acquiring write lock
|
||||
if content, exists = cw.popularContent[key]; !exists {
|
||||
content = &PopularContent{
|
||||
Key: key,
|
||||
AccessCount: 1,
|
||||
LastAccess: time.Now(),
|
||||
Size: size,
|
||||
Priority: 1,
|
||||
}
|
||||
cw.popularContent[key] = content
|
||||
}
|
||||
cw.mu.Unlock()
|
||||
} else {
|
||||
// Lightweight update - just increment counter
|
||||
content.AccessCount++
|
||||
content.LastAccess = time.Now()
|
||||
|
||||
// Only update priority occasionally to reduce overhead
|
||||
if content.AccessCount%5 == 0 {
|
||||
if content.AccessCount > 10 {
|
||||
content.Priority = 3
|
||||
} else if content.AccessCount > 5 {
|
||||
content.Priority = 2
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// GetPopularContent returns the most popular content for warming
|
||||
func (cw *CacheWarmer) GetPopularContent(limit int) []*PopularContent {
|
||||
cw.mu.RLock()
|
||||
defer cw.mu.RUnlock()
|
||||
|
||||
// Sort by access count and return top items
|
||||
popular := make([]*PopularContent, 0, len(cw.popularContent))
|
||||
for _, content := range cw.popularContent {
|
||||
popular = append(popular, content)
|
||||
}
|
||||
|
||||
// Simple sort by access count (in production, use proper sorting)
|
||||
// For now, just return the first 'limit' items
|
||||
if len(popular) > limit {
|
||||
popular = popular[:limit]
|
||||
}
|
||||
|
||||
return popular
|
||||
}
|
||||
|
||||
// RequestWarming requests warming of a specific key
|
||||
func (cw *CacheWarmer) RequestWarming(key string, priority int, reason string, size int64) {
|
||||
select {
|
||||
case cw.warmerQueue <- WarmRequest{
|
||||
Key: key,
|
||||
Priority: priority,
|
||||
Reason: reason,
|
||||
Size: size,
|
||||
RequestedAt: time.Now(),
|
||||
Source: "predictive",
|
||||
}:
|
||||
// Successfully queued
|
||||
default:
|
||||
// Queue full, skip warming
|
||||
}
|
||||
}
|
||||
|
||||
// prefetchWorker processes prefetch requests
|
||||
func (pcm *PredictiveCacheManager) prefetchWorker() {
|
||||
defer pcm.wg.Done()
|
||||
|
||||
for {
|
||||
select {
|
||||
case <-pcm.ctx.Done():
|
||||
return
|
||||
case req := <-pcm.prefetchQueue:
|
||||
// Process prefetch request
|
||||
pcm.processPrefetchRequest(req)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// analysisWorker performs periodic analysis and cache warming
|
||||
func (pcm *PredictiveCacheManager) analysisWorker() {
|
||||
defer pcm.wg.Done()
|
||||
|
||||
ticker := time.NewTicker(30 * time.Second) // Analyze every 30 seconds
|
||||
defer ticker.Stop()
|
||||
|
||||
for {
|
||||
select {
|
||||
case <-pcm.ctx.Done():
|
||||
return
|
||||
case <-ticker.C:
|
||||
pcm.performAnalysis()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// processPrefetchRequest processes a prefetch request
|
||||
func (pcm *PredictiveCacheManager) processPrefetchRequest(req PrefetchRequest) {
|
||||
// In a real implementation, this would:
|
||||
// 1. Check if content is already cached
|
||||
// 2. If not, fetch and cache it
|
||||
// 3. Update statistics
|
||||
|
||||
// For now, just log the prefetch request
|
||||
// In production, integrate with the actual cache system
|
||||
}
|
||||
|
||||
// performAnalysis performs periodic analysis and cache warming
|
||||
func (pcm *PredictiveCacheManager) performAnalysis() {
|
||||
// Get popular content for warming
|
||||
popular := pcm.cacheWarmer.GetPopularContent(10)
|
||||
|
||||
// Request warming for popular content
|
||||
for _, content := range popular {
|
||||
if content.AccessCount > 5 { // Only warm frequently accessed content
|
||||
select {
|
||||
case pcm.cacheWarmer.warmerQueue <- WarmRequest{
|
||||
Key: content.Key,
|
||||
Priority: content.Priority,
|
||||
Reason: "popular_content",
|
||||
}:
|
||||
default:
|
||||
// Queue full, skip
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// GetStats returns predictive caching statistics
|
||||
func (pcm *PredictiveCacheManager) GetStats() *PredictiveStats {
|
||||
pcm.stats.mu.RLock()
|
||||
defer pcm.stats.mu.RUnlock()
|
||||
|
||||
return &PredictiveStats{
|
||||
PrefetchHits: atomic.LoadInt64(&pcm.stats.PrefetchHits),
|
||||
PrefetchMisses: atomic.LoadInt64(&pcm.stats.PrefetchMisses),
|
||||
PrefetchRequests: atomic.LoadInt64(&pcm.stats.PrefetchRequests),
|
||||
CacheWarmHits: atomic.LoadInt64(&pcm.stats.CacheWarmHits),
|
||||
CacheWarmMisses: atomic.LoadInt64(&pcm.stats.CacheWarmMisses),
|
||||
}
|
||||
}
|
||||
|
||||
// Stop stops the predictive cache manager
|
||||
func (pcm *PredictiveCacheManager) Stop() {
|
||||
pcm.cancel()
|
||||
pcm.wg.Wait()
|
||||
}
|
||||
87
vfs/types/types.go
Normal file
87
vfs/types/types.go
Normal file
@@ -0,0 +1,87 @@
|
||||
// vfs/types/types.go
|
||||
package types
|
||||
|
||||
import (
|
||||
"os"
|
||||
"time"
|
||||
)
|
||||
|
||||
// FileInfo contains metadata about a cached file
|
||||
type FileInfo struct {
|
||||
Key string `json:"key"`
|
||||
Size int64 `json:"size"`
|
||||
ATime time.Time `json:"atime"` // Last access time
|
||||
CTime time.Time `json:"ctime"` // Creation time
|
||||
AccessCount int `json:"access_count"`
|
||||
}
|
||||
|
||||
// NewFileInfo creates a new FileInfo with the given key and current timestamp
|
||||
func NewFileInfo(key string, size int64) *FileInfo {
|
||||
now := time.Now()
|
||||
return &FileInfo{
|
||||
Key: key,
|
||||
Size: size,
|
||||
ATime: now,
|
||||
CTime: now,
|
||||
AccessCount: 1,
|
||||
}
|
||||
}
|
||||
|
||||
// NewFileInfoFromOS creates a FileInfo from os.FileInfo
|
||||
func NewFileInfoFromOS(info os.FileInfo, key string) *FileInfo {
|
||||
return &FileInfo{
|
||||
Key: key,
|
||||
Size: info.Size(),
|
||||
ATime: time.Now(), // We don't have access time from os.FileInfo
|
||||
CTime: info.ModTime(),
|
||||
AccessCount: 1,
|
||||
}
|
||||
}
|
||||
|
||||
// UpdateAccess updates the access time and increments the access count
|
||||
func (fi *FileInfo) UpdateAccess() {
|
||||
fi.ATime = time.Now()
|
||||
fi.AccessCount++
|
||||
}
|
||||
|
||||
// BatchedTimeUpdate provides a way to batch time updates for better performance
|
||||
type BatchedTimeUpdate struct {
|
||||
currentTime time.Time
|
||||
lastUpdate time.Time
|
||||
updateInterval time.Duration
|
||||
}
|
||||
|
||||
// NewBatchedTimeUpdate creates a new batched time updater
|
||||
func NewBatchedTimeUpdate(interval time.Duration) *BatchedTimeUpdate {
|
||||
now := time.Now()
|
||||
return &BatchedTimeUpdate{
|
||||
currentTime: now,
|
||||
lastUpdate: now,
|
||||
updateInterval: interval,
|
||||
}
|
||||
}
|
||||
|
||||
// GetTime returns the current cached time, updating it if necessary
|
||||
func (btu *BatchedTimeUpdate) GetTime() time.Time {
|
||||
now := time.Now()
|
||||
if now.Sub(btu.lastUpdate) >= btu.updateInterval {
|
||||
btu.currentTime = now
|
||||
btu.lastUpdate = now
|
||||
}
|
||||
return btu.currentTime
|
||||
}
|
||||
|
||||
// UpdateAccessBatched updates the access time using batched time updates
|
||||
func (fi *FileInfo) UpdateAccessBatched(btu *BatchedTimeUpdate) {
|
||||
fi.ATime = btu.GetTime()
|
||||
fi.AccessCount++
|
||||
}
|
||||
|
||||
// GetTimeDecayedScore calculates a score based on access time and frequency
|
||||
// More recent and frequent accesses get higher scores
|
||||
func (fi *FileInfo) GetTimeDecayedScore() float64 {
|
||||
timeSinceAccess := time.Since(fi.ATime).Hours()
|
||||
decayFactor := 1.0 / (1.0 + timeSinceAccess/24.0) // Decay over days
|
||||
frequencyBonus := float64(fi.AccessCount) * 0.1
|
||||
return decayFactor + frequencyBonus
|
||||
}
|
||||
90
vfs/vfs.go
90
vfs/vfs.go
@@ -3,8 +3,7 @@ package vfs
|
||||
|
||||
import (
|
||||
"io"
|
||||
"os"
|
||||
"time"
|
||||
"s1d3sw1ped/steamcache2/vfs/types"
|
||||
)
|
||||
|
||||
// VFS defines the interface for virtual file systems
|
||||
@@ -19,7 +18,7 @@ type VFS interface {
|
||||
Delete(key string) error
|
||||
|
||||
// Stat returns information about the file at the given key
|
||||
Stat(key string) (*FileInfo, error)
|
||||
Stat(key string) (*types.FileInfo, error)
|
||||
|
||||
// Name returns the name of this VFS
|
||||
Name() string
|
||||
@@ -31,82 +30,17 @@ type VFS interface {
|
||||
Capacity() int64
|
||||
}
|
||||
|
||||
// FileInfo contains metadata about a cached file
|
||||
type FileInfo struct {
|
||||
Key string `json:"key"`
|
||||
Size int64 `json:"size"`
|
||||
ATime time.Time `json:"atime"` // Last access time
|
||||
CTime time.Time `json:"ctime"` // Creation time
|
||||
AccessCount int `json:"access_count"`
|
||||
}
|
||||
// FileInfo is an alias for types.FileInfo for backward compatibility
|
||||
type FileInfo = types.FileInfo
|
||||
|
||||
// NewFileInfo creates a new FileInfo with the given key and current timestamp
|
||||
func NewFileInfo(key string, size int64) *FileInfo {
|
||||
now := time.Now()
|
||||
return &FileInfo{
|
||||
Key: key,
|
||||
Size: size,
|
||||
ATime: now,
|
||||
CTime: now,
|
||||
AccessCount: 1,
|
||||
}
|
||||
}
|
||||
// NewFileInfo is an alias for types.NewFileInfo for backward compatibility
|
||||
var NewFileInfo = types.NewFileInfo
|
||||
|
||||
// NewFileInfoFromOS creates a FileInfo from os.FileInfo
|
||||
func NewFileInfoFromOS(info os.FileInfo, key string) *FileInfo {
|
||||
return &FileInfo{
|
||||
Key: key,
|
||||
Size: info.Size(),
|
||||
ATime: time.Now(), // We don't have access time from os.FileInfo
|
||||
CTime: info.ModTime(),
|
||||
AccessCount: 1,
|
||||
}
|
||||
}
|
||||
// NewFileInfoFromOS is an alias for types.NewFileInfoFromOS for backward compatibility
|
||||
var NewFileInfoFromOS = types.NewFileInfoFromOS
|
||||
|
||||
// UpdateAccess updates the access time and increments the access count
|
||||
func (fi *FileInfo) UpdateAccess() {
|
||||
fi.ATime = time.Now()
|
||||
fi.AccessCount++
|
||||
}
|
||||
// BatchedTimeUpdate is an alias for types.BatchedTimeUpdate for backward compatibility
|
||||
type BatchedTimeUpdate = types.BatchedTimeUpdate
|
||||
|
||||
// BatchedTimeUpdate provides a way to batch time updates for better performance
|
||||
type BatchedTimeUpdate struct {
|
||||
currentTime time.Time
|
||||
lastUpdate time.Time
|
||||
updateInterval time.Duration
|
||||
}
|
||||
|
||||
// NewBatchedTimeUpdate creates a new batched time updater
|
||||
func NewBatchedTimeUpdate(interval time.Duration) *BatchedTimeUpdate {
|
||||
now := time.Now()
|
||||
return &BatchedTimeUpdate{
|
||||
currentTime: now,
|
||||
lastUpdate: now,
|
||||
updateInterval: interval,
|
||||
}
|
||||
}
|
||||
|
||||
// GetTime returns the current cached time, updating it if necessary
|
||||
func (btu *BatchedTimeUpdate) GetTime() time.Time {
|
||||
now := time.Now()
|
||||
if now.Sub(btu.lastUpdate) >= btu.updateInterval {
|
||||
btu.currentTime = now
|
||||
btu.lastUpdate = now
|
||||
}
|
||||
return btu.currentTime
|
||||
}
|
||||
|
||||
// UpdateAccessBatched updates the access time using batched time updates
|
||||
func (fi *FileInfo) UpdateAccessBatched(btu *BatchedTimeUpdate) {
|
||||
fi.ATime = btu.GetTime()
|
||||
fi.AccessCount++
|
||||
}
|
||||
|
||||
// GetTimeDecayedScore calculates a score based on access time and frequency
|
||||
// More recent and frequent accesses get higher scores
|
||||
func (fi *FileInfo) GetTimeDecayedScore() float64 {
|
||||
timeSinceAccess := time.Since(fi.ATime).Hours()
|
||||
decayFactor := 1.0 / (1.0 + timeSinceAccess/24.0) // Decay over days
|
||||
frequencyBonus := float64(fi.AccessCount) * 0.1
|
||||
return decayFactor + frequencyBonus
|
||||
}
|
||||
// NewBatchedTimeUpdate is an alias for types.NewBatchedTimeUpdate for backward compatibility
|
||||
var NewBatchedTimeUpdate = types.NewBatchedTimeUpdate
|
||||
|
||||
Reference in New Issue
Block a user