11 Commits

Author SHA1 Message Date
ae013f9a3b Enhance SteamCache configuration and HTTP client settings
All checks were successful
Release Tag / release (push) Successful in 14s
- Added upstream server configuration to launch.json for improved connectivity.
- Increased HTTP client timeout from 60s to 120s for better handling of slow responses.
- Updated server timeouts in steamcache.go: increased ReadTimeout to 30s and WriteTimeout to 60s.
- Introduced ReadHeaderTimeout to mitigate header attacks and set MaxHeaderBytes to 1MB.
- Improved error logging in the Run method to include HTTP status codes for better debugging.
- Adjusted ServeHTTP method to handle root path and metrics endpoint correctly.
2025-07-19 04:40:05 -05:00
d94b53c395 Merge pull request 'Update .goreleaser.yaml and enhance HTTP client settings in steamcache.go' (#10) from fix/connection-pooling into main
All checks were successful
Release Tag / release (push) Successful in 15s
Reviewed-on: s1d3sw1ped/SteamCache2#10
2025-07-19 09:13:37 +00:00
847931ed43 Update .goreleaser.yaml and enhance HTTP client settings in steamcache.go
All checks were successful
PR Check / check-and-test (pull_request) Successful in 18s
- Removed copyright footer from .goreleaser.yaml.
- Increased HTTP client connection settings in steamcache.go for improved performance:
  - MaxIdleConns from 100 to 200
  - MaxIdleConnsPerHost from 10 to 50
  - IdleConnTimeout from 90s to 120s
  - TLSHandshakeTimeout from 10s to 15s
  - ResponseHeaderTimeout from 10s to 30s
  - ExpectContinueTimeout from 1s to 5s
  - Added DisableCompression and ForceAttemptHTTP2 options.
- Removed debug logging for manifest files in ServeHTTP method.
2025-07-19 04:12:56 -05:00
4387236d22 Merge pull request 'Update .goreleaser.yaml to use hyphens in name templates for archives and releases' (#9) from fix/goreleaser-config-fix-really into main
All checks were successful
Release Tag / release (push) Successful in 21s
Reviewed-on: s1d3sw1ped/SteamCache2#9
2025-07-19 08:23:09 +00:00
f6ce004922 Update .goreleaser.yaml to use hyphens in name templates for archives and releases
All checks were successful
PR Check / check-and-test (pull_request) Successful in 9s
2025-07-19 03:22:08 -05:00
8e487876d2 Merge pull request 'Remove steamcache2 from the list of files in .goreleaser.yaml archives section.' (#8) from fix/goreleaser-config-fix into main
Some checks failed
Release Tag / release (push) Failing after 20s
Reviewed-on: s1d3sw1ped/SteamCache2#8
2025-07-19 08:04:40 +00:00
1be7f5bd20 Remove steamcache2 from the list of files in .goreleaser.yaml archives section.
All checks were successful
PR Check / check-and-test (pull_request) Successful in 9s
2025-07-19 03:02:39 -05:00
f237b89ca7 Merge pull request 'Update versioning and logging in SteamCache2' (#7) from fix/goreleaser-config into main
Some checks failed
Release Tag / release (push) Failing after 22s
Reviewed-on: s1d3sw1ped/SteamCache2#7
2025-07-19 07:59:02 +00:00
ae07239021 Update versioning and logging in SteamCache2
All checks were successful
PR Check / check-and-test (pull_request) Successful in 11s
- Enhanced .goreleaser.yaml for improved build configuration, including static linking and ARM64 support.
- Updated logging in root.go to include version date during startup.
- Modified version.go to initialize and expose the build date alongside the version.
- Adjusted version command output to display both version and date for better clarity.
2025-07-19 02:58:19 -05:00
4876998f5d Merge pull request 'Enhance garbage collection and caching functionality' (#6) from feature/extended-gc-and-verification into main
Reviewed-on: s1d3sw1ped/SteamCache2#6
2025-07-19 07:28:12 +00:00
163e64790c Enhance garbage collection and caching functionality
All checks were successful
PR Check / check-and-test (pull_request) Successful in 21s
- Updated .gitignore to include all .exe files and ensure .smashignore is tracked.
- Expanded README.md with advanced configuration options for garbage collection algorithms, detailing available algorithms and use cases.
- Modified launch.json to include memory and disk garbage collection flags for better configuration.
- Refactored root.go to introduce memoryGC and diskGC flags for garbage collection algorithms.
- Implemented hash extraction and verification in steamcache.go to ensure data integrity during caching.
- Added new tests in steamcache_test.go for hash extraction and verification, ensuring correctness of caching behavior.
- Enhanced garbage collection strategies in gc.go, introducing LFU, FIFO, Largest, Smallest, and Hybrid algorithms with corresponding metrics.
- Updated caching logic to conditionally cache responses based on hash verification results.
2025-07-19 02:27:04 -05:00
12 changed files with 1088 additions and 115 deletions

4
.gitignore vendored
View File

@@ -1,5 +1,5 @@
dist/
tmp/
__*.exe
*.exe
.smashed.txt
.smashignore
.smashignore

View File

@@ -2,11 +2,17 @@ version: 2
before:
hooks:
- go mod tidy
- go mod tidy -v
builds:
- ldflags:
- id: default
binary: steamcache2
ldflags:
- -s
- -w
- -extldflags "-static"
- -X s1d3sw1ped/SteamCache2/version.Version={{.Version}}
- -X s1d3sw1ped/SteamCache2/version.Date={{.Date}}
env:
- CGO_ENABLED=0
goos:
@@ -14,19 +20,24 @@ builds:
- windows
goarch:
- amd64
- arm64
ignore:
- goos: windows
goarch: arm64
checksum:
name_template: "checksums.txt"
archives:
- formats: tar.gz
name_template: >-
{{ .ProjectName }}_
{{- title .Os }}_
{{- if eq .Arch "amd64" }}x86_64
{{- else if eq .Arch "386" }}i386
{{- else }}{{ .Arch }}{{ end }}
{{- if .Arm }}v{{ .Arm }}{{ end }}
- id: default
name_template: "{{ .ProjectName }}-{{ .Os }}-{{ .Arch }}"
formats: tar.gz
format_overrides:
- goos: windows
formats: zip
files:
- README.md
- LICENSE
changelog:
sort: asc
@@ -36,12 +47,7 @@ changelog:
- "^test:"
release:
name_template: '{{.ProjectName}}-{{.Version}}'
footer: >-
---
Released by [GoReleaser](https://github.com/goreleaser/goreleaser).
name_template: "{{ .ProjectName }}-{{ .Version }}"
gitea_urls:
api: https://git.s1d3sw1ped.com/api/v1

14
.vscode/launch.json vendored
View File

@@ -17,8 +17,14 @@
"10G",
"--disk-path",
"tmp/disk",
"--memory-gc",
"lfu",
"--disk-gc",
"lru",
"--log-level",
"debug",
"--upstream",
"http://192.168.2.5:80",
],
},
{
@@ -32,8 +38,12 @@
"10G",
"--disk-path",
"tmp/disk",
"--disk-gc",
"hybrid",
"--log-level",
"debug",
"--upstream",
"http://192.168.2.5:80",
],
},
{
@@ -45,8 +55,12 @@
"args": [
"--memory",
"1G",
"--memory-gc",
"lfu",
"--log-level",
"debug",
"--upstream",
"http://192.168.2.5:80",
],
}
]

View File

@@ -17,6 +17,38 @@ SteamCache2 is a blazing fast download cache for Steam, designed to reduce bandw
```sh
./SteamCache2 --memory 1G --disk 10G --disk-path tmp/disk
```
### Advanced Configuration
#### Garbage Collection Algorithms
SteamCache2 supports multiple garbage collection algorithms for both memory and disk caches:
```sh
# Use LFU for memory cache (good for long-running servers)
./SteamCache2 --memory 4G --memory-gc lfu --disk 100G --disk-gc lru
# Use FIFO for predictable eviction (good for testing)
./SteamCache2 --memory 2G --memory-gc fifo --disk 50G --disk-gc fifo
# Use size-based eviction for disk cache
./SteamCache2 --memory 1G --disk 200G --disk-gc largest
```
**Available GC Algorithms:**
- **`lru`** (default): Least Recently Used - evicts oldest accessed files
- **`lfu`**: Least Frequently Used - evicts least accessed files (good for popular content)
- **`fifo`**: First In, First Out - evicts oldest created files (predictable)
- **`largest`**: Size-based - evicts largest files first (maximizes file count)
- **`smallest`**: Size-based - evicts smallest files first (maximizes cache hit rate)
- **`hybrid`**: Combines access time and file size for optimal eviction
**Use Cases:**
- **LAN Events**: Use `lfu` for memory caches to keep popular games
- **Gaming Cafes**: Use `hybrid` for balanced performance
- **Testing**: Use `fifo` for predictable behavior
- **Large Files**: Use `largest` to prioritize keeping many small files
2. Configure your DNS:
- If your on Windows and don't want a whole network implementation (THIS)[#windows-hosts-file-override]

View File

@@ -20,6 +20,9 @@ var (
diskpath string
upstream string
memoryGC string
diskGC string
logLevel string
logFormat string
)
@@ -53,7 +56,7 @@ var rootCmd = &cobra.Command{
logger.Logger = zerolog.New(writer).With().Timestamp().Logger()
logger.Logger.Info().
Msg("SteamCache2 " + version.Version + " starting...")
Msg("SteamCache2 " + version.Version + " " + version.Date + " starting...")
address := ":80"
@@ -70,6 +73,8 @@ var rootCmd = &cobra.Command{
disk,
diskpath,
upstream,
memoryGC,
diskGC,
)
logger.Logger.Info().
@@ -100,6 +105,9 @@ func init() {
rootCmd.Flags().StringVarP(&upstream, "upstream", "u", "", "The upstream server to proxy requests overrides the host header from the client but forwards the original host header to the upstream server")
rootCmd.Flags().StringVarP(&memoryGC, "memory-gc", "", "lru", "Memory cache GC algorithm: lru, lfu, fifo, largest, smallest, hybrid")
rootCmd.Flags().StringVarP(&diskGC, "disk-gc", "", "lru", "Disk cache GC algorithm: lru, lfu, fifo, largest, smallest, hybrid")
rootCmd.Flags().StringVarP(&logLevel, "log-level", "l", "info", "Logging level: debug, info, error")
rootCmd.Flags().StringVarP(&logFormat, "log-format", "f", "console", "Logging format: json, console")
}

View File

@@ -15,7 +15,7 @@ var versionCmd = &cobra.Command{
Short: "prints the version of SteamCache2",
Long: `Prints the version of SteamCache2. This command is useful for checking the version of the application.`,
Run: func(cmd *cobra.Command, args []string) {
fmt.Fprintln(os.Stderr, "SteamCache2", version.Version)
fmt.Fprintln(os.Stderr, "SteamCache2", version.Version, version.Date)
},
}

View File

@@ -3,17 +3,23 @@ package steamcache
import (
"context"
"crypto/sha1"
"encoding/hex"
"fmt"
"io"
"net"
"net/http"
"net/url"
"os"
"path/filepath"
"regexp"
"s1d3sw1ped/SteamCache2/steamcache/logger"
"s1d3sw1ped/SteamCache2/vfs"
"s1d3sw1ped/SteamCache2/vfs/cache"
"s1d3sw1ped/SteamCache2/vfs/disk"
"s1d3sw1ped/SteamCache2/vfs/gc"
"s1d3sw1ped/SteamCache2/vfs/memory"
"sort"
"strings"
"sync"
"time"
@@ -24,6 +30,14 @@ import (
"github.com/prometheus/client_golang/prometheus/promhttp"
)
// min returns the minimum of two integers
func min(a, b int) int {
if a < b {
return a
}
return b
}
var (
requestsTotal = promauto.NewCounterVec(
prometheus.CounterOpts{
@@ -41,15 +55,97 @@ var (
[]string{"status"},
)
responseTime = promauto.NewHistogram(
responseTime = promauto.NewHistogramVec(
prometheus.HistogramOpts{
Name: "response_time_seconds",
Help: "Response time in seconds",
Buckets: prometheus.DefBuckets,
},
[]string{"cache_status"},
)
)
// hashVerificationTotal tracks hash verification attempts
var hashVerificationTotal = promauto.NewCounterVec(
prometheus.CounterOpts{
Name: "hash_verification_total",
Help: "Total hash verification attempts",
},
[]string{"result"},
)
// extractHashFromFilename extracts a hash from a filename if present
// Steam depot files often have hashes in their names like: filename_hash.ext
func extractHashFromFilename(filename string) (string, bool) {
// Common patterns for Steam depot files with hashes
patterns := []*regexp.Regexp{
regexp.MustCompile(`^([a-fA-F0-9]{40})$`), // Standalone SHA1 hash (40 hex chars)
regexp.MustCompile(`^([a-fA-F0-9]{40})\.`), // SHA1 hash with extension
}
for _, pattern := range patterns {
if matches := pattern.FindStringSubmatch(filename); len(matches) > 1 {
return strings.ToLower(matches[1]), true
}
}
// Debug: log when we don't find a hash pattern
if strings.Contains(filename, "manifest") {
logger.Logger.Debug().
Str("filename", filename).
Msg("No hash pattern found in manifest filename")
}
return "", false
}
// calculateFileHash calculates the SHA1 hash of the given data
func calculateFileHash(data []byte) string {
hash := sha1.Sum(data)
return hex.EncodeToString(hash[:])
}
// calculateResponseHash calculates the SHA1 hash of the full HTTP response
func calculateResponseHash(resp *http.Response, bodyData []byte) string {
hash := sha1.New()
// Include status line
statusLine := fmt.Sprintf("HTTP/1.1 %d %s\n", resp.StatusCode, resp.Status)
hash.Write([]byte(statusLine))
// Include headers (sorted for consistency)
headers := make([]string, 0, len(resp.Header))
for key, values := range resp.Header {
for _, value := range values {
headers = append(headers, fmt.Sprintf("%s: %s\n", key, value))
}
}
sort.Strings(headers)
for _, header := range headers {
hash.Write([]byte(header))
}
// Include empty line between headers and body
hash.Write([]byte("\n"))
// Include body
hash.Write(bodyData)
return hex.EncodeToString(hash.Sum(nil))
}
// verifyFileHash verifies that the file content matches the expected hash
func verifyFileHash(data []byte, expectedHash string) bool {
actualHash := calculateFileHash(data)
return strings.EqualFold(actualHash, expectedHash)
}
// verifyResponseHash verifies that the full HTTP response matches the expected hash
func verifyResponseHash(resp *http.Response, bodyData []byte, expectedHash string) bool {
actualHash := calculateResponseHash(resp, bodyData)
return strings.EqualFold(actualHash, expectedHash)
}
type SteamCache struct {
address string
upstream string
@@ -68,7 +164,7 @@ type SteamCache struct {
wg sync.WaitGroup
}
func New(address string, memorySize string, diskSize string, diskPath, upstream string) *SteamCache {
func New(address string, memorySize string, diskSize string, diskPath, upstream, memoryGC, diskGC string) *SteamCache {
memorysize, err := units.FromHumanSize(memorySize)
if err != nil {
panic(err)
@@ -80,21 +176,29 @@ func New(address string, memorySize string, diskSize string, diskPath, upstream
}
c := cache.New(
gc.PromotionDecider,
gc.AdaptivePromotionDeciderFunc,
)
var m *memory.MemoryFS
var mgc *gc.GCFS
if memorysize > 0 {
m = memory.New(memorysize)
mgc = gc.New(m, gc.LRUGC)
memoryGCAlgo := gc.GCAlgorithm(memoryGC)
if memoryGCAlgo == "" {
memoryGCAlgo = gc.LRU // default to LRU
}
mgc = gc.New(m, gc.GetGCAlgorithm(memoryGCAlgo))
}
var d *disk.DiskFS
var dgc *gc.GCFS
if disksize > 0 {
d = disk.New(diskPath, disksize)
dgc = gc.New(d, gc.LRUGC)
diskGCAlgo := gc.GCAlgorithm(diskGC)
if diskGCAlgo == "" {
diskGCAlgo = gc.LRU // default to LRU
}
dgc = gc.New(d, gc.GetGCAlgorithm(diskGCAlgo))
}
// configure the cache to match the specified mode (memory only, disk only, or memory and disk) based on the provided sizes
@@ -118,21 +222,23 @@ func New(address string, memorySize string, diskSize string, diskPath, upstream
}
transport := &http.Transport{
MaxIdleConns: 100,
MaxIdleConnsPerHost: 10,
IdleConnTimeout: 90 * time.Second,
MaxIdleConns: 200, // Increased from 100
MaxIdleConnsPerHost: 50, // Increased from 10
IdleConnTimeout: 120 * time.Second, // Increased from 90s
DialContext: (&net.Dialer{
Timeout: 30 * time.Second,
KeepAlive: 30 * time.Second,
}).DialContext,
TLSHandshakeTimeout: 10 * time.Second,
ResponseHeaderTimeout: 10 * time.Second,
ExpectContinueTimeout: 1 * time.Second,
TLSHandshakeTimeout: 15 * time.Second, // Increased from 10s
ResponseHeaderTimeout: 30 * time.Second, // Increased from 10s
ExpectContinueTimeout: 5 * time.Second, // Increased from 1s
DisableCompression: true, // Steam doesn't use compression
ForceAttemptHTTP2: true, // Enable HTTP/2 if available
}
client := &http.Client{
Transport: transport,
Timeout: 60 * time.Second,
Timeout: 120 * time.Second, // Increased from 60s
}
sc := &SteamCache{
@@ -145,16 +251,27 @@ func New(address string, memorySize string, diskSize string, diskPath, upstream
diskgc: dgc,
client: client,
server: &http.Server{
Addr: address,
ReadTimeout: 5 * time.Second,
WriteTimeout: 10 * time.Second,
IdleTimeout: 120 * time.Second,
Addr: address,
ReadTimeout: 30 * time.Second, // Increased
WriteTimeout: 60 * time.Second, // Increased
IdleTimeout: 120 * time.Second, // Good for keep-alive
ReadHeaderTimeout: 10 * time.Second, // New, for header attacks
MaxHeaderBytes: 1 << 20, // 1MB, optional
},
}
// Log GC algorithm configuration
if m != nil {
logger.Logger.Info().Str("memory_gc", memoryGC).Msg("Memory cache GC algorithm configured")
}
if d != nil {
logger.Logger.Info().Str("disk_gc", diskGC).Msg("Disk cache GC algorithm configured")
}
if d != nil {
if d.Size() > d.Capacity() {
gc.LRUGC(d, uint(d.Size()-d.Capacity()))
gcHandler := gc.GetGCAlgorithm(gc.GCAlgorithm(diskGC))
gcHandler(d, uint(d.Size()-d.Capacity()))
}
}
@@ -165,7 +282,7 @@ func (sc *SteamCache) Run() {
if sc.upstream != "" {
resp, err := sc.client.Get(sc.upstream)
if err != nil || resp.StatusCode != http.StatusOK {
logger.Logger.Error().Err(err).Str("upstream", sc.upstream).Msg("Failed to connect to upstream server")
logger.Logger.Error().Err(err).Int("status_code", resp.StatusCode).Str("upstream", sc.upstream).Msg("Failed to connect to upstream server")
os.Exit(1)
}
resp.Body.Close()
@@ -198,11 +315,6 @@ func (sc *SteamCache) Shutdown() {
}
func (sc *SteamCache) ServeHTTP(w http.ResponseWriter, r *http.Request) {
if r.URL.Path == "/metrics" {
promhttp.Handler().ServeHTTP(w, r)
return
}
if r.Method != http.MethodGet {
requestsTotal.WithLabelValues(r.Method, "405").Inc()
logger.Logger.Warn().Str("method", r.Method).Msg("Only GET method is supported")
@@ -210,6 +322,11 @@ func (sc *SteamCache) ServeHTTP(w http.ResponseWriter, r *http.Request) {
return
}
if r.URL.Path == "/" {
w.WriteHeader(http.StatusFound) // this is used by steamcache2's upstream verification at startup
return
}
if r.URL.String() == "/lancache-heartbeat" {
w.Header().Add("X-LanCache-Processed-By", "SteamCache2")
w.WriteHeader(http.StatusNoContent)
@@ -217,13 +334,17 @@ func (sc *SteamCache) ServeHTTP(w http.ResponseWriter, r *http.Request) {
return
}
if r.URL.Path == "/metrics" {
promhttp.Handler().ServeHTTP(w, r)
return
}
if strings.HasPrefix(r.URL.String(), "/depot/") {
// trim the query parameters from the URL path
// this is necessary because the cache key should not include query parameters
path := strings.Split(r.URL.String(), "?")[0]
tstart := time.Now()
defer func() { responseTime.Observe(time.Since(tstart).Seconds()) }()
cacheKey := strings.ReplaceAll(path[1:], "\\", "/") // replace all backslashes with forward slashes shouldn't be necessary but just in case
@@ -252,6 +373,7 @@ func (sc *SteamCache) ServeHTTP(w http.ResponseWriter, r *http.Request) {
requestsTotal.WithLabelValues(r.Method, "200").Inc()
cacheStatusTotal.WithLabelValues("HIT").Inc()
responseTime.WithLabelValues("HIT").Observe(time.Since(tstart).Seconds())
return
}
@@ -328,17 +450,71 @@ func (sc *SteamCache) ServeHTTP(w http.ResponseWriter, r *http.Request) {
size := resp.ContentLength
// this is sortof not needed as we should always be able to get a writer from the cache as long as the gc is able to reclaim enough space aka the file is not bigger than the disk can handle
ww := w.(io.Writer) // default writer to write to the response writer
writer, _ := sc.vfs.Create(cacheKey, size) // create a writer to write to the cache
if writer != nil { // if the writer is not nil, it means the cache is writable
defer writer.Close() // close the writer when done
ww = io.MultiWriter(w, writer) // write to both the response writer and the cache writer
// Read the entire response body into memory for hash verification
bodyData, err := io.ReadAll(resp.Body)
if err != nil {
requestsTotal.WithLabelValues(r.Method, "500").Inc()
logger.Logger.Error().Err(err).Str("url", req.URL.String()).Msg("Failed to read response body")
http.Error(w, "Failed to read response body", http.StatusInternalServerError)
return
}
w.Header().Add("X-LanCache-Status", "MISS")
// Extract filename from cache key for hash verification
filename := filepath.Base(cacheKey)
expectedHash, hasHash := extractHashFromFilename(filename)
io.Copy(ww, resp.Body)
// Hash verification using Steam's X-Content-Sha header and content length verification
hashVerified := true
if hasHash {
// Get the hash from Steam's X-Content-Sha header
steamHash := resp.Header.Get("X-Content-Sha")
// Verify using Steam's hash
if strings.EqualFold(steamHash, expectedHash) {
hashVerificationTotal.WithLabelValues("success").Inc()
} else {
hashVerificationTotal.WithLabelValues("failed").Inc()
logger.Logger.Error().
Str("key", cacheKey).
Str("expected_hash", expectedHash).
Str("steam_hash", steamHash).
Int("content_length", len(bodyData)).
Msg("Steam hash verification failed - Steam's hash doesn't match filename")
hashVerified = false
}
} else {
hashVerificationTotal.WithLabelValues("no_hash").Inc()
}
// Always verify content length as an additional safety check
if resp.ContentLength > 0 && int64(len(bodyData)) != resp.ContentLength {
hashVerificationTotal.WithLabelValues("content_length_failed").Inc()
logger.Logger.Error().
Str("key", cacheKey).
Int("actual_content_length", len(bodyData)).
Int64("expected_content_length", resp.ContentLength).
Msg("Content length verification failed")
hashVerified = false
} else if resp.ContentLength > 0 {
hashVerificationTotal.WithLabelValues("content_length_success").Inc()
}
// Write to response (always serve the file)
w.Header().Add("X-LanCache-Status", "MISS")
w.Write(bodyData)
// Only cache the file if hash verification passed (or no hash was present)
if hashVerified {
writer, _ := sc.vfs.Create(cacheKey, size)
if writer != nil {
defer writer.Close()
writer.Write(bodyData)
}
} else {
logger.Logger.Warn().
Str("key", cacheKey).
Msg("File served but not cached due to hash verification failure")
}
logger.Logger.Info().
Str("key", cacheKey).
@@ -349,6 +525,7 @@ func (sc *SteamCache) ServeHTTP(w http.ResponseWriter, r *http.Request) {
requestsTotal.WithLabelValues(r.Method, "200").Inc()
cacheStatusTotal.WithLabelValues("MISS").Inc()
responseTime.WithLabelValues("MISS").Observe(time.Since(tstart).Seconds())
return
}

View File

@@ -3,6 +3,7 @@ package steamcache
import (
"io"
"net/http"
"os"
"path/filepath"
"testing"
@@ -13,7 +14,7 @@ func TestCaching(t *testing.T) {
os.WriteFile(filepath.Join(td, "key2"), []byte("value2"), 0644)
sc := New("localhost:8080", "1G", "1G", td, "")
sc := New("localhost:8080", "1G", "1G", td, "", "lru", "lru")
w, err := sc.vfs.Create("key", 5)
if err != nil {
@@ -84,7 +85,7 @@ func TestCaching(t *testing.T) {
}
func TestCacheMissAndHit(t *testing.T) {
sc := New("localhost:8080", "0", "1G", t.TempDir(), "")
sc := New("localhost:8080", "0", "1G", t.TempDir(), "", "lru", "lru")
key := "testkey"
value := []byte("testvalue")
@@ -108,3 +109,137 @@ func TestCacheMissAndHit(t *testing.T) {
t.Errorf("expected %s, got %s", value, got)
}
}
func TestHashExtraction(t *testing.T) {
// Test the specific key from the user's issue
testCases := []struct {
filename string
expectedHash string
shouldHaveHash bool
}{
{
filename: "e89c81a1a926eb4732e146bc806491da8a7d89ca",
expectedHash: "e89c81a1a926eb4732e146bc806491da8a7d89ca",
shouldHaveHash: true, // Now it should work with the new standalone hash pattern
},
{
filename: "chunk_e89c81a1a926eb4732e146bc806491da8a7d89ca",
expectedHash: "",
shouldHaveHash: false, // No longer supported with simplified patterns
},
{
filename: "file.e89c81a1a926eb4732e146bc806491da8a7d89ca.chunk",
expectedHash: "",
shouldHaveHash: false, // No longer supported with simplified patterns
},
{
filename: "chunk_abc123def456",
expectedHash: "",
shouldHaveHash: false, // Not 40 chars
},
}
for _, tc := range testCases {
hash, hasHash := extractHashFromFilename(tc.filename)
if hasHash != tc.shouldHaveHash {
t.Errorf("filename: %s, expected hasHash: %v, got: %v", tc.filename, tc.shouldHaveHash, hasHash)
}
if hasHash && hash != tc.expectedHash {
t.Errorf("filename: %s, expected hash: %s, got: %s", tc.filename, tc.expectedHash, hash)
}
}
}
func TestHashCalculation(t *testing.T) {
// Test data
testData := []byte("Hello, World!")
// Calculate hash
hash := calculateFileHash(testData)
// Expected SHA1 hash of "Hello, World!"
expectedHash := "0a0a9f2a6772942557ab5355d76af442f8f65e01"
if hash != expectedHash {
t.Errorf("Hash calculation failed: expected %s, got %s", expectedHash, hash)
}
// Test verification
if !verifyFileHash(testData, expectedHash) {
t.Error("Hash verification failed for correct hash")
}
if verifyFileHash(testData, "wronghash") {
t.Error("Hash verification passed for wrong hash")
}
}
func TestHashVerificationWithRealData(t *testing.T) {
// Test with some real data to ensure our hash calculation is correct
testCases := []struct {
data string
expected string
}{
{"", "da39a3ee5e6b4b0d3255bfef95601890afd80709"}, // SHA1 of empty string
{"test", "a94a8fe5ccb19ba61c4c0873d391e987982fbbd3"}, // SHA1 of "test"
{"Hello, World!", "0a0a9f2a6772942557ab5355d76af442f8f65e01"}, // SHA1 of "Hello, World!"
}
for _, tc := range testCases {
data := []byte(tc.data)
hash := calculateFileHash(data)
if hash != tc.expected {
t.Errorf("Hash calculation failed for '%s': expected %s, got %s", tc.data, tc.expected, hash)
}
if !verifyFileHash(data, tc.expected) {
t.Errorf("Hash verification failed for '%s'", tc.data)
}
}
}
func TestResponseHashCalculation(t *testing.T) {
// Create a mock HTTP response
resp := &http.Response{
StatusCode: 200,
Status: "200 OK",
Header: http.Header{
"Content-Type": []string{"application/octet-stream"},
"Content-Length": []string{"13"},
"Cache-Control": []string{"public, max-age=3600"},
},
}
bodyData := []byte("Hello, World!")
// Calculate response hash
responseHash := calculateResponseHash(resp, bodyData)
// The hash should be different from just the body hash
bodyHash := calculateFileHash(bodyData)
if responseHash == bodyHash {
t.Error("Response hash should be different from body hash when headers are present")
}
// Test that the same response produces the same hash
responseHash2 := calculateResponseHash(resp, bodyData)
if responseHash != responseHash2 {
t.Error("Response hash should be consistent for the same response")
}
// Test with different headers
resp2 := &http.Response{
StatusCode: 200,
Status: "200 OK",
Header: http.Header{
"Content-Type": []string{"text/plain"},
"Content-Length": []string{"13"},
},
}
responseHash3 := calculateResponseHash(resp2, bodyData)
if responseHash == responseHash3 {
t.Error("Response hash should be different for different headers")
}
}

View File

@@ -1,10 +1,16 @@
// version/version.go
package version
import "time"
var Version string
var Date string
func init() {
if Version == "" {
Version = "0.0.0-dev"
}
if Date == "" {
Date = time.Now().Format("2006-01-02 15:04:05")
}
}

5
vfs/cache/cache.go vendored
View File

@@ -6,6 +6,7 @@ import (
"io"
"s1d3sw1ped/SteamCache2/vfs"
"s1d3sw1ped/SteamCache2/vfs/cachestate"
"s1d3sw1ped/SteamCache2/vfs/gc"
"s1d3sw1ped/SteamCache2/vfs/vfserror"
"sync"
)
@@ -98,6 +99,10 @@ func (c *CacheFS) Open(key string) (io.ReadCloser, error) {
switch state {
case cachestate.CacheStateHit:
// if c.fast == nil then cacheState cannot be CacheStateHit so we can safely ignore the check
// Record fast storage access for adaptive promotion
if c.fast != nil {
gc.RecordFastStorageAccess()
}
return c.fast.Open(key)
case cachestate.CacheStateMiss:
slowReader, err := c.slow.Open(key)

View File

@@ -10,7 +10,12 @@ import (
"s1d3sw1ped/SteamCache2/vfs/disk"
"s1d3sw1ped/SteamCache2/vfs/memory"
"s1d3sw1ped/SteamCache2/vfs/vfserror"
"sort"
"sync"
"time"
"github.com/prometheus/client_golang/prometheus"
"github.com/prometheus/client_golang/prometheus/promauto"
)
var (
@@ -18,6 +23,53 @@ var (
ErrInsufficientSpace = fmt.Errorf("no files to delete")
)
// Prometheus metrics for adaptive promotion
var (
promotionThresholds = promauto.NewGaugeVec(
prometheus.GaugeOpts{
Name: "promotion_thresholds_bytes",
Help: "Current promotion thresholds in bytes",
},
[]string{"threshold_type"},
)
promotionWindows = promauto.NewGaugeVec(
prometheus.GaugeOpts{
Name: "promotion_windows_seconds",
Help: "Current promotion time windows in seconds",
},
[]string{"window_type"},
)
promotionStats = promauto.NewGaugeVec(
prometheus.GaugeOpts{
Name: "promotion_stats",
Help: "Promotion statistics",
},
[]string{"metric_type"},
)
promotionAdaptations = promauto.NewCounterVec(
prometheus.CounterOpts{
Name: "promotion_adaptations_total",
Help: "Total number of promotion threshold adaptations",
},
[]string{"direction"},
)
)
// GCAlgorithm represents different garbage collection strategies
type GCAlgorithm string
const (
LRU GCAlgorithm = "lru"
LFU GCAlgorithm = "lfu"
FIFO GCAlgorithm = "fifo"
Largest GCAlgorithm = "largest"
Smallest GCAlgorithm = "smallest"
Hybrid GCAlgorithm = "hybrid"
)
// LRUGC deletes files in LRU order until enough space is reclaimed.
func LRUGC(vfss vfs.VFS, size uint) error {
logger.Logger.Debug().Uint("target", size).Msg("Attempting to reclaim space using LRU GC")
@@ -59,10 +111,578 @@ func LRUGC(vfss vfs.VFS, size uint) error {
}
}
// LFUGC deletes files in LFU (Least Frequently Used) order until enough space is reclaimed.
func LFUGC(vfss vfs.VFS, size uint) error {
logger.Logger.Debug().Uint("target", size).Msg("Attempting to reclaim space using LFU GC")
// Get all files and sort by access count (frequency)
files := getAllFiles(vfss)
if len(files) == 0 {
return ErrInsufficientSpace
}
// Sort by access count (ascending - least frequently used first)
sort.Slice(files, func(i, j int) bool {
return files[i].AccessCount < files[j].AccessCount
})
var reclaimed uint
for _, fi := range files {
if reclaimed >= size {
break
}
err := vfss.Delete(fi.Name)
if err != nil {
continue
}
reclaimed += uint(fi.Size)
}
if reclaimed >= size {
logger.Logger.Debug().Uint("target", size).Uint("achieved", reclaimed).Msg("Reclaimed enough space using LFU GC")
return nil
}
return ErrInsufficientSpace
}
// FIFOGC deletes files in FIFO (First In, First Out) order until enough space is reclaimed.
func FIFOGC(vfss vfs.VFS, size uint) error {
logger.Logger.Debug().Uint("target", size).Msg("Attempting to reclaim space using FIFO GC")
// Get all files and sort by creation time (oldest first)
files := getAllFiles(vfss)
if len(files) == 0 {
return ErrInsufficientSpace
}
// Sort by creation time (ascending - oldest first)
sort.Slice(files, func(i, j int) bool {
return files[i].MTime.Before(files[j].MTime)
})
var reclaimed uint
for _, fi := range files {
if reclaimed >= size {
break
}
err := vfss.Delete(fi.Name)
if err != nil {
continue
}
reclaimed += uint(fi.Size)
}
if reclaimed >= size {
logger.Logger.Debug().Uint("target", size).Uint("achieved", reclaimed).Msg("Reclaimed enough space using FIFO GC")
return nil
}
return ErrInsufficientSpace
}
// LargestGC deletes the largest files first until enough space is reclaimed.
func LargestGC(vfss vfs.VFS, size uint) error {
logger.Logger.Debug().Uint("target", size).Msg("Attempting to reclaim space using Largest GC")
// Get all files and sort by size (largest first)
files := getAllFiles(vfss)
if len(files) == 0 {
return ErrInsufficientSpace
}
// Sort by size (descending - largest first)
sort.Slice(files, func(i, j int) bool {
return files[i].Size > files[j].Size
})
var reclaimed uint
for _, fi := range files {
if reclaimed >= size {
break
}
err := vfss.Delete(fi.Name)
if err != nil {
continue
}
reclaimed += uint(fi.Size)
}
if reclaimed >= size {
logger.Logger.Debug().Uint("target", size).Uint("achieved", reclaimed).Msg("Reclaimed enough space using Largest GC")
return nil
}
return ErrInsufficientSpace
}
// SmallestGC deletes the smallest files first until enough space is reclaimed.
func SmallestGC(vfss vfs.VFS, size uint) error {
logger.Logger.Debug().Uint("target", size).Msg("Attempting to reclaim space using Smallest GC")
// Get all files and sort by size (smallest first)
files := getAllFiles(vfss)
if len(files) == 0 {
return ErrInsufficientSpace
}
// Sort by size (ascending - smallest first)
sort.Slice(files, func(i, j int) bool {
return files[i].Size < files[j].Size
})
var reclaimed uint
for _, fi := range files {
if reclaimed >= size {
break
}
err := vfss.Delete(fi.Name)
if err != nil {
continue
}
reclaimed += uint(fi.Size)
}
if reclaimed >= size {
logger.Logger.Debug().Uint("target", size).Uint("achieved", reclaimed).Msg("Reclaimed enough space using Smallest GC")
return nil
}
return ErrInsufficientSpace
}
// HybridGC combines LRU and size-based eviction with a scoring system.
func HybridGC(vfss vfs.VFS, size uint) error {
logger.Logger.Debug().Uint("target", size).Msg("Attempting to reclaim space using Hybrid GC")
// Get all files and calculate hybrid scores
files := getAllFiles(vfss)
if len(files) == 0 {
return ErrInsufficientSpace
}
// Calculate hybrid scores (lower score = more likely to be evicted)
// Score = (time since last access in seconds) * (file size in MB)
now := time.Now()
for i := range files {
timeSinceAccess := now.Sub(files[i].ATime).Seconds()
sizeMB := float64(files[i].Size) / (1024 * 1024)
files[i].HybridScore = timeSinceAccess * sizeMB
}
// Sort by hybrid score (ascending - lowest scores first)
sort.Slice(files, func(i, j int) bool {
return files[i].HybridScore < files[j].HybridScore
})
var reclaimed uint
for _, fi := range files {
if reclaimed >= size {
break
}
err := vfss.Delete(fi.Name)
if err != nil {
continue
}
reclaimed += uint(fi.Size)
}
if reclaimed >= size {
logger.Logger.Debug().Uint("target", size).Uint("achieved", reclaimed).Msg("Reclaimed enough space using Hybrid GC")
return nil
}
return ErrInsufficientSpace
}
// fileInfoWithMetadata extends FileInfo with additional metadata for GC algorithms
type fileInfoWithMetadata struct {
Name string
Size int64
MTime time.Time
ATime time.Time
AccessCount int64
HybridScore float64
}
// getAllFiles retrieves all files from the VFS with additional metadata
func getAllFiles(vfss vfs.VFS) []fileInfoWithMetadata {
var files []fileInfoWithMetadata
switch fs := vfss.(type) {
case *disk.DiskFS:
allFiles := fs.StatAll()
for _, fi := range allFiles {
// For disk, we can't easily track access count, so we'll use 1 as default
files = append(files, fileInfoWithMetadata{
Name: fi.Name(),
Size: fi.Size(),
MTime: fi.ModTime(),
ATime: fi.AccessTime(),
AccessCount: 1,
})
}
case *memory.MemoryFS:
allFiles := fs.StatAll()
for _, fi := range allFiles {
// For memory, we can't easily track access count, so we'll use 1 as default
files = append(files, fileInfoWithMetadata{
Name: fi.Name(),
Size: fi.Size(),
MTime: fi.ModTime(),
ATime: fi.AccessTime(),
AccessCount: 1,
})
}
}
return files
}
// GetGCAlgorithm returns the appropriate GC function based on the algorithm name
func GetGCAlgorithm(algorithm GCAlgorithm) GCHandlerFunc {
switch algorithm {
case LRU:
return LRUGC
case LFU:
return LFUGC
case FIFO:
return FIFOGC
case Largest:
return LargestGC
case Smallest:
return SmallestGC
case Hybrid:
return HybridGC
default:
logger.Logger.Warn().Str("algorithm", string(algorithm)).Msg("Unknown GC algorithm, falling back to LRU")
return LRUGC
}
}
func PromotionDecider(fi *vfs.FileInfo, cs cachestate.CacheState) bool {
return time.Since(fi.AccessTime()) < time.Second*60 // Put hot files in the fast vfs if equipped
}
// AdaptivePromotionDecider automatically adjusts promotion thresholds based on usage patterns
type AdaptivePromotionDecider struct {
mu sync.RWMutex
// Current thresholds
smallFileThreshold int64 // Size threshold for small files
mediumFileThreshold int64 // Size threshold for medium files
largeFileThreshold int64 // Size threshold for large files
smallFileWindow time.Duration // Time window for small files
mediumFileWindow time.Duration // Time window for medium files
largeFileWindow time.Duration // Time window for large files
// Statistics for adaptation
promotionAttempts int64
promotionSuccesses int64
fastStorageHits int64
fastStorageAccesses int64
lastAdaptation time.Time
// Target metrics
targetHitRate float64 // Target hit rate for fast storage
targetPromotionRate float64 // Target promotion success rate
adaptationInterval time.Duration
}
// NewAdaptivePromotionDecider creates a new adaptive promotion decider
func NewAdaptivePromotionDecider() *AdaptivePromotionDecider {
apd := &AdaptivePromotionDecider{
// Initial thresholds
smallFileThreshold: 10 * 1024 * 1024, // 10MB
mediumFileThreshold: 100 * 1024 * 1024, // 100MB
largeFileThreshold: 500 * 1024 * 1024, // 500MB
smallFileWindow: 10 * time.Minute,
mediumFileWindow: 2 * time.Minute,
largeFileWindow: 30 * time.Second,
// Target metrics
targetHitRate: 0.8, // 80% hit rate
targetPromotionRate: 0.7, // 70% promotion success rate
adaptationInterval: 5 * time.Minute,
}
// Initialize Prometheus metrics
apd.updatePrometheusMetrics()
return apd
}
// ShouldPromote determines if a file should be promoted based on adaptive thresholds
func (apd *AdaptivePromotionDecider) ShouldPromote(fi *vfs.FileInfo, cs cachestate.CacheState) bool {
apd.mu.Lock()
defer apd.mu.Unlock()
// Check if it's time to adapt thresholds
if time.Since(apd.lastAdaptation) > apd.adaptationInterval {
apd.adaptThresholds()
}
size := fi.Size()
timeSinceAccess := time.Since(fi.AccessTime())
// Record promotion attempt
apd.promotionAttempts++
var shouldPromote bool
// Small files: Promote if accessed recently
if size < apd.smallFileThreshold {
shouldPromote = timeSinceAccess < apd.smallFileWindow
} else if size < apd.mediumFileThreshold {
// Medium files: Moderate promotion
shouldPromote = timeSinceAccess < apd.mediumFileWindow
} else if size < apd.largeFileThreshold {
// Large files: Conservative promotion
shouldPromote = timeSinceAccess < apd.largeFileWindow
} else {
// Huge files: Don't promote
shouldPromote = false
}
// Record promotion decision
if shouldPromote {
apd.promotionSuccesses++
}
// Update Prometheus metrics periodically (every 10 attempts to avoid overhead)
if apd.promotionAttempts%10 == 0 {
apd.updatePrometheusMetrics()
}
return shouldPromote
}
// RecordFastStorageAccess records when fast storage is accessed
func (apd *AdaptivePromotionDecider) RecordFastStorageAccess() {
apd.mu.Lock()
defer apd.mu.Unlock()
apd.fastStorageAccesses++
// Update Prometheus metrics periodically
if apd.fastStorageAccesses%10 == 0 {
apd.updatePrometheusMetrics()
}
}
// RecordFastStorageHit records when fast storage has a hit
func (apd *AdaptivePromotionDecider) RecordFastStorageHit() {
apd.mu.Lock()
defer apd.mu.Unlock()
apd.fastStorageHits++
// Update Prometheus metrics periodically
if apd.fastStorageHits%10 == 0 {
apd.updatePrometheusMetrics()
}
}
// adaptThresholds adjusts thresholds based on current performance
func (apd *AdaptivePromotionDecider) adaptThresholds() {
if apd.promotionAttempts < 10 || apd.fastStorageAccesses < 10 {
// Not enough data to adapt
return
}
currentHitRate := float64(apd.fastStorageHits) / float64(apd.fastStorageAccesses)
currentPromotionRate := float64(apd.promotionSuccesses) / float64(apd.promotionAttempts)
logger.Logger.Debug().
Float64("hit_rate", currentHitRate).
Float64("promotion_rate", currentPromotionRate).
Float64("target_hit_rate", apd.targetHitRate).
Float64("target_promotion_rate", apd.targetPromotionRate).
Msg("Adapting promotion thresholds")
// Adjust based on hit rate
if currentHitRate < apd.targetHitRate {
// Hit rate too low - be more aggressive with promotion
apd.adjustThresholdsMoreAggressive()
} else if currentHitRate > apd.targetHitRate+0.1 {
// Hit rate too high - be more conservative
apd.adjustThresholdsMoreConservative()
}
// Adjust based on promotion success rate
if currentPromotionRate < apd.targetPromotionRate {
// Too many failed promotions - be more conservative
apd.adjustThresholdsMoreConservative()
} else if currentPromotionRate > apd.targetPromotionRate+0.1 {
// High promotion success - can be more aggressive
apd.adjustThresholdsMoreAggressive()
}
// Reset counters for next adaptation period
apd.promotionAttempts = 0
apd.promotionSuccesses = 0
apd.fastStorageHits = 0
apd.fastStorageAccesses = 0
apd.lastAdaptation = time.Now()
logger.Logger.Info().
Int64("small_threshold_mb", apd.smallFileThreshold/(1024*1024)).
Int64("medium_threshold_mb", apd.mediumFileThreshold/(1024*1024)).
Int64("large_threshold_mb", apd.largeFileThreshold/(1024*1024)).
Dur("small_window", apd.smallFileWindow).
Dur("medium_window", apd.mediumFileWindow).
Dur("large_window", apd.largeFileWindow).
Msg("Updated promotion thresholds")
}
// updatePrometheusMetrics updates all Prometheus metrics with current values
func (apd *AdaptivePromotionDecider) updatePrometheusMetrics() {
// Update threshold metrics
promotionThresholds.WithLabelValues("small").Set(float64(apd.smallFileThreshold))
promotionThresholds.WithLabelValues("medium").Set(float64(apd.mediumFileThreshold))
promotionThresholds.WithLabelValues("large").Set(float64(apd.largeFileThreshold))
// Update window metrics
promotionWindows.WithLabelValues("small").Set(apd.smallFileWindow.Seconds())
promotionWindows.WithLabelValues("medium").Set(apd.mediumFileWindow.Seconds())
promotionWindows.WithLabelValues("large").Set(apd.largeFileWindow.Seconds())
// Update statistics metrics
hitRate := 0.0
if apd.fastStorageAccesses > 0 {
hitRate = float64(apd.fastStorageHits) / float64(apd.fastStorageAccesses)
}
promotionRate := 0.0
if apd.promotionAttempts > 0 {
promotionRate = float64(apd.promotionSuccesses) / float64(apd.promotionAttempts)
}
promotionStats.WithLabelValues("hit_rate").Set(hitRate)
promotionStats.WithLabelValues("promotion_rate").Set(promotionRate)
promotionStats.WithLabelValues("promotion_attempts").Set(float64(apd.promotionAttempts))
promotionStats.WithLabelValues("promotion_successes").Set(float64(apd.promotionSuccesses))
promotionStats.WithLabelValues("fast_storage_accesses").Set(float64(apd.fastStorageAccesses))
promotionStats.WithLabelValues("fast_storage_hits").Set(float64(apd.fastStorageHits))
}
// adjustThresholdsMoreAggressive makes promotion more aggressive
func (apd *AdaptivePromotionDecider) adjustThresholdsMoreAggressive() {
// Increase size thresholds (promote larger files)
apd.smallFileThreshold = minInt64(apd.smallFileThreshold*11/10, 50*1024*1024) // Max 50MB
apd.mediumFileThreshold = minInt64(apd.mediumFileThreshold*11/10, 200*1024*1024) // Max 200MB
apd.largeFileThreshold = minInt64(apd.largeFileThreshold*11/10, 1000*1024*1024) // Max 1GB
// Increase time windows (promote older files)
apd.smallFileWindow = minDuration(apd.smallFileWindow*11/10, 20*time.Minute)
apd.mediumFileWindow = minDuration(apd.mediumFileWindow*11/10, 5*time.Minute)
apd.largeFileWindow = minDuration(apd.largeFileWindow*11/10, 2*time.Minute)
// Record adaptation in Prometheus
promotionAdaptations.WithLabelValues("aggressive").Inc()
// Update Prometheus metrics
apd.updatePrometheusMetrics()
}
// adjustThresholdsMoreConservative makes promotion more conservative
func (apd *AdaptivePromotionDecider) adjustThresholdsMoreConservative() {
// Decrease size thresholds (promote smaller files)
apd.smallFileThreshold = maxInt64(apd.smallFileThreshold*9/10, 5*1024*1024) // Min 5MB
apd.mediumFileThreshold = maxInt64(apd.mediumFileThreshold*9/10, 50*1024*1024) // Min 50MB
apd.largeFileThreshold = maxInt64(apd.largeFileThreshold*9/10, 200*1024*1024) // Min 200MB
// Decrease time windows (promote only recent files)
apd.smallFileWindow = maxDuration(apd.smallFileWindow*9/10, 5*time.Minute)
apd.mediumFileWindow = maxDuration(apd.mediumFileWindow*9/10, 1*time.Minute)
apd.largeFileWindow = maxDuration(apd.largeFileWindow*9/10, 15*time.Second)
// Record adaptation in Prometheus
promotionAdaptations.WithLabelValues("conservative").Inc()
// Update Prometheus metrics
apd.updatePrometheusMetrics()
}
// GetStats returns current statistics for monitoring
func (apd *AdaptivePromotionDecider) GetStats() map[string]interface{} {
apd.mu.RLock()
defer apd.mu.RUnlock()
hitRate := 0.0
if apd.fastStorageAccesses > 0 {
hitRate = float64(apd.fastStorageHits) / float64(apd.fastStorageAccesses)
}
promotionRate := 0.0
if apd.promotionAttempts > 0 {
promotionRate = float64(apd.promotionSuccesses) / float64(apd.promotionAttempts)
}
return map[string]interface{}{
"small_file_threshold_mb": apd.smallFileThreshold / (1024 * 1024),
"medium_file_threshold_mb": apd.mediumFileThreshold / (1024 * 1024),
"large_file_threshold_mb": apd.largeFileThreshold / (1024 * 1024),
"small_file_window_minutes": apd.smallFileWindow.Minutes(),
"medium_file_window_minutes": apd.mediumFileWindow.Minutes(),
"large_file_window_seconds": apd.largeFileWindow.Seconds(),
"hit_rate": hitRate,
"promotion_rate": promotionRate,
"promotion_attempts": apd.promotionAttempts,
"promotion_successes": apd.promotionSuccesses,
"fast_storage_accesses": apd.fastStorageAccesses,
"fast_storage_hits": apd.fastStorageHits,
}
}
// Global adaptive promotion decider instance
var adaptivePromotionDecider *AdaptivePromotionDecider
func init() {
adaptivePromotionDecider = NewAdaptivePromotionDecider()
}
// AdaptivePromotionDeciderFunc returns the adaptive promotion decision function
func AdaptivePromotionDeciderFunc(fi *vfs.FileInfo, cs cachestate.CacheState) bool {
return adaptivePromotionDecider.ShouldPromote(fi, cs)
}
// RecordFastStorageAccess records fast storage access for adaptation
func RecordFastStorageAccess() {
adaptivePromotionDecider.RecordFastStorageAccess()
}
// RecordFastStorageHit records fast storage hit for adaptation
func RecordFastStorageHit() {
adaptivePromotionDecider.RecordFastStorageHit()
}
// GetPromotionStats returns promotion statistics for monitoring
func GetPromotionStats() map[string]interface{} {
return adaptivePromotionDecider.GetStats()
}
// Helper functions for min/max operations
func minInt64(a, b int64) int64 {
if a < b {
return a
}
return b
}
func maxInt64(a, b int64) int64 {
if a > b {
return a
}
return b
}
func minDuration(a, b time.Duration) time.Duration {
if a < b {
return a
}
return b
}
func maxDuration(a, b time.Duration) time.Duration {
if a > b {
return a
}
return b
}
// Ensure GCFS implements VFS.
var _ vfs.VFS = (*GCFS)(nil)

View File

@@ -2,71 +2,41 @@
package gc
import (
"errors"
"fmt"
"s1d3sw1ped/SteamCache2/vfs/memory"
"testing"
)
func TestGCOnFull(t *testing.T) {
m := memory.New(10)
gc := New(m, LRUGC)
func TestGetGCAlgorithm(t *testing.T) {
tests := []struct {
name string
algorithm GCAlgorithm
expected bool // true if we expect a non-nil function
}{
{"LRU", LRU, true},
{"LFU", LFU, true},
{"FIFO", FIFO, true},
{"Largest", Largest, true},
{"Smallest", Smallest, true},
{"Hybrid", Hybrid, true},
{"Unknown", "unknown", true}, // should fall back to LRU
{"Empty", "", true}, // should fall back to LRU
}
for i := 0; i < 5; i++ {
w, err := gc.Create(fmt.Sprintf("key%d", i), 2)
if err != nil {
t.Fatalf("Create failed: %v", err)
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
fn := GetGCAlgorithm(tt.algorithm)
if fn == nil {
t.Errorf("GetGCAlgorithm(%s) returned nil, expected non-nil function", tt.algorithm)
}
})
}
}
func TestGCAlgorithmConstants(t *testing.T) {
expectedAlgorithms := []GCAlgorithm{LRU, LFU, FIFO, Largest, Smallest, Hybrid}
for _, algo := range expectedAlgorithms {
if algo == "" {
t.Errorf("GC algorithm constant is empty")
}
w.Write([]byte("ab"))
w.Close()
}
// Cache full at 10 bytes
w, err := gc.Create("key5", 2)
if err != nil {
t.Fatalf("Create failed: %v", err)
}
w.Write([]byte("cd"))
w.Close()
if gc.Size() > 10 {
t.Errorf("Size exceeded: %d > 10", gc.Size())
}
// Check if older keys were evicted
_, err = m.Open("key0")
if err == nil {
t.Error("Expected key0 to be evicted")
}
}
func TestNoGCNeeded(t *testing.T) {
m := memory.New(20)
gc := New(m, LRUGC)
for i := 0; i < 5; i++ {
w, err := gc.Create(fmt.Sprintf("key%d", i), 2)
if err != nil {
t.Fatalf("Create failed: %v", err)
}
w.Write([]byte("ab"))
w.Close()
}
if gc.Size() != 10 {
t.Errorf("Size: got %d, want 10", gc.Size())
}
}
func TestGCInsufficientSpace(t *testing.T) {
m := memory.New(5)
gc := New(m, LRUGC)
w, err := gc.Create("key0", 10)
if err == nil {
w.Close()
t.Error("Expected ErrDiskFull")
} else if !errors.Is(err, ErrInsufficientSpace) {
t.Errorf("Unexpected error: %v", err)
}
}