s1d3sw1ped/steamcache2
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 19 Wiki Activity

Refactor caching logic and enhance hash generation in steamcache

Browse Source 
- Replaced SHA1 hash calculations with SHA256 for improved security and consistency in cache key generation.
- Introduced a new TestURLHashing function to validate the new cache key generation logic.
- Removed outdated hash calculation tests and streamlined the caching process to focus on URL-based hashing.
- Implemented lightweight validation methods in ServeHTTP to enhance performance and reliability of cached responses.
- Added batched time updates in VFS implementations for better performance during access time tracking.
This commit is contained in:
Justin Harms
2025-09-02 05:45:44 -05:00
parent b9358a0e8d
commit 4a4579b0f3
6 changed files with 621 additions and 324 deletions
Download Patch File Download Diff File Expand all files Collapse all files

224
vfs/disk/disk.go
View File

@@ -10,6 +10,7 @@ import (
"s1d3sw1ped/SteamCache2/steamcache/logger"
"s1d3sw1ped/SteamCache2/vfs"
"s1d3sw1ped/SteamCache2/vfs/vfserror"
"sort"
"strings"
"sync"
"time"
@@ -25,14 +26,18 @@ var _ vfs.VFS = (*DiskFS)(nil)
type DiskFS struct {
root string
info map[string]*vfs.FileInfo
capacity int64
size int64
mu sync.RWMutex
keyLocks sync.Map // map[string]*sync.RWMutex
LRU *lruList
info map[string]*vfs.FileInfo
capacity int64
size int64
mu sync.RWMutex
keyLocks []sync.Map // Sharded lock pools for better concurrency
LRU *lruList
timeUpdater *vfs.BatchedTimeUpdate // Batched time updates for better performance
}
// Number of lock shards for reducing contention
const numLockShards = 32
// lruList for time-decayed LRU eviction
type lruList struct {
list *list.List
@@ -51,12 +56,12 @@ func (l *lruList) Add(key string, fi *vfs.FileInfo) {
l.elem[key] = elem
}
func (l *lruList) MoveToFront(key string) {
func (l *lruList) MoveToFront(key string, timeUpdater *vfs.BatchedTimeUpdate) {
if elem, exists := l.elem[key]; exists {
l.list.MoveToFront(elem)
// Update the FileInfo in the element with new access time
if fi := elem.Value.(*vfs.FileInfo); fi != nil {
fi.UpdateAccess()
fi.UpdateAccessBatched(timeUpdater)
}
}
}
@@ -76,11 +81,8 @@ func (l *lruList) Len() int {
}
// shardPath converts a Steam cache key to a sharded directory path to reduce inode pressure
// Optimized for the steam/{hash} format
func (d *DiskFS) shardPath(key string) string {
// Expect keys in format: steam/{hash}
if !strings.HasPrefix(key, "steam/") {
// Fallback for non-steam keys (shouldn't happen in optimized setup)
return key
}
@@ -103,17 +105,15 @@ func (d *DiskFS) shardPath(key string) string {
}
// extractKeyFromPath reverses the sharding logic to get the original key from a sharded path
// Optimized for steam/{hash} format
func (d *DiskFS) extractKeyFromPath(path string) string {
// Fast path: if no slashes, it's not a sharded path
if !strings.Contains(path, "/") {
return path
}
parts := strings.Split(path, "/")
parts := strings.SplitN(path, "/", 5)
numParts := len(parts)
// Optimized for steam/shard1/shard2/filename format
if numParts >= 4 && parts[0] == "steam" {
lastThree := parts[numParts-3:]
shard1 := lastThree[0]
@@ -150,12 +150,17 @@ func New(root string, capacity int64) *DiskFS {
// Create root directory if it doesn't exist
os.MkdirAll(root, 0755)
// Initialize sharded locks
keyLocks := make([]sync.Map, numLockShards)
d := &DiskFS{
root: root,
info: make(map[string]*vfs.FileInfo),
capacity: capacity,
size: 0,
LRU: newLruList(),
root: root,
info: make(map[string]*vfs.FileInfo),
capacity: capacity,
size: 0,
keyLocks: keyLocks,
LRU: newLruList(),
timeUpdater: vfs.NewBatchedTimeUpdate(100 * time.Millisecond), // Update time every 100ms
}
d.init()
@@ -187,6 +192,8 @@ func (d *DiskFS) init() {
fi := vfs.NewFileInfoFromOS(info, k)
d.info[k] = fi
d.LRU.Add(k, fi)
// Initialize access time with file modification time
fi.UpdateAccessBatched(d.timeUpdater)
d.size += info.Size()
// Track depot files for potential migration
@@ -300,9 +307,23 @@ func (d *DiskFS) Capacity() int64 {
return d.capacity
}
// getKeyLock returns a lock for the given key
// getShardIndex returns the shard index for a given key
func getShardIndex(key string) int {
// Use FNV-1a hash for good distribution
var h uint32 = 2166136261 // FNV offset basis
for i := 0; i < len(key); i++ {
h ^= uint32(key[i])
h *= 16777619 // FNV prime
}
return int(h % numLockShards)
}
// getKeyLock returns a lock for the given key using sharding
func (d *DiskFS) getKeyLock(key string) *sync.RWMutex {
keyLock, _ := d.keyLocks.LoadOrStore(key, &sync.RWMutex{})
shardIndex := getShardIndex(key)
shard := &d.keyLocks[shardIndex]
keyLock, _ := shard.LoadOrStore(key, &sync.RWMutex{})
return keyLock.(*sync.RWMutex)
}
@@ -353,6 +374,8 @@ func (d *DiskFS) Create(key string, size int64) (io.WriteCloser, error) {
d.mu.Lock()
d.info[key] = fi
d.LRU.Add(key, fi)
// Initialize access time with current time
fi.UpdateAccessBatched(d.timeUpdater)
d.size += size
d.mu.Unlock()
@@ -424,8 +447,8 @@ func (d *DiskFS) Open(key string) (io.ReadCloser, error) {
d.mu.Unlock()
return nil, vfserror.ErrNotFound
}
fi.UpdateAccess()
d.LRU.MoveToFront(key)
fi.UpdateAccessBatched(d.timeUpdater)
d.LRU.MoveToFront(key, d.timeUpdater)
d.mu.Unlock()
shardedPath := d.shardPath(key)
@@ -559,3 +582,158 @@ func (d *DiskFS) Stat(key string) (*vfs.FileInfo, error) {
return nil, vfserror.ErrNotFound
}
// EvictLRU evicts the least recently used files to free up space
func (d *DiskFS) EvictLRU(bytesNeeded uint) uint {
d.mu.Lock()
defer d.mu.Unlock()
var evicted uint
// Evict from LRU list until we free enough space
for d.size > d.capacity-int64(bytesNeeded) && d.LRU.Len() > 0 {
// Get the least recently used item
elem := d.LRU.list.Back()
if elem == nil {
break
}
fi := elem.Value.(*vfs.FileInfo)
key := fi.Key
// Remove from LRU
d.LRU.Remove(key)
// Remove from map
delete(d.info, key)
// Remove file from disk
shardedPath := d.shardPath(key)
path := filepath.Join(d.root, shardedPath)
path = strings.ReplaceAll(path, "\\", "/")
if err := os.Remove(path); err != nil {
// Log error but continue
continue
}
// Update size
d.size -= fi.Size
evicted += uint(fi.Size)
// Clean up key lock
shardIndex := getShardIndex(key)
d.keyLocks[shardIndex].Delete(key)
}
return evicted
}
// EvictBySize evicts files by size (ascending = smallest first, descending = largest first)
func (d *DiskFS) EvictBySize(bytesNeeded uint, ascending bool) uint {
d.mu.Lock()
defer d.mu.Unlock()
var evicted uint
var candidates []*vfs.FileInfo
// Collect all files
for _, fi := range d.info {
candidates = append(candidates, fi)
}
// Sort by size
sort.Slice(candidates, func(i, j int) bool {
if ascending {
return candidates[i].Size < candidates[j].Size
}
return candidates[i].Size > candidates[j].Size
})
// Evict files until we free enough space
for _, fi := range candidates {
if d.size <= d.capacity-int64(bytesNeeded) {
break
}
key := fi.Key
// Remove from LRU
d.LRU.Remove(key)
// Remove from map
delete(d.info, key)
// Remove file from disk
shardedPath := d.shardPath(key)
path := filepath.Join(d.root, shardedPath)
path = strings.ReplaceAll(path, "\\", "/")
if err := os.Remove(path); err != nil {
continue
}
// Update size
d.size -= fi.Size
evicted += uint(fi.Size)
// Clean up key lock
shardIndex := getShardIndex(key)
d.keyLocks[shardIndex].Delete(key)
}
return evicted
}
// EvictFIFO evicts files using FIFO (oldest creation time first)
func (d *DiskFS) EvictFIFO(bytesNeeded uint) uint {
d.mu.Lock()
defer d.mu.Unlock()
var evicted uint
var candidates []*vfs.FileInfo
// Collect all files
for _, fi := range d.info {
candidates = append(candidates, fi)
}
// Sort by creation time (oldest first)
sort.Slice(candidates, func(i, j int) bool {
return candidates[i].CTime.Before(candidates[j].CTime)
})
// Evict oldest files until we free enough space
for _, fi := range candidates {
if d.size <= d.capacity-int64(bytesNeeded) {
break
}
key := fi.Key
// Remove from LRU
d.LRU.Remove(key)
// Remove from map
delete(d.info, key)
// Remove file from disk
shardedPath := d.shardPath(key)
path := filepath.Join(d.root, shardedPath)
path = strings.ReplaceAll(path, "\\", "/")
if err := os.Remove(path); err != nil {
continue
}
// Update size
d.size -= fi.Size
evicted += uint(fi.Size)
// Clean up key lock
shardIndex := getShardIndex(key)
d.keyLocks[shardIndex].Delete(key)
}
return evicted
}