Remove plans/ directory (P0/P1/P2 work complete)
This commit is contained in:
+176
-198
@@ -10,6 +10,7 @@ import (
|
||||
"s1d3sw1ped/steamcache2/vfs"
|
||||
"s1d3sw1ped/steamcache2/vfs/locks"
|
||||
"s1d3sw1ped/steamcache2/vfs/lru"
|
||||
"s1d3sw1ped/steamcache2/vfs/types"
|
||||
"s1d3sw1ped/steamcache2/vfs/vfserror"
|
||||
"sort"
|
||||
"strings"
|
||||
@@ -21,6 +22,9 @@ import (
|
||||
"github.com/edsrzf/mmap-go"
|
||||
)
|
||||
|
||||
// maxEvictBatch bounds the candidate snapshot during RLock/Lock collect in Evict* (mirrors memory).
|
||||
const maxEvictBatch = 4096
|
||||
|
||||
// Ensure DiskFS implements VFS.
|
||||
var _ vfs.VFS = (*DiskFS)(nil)
|
||||
|
||||
@@ -61,6 +65,15 @@ func (d *DiskFS) shardPath(key string) string {
|
||||
return filepath.Join("steam", shard1, shard2, hashPart)
|
||||
}
|
||||
|
||||
// pathForKey returns the full on-disk path for a key (sharded + normalized).
|
||||
// Extracted to reduce duplication in Evict*/Delete/Open paths (addresses review nit19; still safe to call under lock for evict).
|
||||
func (d *DiskFS) pathForKey(key string) string {
|
||||
shardedPath := d.shardPath(key)
|
||||
path := filepath.Join(d.root, shardedPath)
|
||||
path = strings.ReplaceAll(path, "\\", "/")
|
||||
return path
|
||||
}
|
||||
|
||||
// New creates a new DiskFS.
|
||||
func New(root string, capacity int64) *DiskFS {
|
||||
if capacity <= 0 {
|
||||
@@ -297,11 +310,9 @@ func (d *DiskFS) Create(key string, size int64) (io.WriteCloser, error) {
|
||||
delete(d.info, key)
|
||||
}
|
||||
|
||||
shardedPath := d.shardPath(key)
|
||||
path := filepath.Join(d.root, shardedPath)
|
||||
path := d.pathForKey(key)
|
||||
d.mu.Unlock()
|
||||
|
||||
path = strings.ReplaceAll(path, "\\", "/")
|
||||
dir := filepath.Dir(path)
|
||||
if err := os.MkdirAll(dir, 0755); err != nil {
|
||||
return nil, err
|
||||
@@ -400,9 +411,7 @@ func (d *DiskFS) Open(key string) (io.ReadCloser, error) {
|
||||
d.LRU.MoveToFront(key, d.timeUpdater)
|
||||
d.mu.Unlock()
|
||||
|
||||
shardedPath := d.shardPath(key)
|
||||
path := filepath.Join(d.root, shardedPath)
|
||||
path = strings.ReplaceAll(path, "\\", "/")
|
||||
path := d.pathForKey(key)
|
||||
|
||||
file, err := os.Open(path)
|
||||
if err != nil {
|
||||
@@ -484,10 +493,7 @@ func (d *DiskFS) Delete(key string) error {
|
||||
delete(d.info, key)
|
||||
d.mu.Unlock()
|
||||
|
||||
shardedPath := d.shardPath(key)
|
||||
path := filepath.Join(d.root, shardedPath)
|
||||
path = strings.ReplaceAll(path, "\\", "/")
|
||||
|
||||
path := d.pathForKey(key)
|
||||
err := os.Remove(path)
|
||||
if err != nil {
|
||||
return err
|
||||
@@ -519,9 +525,7 @@ func (d *DiskFS) Stat(key string) (*vfs.FileInfo, error) {
|
||||
keyMu.RUnlock()
|
||||
|
||||
// Lazy discovery: check if file exists on disk and index it
|
||||
shardedPath := d.shardPath(key)
|
||||
path := filepath.Join(d.root, shardedPath)
|
||||
path = strings.ReplaceAll(path, "\\", "/")
|
||||
path := d.pathForKey(key)
|
||||
|
||||
info, err := os.Stat(path)
|
||||
if err != nil {
|
||||
@@ -552,260 +556,234 @@ func (d *DiskFS) Stat(key string) (*vfs.FileInfo, error) {
|
||||
}
|
||||
|
||||
// EvictLRU evicts the least recently used files to free up space
|
||||
// Collect under short exclusive Lock (to serialize concurrent EvictLRU on LRUList), batch under WLock.
|
||||
func (d *DiskFS) EvictLRU(bytesNeeded uint) uint {
|
||||
d.mu.Lock()
|
||||
defer d.mu.Unlock()
|
||||
|
||||
var evicted uint
|
||||
|
||||
// Evict from LRU list until we free enough space
|
||||
for d.size > d.capacity-int64(bytesNeeded) && d.LRU.Len() > 0 {
|
||||
// Get the least recently used item
|
||||
var toEvict []string
|
||||
need := int64(bytesNeeded)
|
||||
cur := d.size
|
||||
for cur > d.capacity-need && d.LRU.Len() > 0 && len(toEvict) < maxEvictBatch {
|
||||
elem := d.LRU.Back()
|
||||
if elem == nil {
|
||||
break
|
||||
}
|
||||
|
||||
fi := elem.Value.(*vfs.FileInfo)
|
||||
key := fi.Key
|
||||
toEvict = append(toEvict, fi.Key)
|
||||
cur -= fi.Size
|
||||
}
|
||||
d.mu.Unlock()
|
||||
|
||||
// Remove from LRU
|
||||
d.LRU.Remove(key)
|
||||
|
||||
// Remove from map
|
||||
delete(d.info, key)
|
||||
|
||||
// Remove file from disk
|
||||
shardedPath := d.shardPath(key)
|
||||
path := filepath.Join(d.root, shardedPath)
|
||||
path = strings.ReplaceAll(path, "\\", "/")
|
||||
|
||||
if err := os.Remove(path); err != nil {
|
||||
// Log error but continue
|
||||
continue
|
||||
}
|
||||
|
||||
// Update size
|
||||
d.size -= fi.Size
|
||||
evicted += uint(fi.Size)
|
||||
|
||||
// Clean up key lock
|
||||
shardIndex := locks.GetShardIndex(key)
|
||||
d.keyLocks[shardIndex].Delete(key)
|
||||
if len(toEvict) == 0 {
|
||||
return 0
|
||||
}
|
||||
|
||||
d.mu.Lock()
|
||||
var evicted uint
|
||||
for _, key := range toEvict {
|
||||
if fi, exists := d.info[key]; exists {
|
||||
d.LRU.Remove(key)
|
||||
delete(d.info, key)
|
||||
path := d.pathForKey(key)
|
||||
_ = os.Remove(path) // best effort
|
||||
d.size -= fi.Size
|
||||
evicted += uint(fi.Size)
|
||||
shardIndex := locks.GetShardIndex(key)
|
||||
d.keyLocks[shardIndex].Delete(key)
|
||||
}
|
||||
}
|
||||
d.mu.Unlock()
|
||||
return evicted
|
||||
}
|
||||
|
||||
// EvictBySize evicts files by size (ascending = smallest first, descending = largest first)
|
||||
// Scalar snapshot (key+size) under RLock + live re-fetch under WLock for race-free accounting + os.Remove.
|
||||
type evictCandidate struct {
|
||||
key string
|
||||
size int64
|
||||
}
|
||||
|
||||
func (d *DiskFS) EvictBySize(bytesNeeded uint, ascending bool) uint {
|
||||
d.mu.Lock()
|
||||
defer d.mu.Unlock()
|
||||
|
||||
var evicted uint
|
||||
var candidates []*vfs.FileInfo
|
||||
|
||||
// Collect all files
|
||||
for _, fi := range d.info {
|
||||
candidates = append(candidates, fi)
|
||||
d.mu.RLock()
|
||||
var candidates []evictCandidate
|
||||
for key, fi := range d.info {
|
||||
candidates = append(candidates, evictCandidate{key: key, size: fi.Size})
|
||||
}
|
||||
d.mu.RUnlock()
|
||||
|
||||
// Sort by size
|
||||
if len(candidates) == 0 {
|
||||
return 0
|
||||
}
|
||||
sort.Slice(candidates, func(i, j int) bool {
|
||||
if ascending {
|
||||
return candidates[i].Size < candidates[j].Size
|
||||
return candidates[i].size < candidates[j].size
|
||||
}
|
||||
return candidates[i].Size > candidates[j].Size
|
||||
return candidates[i].size > candidates[j].size
|
||||
})
|
||||
|
||||
// Evict files until we free enough space
|
||||
for _, fi := range candidates {
|
||||
d.mu.Lock()
|
||||
var evicted uint
|
||||
for _, c := range candidates {
|
||||
if d.size <= d.capacity-int64(bytesNeeded) {
|
||||
break
|
||||
}
|
||||
|
||||
key := fi.Key
|
||||
|
||||
// Remove from LRU
|
||||
d.LRU.Remove(key)
|
||||
|
||||
// Remove from map
|
||||
delete(d.info, key)
|
||||
|
||||
// Remove file from disk
|
||||
shardedPath := d.shardPath(key)
|
||||
path := filepath.Join(d.root, shardedPath)
|
||||
path = strings.ReplaceAll(path, "\\", "/")
|
||||
|
||||
if err := os.Remove(path); err != nil {
|
||||
continue
|
||||
key := c.key
|
||||
if liveFi, exists := d.info[key]; exists {
|
||||
d.LRU.Remove(key)
|
||||
delete(d.info, key)
|
||||
path := d.pathForKey(key)
|
||||
_ = os.Remove(path)
|
||||
d.size -= liveFi.Size
|
||||
evicted += uint(liveFi.Size)
|
||||
shardIndex := locks.GetShardIndex(key)
|
||||
d.keyLocks[shardIndex].Delete(key)
|
||||
}
|
||||
|
||||
// Update size
|
||||
d.size -= fi.Size
|
||||
evicted += uint(fi.Size)
|
||||
|
||||
// Clean up key lock
|
||||
shardIndex := locks.GetShardIndex(key)
|
||||
d.keyLocks[shardIndex].Delete(key)
|
||||
}
|
||||
|
||||
d.mu.Unlock()
|
||||
return evicted
|
||||
}
|
||||
|
||||
// EvictFIFO evicts files using FIFO (oldest creation time first)
|
||||
// Snapshot ctime under RLock, live re-fetch + remove under WLock.
|
||||
func (d *DiskFS) EvictFIFO(bytesNeeded uint) uint {
|
||||
d.mu.Lock()
|
||||
defer d.mu.Unlock()
|
||||
|
||||
var evicted uint
|
||||
var candidates []*vfs.FileInfo
|
||||
|
||||
// Collect all files
|
||||
for _, fi := range d.info {
|
||||
candidates = append(candidates, fi)
|
||||
d.mu.RLock()
|
||||
var candidates []struct {
|
||||
key string
|
||||
cTime time.Time
|
||||
}
|
||||
for key, fi := range d.info {
|
||||
candidates = append(candidates, struct {
|
||||
key string
|
||||
cTime time.Time
|
||||
}{key: key, cTime: fi.CTime})
|
||||
}
|
||||
d.mu.RUnlock()
|
||||
|
||||
// Sort by creation time (oldest first)
|
||||
if len(candidates) == 0 {
|
||||
return 0
|
||||
}
|
||||
sort.Slice(candidates, func(i, j int) bool {
|
||||
return candidates[i].CTime.Before(candidates[j].CTime)
|
||||
return candidates[i].cTime.Before(candidates[j].cTime)
|
||||
})
|
||||
|
||||
// Evict oldest files until we free enough space
|
||||
for _, fi := range candidates {
|
||||
d.mu.Lock()
|
||||
var evicted uint
|
||||
for _, c := range candidates {
|
||||
if d.size <= d.capacity-int64(bytesNeeded) {
|
||||
break
|
||||
}
|
||||
|
||||
key := fi.Key
|
||||
|
||||
// Remove from LRU
|
||||
d.LRU.Remove(key)
|
||||
|
||||
// Remove from map
|
||||
delete(d.info, key)
|
||||
|
||||
// Remove file from disk
|
||||
shardedPath := d.shardPath(key)
|
||||
path := filepath.Join(d.root, shardedPath)
|
||||
path = strings.ReplaceAll(path, "\\", "/")
|
||||
|
||||
if err := os.Remove(path); err != nil {
|
||||
continue
|
||||
key := c.key
|
||||
if liveFi, exists := d.info[key]; exists {
|
||||
d.LRU.Remove(key)
|
||||
delete(d.info, key)
|
||||
path := d.pathForKey(key)
|
||||
_ = os.Remove(path)
|
||||
d.size -= liveFi.Size
|
||||
evicted += uint(liveFi.Size)
|
||||
shardIndex := locks.GetShardIndex(key)
|
||||
d.keyLocks[shardIndex].Delete(key)
|
||||
}
|
||||
|
||||
// Update size
|
||||
d.size -= fi.Size
|
||||
evicted += uint(fi.Size)
|
||||
|
||||
// Clean up key lock
|
||||
shardIndex := locks.GetShardIndex(key)
|
||||
d.keyLocks[shardIndex].Delete(key)
|
||||
}
|
||||
|
||||
d.mu.Unlock()
|
||||
return evicted
|
||||
}
|
||||
|
||||
// EvictLFU evicts least frequently used files first (by AccessCount asc; P1-03 real LFU using existing field).
|
||||
// Ties broken by ATime (older first).
|
||||
// EvictLFU evicts least frequently used files first (by AccessCount ascending).
|
||||
// Ties broken by ATime (older first). Uses snapshot + live re-fetch under WLock.
|
||||
func (d *DiskFS) EvictLFU(bytesNeeded uint) uint {
|
||||
d.mu.Lock()
|
||||
defer d.mu.Unlock()
|
||||
|
||||
var evicted uint
|
||||
var candidates []*vfs.FileInfo
|
||||
|
||||
// Collect all files
|
||||
for _, fi := range d.info {
|
||||
candidates = append(candidates, fi)
|
||||
d.mu.RLock()
|
||||
var candidates []struct {
|
||||
key string
|
||||
accessCount int
|
||||
aTime time.Time
|
||||
}
|
||||
for key, fi := range d.info {
|
||||
candidates = append(candidates, struct {
|
||||
key string
|
||||
accessCount int
|
||||
aTime time.Time
|
||||
}{key: key, accessCount: fi.AccessCount, aTime: fi.ATime})
|
||||
}
|
||||
d.mu.RUnlock()
|
||||
|
||||
// Sort by access count asc (LFU), then older ATime for ties
|
||||
if len(candidates) == 0 {
|
||||
return 0
|
||||
}
|
||||
sort.Slice(candidates, func(i, j int) bool {
|
||||
if candidates[i].AccessCount != candidates[j].AccessCount {
|
||||
return candidates[i].AccessCount < candidates[j].AccessCount
|
||||
if candidates[i].accessCount != candidates[j].accessCount {
|
||||
return candidates[i].accessCount < candidates[j].accessCount
|
||||
}
|
||||
return candidates[i].ATime.Before(candidates[j].ATime)
|
||||
return candidates[i].aTime.Before(candidates[j].aTime)
|
||||
})
|
||||
|
||||
// Evict until enough space
|
||||
for _, fi := range candidates {
|
||||
d.mu.Lock()
|
||||
var evicted uint
|
||||
for _, c := range candidates {
|
||||
if d.size <= d.capacity-int64(bytesNeeded) {
|
||||
break
|
||||
}
|
||||
|
||||
key := fi.Key
|
||||
|
||||
// Remove from LRU
|
||||
d.LRU.Remove(key)
|
||||
|
||||
// Remove from map
|
||||
delete(d.info, key)
|
||||
|
||||
// Remove file from disk (best effort; sharding not critical for test coverage)
|
||||
shardedPath := d.shardPath(key)
|
||||
path := filepath.Join(d.root, shardedPath)
|
||||
path = strings.ReplaceAll(path, "\\", "/")
|
||||
_ = os.Remove(path)
|
||||
|
||||
// Update size
|
||||
d.size -= fi.Size
|
||||
evicted += uint(fi.Size)
|
||||
|
||||
// Clean up key lock
|
||||
shardIndex := locks.GetShardIndex(key)
|
||||
d.keyLocks[shardIndex].Delete(key)
|
||||
key := c.key
|
||||
if liveFi, exists := d.info[key]; exists {
|
||||
d.LRU.Remove(key)
|
||||
delete(d.info, key)
|
||||
path := d.pathForKey(key)
|
||||
_ = os.Remove(path)
|
||||
d.size -= liveFi.Size
|
||||
evicted += uint(liveFi.Size)
|
||||
shardIndex := locks.GetShardIndex(key)
|
||||
d.keyLocks[shardIndex].Delete(key)
|
||||
}
|
||||
}
|
||||
|
||||
d.mu.Unlock()
|
||||
return evicted
|
||||
}
|
||||
|
||||
// EvictHybrid evicts using time-decayed score (recency + frequency from GetTimeDecayedScore; lower value first).
|
||||
// This makes "hybrid" a meaningful size+recency+freq policy (P1-03).
|
||||
// This makes "hybrid" a meaningful size + recency + frequency policy.
|
||||
// Snapshot + decayed score under the appropriate locks.
|
||||
func (d *DiskFS) EvictHybrid(bytesNeeded uint) uint {
|
||||
d.mu.Lock()
|
||||
defer d.mu.Unlock()
|
||||
|
||||
var evicted uint
|
||||
var candidates []*vfs.FileInfo
|
||||
|
||||
// Collect all files
|
||||
for _, fi := range d.info {
|
||||
candidates = append(candidates, fi)
|
||||
d.mu.RLock()
|
||||
var candidates []struct {
|
||||
key string
|
||||
accessCount int
|
||||
aTime time.Time
|
||||
}
|
||||
for key, fi := range d.info {
|
||||
candidates = append(candidates, struct {
|
||||
key string
|
||||
accessCount int
|
||||
aTime time.Time
|
||||
}{key: key, accessCount: fi.AccessCount, aTime: fi.ATime})
|
||||
}
|
||||
d.mu.RUnlock()
|
||||
|
||||
// Sort by ascending decayed score (least valuable = evict first)
|
||||
if len(candidates) == 0 {
|
||||
return 0
|
||||
}
|
||||
sort.Slice(candidates, func(i, j int) bool {
|
||||
return candidates[i].GetTimeDecayedScore() < candidates[j].GetTimeDecayedScore()
|
||||
// Use shared canonical DecayedScore from types (eliminates dupe with memory + FileInfo method).
|
||||
scoreI := types.DecayedScore(candidates[i].aTime, candidates[i].accessCount)
|
||||
scoreJ := types.DecayedScore(candidates[j].aTime, candidates[j].accessCount)
|
||||
return scoreI < scoreJ
|
||||
})
|
||||
|
||||
// Evict until enough space
|
||||
for _, fi := range candidates {
|
||||
d.mu.Lock()
|
||||
var evicted uint
|
||||
for _, c := range candidates {
|
||||
if d.size <= d.capacity-int64(bytesNeeded) {
|
||||
break
|
||||
}
|
||||
|
||||
key := fi.Key
|
||||
|
||||
// Remove from LRU
|
||||
d.LRU.Remove(key)
|
||||
|
||||
// Remove from map
|
||||
delete(d.info, key)
|
||||
|
||||
shardedPath := d.shardPath(key)
|
||||
path := filepath.Join(d.root, shardedPath)
|
||||
path = strings.ReplaceAll(path, "\\", "/")
|
||||
_ = os.Remove(path)
|
||||
|
||||
// Update size
|
||||
d.size -= fi.Size
|
||||
evicted += uint(fi.Size)
|
||||
|
||||
// Clean up key lock
|
||||
shardIndex := locks.GetShardIndex(key)
|
||||
d.keyLocks[shardIndex].Delete(key)
|
||||
key := c.key
|
||||
if liveFi, exists := d.info[key]; exists {
|
||||
d.LRU.Remove(key)
|
||||
delete(d.info, key)
|
||||
path := d.pathForKey(key)
|
||||
_ = os.Remove(path)
|
||||
d.size -= liveFi.Size
|
||||
evicted += uint(liveFi.Size)
|
||||
shardIndex := locks.GetShardIndex(key)
|
||||
d.keyLocks[shardIndex].Delete(key)
|
||||
}
|
||||
}
|
||||
|
||||
d.mu.Unlock()
|
||||
return evicted
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user