Enhance file metadata tracking and garbage collection logic

- Added AccessCount field to FileInfo struct for improved tracking of file access frequency. - Updated NewFileInfo and NewFileInfoFromOS functions to initialize AccessCount. - Modified DiskFS and MemoryFS to preserve and increment AccessCount during file operations. - Enhanced garbage collection methods (LRU, LFU, FIFO, Largest, Smallest, Hybrid) to utilize AccessCount for more effective space reclamation.
revert 30e804709f
2025-07-19 09:07:49 -05:00 · 2025-07-19 14:02:53 +00:00 · 2025-07-19 08:07:36 -05:00
5 changed files with 106 additions and 130 deletions
--- a/.vscode/launch.json
+++ b/.vscode/launch.json
@@ -23,8 +23,8 @@
                "lru",
                "--log-level",
                "debug",
-                "--upstream",
-                "http://192.168.2.5:80",
+                // "--upstream",
+                // "http://192.168.2.5:80",
            ],
        },
        {
@@ -42,8 +42,8 @@
                "hybrid",
                "--log-level",
                "debug",
-                "--upstream",
-                "http://192.168.2.5:80",
+                // "--upstream",
+                // "http://192.168.2.5:80",
            ],
        },
        {
@@ -56,11 +56,11 @@
                "--memory",
                "1G",
                "--memory-gc",
-                "lfu",
+                "lru",
                "--log-level",
                "debug",
-                "--upstream",
-                "http://192.168.2.5:80",
+                // "--upstream",
+                // "http://192.168.2.5:80",
            ],
        }
    ]
--- a/vfs/disk/disk.go
+++ b/vfs/disk/disk.go
@@ -3,10 +3,8 @@ package disk

 import (
 	"container/list"
-	"encoding/json"
 	"fmt"
 	"io"
-	"io/ioutil"
 	"os"
 	"path/filepath"
 	"s1d3sw1ped/SteamCache2/steamcache/logger"
@@ -58,17 +56,12 @@ var _ vfs.VFS = (*DiskFS)(nil)
 type DiskFS struct {
 	root string

-	info         map[string]*vfs.FileInfo
-	capacity     int64
-	size         int64
-	mu           sync.RWMutex
-	keyLocks     sync.Map // map[string]*sync.RWMutex
-	LRU          *lruList
-	accessCounts map[string]int64 // key: filename, value: access count
-	fileMeta     map[string]struct {
-		AccessCount int64
-		CTime       int64
-	} // key: filename
+	info     map[string]*vfs.FileInfo
+	capacity int64
+	size     int64
+	mu       sync.RWMutex
+	keyLocks sync.Map // map[string]*sync.RWMutex
+	LRU      *lruList
 }

 // lruList for LRU eviction
@@ -136,17 +129,12 @@ func new(root string, capacity int64, skipinit bool) *DiskFS {
 	}

 	dfs := &DiskFS{
-		root:         root,
-		info:         make(map[string]*vfs.FileInfo),
-		capacity:     capacity,
-		mu:           sync.RWMutex{},
-		keyLocks:     sync.Map{},
-		LRU:          newLruList(),
-		accessCounts: make(map[string]int64),
-		fileMeta: make(map[string]struct {
-			AccessCount int64
-			CTime       int64
-		}),
+		root:     root,
+		info:     make(map[string]*vfs.FileInfo),
+		capacity: capacity,
+		mu:       sync.RWMutex{},
+		keyLocks: sync.Map{},
+		LRU:      newLruList(),
 	}

 	os.MkdirAll(dfs.root, 0755)
@@ -254,10 +242,12 @@ func (d *DiskFS) Create(key string, size int64) (io.WriteCloser, error) {

 	// Check again after lock
 	d.mu.Lock()
+	var accessCount int64 = 0
 	if fi, exists := d.info[key]; exists {
 		d.size -= fi.Size()
 		d.LRU.Remove(key)
 		delete(d.info, key)
+		accessCount = fi.AccessCount // preserve access count if overwriting
 		path := filepath.Join(d.root, key)
 		os.Remove(path) // Ignore error, as file might not exist or other issues
 	}
@@ -286,6 +276,7 @@ func (d *DiskFS) Create(key string, size int64) (io.WriteCloser, error) {

 			d.mu.Lock()
 			finfo := vfs.NewFileInfoFromOS(fi, key)
+			finfo.AccessCount = accessCount
 			d.info[key] = finfo
 			d.LRU.Add(key, finfo)
 			d.size += n
@@ -294,15 +285,6 @@ func (d *DiskFS) Create(key string, size int64) (io.WriteCloser, error) {
 			diskWriteBytes.Add(float64(n))
 			diskSizeBytes.Set(float64(d.Size()))

-			// On new file, set access count to 1
-			finfo.AccessCount = 1
-			finfo.CTime = time.Now()
-			d.fileMeta[key] = struct {
-				AccessCount int64
-				CTime       int64
-			}{1, finfo.CTime.Unix()}
-			flushFileMeta(d.root, d.fileMeta)
-
 			return nil
 		},
 		key:  key,
@@ -372,10 +354,6 @@ func (d *DiskFS) Delete(key string) error {

 	diskSizeBytes.Set(float64(d.Size()))

-	delete(d.accessCounts, key)
-	delete(d.fileMeta, key)
-	flushFileMeta(d.root, d.fileMeta)
-
 	return nil
 }

@@ -406,17 +384,10 @@ func (d *DiskFS) Open(key string) (io.ReadCloser, error) {
 		return nil, vfserror.ErrNotFound
 	}
 	fi.ATime = time.Now()
-	fi.AccessCount++ // Increment access count
+	fi.AccessCount++ // Increment access count for LFU
 	d.LRU.MoveToFront(key)
 	d.mu.Unlock()

-	fi.AccessCount++
-	d.fileMeta[key] = struct {
-		AccessCount int64
-		CTime       int64
-	}{fi.AccessCount, fi.CTime.Unix()}
-	flushFileMeta(d.root, d.fileMeta)
-
 	path := filepath.Join(d.root, key)
 	path = strings.ReplaceAll(path, "\\", "/") // Ensure forward slashes for consistency
 	file, err := os.Open(path)
@@ -494,22 +465,3 @@ func (d *DiskFS) StatAll() []*vfs.FileInfo {

 	return files
 }
-
-func flushAccessCounts(root string, counts map[string]int64) {
-	path := filepath.Join(root, "access_counts.json")
-	data, _ := json.MarshalIndent(counts, "", "  ")
-	_ = ioutil.WriteFile(path, data, 0644)
-}
-
-func flushFileMeta(root string, meta map[string]struct {
-	AccessCount int64
-	CTime       int64
-}) {
-	path := filepath.Join(root, "filemeta.json")
-	if len(meta) == 0 {
-		_ = os.Remove(path)
-		return
-	}
-	data, _ := json.MarshalIndent(meta, "", "  ")
-	_ = ioutil.WriteFile(path, data, 0644)
-}
--- a/vfs/fileinfo.go
+++ b/vfs/fileinfo.go
@@ -11,30 +11,25 @@ type FileInfo struct {
 	size        int64
 	MTime       time.Time
 	ATime       time.Time
-	CTime       time.Time // Creation time
-	AccessCount int64
+	AccessCount int64 // Number of times the file has been accessed
 }

 func NewFileInfo(key string, size int64, modTime time.Time) *FileInfo {
-	now := time.Now()
 	return &FileInfo{
 		name:        key,
 		size:        size,
 		MTime:       modTime,
-		ATime:       now,
-		CTime:       now,
+		ATime:       time.Now(),
 		AccessCount: 0,
 	}
 }

 func NewFileInfoFromOS(f os.FileInfo, key string) *FileInfo {
-	now := time.Now()
 	return &FileInfo{
 		name:        key,
 		size:        f.Size(),
 		MTime:       f.ModTime(),
-		ATime:       now,
-		CTime:       now, // Will be overwritten if loaded from disk
+		ATime:       time.Now(),
 		AccessCount: 0,
 	}
 }
--- a/vfs/gc/gc.go
+++ b/vfs/gc/gc.go
@@ -75,12 +75,17 @@ func LRUGC(vfss vfs.VFS, size uint) error {
 	logger.Logger.Debug().Uint("target", size).Msg("Attempting to reclaim space using LRU GC")

 	var reclaimed uint // reclaimed space in bytes
+	deleted := false

 	for {
 		switch fs := vfss.(type) {
 		case *disk.DiskFS:
 			fi := fs.LRU.Back()
 			if fi == nil {
+				if deleted {
+					logger.Logger.Debug().Uint("target", size).Uint("achieved", reclaimed).Msg("Reclaimed enough space using LRU GC (at least one file deleted)")
+					return nil
+				}
 				return ErrInsufficientSpace // No files to delete
 			}
 			sz := uint(fi.Size())
@@ -89,9 +94,14 @@ func LRUGC(vfss vfs.VFS, size uint) error {
 				continue // If delete fails, try the next file
 			}
 			reclaimed += sz
+			deleted = true
 		case *memory.MemoryFS:
 			fi := fs.LRU.Back()
 			if fi == nil {
+				if deleted {
+					logger.Logger.Debug().Uint("target", size).Uint("achieved", reclaimed).Msg("Reclaimed enough space using LRU GC (at least one file deleted)")
+					return nil
+				}
 				return ErrInsufficientSpace // No files to delete
 			}
 			sz := uint(fi.Size())
@@ -100,13 +110,14 @@ func LRUGC(vfss vfs.VFS, size uint) error {
 				continue // If delete fails, try the next file
 			}
 			reclaimed += sz
+			deleted = true
 		default:
 			panic("unreachable: unsupported VFS type for LRU GC") // panic if the VFS is not disk or memory
 		}

-		if reclaimed >= size {
-			logger.Logger.Debug().Uint("target", size).Uint("achieved", reclaimed).Msg("Reclaimed enough space using LRU GC")
-			return nil // stop if enough space is reclaimed
+		if deleted && (size == 0 || reclaimed >= size) {
+			logger.Logger.Debug().Uint("target", size).Uint("achieved", reclaimed).Msg("Reclaimed enough space using LRU GC (at least one file deleted)")
+			return nil // stop if enough space is reclaimed or at least one file deleted for size==0
 		}
 	}
 }
@@ -115,31 +126,32 @@ func LRUGC(vfss vfs.VFS, size uint) error {
 func LFUGC(vfss vfs.VFS, size uint) error {
 	logger.Logger.Debug().Uint("target", size).Msg("Attempting to reclaim space using LFU GC")

-	// Get all files and sort by access count (frequency)
 	files := getAllFiles(vfss)
 	if len(files) == 0 {
 		return ErrInsufficientSpace
 	}

-	// Sort by access count (ascending - least frequently used first)
 	sort.Slice(files, func(i, j int) bool {
 		return files[i].AccessCount < files[j].AccessCount
 	})

 	var reclaimed uint
+	deleted := false
 	for _, fi := range files {
-		if reclaimed >= size {
-			break
-		}
 		err := vfss.Delete(fi.Name)
 		if err != nil {
 			continue
 		}
 		reclaimed += uint(fi.Size)
+		deleted = true
+		if deleted && (size == 0 || reclaimed >= size) {
+			logger.Logger.Debug().Uint("target", size).Uint("achieved", reclaimed).Msg("Reclaimed enough space using LFU GC (at least one file deleted)")
+			return nil
+		}
 	}

-	if reclaimed >= size {
-		logger.Logger.Debug().Uint("target", size).Uint("achieved", reclaimed).Msg("Reclaimed enough space using LFU GC")
+	if deleted {
+		logger.Logger.Debug().Uint("target", size).Uint("achieved", reclaimed).Msg("Reclaimed enough space using LFU GC (at least one file deleted)")
 		return nil
 	}
 	return ErrInsufficientSpace
@@ -149,31 +161,32 @@ func LFUGC(vfss vfs.VFS, size uint) error {
 func FIFOGC(vfss vfs.VFS, size uint) error {
 	logger.Logger.Debug().Uint("target", size).Msg("Attempting to reclaim space using FIFO GC")

-	// Get all files and sort by creation time (oldest first)
 	files := getAllFiles(vfss)
 	if len(files) == 0 {
 		return ErrInsufficientSpace
 	}

-	// Sort by creation time (ascending - oldest first)
 	sort.Slice(files, func(i, j int) bool {
 		return files[i].MTime.Before(files[j].MTime)
 	})

 	var reclaimed uint
+	deleted := false
 	for _, fi := range files {
-		if reclaimed >= size {
-			break
-		}
 		err := vfss.Delete(fi.Name)
 		if err != nil {
 			continue
 		}
 		reclaimed += uint(fi.Size)
+		deleted = true
+		if deleted && (size == 0 || reclaimed >= size) {
+			logger.Logger.Debug().Uint("target", size).Uint("achieved", reclaimed).Msg("Reclaimed enough space using FIFO GC (at least one file deleted)")
+			return nil
+		}
 	}

-	if reclaimed >= size {
-		logger.Logger.Debug().Uint("target", size).Uint("achieved", reclaimed).Msg("Reclaimed enough space using FIFO GC")
+	if deleted {
+		logger.Logger.Debug().Uint("target", size).Uint("achieved", reclaimed).Msg("Reclaimed enough space using FIFO GC (at least one file deleted)")
 		return nil
 	}
 	return ErrInsufficientSpace
@@ -183,31 +196,32 @@ func FIFOGC(vfss vfs.VFS, size uint) error {
 func LargestGC(vfss vfs.VFS, size uint) error {
 	logger.Logger.Debug().Uint("target", size).Msg("Attempting to reclaim space using Largest GC")

-	// Get all files and sort by size (largest first)
 	files := getAllFiles(vfss)
 	if len(files) == 0 {
 		return ErrInsufficientSpace
 	}

-	// Sort by size (descending - largest first)
 	sort.Slice(files, func(i, j int) bool {
 		return files[i].Size > files[j].Size
 	})

 	var reclaimed uint
+	deleted := false
 	for _, fi := range files {
-		if reclaimed >= size {
-			break
-		}
 		err := vfss.Delete(fi.Name)
 		if err != nil {
 			continue
 		}
 		reclaimed += uint(fi.Size)
+		deleted = true
+		if deleted && (size == 0 || reclaimed >= size) {
+			logger.Logger.Debug().Uint("target", size).Uint("achieved", reclaimed).Msg("Reclaimed enough space using Largest GC (at least one file deleted)")
+			return nil
+		}
 	}

-	if reclaimed >= size {
-		logger.Logger.Debug().Uint("target", size).Uint("achieved", reclaimed).Msg("Reclaimed enough space using Largest GC")
+	if deleted {
+		logger.Logger.Debug().Uint("target", size).Uint("achieved", reclaimed).Msg("Reclaimed enough space using Largest GC (at least one file deleted)")
 		return nil
 	}
 	return ErrInsufficientSpace
@@ -217,31 +231,32 @@ func LargestGC(vfss vfs.VFS, size uint) error {
 func SmallestGC(vfss vfs.VFS, size uint) error {
 	logger.Logger.Debug().Uint("target", size).Msg("Attempting to reclaim space using Smallest GC")

-	// Get all files and sort by size (smallest first)
 	files := getAllFiles(vfss)
 	if len(files) == 0 {
 		return ErrInsufficientSpace
 	}

-	// Sort by size (ascending - smallest first)
 	sort.Slice(files, func(i, j int) bool {
 		return files[i].Size < files[j].Size
 	})

 	var reclaimed uint
+	deleted := false
 	for _, fi := range files {
-		if reclaimed >= size {
-			break
-		}
 		err := vfss.Delete(fi.Name)
 		if err != nil {
 			continue
 		}
 		reclaimed += uint(fi.Size)
+		deleted = true
+		if deleted && (size == 0 || reclaimed >= size) {
+			logger.Logger.Debug().Uint("target", size).Uint("achieved", reclaimed).Msg("Reclaimed enough space using Smallest GC (at least one file deleted)")
+			return nil
+		}
 	}

-	if reclaimed >= size {
-		logger.Logger.Debug().Uint("target", size).Uint("achieved", reclaimed).Msg("Reclaimed enough space using Smallest GC")
+	if deleted {
+		logger.Logger.Debug().Uint("target", size).Uint("achieved", reclaimed).Msg("Reclaimed enough space using Smallest GC (at least one file deleted)")
 		return nil
 	}
 	return ErrInsufficientSpace
@@ -251,14 +266,11 @@ func SmallestGC(vfss vfs.VFS, size uint) error {
 func HybridGC(vfss vfs.VFS, size uint) error {
 	logger.Logger.Debug().Uint("target", size).Msg("Attempting to reclaim space using Hybrid GC")

-	// Get all files and calculate hybrid scores
 	files := getAllFiles(vfss)
 	if len(files) == 0 {
 		return ErrInsufficientSpace
 	}

-	// Calculate hybrid scores (lower score = more likely to be evicted)
-	// Score = (time since last access in seconds) * (file size in MB)
 	now := time.Now()
 	for i := range files {
 		timeSinceAccess := now.Sub(files[i].ATime).Seconds()
@@ -266,25 +278,27 @@ func HybridGC(vfss vfs.VFS, size uint) error {
 		files[i].HybridScore = timeSinceAccess * sizeMB
 	}

-	// Sort by hybrid score (ascending - lowest scores first)
 	sort.Slice(files, func(i, j int) bool {
 		return files[i].HybridScore < files[j].HybridScore
 	})

 	var reclaimed uint
+	deleted := false
 	for _, fi := range files {
-		if reclaimed >= size {
-			break
-		}
 		err := vfss.Delete(fi.Name)
 		if err != nil {
 			continue
 		}
 		reclaimed += uint(fi.Size)
+		deleted = true
+		if deleted && (size == 0 || reclaimed >= size) {
+			logger.Logger.Debug().Uint("target", size).Uint("achieved", reclaimed).Msg("Reclaimed enough space using Hybrid GC (at least one file deleted)")
+			return nil
+		}
 	}

-	if reclaimed >= size {
-		logger.Logger.Debug().Uint("target", size).Uint("achieved", reclaimed).Msg("Reclaimed enough space using Hybrid GC")
+	if deleted {
+		logger.Logger.Debug().Uint("target", size).Uint("achieved", reclaimed).Msg("Reclaimed enough space using Hybrid GC (at least one file deleted)")
 		return nil
 	}
 	return ErrInsufficientSpace
@@ -313,7 +327,7 @@ func getAllFiles(vfss vfs.VFS) []fileInfoWithMetadata {
 				Size:        fi.Size(),
 				MTime:       fi.ModTime(),
 				ATime:       fi.AccessTime(),
-				AccessCount: fi.AccessCount, // Use real access count
+				AccessCount: fi.AccessCount,
 			})
 		}
 	case *memory.MemoryFS:
@@ -324,7 +338,7 @@ func getAllFiles(vfss vfs.VFS) []fileInfoWithMetadata {
 				Size:        fi.Size(),
 				MTime:       fi.ModTime(),
 				ATime:       fi.AccessTime(),
-				AccessCount: fi.AccessCount, // Use real access count
+				AccessCount: fi.AccessCount,
 			})
 		}
 	}
@@ -703,13 +717,17 @@ func New(vfs vfs.VFS, gcHandlerFunc GCHandlerFunc) *GCFS {

 // Create overrides the Create method of the VFS interface. It tries to create the key, if it fails due to disk full error, it calls the GC handler and tries again. If it still fails it returns the error.
 func (g *GCFS) Create(key string, size int64) (io.WriteCloser, error) {
-	w, err := g.VFS.Create(key, size)                          // try to create the key
-	for err == vfserror.ErrDiskFull && g.gcHanderFunc != nil { // if the error is disk full and there is a GC handler
-		errr := g.gcHanderFunc(g.VFS, uint(size)) // call the GC handler
-		if errr == ErrInsufficientSpace {
-			return nil, errr // if the GC handler returns no files to delete, return the error
+	w, err := g.VFS.Create(key, size) // try to create the key
+	for err == vfserror.ErrDiskFull && g.gcHanderFunc != nil {
+		errGC := g.gcHanderFunc(g.VFS, uint(size)) // call the GC handler
+		if errGC == ErrInsufficientSpace {
+			return nil, errGC // if the GC handler returns no files to delete, return the error
 		}
 		w, err = g.VFS.Create(key, size)
+		if err == vfserror.ErrDiskFull {
+			// GC handler did not free enough space, avoid infinite loop
+			return nil, ErrInsufficientSpace
+		}
 	}

 	if err != nil {
--- a/vfs/memory/memory.go
+++ b/vfs/memory/memory.go
@@ -149,6 +149,15 @@ func (m *MemoryFS) getKeyLock(key string) *sync.RWMutex {
 }

 func (m *MemoryFS) Create(key string, size int64) (io.WriteCloser, error) {
+	m.mu.RLock()
+	if m.capacity > 0 {
+		if m.size+size > m.capacity {
+			m.mu.RUnlock()
+			return nil, vfserror.ErrDiskFull
+		}
+	}
+	m.mu.RUnlock()
+
 	keyMu := m.getKeyLock(key)
 	keyMu.Lock()
 	defer keyMu.Unlock()
@@ -160,19 +169,21 @@ func (m *MemoryFS) Create(key string, size int64) (io.WriteCloser, error) {
 		onClose: func() error {
 			data := buf.Bytes()
 			m.mu.Lock()
-			defer m.mu.Unlock()
+			var accessCount int64 = 0
 			if f, exists := m.files[key]; exists {
 				m.size -= int64(len(f.data))
 				m.LRU.Remove(key)
+				accessCount = f.fileinfo.AccessCount // preserve access count if overwriting
 			}
 			fi := vfs.NewFileInfo(key, int64(len(data)), time.Now())
-			fi.CTime = time.Now() // Set creation time
+			fi.AccessCount = accessCount
 			m.files[key] = &file{
-				data:     data,
 				fileinfo: fi,
+				data:     data,
 			}
 			m.LRU.Add(key, fi)
 			m.size += int64(len(data))
+			m.mu.Unlock()

 			memoryWriteBytes.Add(float64(len(data)))
 			memorySizeBytes.Set(float64(m.Size()))
@@ -224,7 +235,7 @@ func (m *MemoryFS) Open(key string) (io.ReadCloser, error) {
 		return nil, vfserror.ErrNotFound
 	}
 	f.fileinfo.ATime = time.Now()
-	f.fileinfo.AccessCount++ // Increment access count
+	f.fileinfo.AccessCount++ // Increment access count for LFU
 	m.LRU.MoveToFront(key)
 	dataCopy := make([]byte, len(f.data))
 	copy(dataCopy, f.data)