diff --git a/.vscode/launch.json b/.vscode/launch.json index 48c710f..480c642 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -23,8 +23,8 @@ "lru", "--log-level", "debug", - "--upstream", - "http://192.168.2.5:80", + // "--upstream", + // "http://192.168.2.5:80", ], }, { @@ -42,8 +42,8 @@ "hybrid", "--log-level", "debug", - "--upstream", - "http://192.168.2.5:80", + // "--upstream", + // "http://192.168.2.5:80", ], }, { diff --git a/vfs/disk/disk.go b/vfs/disk/disk.go index e45c7b8..dd16352 100644 --- a/vfs/disk/disk.go +++ b/vfs/disk/disk.go @@ -242,10 +242,12 @@ func (d *DiskFS) Create(key string, size int64) (io.WriteCloser, error) { // Check again after lock d.mu.Lock() + var accessCount int64 = 0 if fi, exists := d.info[key]; exists { d.size -= fi.Size() d.LRU.Remove(key) delete(d.info, key) + accessCount = fi.AccessCount // preserve access count if overwriting path := filepath.Join(d.root, key) os.Remove(path) // Ignore error, as file might not exist or other issues } @@ -274,6 +276,7 @@ func (d *DiskFS) Create(key string, size int64) (io.WriteCloser, error) { d.mu.Lock() finfo := vfs.NewFileInfoFromOS(fi, key) + finfo.AccessCount = accessCount d.info[key] = finfo d.LRU.Add(key, finfo) d.size += n @@ -381,6 +384,7 @@ func (d *DiskFS) Open(key string) (io.ReadCloser, error) { return nil, vfserror.ErrNotFound } fi.ATime = time.Now() + fi.AccessCount++ // Increment access count for LFU d.LRU.MoveToFront(key) d.mu.Unlock() diff --git a/vfs/fileinfo.go b/vfs/fileinfo.go index 1d7f940..8e66e67 100644 --- a/vfs/fileinfo.go +++ b/vfs/fileinfo.go @@ -7,27 +7,30 @@ import ( ) type FileInfo struct { - name string - size int64 - MTime time.Time - ATime time.Time + name string + size int64 + MTime time.Time + ATime time.Time + AccessCount int64 // Number of times the file has been accessed } func NewFileInfo(key string, size int64, modTime time.Time) *FileInfo { return &FileInfo{ - name: key, - size: size, - MTime: modTime, - ATime: time.Now(), + name: key, + size: size, + MTime: modTime, + ATime: time.Now(), + AccessCount: 0, } } func NewFileInfoFromOS(f os.FileInfo, key string) *FileInfo { return &FileInfo{ - name: key, - size: f.Size(), - MTime: f.ModTime(), - ATime: time.Now(), + name: key, + size: f.Size(), + MTime: f.ModTime(), + ATime: time.Now(), + AccessCount: 0, } } diff --git a/vfs/gc/gc.go b/vfs/gc/gc.go index 24d715b..a459b8d 100644 --- a/vfs/gc/gc.go +++ b/vfs/gc/gc.go @@ -75,12 +75,17 @@ func LRUGC(vfss vfs.VFS, size uint) error { logger.Logger.Debug().Uint("target", size).Msg("Attempting to reclaim space using LRU GC") var reclaimed uint // reclaimed space in bytes + deleted := false for { switch fs := vfss.(type) { case *disk.DiskFS: fi := fs.LRU.Back() if fi == nil { + if deleted { + logger.Logger.Debug().Uint("target", size).Uint("achieved", reclaimed).Msg("Reclaimed enough space using LRU GC (at least one file deleted)") + return nil + } return ErrInsufficientSpace // No files to delete } sz := uint(fi.Size()) @@ -89,9 +94,14 @@ func LRUGC(vfss vfs.VFS, size uint) error { continue // If delete fails, try the next file } reclaimed += sz + deleted = true case *memory.MemoryFS: fi := fs.LRU.Back() if fi == nil { + if deleted { + logger.Logger.Debug().Uint("target", size).Uint("achieved", reclaimed).Msg("Reclaimed enough space using LRU GC (at least one file deleted)") + return nil + } return ErrInsufficientSpace // No files to delete } sz := uint(fi.Size()) @@ -100,13 +110,14 @@ func LRUGC(vfss vfs.VFS, size uint) error { continue // If delete fails, try the next file } reclaimed += sz + deleted = true default: panic("unreachable: unsupported VFS type for LRU GC") // panic if the VFS is not disk or memory } - if reclaimed >= size { - logger.Logger.Debug().Uint("target", size).Uint("achieved", reclaimed).Msg("Reclaimed enough space using LRU GC") - return nil // stop if enough space is reclaimed + if deleted && (size == 0 || reclaimed >= size) { + logger.Logger.Debug().Uint("target", size).Uint("achieved", reclaimed).Msg("Reclaimed enough space using LRU GC (at least one file deleted)") + return nil // stop if enough space is reclaimed or at least one file deleted for size==0 } } } @@ -115,31 +126,32 @@ func LRUGC(vfss vfs.VFS, size uint) error { func LFUGC(vfss vfs.VFS, size uint) error { logger.Logger.Debug().Uint("target", size).Msg("Attempting to reclaim space using LFU GC") - // Get all files and sort by access count (frequency) files := getAllFiles(vfss) if len(files) == 0 { return ErrInsufficientSpace } - // Sort by access count (ascending - least frequently used first) sort.Slice(files, func(i, j int) bool { return files[i].AccessCount < files[j].AccessCount }) var reclaimed uint + deleted := false for _, fi := range files { - if reclaimed >= size { - break - } err := vfss.Delete(fi.Name) if err != nil { continue } reclaimed += uint(fi.Size) + deleted = true + if deleted && (size == 0 || reclaimed >= size) { + logger.Logger.Debug().Uint("target", size).Uint("achieved", reclaimed).Msg("Reclaimed enough space using LFU GC (at least one file deleted)") + return nil + } } - if reclaimed >= size { - logger.Logger.Debug().Uint("target", size).Uint("achieved", reclaimed).Msg("Reclaimed enough space using LFU GC") + if deleted { + logger.Logger.Debug().Uint("target", size).Uint("achieved", reclaimed).Msg("Reclaimed enough space using LFU GC (at least one file deleted)") return nil } return ErrInsufficientSpace @@ -149,31 +161,32 @@ func LFUGC(vfss vfs.VFS, size uint) error { func FIFOGC(vfss vfs.VFS, size uint) error { logger.Logger.Debug().Uint("target", size).Msg("Attempting to reclaim space using FIFO GC") - // Get all files and sort by creation time (oldest first) files := getAllFiles(vfss) if len(files) == 0 { return ErrInsufficientSpace } - // Sort by creation time (ascending - oldest first) sort.Slice(files, func(i, j int) bool { return files[i].MTime.Before(files[j].MTime) }) var reclaimed uint + deleted := false for _, fi := range files { - if reclaimed >= size { - break - } err := vfss.Delete(fi.Name) if err != nil { continue } reclaimed += uint(fi.Size) + deleted = true + if deleted && (size == 0 || reclaimed >= size) { + logger.Logger.Debug().Uint("target", size).Uint("achieved", reclaimed).Msg("Reclaimed enough space using FIFO GC (at least one file deleted)") + return nil + } } - if reclaimed >= size { - logger.Logger.Debug().Uint("target", size).Uint("achieved", reclaimed).Msg("Reclaimed enough space using FIFO GC") + if deleted { + logger.Logger.Debug().Uint("target", size).Uint("achieved", reclaimed).Msg("Reclaimed enough space using FIFO GC (at least one file deleted)") return nil } return ErrInsufficientSpace @@ -183,31 +196,32 @@ func FIFOGC(vfss vfs.VFS, size uint) error { func LargestGC(vfss vfs.VFS, size uint) error { logger.Logger.Debug().Uint("target", size).Msg("Attempting to reclaim space using Largest GC") - // Get all files and sort by size (largest first) files := getAllFiles(vfss) if len(files) == 0 { return ErrInsufficientSpace } - // Sort by size (descending - largest first) sort.Slice(files, func(i, j int) bool { return files[i].Size > files[j].Size }) var reclaimed uint + deleted := false for _, fi := range files { - if reclaimed >= size { - break - } err := vfss.Delete(fi.Name) if err != nil { continue } reclaimed += uint(fi.Size) + deleted = true + if deleted && (size == 0 || reclaimed >= size) { + logger.Logger.Debug().Uint("target", size).Uint("achieved", reclaimed).Msg("Reclaimed enough space using Largest GC (at least one file deleted)") + return nil + } } - if reclaimed >= size { - logger.Logger.Debug().Uint("target", size).Uint("achieved", reclaimed).Msg("Reclaimed enough space using Largest GC") + if deleted { + logger.Logger.Debug().Uint("target", size).Uint("achieved", reclaimed).Msg("Reclaimed enough space using Largest GC (at least one file deleted)") return nil } return ErrInsufficientSpace @@ -217,31 +231,32 @@ func LargestGC(vfss vfs.VFS, size uint) error { func SmallestGC(vfss vfs.VFS, size uint) error { logger.Logger.Debug().Uint("target", size).Msg("Attempting to reclaim space using Smallest GC") - // Get all files and sort by size (smallest first) files := getAllFiles(vfss) if len(files) == 0 { return ErrInsufficientSpace } - // Sort by size (ascending - smallest first) sort.Slice(files, func(i, j int) bool { return files[i].Size < files[j].Size }) var reclaimed uint + deleted := false for _, fi := range files { - if reclaimed >= size { - break - } err := vfss.Delete(fi.Name) if err != nil { continue } reclaimed += uint(fi.Size) + deleted = true + if deleted && (size == 0 || reclaimed >= size) { + logger.Logger.Debug().Uint("target", size).Uint("achieved", reclaimed).Msg("Reclaimed enough space using Smallest GC (at least one file deleted)") + return nil + } } - if reclaimed >= size { - logger.Logger.Debug().Uint("target", size).Uint("achieved", reclaimed).Msg("Reclaimed enough space using Smallest GC") + if deleted { + logger.Logger.Debug().Uint("target", size).Uint("achieved", reclaimed).Msg("Reclaimed enough space using Smallest GC (at least one file deleted)") return nil } return ErrInsufficientSpace @@ -251,14 +266,11 @@ func SmallestGC(vfss vfs.VFS, size uint) error { func HybridGC(vfss vfs.VFS, size uint) error { logger.Logger.Debug().Uint("target", size).Msg("Attempting to reclaim space using Hybrid GC") - // Get all files and calculate hybrid scores files := getAllFiles(vfss) if len(files) == 0 { return ErrInsufficientSpace } - // Calculate hybrid scores (lower score = more likely to be evicted) - // Score = (time since last access in seconds) * (file size in MB) now := time.Now() for i := range files { timeSinceAccess := now.Sub(files[i].ATime).Seconds() @@ -266,25 +278,27 @@ func HybridGC(vfss vfs.VFS, size uint) error { files[i].HybridScore = timeSinceAccess * sizeMB } - // Sort by hybrid score (ascending - lowest scores first) sort.Slice(files, func(i, j int) bool { return files[i].HybridScore < files[j].HybridScore }) var reclaimed uint + deleted := false for _, fi := range files { - if reclaimed >= size { - break - } err := vfss.Delete(fi.Name) if err != nil { continue } reclaimed += uint(fi.Size) + deleted = true + if deleted && (size == 0 || reclaimed >= size) { + logger.Logger.Debug().Uint("target", size).Uint("achieved", reclaimed).Msg("Reclaimed enough space using Hybrid GC (at least one file deleted)") + return nil + } } - if reclaimed >= size { - logger.Logger.Debug().Uint("target", size).Uint("achieved", reclaimed).Msg("Reclaimed enough space using Hybrid GC") + if deleted { + logger.Logger.Debug().Uint("target", size).Uint("achieved", reclaimed).Msg("Reclaimed enough space using Hybrid GC (at least one file deleted)") return nil } return ErrInsufficientSpace @@ -308,25 +322,23 @@ func getAllFiles(vfss vfs.VFS) []fileInfoWithMetadata { case *disk.DiskFS: allFiles := fs.StatAll() for _, fi := range allFiles { - // For disk, we can't easily track access count, so we'll use 1 as default files = append(files, fileInfoWithMetadata{ Name: fi.Name(), Size: fi.Size(), MTime: fi.ModTime(), ATime: fi.AccessTime(), - AccessCount: 1, + AccessCount: fi.AccessCount, }) } case *memory.MemoryFS: allFiles := fs.StatAll() for _, fi := range allFiles { - // For memory, we can't easily track access count, so we'll use 1 as default files = append(files, fileInfoWithMetadata{ Name: fi.Name(), Size: fi.Size(), MTime: fi.ModTime(), ATime: fi.AccessTime(), - AccessCount: 1, + AccessCount: fi.AccessCount, }) } } @@ -705,13 +717,17 @@ func New(vfs vfs.VFS, gcHandlerFunc GCHandlerFunc) *GCFS { // Create overrides the Create method of the VFS interface. It tries to create the key, if it fails due to disk full error, it calls the GC handler and tries again. If it still fails it returns the error. func (g *GCFS) Create(key string, size int64) (io.WriteCloser, error) { - w, err := g.VFS.Create(key, size) // try to create the key - for err == vfserror.ErrDiskFull && g.gcHanderFunc != nil { // if the error is disk full and there is a GC handler - errr := g.gcHanderFunc(g.VFS, uint(size)) // call the GC handler - if errr == ErrInsufficientSpace { - return nil, errr // if the GC handler returns no files to delete, return the error + w, err := g.VFS.Create(key, size) // try to create the key + for err == vfserror.ErrDiskFull && g.gcHanderFunc != nil { + errGC := g.gcHanderFunc(g.VFS, uint(size)) // call the GC handler + if errGC == ErrInsufficientSpace { + return nil, errGC // if the GC handler returns no files to delete, return the error } w, err = g.VFS.Create(key, size) + if err == vfserror.ErrDiskFull { + // GC handler did not free enough space, avoid infinite loop + return nil, ErrInsufficientSpace + } } if err != nil { diff --git a/vfs/memory/memory.go b/vfs/memory/memory.go index 01ccf40..0aad37d 100644 --- a/vfs/memory/memory.go +++ b/vfs/memory/memory.go @@ -169,11 +169,14 @@ func (m *MemoryFS) Create(key string, size int64) (io.WriteCloser, error) { onClose: func() error { data := buf.Bytes() m.mu.Lock() + var accessCount int64 = 0 if f, exists := m.files[key]; exists { m.size -= int64(len(f.data)) m.LRU.Remove(key) + accessCount = f.fileinfo.AccessCount // preserve access count if overwriting } fi := vfs.NewFileInfo(key, int64(len(data)), time.Now()) + fi.AccessCount = accessCount m.files[key] = &file{ fileinfo: fi, data: data, @@ -232,6 +235,7 @@ func (m *MemoryFS) Open(key string) (io.ReadCloser, error) { return nil, vfserror.ErrNotFound } f.fileinfo.ATime = time.Now() + f.fileinfo.AccessCount++ // Increment access count for LFU m.LRU.MoveToFront(key) dataCopy := make([]byte, len(f.data)) copy(dataCopy, f.data)