Refactor VFS implementation to use Create and Open methods

- Updated disk_test.go to replace Set and Get with Create and Open methods for better clarity and functionality.
- Modified fileinfo.go to include package comment.
- Refactored gc.go to streamline garbage collection handling and removed unused statistics.
- Updated gc_test.go to comment out large random tests for future implementation.
- Enhanced memory.go to implement LRU caching and metrics for memory usage.
- Updated memory_test.go to replace Set and Get with Create and Open methods.
- Removed sync.go as it was redundant and not utilized.
- Updated vfs.go to reflect changes in the VFS interface, replacing Set and Get with Create and Open.
- Added package comments to vfserror.go for consistency.

vfs/memory/memory.go

@@ -1,6 +1,10 @@
+// vfs/memory/memory.go
 package memory
 
 import (
+	"bytes"
+	"container/list"
+	"io"
 	"s1d3sw1ped/SteamCache2/steamcache/logger"
 	"s1d3sw1ped/SteamCache2/vfs"
 	"s1d3sw1ped/SteamCache2/vfs/vfserror"
@@ -8,6 +12,38 @@ import (
 	"time"
 
 	"github.com/docker/go-units"
+	"github.com/prometheus/client_golang/prometheus"
+	"github.com/prometheus/client_golang/prometheus/promauto"
+)
+
+var (
+	memoryCapacityBytes = promauto.NewGauge(
+		prometheus.GaugeOpts{
+			Name: "memory_cache_capacity_bytes",
+			Help: "Total capacity of the memory cache in bytes",
+		},
+	)
+
+	memorySizeBytes = promauto.NewGauge(
+		prometheus.GaugeOpts{
+			Name: "memory_cache_size_bytes",
+			Help: "Total size of the memory cache in bytes",
+		},
+	)
+
+	memoryReadBytes = promauto.NewCounter(
+		prometheus.CounterOpts{
+			Name: "memory_cache_read_bytes_total",
+			Help: "Total number of bytes read from the memory cache",
+		},
+	)
+
+	memoryWriteBytes = promauto.NewCounter(
+		prometheus.CounterOpts{
+			Name: "memory_cache_write_bytes_total",
+			Help: "Total number of bytes written to the memory cache",
+		},
+	)
 )
 
 // Ensure MemoryFS implements VFS.
@@ -23,9 +59,49 @@ type file struct {
 type MemoryFS struct {
 	files    map[string]*file
 	capacity int64
-	mu       sync.Mutex
+	size     int64
+	mu       sync.RWMutex
+	keyLocks sync.Map // map[string]*sync.RWMutex
+	LRU      *lruList
+}
 
-	bytePool sync.Pool // Pool for []byte slices
+// lruList for LRU eviction
+type lruList struct {
+	list *list.List
+	elem map[string]*list.Element
+}
+
+func newLruList() *lruList {
+	return &lruList{
+		list: list.New(),
+		elem: make(map[string]*list.Element),
+	}
+}
+
+func (l *lruList) MoveToFront(key string) {
+	if e, ok := l.elem[key]; ok {
+		l.list.MoveToFront(e)
+	}
+}
+
+func (l *lruList) Add(key string, fi *vfs.FileInfo) *list.Element {
+	e := l.list.PushFront(fi)
+	l.elem[key] = e
+	return e
+}
+
+func (l *lruList) Remove(key string) {
+	if e, ok := l.elem[key]; ok {
+		l.list.Remove(e)
+		delete(l.elem, key)
+	}
+}
+
+func (l *lruList) Back() *vfs.FileInfo {
+	if e := l.list.Back(); e != nil {
+		return e.Value.(*vfs.FileInfo)
+	}
+	return nil
 }
 
 // New creates a new MemoryFS.
@@ -39,14 +115,18 @@ func New(capacity int64) *MemoryFS {
 		Str("capacity", units.HumanSize(float64(capacity))).
 		Msg("init")
 
-	return &MemoryFS{
+	mfs := &MemoryFS{
 		files:    make(map[string]*file),
 		capacity: capacity,
-		mu:       sync.Mutex{},
-		bytePool: sync.Pool{
-			New: func() interface{} { return make([]byte, 0) }, // Initial capacity for pooled slices
-		},
+		mu:       sync.RWMutex{},
+		keyLocks: sync.Map{},
+		LRU:      newLruList(),
 	}
+
+	memoryCapacityBytes.Set(float64(capacity))
+	memorySizeBytes.Set(float64(mfs.Size()))
+
+	return mfs
 }
 
 func (m *MemoryFS) Capacity() int64 {
@@ -58,93 +138,118 @@ func (m *MemoryFS) Name() string {
 }
 
 func (m *MemoryFS) Size() int64 {
-	var size int64
-
-	m.mu.Lock()
-	defer m.mu.Unlock()
-
-	for _, v := range m.files {
-		size += int64(len(v.data))
-	}
-
-	return size
+	m.mu.RLock()
+	defer m.mu.RUnlock()
+	return m.size
 }
 
-func (m *MemoryFS) Set(key string, src []byte) error {
+func (m *MemoryFS) getKeyLock(key string) *sync.RWMutex {
+	mu, _ := m.keyLocks.LoadOrStore(key, &sync.RWMutex{})
+	return mu.(*sync.RWMutex)
+}
+
+func (m *MemoryFS) Create(key string, size int64) (io.WriteCloser, error) {
+	m.mu.RLock()
 	if m.capacity > 0 {
-		if size := m.Size() + int64(len(src)); size > m.capacity {
-			return vfserror.ErrDiskFull
+		if m.size+size > m.capacity {
+			m.mu.RUnlock()
+			return nil, vfserror.ErrDiskFull
 		}
 	}
+	m.mu.RUnlock()
 
-	m.mu.Lock()
-	defer m.mu.Unlock()
+	keyMu := m.getKeyLock(key)
+	keyMu.Lock()
+	defer keyMu.Unlock()
 
-	// Use pooled slice
-	data := m.bytePool.Get().([]byte)
-	if cap(data) < len(src) {
-		data = make([]byte, len(src)) // expand the slice if the pool slice is too small
-	} else {
-		data = data[:len(src)] // reuse the pool slice, but resize it to fit
-	}
-	copy(data, src)
+	buf := &bytes.Buffer{}
 
-	m.files[key] = &file{
-		fileinfo: vfs.NewFileInfo(
-			key,
-			int64(len(src)),
-			time.Now(),
-		),
-		data: data,
-	}
+	return &memWriteCloser{
+		Writer: buf,
+		onClose: func() error {
+			data := buf.Bytes()
+			m.mu.Lock()
+			if f, exists := m.files[key]; exists {
+				m.size -= int64(len(f.data))
+				m.LRU.Remove(key)
+			}
+			fi := vfs.NewFileInfo(key, int64(len(data)), time.Now())
+			m.files[key] = &file{
+				fileinfo: fi,
+				data:     data,
+			}
+			m.LRU.Add(key, fi)
+			m.size += int64(len(data))
+			m.mu.Unlock()
 
-	return nil
+			memoryWriteBytes.Add(float64(len(data)))
+			memorySizeBytes.Set(float64(m.Size()))
+
+			return nil
+		},
+	}, nil
+}
+
+type memWriteCloser struct {
+	io.Writer
+	onClose func() error
+}
+
+func (wc *memWriteCloser) Close() error {
+	return wc.onClose()
 }
 
 func (m *MemoryFS) Delete(key string) error {
-	_, err := m.Stat(key)
-	if err != nil {
-		return err
-	}
+	keyMu := m.getKeyLock(key)
+	keyMu.Lock()
+	defer keyMu.Unlock()
 
 	m.mu.Lock()
-	defer m.mu.Unlock()
-
-	// Return data to pool
-	if f, ok := m.files[key]; ok {
-		m.bytePool.Put(f.data)
+	f, exists := m.files[key]
+	if !exists {
+		m.mu.Unlock()
+		return vfserror.ErrNotFound
 	}
-
+	m.size -= int64(len(f.data))
+	m.LRU.Remove(key)
 	delete(m.files, key)
+	m.mu.Unlock()
+
+	memorySizeBytes.Set(float64(m.Size()))
 
 	return nil
 }
 
-func (m *MemoryFS) Get(key string) ([]byte, error) {
-	_, err := m.Stat(key)
-	if err != nil {
-		return nil, err
-	}
+func (m *MemoryFS) Open(key string) (io.ReadCloser, error) {
+	keyMu := m.getKeyLock(key)
+	keyMu.RLock()
+	defer keyMu.RUnlock()
 
 	m.mu.Lock()
-	defer m.mu.Unlock()
+	f, exists := m.files[key]
+	if !exists {
+		m.mu.Unlock()
+		return nil, vfserror.ErrNotFound
+	}
+	f.fileinfo.ATime = time.Now()
+	m.LRU.MoveToFront(key)
+	dataCopy := make([]byte, len(f.data))
+	copy(dataCopy, f.data)
+	m.mu.Unlock()
 
-	m.files[key].fileinfo.ATime = time.Now()
-	dst := make([]byte, len(m.files[key].data))
-	copy(dst, m.files[key].data)
+	memoryReadBytes.Add(float64(len(dataCopy)))
+	memorySizeBytes.Set(float64(m.Size()))
 
-	logger.Logger.Debug().
-		Str("name", key).
-		Str("status", "GET").
-		Int64("size", int64(len(dst))).
-		Msg("get file from memory")
-
-	return dst, nil
+	return io.NopCloser(bytes.NewReader(dataCopy)), nil
 }
 
 func (m *MemoryFS) Stat(key string) (*vfs.FileInfo, error) {
-	m.mu.Lock()
-	defer m.mu.Unlock()
+	keyMu := m.getKeyLock(key)
+	keyMu.RLock()
+	defer keyMu.RUnlock()
+
+	m.mu.RLock()
+	defer m.mu.RUnlock()
 
 	f, ok := m.files[key]
 	if !ok {
@@ -155,8 +260,8 @@ func (m *MemoryFS) Stat(key string) (*vfs.FileInfo, error) {
 }
 
 func (m *MemoryFS) StatAll() []*vfs.FileInfo {
-	m.mu.Lock()
-	defer m.mu.Unlock()
+	m.mu.RLock()
+	defer m.mu.RUnlock()
 
 	// hard copy the file info to prevent modification of the original file info or the other way around
 	files := make([]*vfs.FileInfo, 0, len(m.files))