Refactor configuration management and enhance build process
- Introduced a YAML-based configuration system, allowing for automatic generation of a default `config.yaml` file. - Updated the application to load configuration settings from the YAML file, improving flexibility and ease of use. - Added a Makefile to streamline development tasks, including running the application, testing, and managing dependencies. - Enhanced `.gitignore` to include build artifacts and configuration files. - Removed unused Prometheus metrics and related code to simplify the codebase. - Updated dependencies in `go.mod` and `go.sum` for improved functionality and performance.
This commit is contained in:
@@ -5,59 +5,19 @@ import (
|
||||
"bytes"
|
||||
"container/list"
|
||||
"io"
|
||||
"s1d3sw1ped/SteamCache2/steamcache/logger"
|
||||
"s1d3sw1ped/SteamCache2/vfs"
|
||||
"s1d3sw1ped/SteamCache2/vfs/vfserror"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/docker/go-units"
|
||||
"github.com/prometheus/client_golang/prometheus"
|
||||
"github.com/prometheus/client_golang/prometheus/promauto"
|
||||
)
|
||||
|
||||
var (
|
||||
memoryCapacityBytes = promauto.NewGauge(
|
||||
prometheus.GaugeOpts{
|
||||
Name: "memory_cache_capacity_bytes",
|
||||
Help: "Total capacity of the memory cache in bytes",
|
||||
},
|
||||
)
|
||||
|
||||
memorySizeBytes = promauto.NewGauge(
|
||||
prometheus.GaugeOpts{
|
||||
Name: "memory_cache_size_bytes",
|
||||
Help: "Total size of the memory cache in bytes",
|
||||
},
|
||||
)
|
||||
|
||||
memoryReadBytes = promauto.NewCounter(
|
||||
prometheus.CounterOpts{
|
||||
Name: "memory_cache_read_bytes_total",
|
||||
Help: "Total number of bytes read from the memory cache",
|
||||
},
|
||||
)
|
||||
|
||||
memoryWriteBytes = promauto.NewCounter(
|
||||
prometheus.CounterOpts{
|
||||
Name: "memory_cache_write_bytes_total",
|
||||
Help: "Total number of bytes written to the memory cache",
|
||||
},
|
||||
)
|
||||
)
|
||||
|
||||
// Ensure MemoryFS implements VFS.
|
||||
var _ vfs.VFS = (*MemoryFS)(nil)
|
||||
|
||||
// file represents a file in memory.
|
||||
type file struct {
|
||||
fileinfo *vfs.FileInfo
|
||||
data []byte
|
||||
}
|
||||
|
||||
// MemoryFS is a virtual file system that stores files in memory.
|
||||
// MemoryFS is an in-memory virtual file system
|
||||
type MemoryFS struct {
|
||||
files map[string]*file
|
||||
data map[string]*bytes.Buffer
|
||||
info map[string]*vfs.FileInfo
|
||||
capacity int64
|
||||
size int64
|
||||
mu sync.RWMutex
|
||||
@@ -65,7 +25,7 @@ type MemoryFS struct {
|
||||
LRU *lruList
|
||||
}
|
||||
|
||||
// lruList for LRU eviction
|
||||
// lruList for time-decayed LRU eviction
|
||||
type lruList struct {
|
||||
list *list.List
|
||||
elem map[string]*list.Element
|
||||
@@ -78,176 +38,239 @@ func newLruList() *lruList {
|
||||
}
|
||||
}
|
||||
|
||||
func (l *lruList) Add(key string, fi *vfs.FileInfo) {
|
||||
elem := l.list.PushFront(fi)
|
||||
l.elem[key] = elem
|
||||
}
|
||||
|
||||
func (l *lruList) MoveToFront(key string) {
|
||||
if e, ok := l.elem[key]; ok {
|
||||
l.list.MoveToFront(e)
|
||||
if elem, exists := l.elem[key]; exists {
|
||||
l.list.MoveToFront(elem)
|
||||
// Update the FileInfo in the element with new access time
|
||||
if fi := elem.Value.(*vfs.FileInfo); fi != nil {
|
||||
fi.UpdateAccess()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (l *lruList) Add(key string, fi *vfs.FileInfo) *list.Element {
|
||||
e := l.list.PushFront(fi)
|
||||
l.elem[key] = e
|
||||
return e
|
||||
}
|
||||
|
||||
func (l *lruList) Remove(key string) {
|
||||
if e, ok := l.elem[key]; ok {
|
||||
l.list.Remove(e)
|
||||
func (l *lruList) Remove(key string) *vfs.FileInfo {
|
||||
if elem, exists := l.elem[key]; exists {
|
||||
delete(l.elem, key)
|
||||
}
|
||||
}
|
||||
|
||||
func (l *lruList) Back() *vfs.FileInfo {
|
||||
if e := l.list.Back(); e != nil {
|
||||
return e.Value.(*vfs.FileInfo)
|
||||
if fi := l.list.Remove(elem).(*vfs.FileInfo); fi != nil {
|
||||
return fi
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// New creates a new MemoryFS.
|
||||
func (l *lruList) Len() int {
|
||||
return l.list.Len()
|
||||
}
|
||||
|
||||
// New creates a new MemoryFS
|
||||
func New(capacity int64) *MemoryFS {
|
||||
if capacity <= 0 {
|
||||
panic("memory capacity must be greater than 0") // panic if the capacity is less than or equal to 0
|
||||
panic("memory capacity must be greater than 0")
|
||||
}
|
||||
|
||||
logger.Logger.Info().
|
||||
Str("name", "MemoryFS").
|
||||
Str("capacity", units.HumanSize(float64(capacity))).
|
||||
Msg("init")
|
||||
|
||||
mfs := &MemoryFS{
|
||||
files: make(map[string]*file),
|
||||
return &MemoryFS{
|
||||
data: make(map[string]*bytes.Buffer),
|
||||
info: make(map[string]*vfs.FileInfo),
|
||||
capacity: capacity,
|
||||
mu: sync.RWMutex{},
|
||||
keyLocks: sync.Map{},
|
||||
size: 0,
|
||||
LRU: newLruList(),
|
||||
}
|
||||
|
||||
memoryCapacityBytes.Set(float64(capacity))
|
||||
memorySizeBytes.Set(float64(mfs.Size()))
|
||||
|
||||
return mfs
|
||||
}
|
||||
|
||||
func (m *MemoryFS) Capacity() int64 {
|
||||
return m.capacity
|
||||
}
|
||||
|
||||
// Name returns the name of this VFS
|
||||
func (m *MemoryFS) Name() string {
|
||||
return "MemoryFS"
|
||||
}
|
||||
|
||||
// Size returns the current size
|
||||
func (m *MemoryFS) Size() int64 {
|
||||
m.mu.RLock()
|
||||
defer m.mu.RUnlock()
|
||||
return m.size
|
||||
}
|
||||
|
||||
// Capacity returns the maximum capacity
|
||||
func (m *MemoryFS) Capacity() int64 {
|
||||
return m.capacity
|
||||
}
|
||||
|
||||
// getKeyLock returns a lock for the given key
|
||||
func (m *MemoryFS) getKeyLock(key string) *sync.RWMutex {
|
||||
mu, _ := m.keyLocks.LoadOrStore(key, &sync.RWMutex{})
|
||||
return mu.(*sync.RWMutex)
|
||||
keyLock, _ := m.keyLocks.LoadOrStore(key, &sync.RWMutex{})
|
||||
return keyLock.(*sync.RWMutex)
|
||||
}
|
||||
|
||||
// Create creates a new file
|
||||
func (m *MemoryFS) Create(key string, size int64) (io.WriteCloser, error) {
|
||||
m.mu.RLock()
|
||||
if m.capacity > 0 {
|
||||
if m.size+size > m.capacity {
|
||||
m.mu.RUnlock()
|
||||
return nil, vfserror.ErrDiskFull
|
||||
}
|
||||
if key == "" {
|
||||
return nil, vfserror.ErrInvalidKey
|
||||
}
|
||||
if key[0] == '/' {
|
||||
return nil, vfserror.ErrInvalidKey
|
||||
}
|
||||
m.mu.RUnlock()
|
||||
|
||||
keyMu := m.getKeyLock(key)
|
||||
keyMu.Lock()
|
||||
defer keyMu.Unlock()
|
||||
// Sanitize key to prevent path traversal
|
||||
if strings.Contains(key, "..") {
|
||||
return nil, vfserror.ErrInvalidKey
|
||||
}
|
||||
|
||||
buf := &bytes.Buffer{}
|
||||
|
||||
return &memWriteCloser{
|
||||
Writer: buf,
|
||||
onClose: func() error {
|
||||
data := buf.Bytes()
|
||||
m.mu.Lock()
|
||||
var accessCount int64 = 0
|
||||
if f, exists := m.files[key]; exists {
|
||||
m.size -= int64(len(f.data))
|
||||
m.LRU.Remove(key)
|
||||
accessCount = f.fileinfo.AccessCount // preserve access count if overwriting
|
||||
}
|
||||
fi := vfs.NewFileInfo(key, int64(len(data)), time.Now())
|
||||
fi.AccessCount = accessCount
|
||||
m.files[key] = &file{
|
||||
fileinfo: fi,
|
||||
data: data,
|
||||
}
|
||||
m.LRU.Add(key, fi)
|
||||
m.size += int64(len(data))
|
||||
m.mu.Unlock()
|
||||
|
||||
memoryWriteBytes.Add(float64(len(data)))
|
||||
memorySizeBytes.Set(float64(m.Size()))
|
||||
|
||||
return nil
|
||||
},
|
||||
}, nil
|
||||
}
|
||||
|
||||
type memWriteCloser struct {
|
||||
io.Writer
|
||||
onClose func() error
|
||||
}
|
||||
|
||||
func (wc *memWriteCloser) Close() error {
|
||||
return wc.onClose()
|
||||
}
|
||||
|
||||
func (m *MemoryFS) Delete(key string) error {
|
||||
keyMu := m.getKeyLock(key)
|
||||
keyMu.Lock()
|
||||
defer keyMu.Unlock()
|
||||
|
||||
m.mu.Lock()
|
||||
f, exists := m.files[key]
|
||||
// Check if file already exists and handle overwrite
|
||||
if fi, exists := m.info[key]; exists {
|
||||
m.size -= fi.Size
|
||||
m.LRU.Remove(key)
|
||||
delete(m.info, key)
|
||||
delete(m.data, key)
|
||||
}
|
||||
|
||||
buffer := &bytes.Buffer{}
|
||||
m.data[key] = buffer
|
||||
fi := vfs.NewFileInfo(key, size)
|
||||
m.info[key] = fi
|
||||
m.LRU.Add(key, fi)
|
||||
m.size += size
|
||||
m.mu.Unlock()
|
||||
|
||||
return &memoryWriteCloser{
|
||||
buffer: buffer,
|
||||
memory: m,
|
||||
key: key,
|
||||
}, nil
|
||||
}
|
||||
|
||||
// memoryWriteCloser implements io.WriteCloser for memory files
|
||||
type memoryWriteCloser struct {
|
||||
buffer *bytes.Buffer
|
||||
memory *MemoryFS
|
||||
key string
|
||||
}
|
||||
|
||||
func (mwc *memoryWriteCloser) Write(p []byte) (n int, err error) {
|
||||
return mwc.buffer.Write(p)
|
||||
}
|
||||
|
||||
func (mwc *memoryWriteCloser) Close() error {
|
||||
// Update the actual size in FileInfo
|
||||
mwc.memory.mu.Lock()
|
||||
if fi, exists := mwc.memory.info[mwc.key]; exists {
|
||||
actualSize := int64(mwc.buffer.Len())
|
||||
sizeDiff := actualSize - fi.Size
|
||||
fi.Size = actualSize
|
||||
mwc.memory.size += sizeDiff
|
||||
}
|
||||
mwc.memory.mu.Unlock()
|
||||
return nil
|
||||
}
|
||||
|
||||
// Open opens a file for reading
|
||||
func (m *MemoryFS) Open(key string) (io.ReadCloser, error) {
|
||||
if key == "" {
|
||||
return nil, vfserror.ErrInvalidKey
|
||||
}
|
||||
if key[0] == '/' {
|
||||
return nil, vfserror.ErrInvalidKey
|
||||
}
|
||||
|
||||
if strings.Contains(key, "..") {
|
||||
return nil, vfserror.ErrInvalidKey
|
||||
}
|
||||
|
||||
keyMu := m.getKeyLock(key)
|
||||
keyMu.RLock()
|
||||
defer keyMu.RUnlock()
|
||||
|
||||
m.mu.Lock()
|
||||
fi, exists := m.info[key]
|
||||
if !exists {
|
||||
m.mu.Unlock()
|
||||
return nil, vfserror.ErrNotFound
|
||||
}
|
||||
fi.UpdateAccess()
|
||||
m.LRU.MoveToFront(key)
|
||||
|
||||
buffer, exists := m.data[key]
|
||||
if !exists {
|
||||
m.mu.Unlock()
|
||||
return nil, vfserror.ErrNotFound
|
||||
}
|
||||
|
||||
// Create a copy of the buffer for reading
|
||||
data := make([]byte, buffer.Len())
|
||||
copy(data, buffer.Bytes())
|
||||
m.mu.Unlock()
|
||||
|
||||
return &memoryReadCloser{
|
||||
reader: bytes.NewReader(data),
|
||||
}, nil
|
||||
}
|
||||
|
||||
// memoryReadCloser implements io.ReadCloser for memory files
|
||||
type memoryReadCloser struct {
|
||||
reader *bytes.Reader
|
||||
}
|
||||
|
||||
func (mrc *memoryReadCloser) Read(p []byte) (n int, err error) {
|
||||
return mrc.reader.Read(p)
|
||||
}
|
||||
|
||||
func (mrc *memoryReadCloser) Close() error {
|
||||
return nil
|
||||
}
|
||||
|
||||
// Delete removes a file
|
||||
func (m *MemoryFS) Delete(key string) error {
|
||||
if key == "" {
|
||||
return vfserror.ErrInvalidKey
|
||||
}
|
||||
if key[0] == '/' {
|
||||
return vfserror.ErrInvalidKey
|
||||
}
|
||||
|
||||
if strings.Contains(key, "..") {
|
||||
return vfserror.ErrInvalidKey
|
||||
}
|
||||
|
||||
keyMu := m.getKeyLock(key)
|
||||
keyMu.Lock()
|
||||
defer keyMu.Unlock()
|
||||
|
||||
m.mu.Lock()
|
||||
fi, exists := m.info[key]
|
||||
if !exists {
|
||||
m.mu.Unlock()
|
||||
return vfserror.ErrNotFound
|
||||
}
|
||||
m.size -= int64(len(f.data))
|
||||
m.size -= fi.Size
|
||||
m.LRU.Remove(key)
|
||||
delete(m.files, key)
|
||||
delete(m.info, key)
|
||||
delete(m.data, key)
|
||||
m.mu.Unlock()
|
||||
|
||||
memorySizeBytes.Set(float64(m.Size()))
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (m *MemoryFS) Open(key string) (io.ReadCloser, error) {
|
||||
keyMu := m.getKeyLock(key)
|
||||
keyMu.RLock()
|
||||
defer keyMu.RUnlock()
|
||||
|
||||
m.mu.Lock()
|
||||
f, exists := m.files[key]
|
||||
if !exists {
|
||||
m.mu.Unlock()
|
||||
return nil, vfserror.ErrNotFound
|
||||
}
|
||||
f.fileinfo.ATime = time.Now()
|
||||
f.fileinfo.AccessCount++ // Increment access count for LFU
|
||||
m.LRU.MoveToFront(key)
|
||||
dataCopy := make([]byte, len(f.data))
|
||||
copy(dataCopy, f.data)
|
||||
m.mu.Unlock()
|
||||
|
||||
memoryReadBytes.Add(float64(len(dataCopy)))
|
||||
memorySizeBytes.Set(float64(m.Size()))
|
||||
|
||||
return io.NopCloser(bytes.NewReader(dataCopy)), nil
|
||||
}
|
||||
|
||||
// Stat returns file information
|
||||
func (m *MemoryFS) Stat(key string) (*vfs.FileInfo, error) {
|
||||
if key == "" {
|
||||
return nil, vfserror.ErrInvalidKey
|
||||
}
|
||||
if key[0] == '/' {
|
||||
return nil, vfserror.ErrInvalidKey
|
||||
}
|
||||
|
||||
if strings.Contains(key, "..") {
|
||||
return nil, vfserror.ErrInvalidKey
|
||||
}
|
||||
|
||||
keyMu := m.getKeyLock(key)
|
||||
keyMu.RLock()
|
||||
defer keyMu.RUnlock()
|
||||
@@ -255,24 +278,9 @@ func (m *MemoryFS) Stat(key string) (*vfs.FileInfo, error) {
|
||||
m.mu.RLock()
|
||||
defer m.mu.RUnlock()
|
||||
|
||||
f, ok := m.files[key]
|
||||
if !ok {
|
||||
return nil, vfserror.ErrNotFound
|
||||
if fi, ok := m.info[key]; ok {
|
||||
return fi, nil
|
||||
}
|
||||
|
||||
return f.fileinfo, nil
|
||||
}
|
||||
|
||||
func (m *MemoryFS) StatAll() []*vfs.FileInfo {
|
||||
m.mu.RLock()
|
||||
defer m.mu.RUnlock()
|
||||
|
||||
// hard copy the file info to prevent modification of the original file info or the other way around
|
||||
files := make([]*vfs.FileInfo, 0, len(m.files))
|
||||
for _, v := range m.files {
|
||||
fi := *v.fileinfo
|
||||
files = append(files, &fi)
|
||||
}
|
||||
|
||||
return files
|
||||
return nil, vfserror.ErrNotFound
|
||||
}
|
||||
|
||||
@@ -1,129 +0,0 @@
|
||||
// vfs/memory/memory_test.go
|
||||
package memory
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"s1d3sw1ped/SteamCache2/vfs/vfserror"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestCreateAndOpen(t *testing.T) {
|
||||
m := New(1024)
|
||||
key := "key"
|
||||
value := []byte("value")
|
||||
|
||||
w, err := m.Create(key, int64(len(value)))
|
||||
if err != nil {
|
||||
t.Fatalf("Create failed: %v", err)
|
||||
}
|
||||
w.Write(value)
|
||||
w.Close()
|
||||
|
||||
rc, err := m.Open(key)
|
||||
if err != nil {
|
||||
t.Fatalf("Open failed: %v", err)
|
||||
}
|
||||
got, _ := io.ReadAll(rc)
|
||||
rc.Close()
|
||||
|
||||
if string(got) != string(value) {
|
||||
t.Fatalf("expected %s, got %s", value, got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestOverwrite(t *testing.T) {
|
||||
m := New(1024)
|
||||
key := "key"
|
||||
value1 := []byte("value1")
|
||||
value2 := []byte("value2")
|
||||
|
||||
w, err := m.Create(key, int64(len(value1)))
|
||||
if err != nil {
|
||||
t.Fatalf("Create failed: %v", err)
|
||||
}
|
||||
w.Write(value1)
|
||||
w.Close()
|
||||
|
||||
w, err = m.Create(key, int64(len(value2)))
|
||||
if err != nil {
|
||||
t.Fatalf("Create failed: %v", err)
|
||||
}
|
||||
w.Write(value2)
|
||||
w.Close()
|
||||
|
||||
rc, err := m.Open(key)
|
||||
if err != nil {
|
||||
t.Fatalf("Open failed: %v", err)
|
||||
}
|
||||
got, _ := io.ReadAll(rc)
|
||||
rc.Close()
|
||||
|
||||
if string(got) != string(value2) {
|
||||
t.Fatalf("expected %s, got %s", value2, got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDelete(t *testing.T) {
|
||||
m := New(1024)
|
||||
key := "key"
|
||||
value := []byte("value")
|
||||
|
||||
w, err := m.Create(key, int64(len(value)))
|
||||
if err != nil {
|
||||
t.Fatalf("Create failed: %v", err)
|
||||
}
|
||||
w.Write(value)
|
||||
w.Close()
|
||||
|
||||
if err := m.Delete(key); err != nil {
|
||||
t.Fatalf("Delete failed: %v", err)
|
||||
}
|
||||
|
||||
_, err = m.Open(key)
|
||||
if !errors.Is(err, vfserror.ErrNotFound) {
|
||||
t.Fatalf("expected %v, got %v", vfserror.ErrNotFound, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCapacityLimit(t *testing.T) {
|
||||
m := New(10)
|
||||
for i := 0; i < 11; i++ {
|
||||
w, err := m.Create(fmt.Sprintf("key%d", i), 1)
|
||||
if err != nil && i < 10 {
|
||||
t.Errorf("Create failed: %v", err)
|
||||
} else if i == 10 && err == nil {
|
||||
t.Errorf("Create succeeded: got nil, want %v", vfserror.ErrDiskFull)
|
||||
}
|
||||
if i < 10 {
|
||||
w.Write([]byte("1"))
|
||||
w.Close()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestStat(t *testing.T) {
|
||||
m := New(1024)
|
||||
key := "key"
|
||||
value := []byte("value")
|
||||
|
||||
w, err := m.Create(key, int64(len(value)))
|
||||
if err != nil {
|
||||
t.Fatalf("Create failed: %v", err)
|
||||
}
|
||||
w.Write(value)
|
||||
w.Close()
|
||||
|
||||
info, err := m.Stat(key)
|
||||
if err != nil {
|
||||
t.Fatalf("unexpected error: %v", err)
|
||||
}
|
||||
|
||||
if info == nil {
|
||||
t.Fatal("expected file info to be non-nil")
|
||||
}
|
||||
if info.Size() != int64(len(value)) {
|
||||
t.Errorf("expected size %d, got %d", len(value), info.Size())
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user