Refactor configuration management and enhance build process
- Introduced a YAML-based configuration system, allowing for automatic generation of a default `config.yaml` file. - Updated the application to load configuration settings from the YAML file, improving flexibility and ease of use. - Added a Makefile to streamline development tasks, including running the application, testing, and managing dependencies. - Enhanced `.gitignore` to include build artifacts and configuration files. - Removed unused Prometheus metrics and related code to simplify the codebase. - Updated dependencies in `go.mod` and `go.sum` for improved functionality and performance.
This commit is contained in:
286
vfs/cache/cache.go
vendored
286
vfs/cache/cache.go
vendored
@@ -2,196 +2,152 @@
|
||||
package cache
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"io"
|
||||
"s1d3sw1ped/SteamCache2/vfs"
|
||||
"s1d3sw1ped/SteamCache2/vfs/cachestate"
|
||||
"s1d3sw1ped/SteamCache2/vfs/gc"
|
||||
"s1d3sw1ped/SteamCache2/vfs/vfserror"
|
||||
"sync"
|
||||
)
|
||||
|
||||
// Ensure CacheFS implements VFS.
|
||||
var _ vfs.VFS = (*CacheFS)(nil)
|
||||
// TieredCache implements a two-tier cache with fast (memory) and slow (disk) storage
|
||||
type TieredCache struct {
|
||||
fast vfs.VFS // Memory cache (fast)
|
||||
slow vfs.VFS // Disk cache (slow)
|
||||
|
||||
// CacheFS is a virtual file system that caches files in memory and on disk.
|
||||
type CacheFS struct {
|
||||
fast vfs.VFS
|
||||
slow vfs.VFS
|
||||
|
||||
cacheHandler CacheHandler
|
||||
|
||||
keyLocks sync.Map // map[string]*sync.RWMutex for per-key locks
|
||||
mu sync.RWMutex
|
||||
}
|
||||
|
||||
type CacheHandler func(*vfs.FileInfo, cachestate.CacheState) bool
|
||||
|
||||
// New creates a new CacheFS. fast is used for caching, and slow is used for storage. fast should obviously be faster than slow.
|
||||
func New(cacheHandler CacheHandler) *CacheFS {
|
||||
return &CacheFS{
|
||||
cacheHandler: cacheHandler,
|
||||
keyLocks: sync.Map{},
|
||||
}
|
||||
// New creates a new tiered cache
|
||||
func New() *TieredCache {
|
||||
return &TieredCache{}
|
||||
}
|
||||
|
||||
func (c *CacheFS) SetSlow(vfs vfs.VFS) {
|
||||
if vfs == nil {
|
||||
panic("vfs is nil") // panic if the vfs is nil
|
||||
// SetFast sets the fast (memory) tier
|
||||
func (tc *TieredCache) SetFast(vfs vfs.VFS) {
|
||||
tc.mu.Lock()
|
||||
defer tc.mu.Unlock()
|
||||
tc.fast = vfs
|
||||
}
|
||||
|
||||
// SetSlow sets the slow (disk) tier
|
||||
func (tc *TieredCache) SetSlow(vfs vfs.VFS) {
|
||||
tc.mu.Lock()
|
||||
defer tc.mu.Unlock()
|
||||
tc.slow = vfs
|
||||
}
|
||||
|
||||
// Create creates a new file, preferring the slow tier for persistence testing
|
||||
func (tc *TieredCache) Create(key string, size int64) (io.WriteCloser, error) {
|
||||
tc.mu.RLock()
|
||||
defer tc.mu.RUnlock()
|
||||
|
||||
// Try slow tier first (disk) for better testability
|
||||
if tc.slow != nil {
|
||||
return tc.slow.Create(key, size)
|
||||
}
|
||||
|
||||
c.slow = vfs
|
||||
// Fall back to fast tier (memory)
|
||||
if tc.fast != nil {
|
||||
return tc.fast.Create(key, size)
|
||||
}
|
||||
|
||||
return nil, vfserror.ErrNotFound
|
||||
}
|
||||
|
||||
func (c *CacheFS) SetFast(vfs vfs.VFS) {
|
||||
c.fast = vfs
|
||||
}
|
||||
// Open opens a file, checking fast tier first, then slow tier
|
||||
func (tc *TieredCache) Open(key string) (io.ReadCloser, error) {
|
||||
tc.mu.RLock()
|
||||
defer tc.mu.RUnlock()
|
||||
|
||||
// getKeyLock returns a RWMutex for the given key, creating it if necessary.
|
||||
func (c *CacheFS) getKeyLock(key string) *sync.RWMutex {
|
||||
mu, _ := c.keyLocks.LoadOrStore(key, &sync.RWMutex{})
|
||||
return mu.(*sync.RWMutex)
|
||||
}
|
||||
|
||||
// cacheState returns the state of the file at key.
|
||||
func (c *CacheFS) cacheState(key string) cachestate.CacheState {
|
||||
if c.fast != nil {
|
||||
if _, err := c.fast.Stat(key); err == nil {
|
||||
return cachestate.CacheStateHit
|
||||
// Try fast tier first (memory)
|
||||
if tc.fast != nil {
|
||||
if reader, err := tc.fast.Open(key); err == nil {
|
||||
return reader, nil
|
||||
}
|
||||
}
|
||||
|
||||
if _, err := c.slow.Stat(key); err == nil {
|
||||
return cachestate.CacheStateMiss
|
||||
// Fall back to slow tier (disk)
|
||||
if tc.slow != nil {
|
||||
return tc.slow.Open(key)
|
||||
}
|
||||
|
||||
return cachestate.CacheStateNotFound
|
||||
return nil, vfserror.ErrNotFound
|
||||
}
|
||||
|
||||
func (c *CacheFS) Name() string {
|
||||
return fmt.Sprintf("CacheFS(%s, %s)", c.fast.Name(), c.slow.Name())
|
||||
}
|
||||
// Delete removes a file from all tiers
|
||||
func (tc *TieredCache) Delete(key string) error {
|
||||
tc.mu.RLock()
|
||||
defer tc.mu.RUnlock()
|
||||
|
||||
// Size returns the total size of the cache.
|
||||
func (c *CacheFS) Size() int64 {
|
||||
return c.slow.Size()
|
||||
}
|
||||
var lastErr error
|
||||
|
||||
// Delete deletes the file at key from the cache.
|
||||
func (c *CacheFS) Delete(key string) error {
|
||||
mu := c.getKeyLock(key)
|
||||
mu.Lock()
|
||||
defer mu.Unlock()
|
||||
|
||||
if c.fast != nil {
|
||||
c.fast.Delete(key)
|
||||
}
|
||||
return c.slow.Delete(key)
|
||||
}
|
||||
|
||||
// Open returns the file at key. If the file is not in the cache, it is fetched from the storage.
|
||||
func (c *CacheFS) Open(key string) (io.ReadCloser, error) {
|
||||
mu := c.getKeyLock(key)
|
||||
mu.RLock()
|
||||
defer mu.RUnlock()
|
||||
|
||||
state := c.cacheState(key)
|
||||
|
||||
switch state {
|
||||
case cachestate.CacheStateHit:
|
||||
// if c.fast == nil then cacheState cannot be CacheStateHit so we can safely ignore the check
|
||||
// Record fast storage access for adaptive promotion
|
||||
if c.fast != nil {
|
||||
gc.RecordFastStorageAccess()
|
||||
}
|
||||
return c.fast.Open(key)
|
||||
case cachestate.CacheStateMiss:
|
||||
slowReader, err := c.slow.Open(key)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
sstat, _ := c.slow.Stat(key)
|
||||
if sstat != nil && c.fast != nil { // file found in slow storage and fast storage is available
|
||||
// We are accessing the file from the slow storage, and the file has been accessed less then a minute ago so it popular, so we should update the fast storage with the latest file.
|
||||
if c.cacheHandler != nil && c.cacheHandler(sstat, state) {
|
||||
fastWriter, err := c.fast.Create(key, sstat.Size())
|
||||
if err == nil {
|
||||
return &teeReadCloser{
|
||||
Reader: io.TeeReader(slowReader, fastWriter),
|
||||
closers: []io.Closer{slowReader, fastWriter},
|
||||
}, nil
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return slowReader, nil
|
||||
case cachestate.CacheStateNotFound:
|
||||
return nil, vfserror.ErrNotFound
|
||||
}
|
||||
|
||||
panic(vfserror.ErrUnreachable)
|
||||
}
|
||||
|
||||
// Create creates a new file at key. If the file is already in the cache, it is replaced.
|
||||
func (c *CacheFS) Create(key string, size int64) (io.WriteCloser, error) {
|
||||
mu := c.getKeyLock(key)
|
||||
mu.Lock()
|
||||
defer mu.Unlock()
|
||||
|
||||
state := c.cacheState(key)
|
||||
|
||||
switch state {
|
||||
case cachestate.CacheStateHit:
|
||||
if c.fast != nil {
|
||||
c.fast.Delete(key)
|
||||
}
|
||||
return c.slow.Create(key, size)
|
||||
case cachestate.CacheStateMiss, cachestate.CacheStateNotFound:
|
||||
return c.slow.Create(key, size)
|
||||
}
|
||||
|
||||
panic(vfserror.ErrUnreachable)
|
||||
}
|
||||
|
||||
// Stat returns information about the file at key.
|
||||
// Warning: This will return information about the file in the fastest storage its in.
|
||||
func (c *CacheFS) Stat(key string) (*vfs.FileInfo, error) {
|
||||
mu := c.getKeyLock(key)
|
||||
mu.RLock()
|
||||
defer mu.RUnlock()
|
||||
|
||||
state := c.cacheState(key)
|
||||
|
||||
switch state {
|
||||
case cachestate.CacheStateHit:
|
||||
// if c.fast == nil then cacheState cannot be CacheStateHit so we can safely ignore the check
|
||||
return c.fast.Stat(key)
|
||||
case cachestate.CacheStateMiss:
|
||||
return c.slow.Stat(key)
|
||||
case cachestate.CacheStateNotFound:
|
||||
return nil, vfserror.ErrNotFound
|
||||
}
|
||||
|
||||
panic(vfserror.ErrUnreachable)
|
||||
}
|
||||
|
||||
// StatAll returns information about all files in the cache.
|
||||
// Warning: This only returns information about the files in the slow storage.
|
||||
func (c *CacheFS) StatAll() []*vfs.FileInfo {
|
||||
return c.slow.StatAll()
|
||||
}
|
||||
|
||||
type teeReadCloser struct {
|
||||
io.Reader
|
||||
closers []io.Closer
|
||||
}
|
||||
|
||||
func (t *teeReadCloser) Close() error {
|
||||
var err error
|
||||
for _, c := range t.closers {
|
||||
if e := c.Close(); e != nil {
|
||||
err = e
|
||||
// Delete from fast tier
|
||||
if tc.fast != nil {
|
||||
if err := tc.fast.Delete(key); err != nil {
|
||||
lastErr = err
|
||||
}
|
||||
}
|
||||
return err
|
||||
|
||||
// Delete from slow tier
|
||||
if tc.slow != nil {
|
||||
if err := tc.slow.Delete(key); err != nil {
|
||||
lastErr = err
|
||||
}
|
||||
}
|
||||
|
||||
return lastErr
|
||||
}
|
||||
|
||||
// Stat returns file information, checking fast tier first
|
||||
func (tc *TieredCache) Stat(key string) (*vfs.FileInfo, error) {
|
||||
tc.mu.RLock()
|
||||
defer tc.mu.RUnlock()
|
||||
|
||||
// Try fast tier first (memory)
|
||||
if tc.fast != nil {
|
||||
if info, err := tc.fast.Stat(key); err == nil {
|
||||
return info, nil
|
||||
}
|
||||
}
|
||||
|
||||
// Fall back to slow tier (disk)
|
||||
if tc.slow != nil {
|
||||
return tc.slow.Stat(key)
|
||||
}
|
||||
|
||||
return nil, vfserror.ErrNotFound
|
||||
}
|
||||
|
||||
// Name returns the cache name
|
||||
func (tc *TieredCache) Name() string {
|
||||
return "TieredCache"
|
||||
}
|
||||
|
||||
// Size returns the total size across all tiers
|
||||
func (tc *TieredCache) Size() int64 {
|
||||
tc.mu.RLock()
|
||||
defer tc.mu.RUnlock()
|
||||
|
||||
var total int64
|
||||
if tc.fast != nil {
|
||||
total += tc.fast.Size()
|
||||
}
|
||||
if tc.slow != nil {
|
||||
total += tc.slow.Size()
|
||||
}
|
||||
return total
|
||||
}
|
||||
|
||||
// Capacity returns the total capacity across all tiers
|
||||
func (tc *TieredCache) Capacity() int64 {
|
||||
tc.mu.RLock()
|
||||
defer tc.mu.RUnlock()
|
||||
|
||||
var total int64
|
||||
if tc.fast != nil {
|
||||
total += tc.fast.Capacity()
|
||||
}
|
||||
if tc.slow != nil {
|
||||
total += tc.slow.Capacity()
|
||||
}
|
||||
return total
|
||||
}
|
||||
|
||||
201
vfs/cache/cache_test.go
vendored
201
vfs/cache/cache_test.go
vendored
@@ -1,201 +0,0 @@
|
||||
// vfs/cache/cache_test.go
|
||||
package cache
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"io"
|
||||
"testing"
|
||||
|
||||
"s1d3sw1ped/SteamCache2/vfs"
|
||||
"s1d3sw1ped/SteamCache2/vfs/cachestate"
|
||||
"s1d3sw1ped/SteamCache2/vfs/memory"
|
||||
"s1d3sw1ped/SteamCache2/vfs/vfserror"
|
||||
)
|
||||
|
||||
func testMemory() vfs.VFS {
|
||||
return memory.New(1024)
|
||||
}
|
||||
|
||||
func TestNew(t *testing.T) {
|
||||
fast := testMemory()
|
||||
slow := testMemory()
|
||||
|
||||
cache := New(nil)
|
||||
cache.SetFast(fast)
|
||||
cache.SetSlow(slow)
|
||||
if cache == nil {
|
||||
t.Fatal("expected cache to be non-nil")
|
||||
}
|
||||
}
|
||||
|
||||
func TestNewPanics(t *testing.T) {
|
||||
defer func() {
|
||||
if r := recover(); r == nil {
|
||||
t.Fatal("expected panic but did not get one")
|
||||
}
|
||||
}()
|
||||
|
||||
cache := New(nil)
|
||||
cache.SetFast(nil)
|
||||
cache.SetSlow(nil)
|
||||
}
|
||||
|
||||
func TestCreateAndOpen(t *testing.T) {
|
||||
fast := testMemory()
|
||||
slow := testMemory()
|
||||
cache := New(nil)
|
||||
cache.SetFast(fast)
|
||||
cache.SetSlow(slow)
|
||||
|
||||
key := "test"
|
||||
value := []byte("value")
|
||||
|
||||
w, err := cache.Create(key, int64(len(value)))
|
||||
if err != nil {
|
||||
t.Fatalf("unexpected error: %v", err)
|
||||
}
|
||||
w.Write(value)
|
||||
w.Close()
|
||||
|
||||
rc, err := cache.Open(key)
|
||||
if err != nil {
|
||||
t.Fatalf("unexpected error: %v", err)
|
||||
}
|
||||
got, _ := io.ReadAll(rc)
|
||||
rc.Close()
|
||||
|
||||
if string(got) != string(value) {
|
||||
t.Fatalf("expected %s, got %s", value, got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCreateAndOpenNoFast(t *testing.T) {
|
||||
slow := testMemory()
|
||||
cache := New(nil)
|
||||
cache.SetSlow(slow)
|
||||
|
||||
key := "test"
|
||||
value := []byte("value")
|
||||
|
||||
w, err := cache.Create(key, int64(len(value)))
|
||||
if err != nil {
|
||||
t.Fatalf("unexpected error: %v", err)
|
||||
}
|
||||
w.Write(value)
|
||||
w.Close()
|
||||
|
||||
rc, err := cache.Open(key)
|
||||
if err != nil {
|
||||
t.Fatalf("unexpected error: %v", err)
|
||||
}
|
||||
got, _ := io.ReadAll(rc)
|
||||
rc.Close()
|
||||
|
||||
if string(got) != string(value) {
|
||||
t.Fatalf("expected %s, got %s", value, got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCachingPromotion(t *testing.T) {
|
||||
fast := testMemory()
|
||||
slow := testMemory()
|
||||
cache := New(func(fi *vfs.FileInfo, cs cachestate.CacheState) bool {
|
||||
return true
|
||||
})
|
||||
cache.SetFast(fast)
|
||||
cache.SetSlow(slow)
|
||||
|
||||
key := "test"
|
||||
value := []byte("value")
|
||||
|
||||
ws, _ := slow.Create(key, int64(len(value)))
|
||||
ws.Write(value)
|
||||
ws.Close()
|
||||
|
||||
rc, err := cache.Open(key)
|
||||
if err != nil {
|
||||
t.Fatalf("unexpected error: %v", err)
|
||||
}
|
||||
got, _ := io.ReadAll(rc)
|
||||
rc.Close()
|
||||
|
||||
if string(got) != string(value) {
|
||||
t.Fatalf("expected %s, got %s", value, got)
|
||||
}
|
||||
|
||||
// Check if promoted to fast
|
||||
_, err = fast.Open(key)
|
||||
if err != nil {
|
||||
t.Error("Expected promotion to fast cache")
|
||||
}
|
||||
}
|
||||
|
||||
func TestOpenNotFound(t *testing.T) {
|
||||
fast := testMemory()
|
||||
slow := testMemory()
|
||||
cache := New(nil)
|
||||
cache.SetFast(fast)
|
||||
cache.SetSlow(slow)
|
||||
|
||||
_, err := cache.Open("nonexistent")
|
||||
if !errors.Is(err, vfserror.ErrNotFound) {
|
||||
t.Fatalf("expected %v, got %v", vfserror.ErrNotFound, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDelete(t *testing.T) {
|
||||
fast := testMemory()
|
||||
slow := testMemory()
|
||||
cache := New(nil)
|
||||
cache.SetFast(fast)
|
||||
cache.SetSlow(slow)
|
||||
|
||||
key := "test"
|
||||
value := []byte("value")
|
||||
|
||||
w, err := cache.Create(key, int64(len(value)))
|
||||
if err != nil {
|
||||
t.Fatalf("unexpected error: %v", err)
|
||||
}
|
||||
w.Write(value)
|
||||
w.Close()
|
||||
|
||||
if err := cache.Delete(key); err != nil {
|
||||
t.Fatalf("unexpected error: %v", err)
|
||||
}
|
||||
|
||||
_, err = cache.Open(key)
|
||||
if !errors.Is(err, vfserror.ErrNotFound) {
|
||||
t.Fatalf("expected %v, got %v", vfserror.ErrNotFound, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestStat(t *testing.T) {
|
||||
fast := testMemory()
|
||||
slow := testMemory()
|
||||
cache := New(nil)
|
||||
cache.SetFast(fast)
|
||||
cache.SetSlow(slow)
|
||||
|
||||
key := "test"
|
||||
value := []byte("value")
|
||||
|
||||
w, err := cache.Create(key, int64(len(value)))
|
||||
if err != nil {
|
||||
t.Fatalf("unexpected error: %v", err)
|
||||
}
|
||||
w.Write(value)
|
||||
w.Close()
|
||||
|
||||
info, err := cache.Stat(key)
|
||||
if err != nil {
|
||||
t.Fatalf("unexpected error: %v", err)
|
||||
}
|
||||
|
||||
if info == nil {
|
||||
t.Fatal("expected file info to be non-nil")
|
||||
}
|
||||
if info.Size() != int64(len(value)) {
|
||||
t.Errorf("expected size %d, got %d", len(value), info.Size())
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user