initial commit

vfs/gc/gc.go

@@ -0,0 +1,44 @@
+package gc
+
+import (
+	"fmt"
+	"s1d3sw1ped/SteamCache2/vfs"
+	"s1d3sw1ped/SteamCache2/vfs/vfserror"
+)
+
+// Ensure GCFS implements VFS.
+var _ vfs.VFS = (*GCFS)(nil)
+
+// GCFS is a virtual file system that calls a GC handler when the disk is full. The GC handler is responsible for freeing up space on the disk. The GCFS is a wrapper around another VFS.
+type GCFS struct {
+	vfs.VFS
+	multiplier   int
+	gcHanderFunc GCHandlerFunc
+}
+
+// GCHandlerFunc is a function that is called when the disk is full and the GCFS needs to free up space. It is passed the VFS and the size of the file that needs to be written. Its up to the implementation to free up space. How much space is freed is also up to the implementation.
+type GCHandlerFunc func(vfs vfs.VFS, size int)
+
+func New(vfs vfs.VFS, multiplier int, gcHandlerFunc GCHandlerFunc) *GCFS {
+	return &GCFS{
+		VFS:          vfs,
+		multiplier:   multiplier,
+		gcHanderFunc: gcHandlerFunc,
+	}
+}
+
+// Set overrides the Set method of the VFS interface. It tries to set the key and src, if it fails due to disk full error, it calls the GC handler and tries again. If it still fails it returns the error.
+func (g *GCFS) Set(key string, src []byte) error {
+	err := g.VFS.Set(key, src) // try to set the key and src
+
+	if err == vfserror.ErrDiskFull && g.gcHanderFunc != nil { // if the error is disk full and there is a GC handler
+		g.gcHanderFunc(g.VFS, len(src)*g.multiplier) // call the GC handler
+		err = g.VFS.Set(key, src)                    // try again after GC if it still fails return the error
+	}
+
+	return err
+}
+
+func (g *GCFS) Name() string {
+	return fmt.Sprintf("GCFS(%s)", g.VFS.Name()) // wrap the name of the VFS with GCFS so we can see that its a GCFS
+}

vfs/gc/gc_test.go

@@ -0,0 +1,111 @@
+package gc
+
+import (
+	"fmt"
+	"s1d3sw1ped/SteamCache2/vfs"
+	"s1d3sw1ped/SteamCache2/vfs/memory"
+	"sort"
+	"testing"
+	"time"
+
+	"golang.org/x/exp/rand"
+)
+
+func TestGCSmallRandom(t *testing.T) {
+	t.Parallel()
+
+	m := memory.New(1024 * 1024 * 16)
+	gc := New(m, 10, func(vfs vfs.VFS, size int) {
+		tstart := time.Now()
+		deletions := 0
+		targetreclaim := int64(size)
+		var reclaimed int64
+
+		t.Logf("GC starting to reclaim %d bytes", targetreclaim)
+
+		stats := vfs.StatAll()
+		sort.Slice(stats, func(i, j int) bool {
+			// Sort by access time so we can remove the oldest files first.
+			return stats[i].AccessTime().Before(stats[j].AccessTime())
+		})
+
+		// Delete the oldest files until we've reclaimed enough space.
+		for _, s := range stats {
+			sz := s.Size() // Get the size of the file
+			err := vfs.Delete(s.Name())
+			if err != nil {
+				panic(err)
+			}
+			reclaimed += sz // Track how much space we've reclaimed
+			deletions++     // Track how many files we've deleted
+
+			// t.Logf("GC deleting %s, %v", s.Name(), s.AccessTime().Format(time.RFC3339Nano))
+
+			if reclaimed >= targetreclaim { // We've reclaimed enough space
+				break
+			}
+		}
+
+		t.Logf("GC took %v to reclaim %d bytes by deleting %d files", time.Since(tstart), reclaimed, deletions)
+	})
+
+	for i := 0; i < 10000; i++ {
+		if err := gc.Set(fmt.Sprintf("key:%d", i), genRandomData(1024*1, 1024*4)); err != nil {
+			t.Errorf("Set failed: %v", err)
+		}
+	}
+
+	if gc.Size() > 1024*1024*16 {
+		t.Errorf("MemoryFS size is %d, want <= 1024", m.Size())
+	}
+}
+
+func genRandomData(min int, max int) []byte {
+	data := make([]byte, rand.Intn(max-min)+min)
+	rand.Read(data)
+	return data
+}
+
+func TestGCLargeRandom(t *testing.T) {
+	t.Parallel()
+
+	m := memory.New(1024 * 1024 * 16) // 16MB
+	gc := New(m, 10, func(vfs vfs.VFS, size int) {
+		tstart := time.Now()
+		deletions := 0
+		targetreclaim := int64(size)
+		var reclaimed int64
+
+		t.Logf("GC starting to reclaim %d bytes", targetreclaim)
+
+		stats := vfs.StatAll()
+		sort.Slice(stats, func(i, j int) bool {
+			// Sort by access time so we can remove the oldest files first.
+			return stats[i].AccessTime().Before(stats[j].AccessTime())
+		})
+
+		// Delete the oldest files until we've reclaimed enough space.
+		for _, s := range stats {
+			sz := s.Size() // Get the size of the file
+			vfs.Delete(s.Name())
+			reclaimed += sz // Track how much space we've reclaimed
+			deletions++     // Track how many files we've deleted
+
+			if reclaimed >= targetreclaim { // We've reclaimed enough space
+				break
+			}
+		}
+
+		t.Logf("GC took %v to reclaim %d bytes by deleting %d files", time.Since(tstart), reclaimed, deletions)
+	})
+
+	for i := 0; i < 10000; i++ {
+		if err := gc.Set(fmt.Sprintf("key:%d", i), genRandomData(1024, 1024*1024)); err != nil {
+			t.Errorf("Set failed: %v", err)
+		}
+	}
+
+	if gc.Size() > 1024*1024*16 {
+		t.Errorf("MemoryFS size is %d, want <= 1024", m.Size())
+	}
+}