diff --git a/README.md b/README.md index 75c8726..0a0cacd 100644 --- a/README.md +++ b/README.md @@ -88,6 +88,9 @@ curl -s -i http://localhost/lancache-heartbeat | `total_requests` / `errors` | Volume and failures | | `upstream_errors` / `cache_write_failures` / `rate_limited` | Upstream pipe, cache write, and rate-limit pressure (Quick check highlights these next to hit/miss) | | `disk_tier_ready` | `0` while disk slow-tier attach pending; `1` when attached, or when no disk configured (N/A — not waiting) | +| `memory_cache_size` / `disk_cache_size` | Current cache occupancy per tier (bytes) | +| `memory_cache_capacity` / `disk_cache_capacity` | Configured capacity per tier (bytes); `disk_cache_capacity` is `0` when no disk is configured | +| `disk_cache_full_ratio` | `disk_cache_size / disk_cache_capacity` in [0,1]; 0 when no disk is configured or capacity is 0. Tells you "95% full" vs "barely filled" without reading the filesystem | | `capacity_pressure_events` | Soft eviction under the memory or disk cap, and/or disk Create/Write/Mkdir hitting ENOSPC (volume full). Distinct from cold-cache misses and from the existing `evictions` counter. Logs `tier` (memory or disk) and `reason` (eviction or enospc). | A first pass through new content is mostly misses (`hit_rate` near 0). Repeat the same content and `cache_hits` / `hit_rate` should rise. @@ -311,6 +314,7 @@ See `config.Validate()` and `steamcache.New` error paths. This ensures the LAN a - Godoc on `disk.New` and `DiskFS.Size` expanded with the barrier/attach behavior. - Startup logs: Info "Disk slow tier attach pending..." then later "Disk slow tier attached (...)" for disk-only and mixed modes. - `/metrics` exposes `disk_tier_ready` 0/1 and stays responsive during attach (GetMetrics does not block on Size while pending). +- `/metrics` tier occupancy: `memory_cache_size` / `disk_cache_size` (bytes in use) next to `memory_cache_capacity` / `disk_cache_capacity` (configured capacity; `disk_cache_capacity` is 0 when no disk is configured), plus `disk_cache_full_ratio` (size/capacity in [0,1]). Capacity is a config read, so it is reported even while the disk attach is pending (size stays 0 until attach). - `/lancache-heartbeat` header `X-SteamCache-Disk-Tier` mirrors that state. - `/metrics` `capacity_pressure_events` counts times the cache dropped data under capacity pressure (soft eviction at the memory or disk cap, or disk Create/Write/Mkdir returning ENOSPC). Logs include `tier=memory|disk` and `reason=eviction|enospc` so operators can grep and tell this apart from a cold cache. The existing `evictions` counter is unchanged. diff --git a/steamcache/metrics/metrics.go b/steamcache/metrics/metrics.go index e5dc52e..efab1d0 100644 --- a/steamcache/metrics/metrics.go +++ b/steamcache/metrics/metrics.go @@ -32,6 +32,8 @@ type Metrics struct { // Cache metrics MemoryCacheSize int64 DiskCacheSize int64 + MemoryCacheCapacity int64 // configured memory capacity (bytes) + DiskCacheCapacity int64 // configured disk capacity (bytes); 0 when no disk MemoryCacheHits int64 DiskCacheHits int64 Promotions int64 @@ -137,6 +139,17 @@ func (m *Metrics) SetDiskCacheSize(size int64) { atomic.StoreInt64(&m.DiskCacheSize, size) } +// SetMemoryCacheCapacity sets the configured memory cache capacity in bytes. +func (m *Metrics) SetMemoryCacheCapacity(capacity int64) { + atomic.StoreInt64(&m.MemoryCacheCapacity, capacity) +} + +// SetDiskCacheCapacity sets the configured disk cache capacity in bytes +// (0 when no disk is configured). +func (m *Metrics) SetDiskCacheCapacity(capacity int64) { + atomic.StoreInt64(&m.DiskCacheCapacity, capacity) +} + // SetDiskTierReady sets whether the disk slow tier is attached (1) or still pending (0). // Memory-only (no disk) also uses 1 — meaning "not waiting on disk attach". Reset does not clear this. func (m *Metrics) SetDiskTierReady(ready int64) { @@ -248,6 +261,11 @@ func (m *Metrics) GetStats() *Stats { serviceErrors[k] = v } + memoryCacheSize := atomic.LoadInt64(&m.MemoryCacheSize) + diskCacheSize := atomic.LoadInt64(&m.DiskCacheSize) + memoryCacheCapacity := atomic.LoadInt64(&m.MemoryCacheCapacity) + diskCacheCapacity := atomic.LoadInt64(&m.DiskCacheCapacity) + return &Stats{ TotalRequests: totalRequests, CacheHits: cacheHits, @@ -262,8 +280,11 @@ func (m *Metrics) GetStats() *Stats { AvgResponseTime: avgResponseTime, TotalBytesServed: atomic.LoadInt64(&m.TotalBytesServed), TotalBytesSaved: atomic.LoadInt64(&m.TotalBytesSaved), - MemoryCacheSize: atomic.LoadInt64(&m.MemoryCacheSize), - DiskCacheSize: atomic.LoadInt64(&m.DiskCacheSize), + MemoryCacheSize: memoryCacheSize, + DiskCacheSize: diskCacheSize, + MemoryCacheCapacity: memoryCacheCapacity, + DiskCacheCapacity: diskCacheCapacity, + DiskCacheFullRatio: diskFullRatio(diskCacheSize, diskCacheCapacity), DiskTierReady: atomic.LoadInt64(&m.DiskTierReady), MemoryCacheHits: atomic.LoadInt64(&m.MemoryCacheHits), DiskCacheHits: atomic.LoadInt64(&m.DiskCacheHits), @@ -279,6 +300,19 @@ func (m *Metrics) GetStats() *Stats { } } +// diskFullRatio is size / capacity clamped to [0,1]. +// It is 0 when no disk is configured or the capacity is 0. +func diskFullRatio(size, capacity int64) float64 { + if size <= 0 || capacity <= 0 { + return 0 + } + ratio := float64(size) / float64(capacity) + if ratio > 1 { + return 1 + } + return ratio +} + // Reset resets all metrics to zero func (m *Metrics) Reset() { atomic.StoreInt64(&m.TotalRequests, 0) @@ -330,6 +364,9 @@ type Stats struct { MemoryCacheSize int64 DiskCacheSize int64 + MemoryCacheCapacity int64 // configured memory capacity (bytes) + DiskCacheCapacity int64 // configured disk capacity (bytes); 0 when no disk + DiskCacheFullRatio float64 // disk_cache_size / disk_cache_capacity, clamped to [0,1]; 0 when no disk or capacity is 0 DiskTierReady int64 MemoryCacheHits int64 DiskCacheHits int64 @@ -382,7 +419,10 @@ func WriteText(w http.ResponseWriter, stats *Stats) { writeInt(w, "total_bytes_served", "Total bytes sent to clients.", "counter", stats.TotalBytesServed) writeInt(w, "total_bytes_saved", "Bytes served from cache instead of being re-downloaded from upstream.", "counter", stats.TotalBytesSaved) writeInt(w, "memory_cache_size", "Current memory cache size in bytes.", "gauge", stats.MemoryCacheSize) + writeInt(w, "memory_cache_capacity", "Configured memory cache capacity in bytes.", "gauge", stats.MemoryCacheCapacity) writeInt(w, "disk_cache_size", "Current disk cache size in bytes.", "gauge", stats.DiskCacheSize) + writeInt(w, "disk_cache_capacity", "Configured disk cache capacity in bytes; 0 when no disk is configured.", "gauge", stats.DiskCacheCapacity) + writeFloat(w, "disk_cache_full_ratio", "disk_cache_size / disk_cache_capacity in [0,1]; 0 when no disk or capacity is 0.", "gauge", "%.4f", stats.DiskCacheFullRatio) writeInt(w, "disk_tier_ready", "1 if the disk tier is attached or no disk is configured; 0 while attach is pending.", "gauge", stats.DiskTierReady) writeFloat(w, "uptime_seconds", "Process uptime in seconds.", "gauge", "%.2f", stats.Uptime.Seconds()) } diff --git a/steamcache/metrics/metrics_test.go b/steamcache/metrics/metrics_test.go index 058c9ac..b1c8d02 100644 --- a/steamcache/metrics/metrics_test.go +++ b/steamcache/metrics/metrics_test.go @@ -100,6 +100,9 @@ func TestWriteTextPrometheusExposition(t *testing.T) { TotalBytesSaved: 50, MemoryCacheSize: 8, DiskCacheSize: 16, + MemoryCacheCapacity: 8, + DiskCacheCapacity: 32, + DiskCacheFullRatio: 0.5, DiskTierReady: 1, Uptime: 3 * time.Second, } @@ -120,6 +123,7 @@ func TestWriteTextPrometheusExposition(t *testing.T) { } gauges := []string{ "hit_rate", "avg_response_time_ms", "memory_cache_size", "disk_cache_size", + "memory_cache_capacity", "disk_cache_capacity", "disk_cache_full_ratio", "disk_tier_ready", "uptime_seconds", } for _, name := range counters { @@ -138,6 +142,15 @@ func TestWriteTextPrometheusExposition(t *testing.T) { if !strings.Contains(body, "disk_tier_ready 1\n") { t.Errorf("missing disk_tier_ready 1 sample: %q", body) } + if !strings.Contains(body, "memory_cache_capacity 8\n") { + t.Errorf("missing memory_cache_capacity 8 sample: %q", body) + } + if !strings.Contains(body, "disk_cache_capacity 32\n") { + t.Errorf("missing disk_cache_capacity 32 sample: %q", body) + } + if !strings.Contains(body, "disk_cache_full_ratio 0.5000\n") { + t.Errorf("missing disk_cache_full_ratio 0.5000 sample: %q", body) + } if !strings.Contains(body, "range_cache 3\n") { t.Errorf("missing range_cache 3 sample: %q", body) } @@ -146,6 +159,54 @@ func TestWriteTextPrometheusExposition(t *testing.T) { } } +func TestDiskCacheFullRatioInGetStats(t *testing.T) { + t.Parallel() + cases := []struct { + name string + size int64 + capacity int64 + wantRatio float64 + wantCapacity int64 + }{ + {"no disk (capacity 0)", 0, 0, 0, 0}, + {"zero size with capacity", 0, 1024, 0, 1024}, + {"half full", 512, 1024, 0.5, 1024}, + {"exact full", 1024, 1024, 1, 1024}, + {"size above capacity clamps to 1", 2048, 1024, 1, 1024}, + } + for _, tc := range cases { + tc := tc + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + m := NewMetrics() + m.SetDiskCacheSize(tc.size) + m.SetDiskCacheCapacity(tc.capacity) + st := m.GetStats() + if st.DiskCacheCapacity != tc.wantCapacity { + t.Fatalf("DiskCacheCapacity=%d, want %d", st.DiskCacheCapacity, tc.wantCapacity) + } + if st.DiskCacheFullRatio != tc.wantRatio { + t.Fatalf("DiskCacheFullRatio=%v, want %v", st.DiskCacheFullRatio, tc.wantRatio) + } + if st.DiskCacheFullRatio < 0 || st.DiskCacheFullRatio > 1 { + t.Fatalf("DiskCacheFullRatio=%v outside [0,1]", st.DiskCacheFullRatio) + } + }) + } + + // Memory capacity is a plain passthrough, and capacity survives Reset + // (re-derived by GetMetrics, like MemoryCacheSize/DiskTierReady). + m := NewMetrics() + m.SetMemoryCacheCapacity(4096) + if got := m.GetStats().MemoryCacheCapacity; got != 4096 { + t.Fatalf("MemoryCacheCapacity=%d, want 4096", got) + } + m.Reset() + if got := m.GetStats().MemoryCacheCapacity; got != 4096 { + t.Fatalf("MemoryCacheCapacity=%d after Reset, want 4096 (config snapshot, like size gauges)", got) + } +} + func assertHelpType(t *testing.T, body, name, typ string) { t.Helper() help := "# HELP " + name + " " diff --git a/steamcache/steamcache.go b/steamcache/steamcache.go index 7feee83..c6fcdbc 100644 --- a/steamcache/steamcache.go +++ b/steamcache/steamcache.go @@ -352,6 +352,11 @@ func (sc *SteamCache) Shutdown() { func (sc *SteamCache) GetMetrics() *metrics.Stats { if sc.memory != nil { sc.metrics.SetMemoryCacheSize(sc.memory.Size()) + sc.metrics.SetMemoryCacheCapacity(sc.memory.Capacity()) + } + if sc.disk != nil { + // Capacity() is a plain config field — safe to read even while disk attach is pending. + sc.metrics.SetDiskCacheCapacity(sc.disk.Capacity()) } // Skip disk.Size() while attach pending — Size() blocks on initDone and would hang /metrics. if sc.disk != nil && sc.metrics.GetDiskTierReady() == 1 { diff --git a/steamcache/steamcache_test.go b/steamcache/steamcache_test.go index 72c9338..5c0fb8d 100644 --- a/steamcache/steamcache_test.go +++ b/steamcache/steamcache_test.go @@ -1497,3 +1497,131 @@ func TestCacheKeySharedAcrossCDNHostAliases(t *testing.T) { t.Errorf("upstream fetched %d times across host aliases, want 1", got) } } + +// TestGetMetricsCapacityGauges covers the tier-occupancy gauges: GetMetrics +// sets memory/disk capacity from the configured sizes, and WriteText emits +// memory_cache_capacity / disk_cache_capacity / disk_cache_full_ratio. During a +// pending disk attach, GetMetrics must return quickly (no disk.Size() call) and +// still report the configured disk capacity. +func TestGetMetricsCapacityGauges(t *testing.T) { + t.Run("memory-only", func(t *testing.T) { + sc, err := New("127.0.0.1:0", "1MB", "0", t.TempDir(), "", "lru", "lru", 10, 5, "0", nil, "") + if err != nil { + t.Fatalf("New memory-only: %v", err) + } + t.Cleanup(func() { sc.Shutdown() }) + + st := sc.GetMetrics() + if st.MemoryCacheCapacity != 1000000 { + t.Errorf("MemoryCacheCapacity=%d, want 1000000 (configured 1MB)", st.MemoryCacheCapacity) + } + if st.DiskCacheCapacity != 0 { + t.Errorf("DiskCacheCapacity=%d, want 0 (no disk configured)", st.DiskCacheCapacity) + } + if st.DiskCacheFullRatio != 0 { + t.Errorf("DiskCacheFullRatio=%v, want 0 (no disk)", st.DiskCacheFullRatio) + } + + rec := httptest.NewRecorder() + metrics.WriteText(rec, sc.GetMetrics()) + body := rec.Body.String() + if !strings.Contains(body, "memory_cache_capacity 1000000\n") { + t.Errorf("WriteText missing memory_cache_capacity 1000000: %q", body) + } + if !strings.Contains(body, "disk_cache_capacity 0\n") { + t.Errorf("WriteText missing disk_cache_capacity 0: %q", body) + } + if !strings.Contains(body, "# TYPE disk_cache_full_ratio gauge") { + t.Errorf("WriteText missing # TYPE disk_cache_full_ratio gauge: %q", body) + } + }) + + t.Run("mixed pending attach reports capacity without Size", func(t *testing.T) { + td := t.TempDir() + diskPath := filepath.Join(td, "disk") + if err := os.MkdirAll(diskPath, 0755); err != nil { + t.Fatal(err) + } + hold := make(chan struct{}) + var holdOnce sync.Once + closeHold := func() { holdOnce.Do(func() { close(hold) }) } + disk.RegisterInitHold(diskPath, hold) + t.Cleanup(func() { + closeHold() + disk.ClearInitHold(diskPath) + }) + + sc, err := New("127.0.0.1:0", "1MB", "10MB", diskPath, "", "lru", "lru", 10, 1, "0", nil, "") + if err != nil { + t.Fatalf("New mixed: %v", err) + } + t.Cleanup(func() { sc.Shutdown() }) + t.Cleanup(closeHold) // before Shutdown: attach is blocked in Size() until the hold closes + + // Pending window is held open; if GetMetrics called disk.Size() it would + // block on the barrier, so a bounded wait proves non-blocking behavior. + done := make(chan *metrics.Stats, 1) + go func() { done <- sc.GetMetrics() }() + select { + case st := <-done: + if got := st.DiskTierReady; got != 0 { + t.Fatalf("immediate DiskTierReady=%d, want 0 (pending)", got) + } + if st.DiskCacheCapacity != 10000000 { + t.Errorf("pending DiskCacheCapacity=%d, want 10000000 (configured 10MB)", st.DiskCacheCapacity) + } + if st.MemoryCacheCapacity != 1000000 { + t.Errorf("pending MemoryCacheCapacity=%d, want 1000000 (configured 1MB)", st.MemoryCacheCapacity) + } + if st.DiskCacheFullRatio != 0 { + t.Errorf("pending DiskCacheFullRatio=%v, want 0 (size not reported while pending)", st.DiskCacheFullRatio) + } + case <-time.After(2 * time.Second): + t.Fatal("GetMetrics blocked during pending attach (must not call disk.Size())") + } + + closeHold() + _ = sc.disk.Size() + deadline := time.Now().Add(2 * time.Second) + for time.Now().Before(deadline) { + if sc.GetMetrics().DiskTierReady == 1 { + break + } + time.Sleep(1 * time.Millisecond) + } + if got := sc.GetMetrics().DiskTierReady; got != 1 { + t.Fatalf("DiskTierReady=%d after barrier, want 1 (ready)", got) + } + + // Post-attach writes prefer the slow (disk) tier, so a write produces a + // non-zero disk size and hence a non-zero occupancy ratio. + w, err := sc.vfs.Create("occupancy-key", 128) + if err != nil { + t.Fatalf("Create failed after attach: %v", err) + } + if _, err := w.Write(make([]byte, 128)); err != nil { + t.Fatalf("Write failed: %v", err) + } + if err := w.Close(); err != nil { + t.Fatalf("Close failed: %v", err) + } + + st := sc.GetMetrics() + if st.DiskCacheCapacity != 10000000 { + t.Errorf("post-attach DiskCacheCapacity=%d, want 10000000", st.DiskCacheCapacity) + } + if st.DiskCacheSize <= 0 { + t.Errorf("post-attach DiskCacheSize=%d, want > 0 after a write", st.DiskCacheSize) + } + if st.DiskCacheFullRatio <= 0 || st.DiskCacheFullRatio > 1 { + t.Errorf("post-attach DiskCacheFullRatio=%v, want in (0,1]", st.DiskCacheFullRatio) + } + + rec := httptest.NewRecorder() + metrics.WriteText(rec, st) + body := rec.Body.String() + if !strings.Contains(body, "disk_cache_capacity 10000000\n") { + t.Errorf("WriteText missing disk_cache_capacity 10000000: %q", body) + } + }) +}