s1d3sw1ped/steamcache2
1
0
Fork 0
Code Issues 1 Pull Requests Actions Packages Projects Releases 32 Wiki Activity

ops: Signal disk-full and eviction capacity pressure

Browse Source 
When the disk (or memory) tier is at cap or the volume returns ENOSPC,
ops currently look like random misses with no clear "we are dropping
data." Count those events as capacity_pressure_events on /metrics and
log tier plus reason so operators can tell capacity pressure from a
cold cache, without changing the existing evictions counter.

Link: #36
This commit is contained in:
ash
2026-09-07 19:23:55 +00:00
parent a0e929c525
commit de43a71929
12 changed files with 382 additions and 78 deletions
Download Patch File Download Diff File Expand all files Collapse all files
steamcache/metrics/metrics.go
+85 -47
View File
@@ -7,6 +7,8 @@ import (
"sync"
"sync/atomic"
"time"
"s1d3sw1ped/steamcache2/steamcache/logger"
)
// Metrics tracks various performance and operational metrics
@@ -28,13 +30,14 @@ type Metrics struct {
TotalBytesSaved int64 // bytes served from cache instead of being re-downloaded from upstream
// Cache metrics
MemoryCacheSize int64
DiskCacheSize int64
MemoryCacheHits int64
DiskCacheHits int64
Promotions int64
Evictions int64
DiskTierReady int64 // 0=pending (or unset), 1=ready or no-disk (N/A)
MemoryCacheSize int64
DiskCacheSize int64
MemoryCacheHits int64
DiskCacheHits int64
Promotions int64
Evictions int64
CapacityPressureEvents int64 // soft eviction under cap and/or disk ENOSPC
DiskTierReady int64 // 0=pending (or unset), 1=ready or no-disk (N/A)
// Expanded observability (upstream breakdowns, cache write failures, per-service errors)
UpstreamErrors int64
@@ -169,8 +172,39 @@ func (m *Metrics) GetServiceRequests(service string) int64 {
return m.ServiceRequests[service]
}
func (m *Metrics) IncrementPromotions() { atomic.AddInt64(&m.Promotions, 1) }
func (m *Metrics) IncrementEvictions() { atomic.AddInt64(&m.Evictions, 1) }
func (m *Metrics) IncrementPromotions() { atomic.AddInt64(&m.Promotions, 1) }
func (m *Metrics) IncrementEvictions() { atomic.AddInt64(&m.Evictions, 1) }
func (m *Metrics) IncrementCapacityPressureEvents() { atomic.AddInt64(&m.CapacityPressureEvents, 1) }
// NoteSoftEviction records one cap-pressure eviction batch that freed bytes.
// Keeps the existing evictions counter and also increments capacity_pressure_events.
// Nil m is safe: the log still fires so ops can grep without metrics wired.
func NoteSoftEviction(m *Metrics, tier string, evicted uint) {
if evicted == 0 {
return
}
if m != nil {
m.IncrementEvictions()
m.IncrementCapacityPressureEvents()
}
logger.Logger.Info().
Str("tier", tier).
Str("reason", "eviction").
Uint("bytes_evicted", evicted).
Msg("cache capacity pressure")
}
// NoteNoSpace records a disk Create/Write/Mkdir ENOSPC (or equivalent) event.
func NoteNoSpace(m *Metrics, err error) {
if m != nil {
m.IncrementCapacityPressureEvents()
}
logger.Logger.Warn().
Str("tier", "disk").
Str("reason", "enospc").
Err(err).
Msg("cache capacity pressure")
}
// Additional observability counters
func (m *Metrics) IncrementUpstreamErrors() { atomic.AddInt64(&m.UpstreamErrors, 1) }
@@ -215,32 +249,33 @@ func (m *Metrics) GetStats() *Stats {
}
return &Stats{
TotalRequests: totalRequests,
CacheHits: cacheHits,
CacheMisses: cacheMisses,
CacheCoalesced: atomic.LoadInt64(&m.CacheCoalesced),
NegativeCacheHits: atomic.LoadInt64(&m.NegativeCacheHits),
RangeCache: atomic.LoadInt64(&m.RangeCache),
RangeUpstream: atomic.LoadInt64(&m.RangeUpstream),
Errors: atomic.LoadInt64(&m.Errors),
RateLimited: atomic.LoadInt64(&m.RateLimited),
HitRate: hitRate,
AvgResponseTime: avgResponseTime,
TotalBytesServed: atomic.LoadInt64(&m.TotalBytesServed),
TotalBytesSaved: atomic.LoadInt64(&m.TotalBytesSaved),
MemoryCacheSize: atomic.LoadInt64(&m.MemoryCacheSize),
DiskCacheSize: atomic.LoadInt64(&m.DiskCacheSize),
DiskTierReady: atomic.LoadInt64(&m.DiskTierReady),
MemoryCacheHits: atomic.LoadInt64(&m.MemoryCacheHits),
DiskCacheHits: atomic.LoadInt64(&m.DiskCacheHits),
Promotions: atomic.LoadInt64(&m.Promotions),
Evictions: atomic.LoadInt64(&m.Evictions),
ServiceRequests: serviceRequests,
UpstreamErrors: atomic.LoadInt64(&m.UpstreamErrors),
CacheWriteFailures: atomic.LoadInt64(&m.CacheWriteFailures),
ServiceErrors: serviceErrors,
Uptime: time.Since(m.StartTime),
LastResetTime: m.LastResetTime,
TotalRequests: totalRequests,
CacheHits: cacheHits,
CacheMisses: cacheMisses,
CacheCoalesced: atomic.LoadInt64(&m.CacheCoalesced),
NegativeCacheHits: atomic.LoadInt64(&m.NegativeCacheHits),
RangeCache: atomic.LoadInt64(&m.RangeCache),
RangeUpstream: atomic.LoadInt64(&m.RangeUpstream),
Errors: atomic.LoadInt64(&m.Errors),
RateLimited: atomic.LoadInt64(&m.RateLimited),
HitRate: hitRate,
AvgResponseTime: avgResponseTime,
TotalBytesServed: atomic.LoadInt64(&m.TotalBytesServed),
TotalBytesSaved: atomic.LoadInt64(&m.TotalBytesSaved),
MemoryCacheSize: atomic.LoadInt64(&m.MemoryCacheSize),
DiskCacheSize: atomic.LoadInt64(&m.DiskCacheSize),
DiskTierReady: atomic.LoadInt64(&m.DiskTierReady),
MemoryCacheHits: atomic.LoadInt64(&m.MemoryCacheHits),
DiskCacheHits: atomic.LoadInt64(&m.DiskCacheHits),
Promotions: atomic.LoadInt64(&m.Promotions),
Evictions: atomic.LoadInt64(&m.Evictions),
CapacityPressureEvents: atomic.LoadInt64(&m.CapacityPressureEvents),
ServiceRequests: serviceRequests,
UpstreamErrors: atomic.LoadInt64(&m.UpstreamErrors),
CacheWriteFailures: atomic.LoadInt64(&m.CacheWriteFailures),
ServiceErrors: serviceErrors,
Uptime: time.Since(m.StartTime),
LastResetTime: m.LastResetTime,
}
}
@@ -262,6 +297,7 @@ func (m *Metrics) Reset() {
atomic.StoreInt64(&m.DiskCacheHits, 0)
atomic.StoreInt64(&m.Promotions, 0)
atomic.StoreInt64(&m.Evictions, 0)
atomic.StoreInt64(&m.CapacityPressureEvents, 0)
atomic.StoreInt64(&m.UpstreamErrors, 0)
atomic.StoreInt64(&m.CacheWriteFailures, 0)
@@ -293,18 +329,19 @@ type Stats struct {
TotalBytesSaved int64
MemoryCacheSize int64
DiskCacheSize int64
DiskTierReady int64
MemoryCacheHits int64
DiskCacheHits int64
Promotions int64
Evictions int64
UpstreamErrors int64
CacheWriteFailures int64
ServiceErrors map[string]int64
ServiceRequests map[string]int64
Uptime time.Duration
LastResetTime time.Time
DiskCacheSize int64
DiskTierReady int64
MemoryCacheHits int64
DiskCacheHits int64
Promotions int64
Evictions int64
CapacityPressureEvents int64
UpstreamErrors int64
CacheWriteFailures int64
ServiceErrors map[string]int64
ServiceRequests map[string]int64
Uptime time.Duration
LastResetTime time.Time
}
// WriteText emits the Prometheus-style text metrics to the ResponseWriter.
@@ -329,6 +366,7 @@ func WriteText(w http.ResponseWriter, stats *Stats) {
_, _ = fmt.Fprintf(w, "disk_cache_hits %d\n", stats.DiskCacheHits)
_, _ = fmt.Fprintf(w, "promotions %d\n", stats.Promotions)
_, _ = fmt.Fprintf(w, "evictions %d\n", stats.Evictions)
_, _ = fmt.Fprintf(w, "capacity_pressure_events %d\n", stats.CapacityPressureEvents)
for svc, cnt := range stats.ServiceErrors {
_, _ = fmt.Fprintf(w, "service_errors{service=%q} %d\n", svc, cnt)
}
steamcache/metrics/metrics_test.go
+63
View File
@@ -0,0 +1,63 @@
package metrics
import (
"bytes"
"errors"
"net/http/httptest"
"testing"
)
func TestCapacityPressureEventsWriteTextAndReset(t *testing.T) {
t.Parallel()
m := NewMetrics()
if got := m.GetStats().CapacityPressureEvents; got != 0 {
t.Fatalf("initial CapacityPressureEvents=%d, want 0", got)
}
NoteSoftEviction(m, "memory", 0)
if got := m.GetStats().CapacityPressureEvents; got != 0 {
t.Fatalf("zero-byte eviction counted: %d", got)
}
NoteSoftEviction(m, "memory", 128)
st := m.GetStats()
if st.CapacityPressureEvents != 1 {
t.Fatalf("after memory eviction, CapacityPressureEvents=%d, want 1", st.CapacityPressureEvents)
}
if st.Evictions != 1 {
t.Fatalf("after memory eviction, Evictions=%d, want 1 (existing counter kept)", st.Evictions)
}
NoteSoftEviction(m, "disk", 64)
NoteNoSpace(m, errors.New("no space left on device"))
st = m.GetStats()
if st.CapacityPressureEvents != 3 {
t.Fatalf("after disk eviction + ENOSPC, CapacityPressureEvents=%d, want 3", st.CapacityPressureEvents)
}
if st.Evictions != 2 {
t.Fatalf("ENOSPC must not increment evictions; Evictions=%d, want 2", st.Evictions)
}
rec := httptest.NewRecorder()
WriteText(rec, st)
body := rec.Body.Bytes()
if !bytes.Contains(body, []byte("capacity_pressure_events 3")) {
t.Errorf("WriteText missing capacity_pressure_events 3: %q", rec.Body.String())
}
if !bytes.Contains(body, []byte("evictions 2")) {
t.Errorf("WriteText missing evictions 2: %q", rec.Body.String())
}
m.Reset()
st = m.GetStats()
if st.CapacityPressureEvents != 0 || st.Evictions != 0 {
t.Errorf("after Reset, CapacityPressureEvents=%d Evictions=%d, want 0", st.CapacityPressureEvents, st.Evictions)
}
}
func TestNoteSoftEvictionNilMetrics(t *testing.T) {
t.Parallel()
// Must not panic when metrics are not wired (unit tests / early init).
NoteSoftEviction(nil, "memory", 10)
NoteNoSpace(nil, errors.New("ENOSPC"))
}