ops: Signal disk-full and eviction capacity pressure

When the disk (or memory) tier is at cap or the volume returns ENOSPC,
ops currently look like random misses with no clear "we are dropping
data." Count those events as capacity_pressure_events on /metrics and
log tier plus reason so operators can tell capacity pressure from a
cold cache, without changing the existing evictions counter.

Link: #36
This commit is contained in:
ash
2026-09-07 19:23:55 +00:00
parent a0e929c525
commit de43a71929
12 changed files with 382 additions and 78 deletions
+63
View File
@@ -0,0 +1,63 @@
package metrics
import (
"bytes"
"errors"
"net/http/httptest"
"testing"
)
func TestCapacityPressureEventsWriteTextAndReset(t *testing.T) {
t.Parallel()
m := NewMetrics()
if got := m.GetStats().CapacityPressureEvents; got != 0 {
t.Fatalf("initial CapacityPressureEvents=%d, want 0", got)
}
NoteSoftEviction(m, "memory", 0)
if got := m.GetStats().CapacityPressureEvents; got != 0 {
t.Fatalf("zero-byte eviction counted: %d", got)
}
NoteSoftEviction(m, "memory", 128)
st := m.GetStats()
if st.CapacityPressureEvents != 1 {
t.Fatalf("after memory eviction, CapacityPressureEvents=%d, want 1", st.CapacityPressureEvents)
}
if st.Evictions != 1 {
t.Fatalf("after memory eviction, Evictions=%d, want 1 (existing counter kept)", st.Evictions)
}
NoteSoftEviction(m, "disk", 64)
NoteNoSpace(m, errors.New("no space left on device"))
st = m.GetStats()
if st.CapacityPressureEvents != 3 {
t.Fatalf("after disk eviction + ENOSPC, CapacityPressureEvents=%d, want 3", st.CapacityPressureEvents)
}
if st.Evictions != 2 {
t.Fatalf("ENOSPC must not increment evictions; Evictions=%d, want 2", st.Evictions)
}
rec := httptest.NewRecorder()
WriteText(rec, st)
body := rec.Body.Bytes()
if !bytes.Contains(body, []byte("capacity_pressure_events 3")) {
t.Errorf("WriteText missing capacity_pressure_events 3: %q", rec.Body.String())
}
if !bytes.Contains(body, []byte("evictions 2")) {
t.Errorf("WriteText missing evictions 2: %q", rec.Body.String())
}
m.Reset()
st = m.GetStats()
if st.CapacityPressureEvents != 0 || st.Evictions != 0 {
t.Errorf("after Reset, CapacityPressureEvents=%d Evictions=%d, want 0", st.CapacityPressureEvents, st.Evictions)
}
}
func TestNoteSoftEvictionNilMetrics(t *testing.T) {
t.Parallel()
// Must not panic when metrics are not wired (unit tests / early init).
NoteSoftEviction(nil, "memory", 10)
NoteNoSpace(nil, errors.New("ENOSPC"))
}