ops: Signal disk-full and eviction capacity pressure
CI / vulncheck (pull_request) Successful in 14s
CI / check-and-test (pull_request) Failing after 39s

When the disk (or memory) tier is at cap or the volume returns ENOSPC,
ops currently look like random misses with no clear "we are dropping
data." Count those events as capacity_pressure_events on /metrics and
log tier plus reason so operators can tell capacity pressure from a
cold cache, without changing the existing evictions counter.

Link: #36
This commit is contained in:
ash
2026-09-07 19:23:55 +00:00
parent 8cebc1f96c
commit d7af699e84
12 changed files with 381 additions and 77 deletions
+63
View File
@@ -0,0 +1,63 @@
package metrics
import (
"bytes"
"errors"
"net/http/httptest"
"testing"
)
func TestCapacityPressureEventsWriteTextAndReset(t *testing.T) {
t.Parallel()
m := NewMetrics()
if got := m.GetStats().CapacityPressureEvents; got != 0 {
t.Fatalf("initial CapacityPressureEvents=%d, want 0", got)
}
NoteSoftEviction(m, "memory", 0)
if got := m.GetStats().CapacityPressureEvents; got != 0 {
t.Fatalf("zero-byte eviction counted: %d", got)
}
NoteSoftEviction(m, "memory", 128)
st := m.GetStats()
if st.CapacityPressureEvents != 1 {
t.Fatalf("after memory eviction, CapacityPressureEvents=%d, want 1", st.CapacityPressureEvents)
}
if st.Evictions != 1 {
t.Fatalf("after memory eviction, Evictions=%d, want 1 (existing counter kept)", st.Evictions)
}
NoteSoftEviction(m, "disk", 64)
NoteNoSpace(m, errors.New("no space left on device"))
st = m.GetStats()
if st.CapacityPressureEvents != 3 {
t.Fatalf("after disk eviction + ENOSPC, CapacityPressureEvents=%d, want 3", st.CapacityPressureEvents)
}
if st.Evictions != 2 {
t.Fatalf("ENOSPC must not increment evictions; Evictions=%d, want 2", st.Evictions)
}
rec := httptest.NewRecorder()
WriteText(rec, st)
body := rec.Body.Bytes()
if !bytes.Contains(body, []byte("capacity_pressure_events 3")) {
t.Errorf("WriteText missing capacity_pressure_events 3: %q", rec.Body.String())
}
if !bytes.Contains(body, []byte("evictions 2")) {
t.Errorf("WriteText missing evictions 2: %q", rec.Body.String())
}
m.Reset()
st = m.GetStats()
if st.CapacityPressureEvents != 0 || st.Evictions != 0 {
t.Errorf("after Reset, CapacityPressureEvents=%d Evictions=%d, want 0", st.CapacityPressureEvents, st.Evictions)
}
}
func TestNoteSoftEvictionNilMetrics(t *testing.T) {
t.Parallel()
// Must not panic when metrics are not wired (unit tests / early init).
NoteSoftEviction(nil, "memory", 10)
NoteNoSpace(nil, errors.New("ENOSPC"))
}