50ca0a071c
Operators could see attach-ready and capacity-pressure events but not how full the configured disk (or memory) tier was without reading filesystems. Expose size/capacity gauges and disk_cache_full_ratio next to disk_tier_ready. Capacity is a config read, so it stays available while attach is pending.
241 lines
7.6 KiB
Go
241 lines
7.6 KiB
Go
package metrics
|
|
|
|
import (
|
|
"bytes"
|
|
"errors"
|
|
"net/http/httptest"
|
|
"strings"
|
|
"testing"
|
|
"time"
|
|
)
|
|
|
|
func TestCapacityPressureEventsWriteTextAndReset(t *testing.T) {
|
|
t.Parallel()
|
|
m := NewMetrics()
|
|
if got := m.GetStats().CapacityPressureEvents; got != 0 {
|
|
t.Fatalf("initial CapacityPressureEvents=%d, want 0", got)
|
|
}
|
|
|
|
NoteSoftEviction(m, "memory", 0)
|
|
if got := m.GetStats().CapacityPressureEvents; got != 0 {
|
|
t.Fatalf("zero-byte eviction counted: %d", got)
|
|
}
|
|
|
|
NoteSoftEviction(m, "memory", 128)
|
|
st := m.GetStats()
|
|
if st.CapacityPressureEvents != 1 {
|
|
t.Fatalf("after memory eviction, CapacityPressureEvents=%d, want 1", st.CapacityPressureEvents)
|
|
}
|
|
if st.Evictions != 1 {
|
|
t.Fatalf("after memory eviction, Evictions=%d, want 1 (existing counter kept)", st.Evictions)
|
|
}
|
|
|
|
NoteSoftEviction(m, "disk", 64)
|
|
NoteNoSpace(m, errors.New("no space left on device"))
|
|
st = m.GetStats()
|
|
if st.CapacityPressureEvents != 3 {
|
|
t.Fatalf("after disk eviction + ENOSPC, CapacityPressureEvents=%d, want 3", st.CapacityPressureEvents)
|
|
}
|
|
if st.Evictions != 2 {
|
|
t.Fatalf("ENOSPC must not increment evictions; Evictions=%d, want 2", st.Evictions)
|
|
}
|
|
|
|
rec := httptest.NewRecorder()
|
|
WriteText(rec, st)
|
|
body := rec.Body.Bytes()
|
|
if !bytes.Contains(body, []byte("capacity_pressure_events 3")) {
|
|
t.Errorf("WriteText missing capacity_pressure_events 3: %q", rec.Body.String())
|
|
}
|
|
if !bytes.Contains(body, []byte("evictions 2")) {
|
|
t.Errorf("WriteText missing evictions 2: %q", rec.Body.String())
|
|
}
|
|
if !bytes.Contains(body, []byte("# HELP capacity_pressure_events")) {
|
|
t.Errorf("WriteText missing # HELP capacity_pressure_events: %q", rec.Body.String())
|
|
}
|
|
if !bytes.Contains(body, []byte("# TYPE capacity_pressure_events counter")) {
|
|
t.Errorf("WriteText missing # TYPE capacity_pressure_events counter: %q", rec.Body.String())
|
|
}
|
|
if !bytes.Contains(body, []byte("# TYPE evictions counter")) {
|
|
t.Errorf("WriteText missing # TYPE evictions counter: %q", rec.Body.String())
|
|
}
|
|
|
|
m.Reset()
|
|
st = m.GetStats()
|
|
if st.CapacityPressureEvents != 0 || st.Evictions != 0 {
|
|
t.Errorf("after Reset, CapacityPressureEvents=%d Evictions=%d, want 0", st.CapacityPressureEvents, st.Evictions)
|
|
}
|
|
}
|
|
|
|
func TestNoteSoftEvictionNilMetrics(t *testing.T) {
|
|
t.Parallel()
|
|
// Must not panic when metrics are not wired (unit tests / early init).
|
|
NoteSoftEviction(nil, "memory", 10)
|
|
NoteNoSpace(nil, errors.New("ENOSPC"))
|
|
}
|
|
|
|
func TestWriteTextPrometheusExposition(t *testing.T) {
|
|
t.Parallel()
|
|
st := &Stats{
|
|
TotalRequests: 10,
|
|
CacheHits: 4,
|
|
CacheMisses: 6,
|
|
NegativeCacheHits: 1,
|
|
CacheCoalesced: 2,
|
|
RangeCache: 3,
|
|
RangeUpstream: 5,
|
|
Errors: 1,
|
|
RateLimited: 1,
|
|
UpstreamErrors: 1,
|
|
CacheWriteFailures: 1,
|
|
MemoryCacheHits: 2,
|
|
DiskCacheHits: 2,
|
|
Promotions: 1,
|
|
Evictions: 1,
|
|
CapacityPressureEvents: 1,
|
|
ServiceErrors: map[string]int64{"steam": 2},
|
|
ServiceRequests: map[string]int64{"steam": 7},
|
|
HitRate: 0.4,
|
|
AvgResponseTime: 2 * time.Millisecond,
|
|
TotalBytesServed: 100,
|
|
TotalBytesSaved: 50,
|
|
MemoryCacheSize: 8,
|
|
DiskCacheSize: 16,
|
|
MemoryCacheCapacity: 8,
|
|
DiskCacheCapacity: 32,
|
|
DiskCacheFullRatio: 0.5,
|
|
DiskTierReady: 1,
|
|
Uptime: 3 * time.Second,
|
|
}
|
|
rec := httptest.NewRecorder()
|
|
WriteText(rec, st)
|
|
body := rec.Body.String()
|
|
|
|
if strings.Contains(body, "# SteamCache2 Metrics") {
|
|
t.Error("non-standard # SteamCache2 Metrics banner must not be present")
|
|
}
|
|
|
|
counters := []string{
|
|
"total_requests", "cache_hits", "cache_misses", "negative_cache_hits",
|
|
"cache_coalesced", "range_cache", "range_upstream", "errors", "rate_limited",
|
|
"upstream_errors", "cache_write_failures", "memory_cache_hits", "disk_cache_hits",
|
|
"promotions", "evictions", "capacity_pressure_events", "service_errors",
|
|
"service_requests", "total_bytes_served", "total_bytes_saved",
|
|
}
|
|
gauges := []string{
|
|
"hit_rate", "avg_response_time_ms", "memory_cache_size", "disk_cache_size",
|
|
"memory_cache_capacity", "disk_cache_capacity", "disk_cache_full_ratio",
|
|
"disk_tier_ready", "uptime_seconds",
|
|
}
|
|
for _, name := range counters {
|
|
assertHelpType(t, body, name, "counter")
|
|
}
|
|
for _, name := range gauges {
|
|
assertHelpType(t, body, name, "gauge")
|
|
}
|
|
|
|
if !strings.Contains(body, `service_errors{service="steam"} 2`) {
|
|
t.Errorf("missing labeled service_errors sample: %q", body)
|
|
}
|
|
if !strings.Contains(body, `service_requests{service="steam"} 7`) {
|
|
t.Errorf("missing labeled service_requests sample: %q", body)
|
|
}
|
|
if !strings.Contains(body, "disk_tier_ready 1\n") {
|
|
t.Errorf("missing disk_tier_ready 1 sample: %q", body)
|
|
}
|
|
if !strings.Contains(body, "memory_cache_capacity 8\n") {
|
|
t.Errorf("missing memory_cache_capacity 8 sample: %q", body)
|
|
}
|
|
if !strings.Contains(body, "disk_cache_capacity 32\n") {
|
|
t.Errorf("missing disk_cache_capacity 32 sample: %q", body)
|
|
}
|
|
if !strings.Contains(body, "disk_cache_full_ratio 0.5000\n") {
|
|
t.Errorf("missing disk_cache_full_ratio 0.5000 sample: %q", body)
|
|
}
|
|
if !strings.Contains(body, "range_cache 3\n") {
|
|
t.Errorf("missing range_cache 3 sample: %q", body)
|
|
}
|
|
if !strings.Contains(body, "negative_cache_hits 1\n") {
|
|
t.Errorf("missing negative_cache_hits 1 sample: %q", body)
|
|
}
|
|
}
|
|
|
|
func TestDiskCacheFullRatioInGetStats(t *testing.T) {
|
|
t.Parallel()
|
|
cases := []struct {
|
|
name string
|
|
size int64
|
|
capacity int64
|
|
wantRatio float64
|
|
wantCapacity int64
|
|
}{
|
|
{"no disk (capacity 0)", 0, 0, 0, 0},
|
|
{"zero size with capacity", 0, 1024, 0, 1024},
|
|
{"half full", 512, 1024, 0.5, 1024},
|
|
{"exact full", 1024, 1024, 1, 1024},
|
|
{"size above capacity clamps to 1", 2048, 1024, 1, 1024},
|
|
}
|
|
for _, tc := range cases {
|
|
tc := tc
|
|
t.Run(tc.name, func(t *testing.T) {
|
|
t.Parallel()
|
|
m := NewMetrics()
|
|
m.SetDiskCacheSize(tc.size)
|
|
m.SetDiskCacheCapacity(tc.capacity)
|
|
st := m.GetStats()
|
|
if st.DiskCacheCapacity != tc.wantCapacity {
|
|
t.Fatalf("DiskCacheCapacity=%d, want %d", st.DiskCacheCapacity, tc.wantCapacity)
|
|
}
|
|
if st.DiskCacheFullRatio != tc.wantRatio {
|
|
t.Fatalf("DiskCacheFullRatio=%v, want %v", st.DiskCacheFullRatio, tc.wantRatio)
|
|
}
|
|
if st.DiskCacheFullRatio < 0 || st.DiskCacheFullRatio > 1 {
|
|
t.Fatalf("DiskCacheFullRatio=%v outside [0,1]", st.DiskCacheFullRatio)
|
|
}
|
|
})
|
|
}
|
|
|
|
// Memory capacity is a plain passthrough, and capacity survives Reset
|
|
// (re-derived by GetMetrics, like MemoryCacheSize/DiskTierReady).
|
|
m := NewMetrics()
|
|
m.SetMemoryCacheCapacity(4096)
|
|
if got := m.GetStats().MemoryCacheCapacity; got != 4096 {
|
|
t.Fatalf("MemoryCacheCapacity=%d, want 4096", got)
|
|
}
|
|
m.Reset()
|
|
if got := m.GetStats().MemoryCacheCapacity; got != 4096 {
|
|
t.Fatalf("MemoryCacheCapacity=%d after Reset, want 4096 (config snapshot, like size gauges)", got)
|
|
}
|
|
}
|
|
|
|
func assertHelpType(t *testing.T, body, name, typ string) {
|
|
t.Helper()
|
|
help := "# HELP " + name + " "
|
|
typeLine := "# TYPE " + name + " " + typ
|
|
iHelp := strings.Index(body, help)
|
|
if iHelp < 0 {
|
|
t.Errorf("missing %q", help)
|
|
return
|
|
}
|
|
iType := strings.Index(body[iHelp:], typeLine)
|
|
if iType < 0 {
|
|
t.Errorf("missing %q after HELP for %s", typeLine, name)
|
|
return
|
|
}
|
|
afterType := body[iHelp+iType+len(typeLine):]
|
|
if !strings.HasPrefix(afterType, "\n") {
|
|
t.Errorf("# TYPE %s not followed by newline", name)
|
|
return
|
|
}
|
|
sample := afterType[1:]
|
|
if !strings.HasPrefix(sample, name+" ") && !strings.HasPrefix(sample, name+"{") {
|
|
t.Errorf("sample for %s does not follow TYPE; next line starts %q", name, firstLine(sample))
|
|
}
|
|
}
|
|
|
|
func firstLine(s string) string {
|
|
if i := strings.IndexByte(s, '\n'); i >= 0 {
|
|
return s[:i]
|
|
}
|
|
return s
|
|
}
|