ops: Disk tier occupancy on /metrics
Operators could see attach-ready and capacity-pressure events but not how full the configured disk (or memory) tier was without reading filesystems. Expose size/capacity gauges and disk_cache_full_ratio next to disk_tier_ready. Capacity is a config read, so it stays available while attach is pending.
This commit was merged in pull request #53.
This commit is contained in:
@@ -32,6 +32,8 @@ type Metrics struct {
|
||||
// Cache metrics
|
||||
MemoryCacheSize int64
|
||||
DiskCacheSize int64
|
||||
MemoryCacheCapacity int64 // configured memory capacity (bytes)
|
||||
DiskCacheCapacity int64 // configured disk capacity (bytes); 0 when no disk
|
||||
MemoryCacheHits int64
|
||||
DiskCacheHits int64
|
||||
Promotions int64
|
||||
@@ -137,6 +139,17 @@ func (m *Metrics) SetDiskCacheSize(size int64) {
|
||||
atomic.StoreInt64(&m.DiskCacheSize, size)
|
||||
}
|
||||
|
||||
// SetMemoryCacheCapacity sets the configured memory cache capacity in bytes.
|
||||
func (m *Metrics) SetMemoryCacheCapacity(capacity int64) {
|
||||
atomic.StoreInt64(&m.MemoryCacheCapacity, capacity)
|
||||
}
|
||||
|
||||
// SetDiskCacheCapacity sets the configured disk cache capacity in bytes
|
||||
// (0 when no disk is configured).
|
||||
func (m *Metrics) SetDiskCacheCapacity(capacity int64) {
|
||||
atomic.StoreInt64(&m.DiskCacheCapacity, capacity)
|
||||
}
|
||||
|
||||
// SetDiskTierReady sets whether the disk slow tier is attached (1) or still pending (0).
|
||||
// Memory-only (no disk) also uses 1 — meaning "not waiting on disk attach". Reset does not clear this.
|
||||
func (m *Metrics) SetDiskTierReady(ready int64) {
|
||||
@@ -248,6 +261,11 @@ func (m *Metrics) GetStats() *Stats {
|
||||
serviceErrors[k] = v
|
||||
}
|
||||
|
||||
memoryCacheSize := atomic.LoadInt64(&m.MemoryCacheSize)
|
||||
diskCacheSize := atomic.LoadInt64(&m.DiskCacheSize)
|
||||
memoryCacheCapacity := atomic.LoadInt64(&m.MemoryCacheCapacity)
|
||||
diskCacheCapacity := atomic.LoadInt64(&m.DiskCacheCapacity)
|
||||
|
||||
return &Stats{
|
||||
TotalRequests: totalRequests,
|
||||
CacheHits: cacheHits,
|
||||
@@ -262,8 +280,11 @@ func (m *Metrics) GetStats() *Stats {
|
||||
AvgResponseTime: avgResponseTime,
|
||||
TotalBytesServed: atomic.LoadInt64(&m.TotalBytesServed),
|
||||
TotalBytesSaved: atomic.LoadInt64(&m.TotalBytesSaved),
|
||||
MemoryCacheSize: atomic.LoadInt64(&m.MemoryCacheSize),
|
||||
DiskCacheSize: atomic.LoadInt64(&m.DiskCacheSize),
|
||||
MemoryCacheSize: memoryCacheSize,
|
||||
DiskCacheSize: diskCacheSize,
|
||||
MemoryCacheCapacity: memoryCacheCapacity,
|
||||
DiskCacheCapacity: diskCacheCapacity,
|
||||
DiskCacheFullRatio: diskFullRatio(diskCacheSize, diskCacheCapacity),
|
||||
DiskTierReady: atomic.LoadInt64(&m.DiskTierReady),
|
||||
MemoryCacheHits: atomic.LoadInt64(&m.MemoryCacheHits),
|
||||
DiskCacheHits: atomic.LoadInt64(&m.DiskCacheHits),
|
||||
@@ -279,6 +300,19 @@ func (m *Metrics) GetStats() *Stats {
|
||||
}
|
||||
}
|
||||
|
||||
// diskFullRatio is size / capacity clamped to [0,1].
|
||||
// It is 0 when no disk is configured or the capacity is 0.
|
||||
func diskFullRatio(size, capacity int64) float64 {
|
||||
if size <= 0 || capacity <= 0 {
|
||||
return 0
|
||||
}
|
||||
ratio := float64(size) / float64(capacity)
|
||||
if ratio > 1 {
|
||||
return 1
|
||||
}
|
||||
return ratio
|
||||
}
|
||||
|
||||
// Reset resets all metrics to zero
|
||||
func (m *Metrics) Reset() {
|
||||
atomic.StoreInt64(&m.TotalRequests, 0)
|
||||
@@ -330,6 +364,9 @@ type Stats struct {
|
||||
MemoryCacheSize int64
|
||||
|
||||
DiskCacheSize int64
|
||||
MemoryCacheCapacity int64 // configured memory capacity (bytes)
|
||||
DiskCacheCapacity int64 // configured disk capacity (bytes); 0 when no disk
|
||||
DiskCacheFullRatio float64 // disk_cache_size / disk_cache_capacity, clamped to [0,1]; 0 when no disk or capacity is 0
|
||||
DiskTierReady int64
|
||||
MemoryCacheHits int64
|
||||
DiskCacheHits int64
|
||||
@@ -382,7 +419,10 @@ func WriteText(w http.ResponseWriter, stats *Stats) {
|
||||
writeInt(w, "total_bytes_served", "Total bytes sent to clients.", "counter", stats.TotalBytesServed)
|
||||
writeInt(w, "total_bytes_saved", "Bytes served from cache instead of being re-downloaded from upstream.", "counter", stats.TotalBytesSaved)
|
||||
writeInt(w, "memory_cache_size", "Current memory cache size in bytes.", "gauge", stats.MemoryCacheSize)
|
||||
writeInt(w, "memory_cache_capacity", "Configured memory cache capacity in bytes.", "gauge", stats.MemoryCacheCapacity)
|
||||
writeInt(w, "disk_cache_size", "Current disk cache size in bytes.", "gauge", stats.DiskCacheSize)
|
||||
writeInt(w, "disk_cache_capacity", "Configured disk cache capacity in bytes; 0 when no disk is configured.", "gauge", stats.DiskCacheCapacity)
|
||||
writeFloat(w, "disk_cache_full_ratio", "disk_cache_size / disk_cache_capacity in [0,1]; 0 when no disk or capacity is 0.", "gauge", "%.4f", stats.DiskCacheFullRatio)
|
||||
writeInt(w, "disk_tier_ready", "1 if the disk tier is attached or no disk is configured; 0 while attach is pending.", "gauge", stats.DiskTierReady)
|
||||
writeFloat(w, "uptime_seconds", "Process uptime in seconds.", "gauge", "%.2f", stats.Uptime.Seconds())
|
||||
}
|
||||
|
||||
@@ -100,6 +100,9 @@ func TestWriteTextPrometheusExposition(t *testing.T) {
|
||||
TotalBytesSaved: 50,
|
||||
MemoryCacheSize: 8,
|
||||
DiskCacheSize: 16,
|
||||
MemoryCacheCapacity: 8,
|
||||
DiskCacheCapacity: 32,
|
||||
DiskCacheFullRatio: 0.5,
|
||||
DiskTierReady: 1,
|
||||
Uptime: 3 * time.Second,
|
||||
}
|
||||
@@ -120,6 +123,7 @@ func TestWriteTextPrometheusExposition(t *testing.T) {
|
||||
}
|
||||
gauges := []string{
|
||||
"hit_rate", "avg_response_time_ms", "memory_cache_size", "disk_cache_size",
|
||||
"memory_cache_capacity", "disk_cache_capacity", "disk_cache_full_ratio",
|
||||
"disk_tier_ready", "uptime_seconds",
|
||||
}
|
||||
for _, name := range counters {
|
||||
@@ -138,6 +142,15 @@ func TestWriteTextPrometheusExposition(t *testing.T) {
|
||||
if !strings.Contains(body, "disk_tier_ready 1\n") {
|
||||
t.Errorf("missing disk_tier_ready 1 sample: %q", body)
|
||||
}
|
||||
if !strings.Contains(body, "memory_cache_capacity 8\n") {
|
||||
t.Errorf("missing memory_cache_capacity 8 sample: %q", body)
|
||||
}
|
||||
if !strings.Contains(body, "disk_cache_capacity 32\n") {
|
||||
t.Errorf("missing disk_cache_capacity 32 sample: %q", body)
|
||||
}
|
||||
if !strings.Contains(body, "disk_cache_full_ratio 0.5000\n") {
|
||||
t.Errorf("missing disk_cache_full_ratio 0.5000 sample: %q", body)
|
||||
}
|
||||
if !strings.Contains(body, "range_cache 3\n") {
|
||||
t.Errorf("missing range_cache 3 sample: %q", body)
|
||||
}
|
||||
@@ -146,6 +159,54 @@ func TestWriteTextPrometheusExposition(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestDiskCacheFullRatioInGetStats(t *testing.T) {
|
||||
t.Parallel()
|
||||
cases := []struct {
|
||||
name string
|
||||
size int64
|
||||
capacity int64
|
||||
wantRatio float64
|
||||
wantCapacity int64
|
||||
}{
|
||||
{"no disk (capacity 0)", 0, 0, 0, 0},
|
||||
{"zero size with capacity", 0, 1024, 0, 1024},
|
||||
{"half full", 512, 1024, 0.5, 1024},
|
||||
{"exact full", 1024, 1024, 1, 1024},
|
||||
{"size above capacity clamps to 1", 2048, 1024, 1, 1024},
|
||||
}
|
||||
for _, tc := range cases {
|
||||
tc := tc
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
t.Parallel()
|
||||
m := NewMetrics()
|
||||
m.SetDiskCacheSize(tc.size)
|
||||
m.SetDiskCacheCapacity(tc.capacity)
|
||||
st := m.GetStats()
|
||||
if st.DiskCacheCapacity != tc.wantCapacity {
|
||||
t.Fatalf("DiskCacheCapacity=%d, want %d", st.DiskCacheCapacity, tc.wantCapacity)
|
||||
}
|
||||
if st.DiskCacheFullRatio != tc.wantRatio {
|
||||
t.Fatalf("DiskCacheFullRatio=%v, want %v", st.DiskCacheFullRatio, tc.wantRatio)
|
||||
}
|
||||
if st.DiskCacheFullRatio < 0 || st.DiskCacheFullRatio > 1 {
|
||||
t.Fatalf("DiskCacheFullRatio=%v outside [0,1]", st.DiskCacheFullRatio)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
// Memory capacity is a plain passthrough, and capacity survives Reset
|
||||
// (re-derived by GetMetrics, like MemoryCacheSize/DiskTierReady).
|
||||
m := NewMetrics()
|
||||
m.SetMemoryCacheCapacity(4096)
|
||||
if got := m.GetStats().MemoryCacheCapacity; got != 4096 {
|
||||
t.Fatalf("MemoryCacheCapacity=%d, want 4096", got)
|
||||
}
|
||||
m.Reset()
|
||||
if got := m.GetStats().MemoryCacheCapacity; got != 4096 {
|
||||
t.Fatalf("MemoryCacheCapacity=%d after Reset, want 4096 (config snapshot, like size gauges)", got)
|
||||
}
|
||||
}
|
||||
|
||||
func assertHelpType(t *testing.T, body, name, typ string) {
|
||||
t.Helper()
|
||||
help := "# HELP " + name + " "
|
||||
|
||||
Reference in New Issue
Block a user