ops: Disk tier occupancy on /metrics
Operators could see attach-ready and capacity-pressure events but not how full the configured disk (or memory) tier was without reading filesystems. Expose size/capacity gauges and disk_cache_full_ratio next to disk_tier_ready. Capacity is a config read, so it stays available while attach is pending.
This commit was merged in pull request #53.
This commit is contained in:
@@ -1497,3 +1497,131 @@ func TestCacheKeySharedAcrossCDNHostAliases(t *testing.T) {
|
||||
t.Errorf("upstream fetched %d times across host aliases, want 1", got)
|
||||
}
|
||||
}
|
||||
|
||||
// TestGetMetricsCapacityGauges covers the tier-occupancy gauges: GetMetrics
|
||||
// sets memory/disk capacity from the configured sizes, and WriteText emits
|
||||
// memory_cache_capacity / disk_cache_capacity / disk_cache_full_ratio. During a
|
||||
// pending disk attach, GetMetrics must return quickly (no disk.Size() call) and
|
||||
// still report the configured disk capacity.
|
||||
func TestGetMetricsCapacityGauges(t *testing.T) {
|
||||
t.Run("memory-only", func(t *testing.T) {
|
||||
sc, err := New("127.0.0.1:0", "1MB", "0", t.TempDir(), "", "lru", "lru", 10, 5, "0", nil, "")
|
||||
if err != nil {
|
||||
t.Fatalf("New memory-only: %v", err)
|
||||
}
|
||||
t.Cleanup(func() { sc.Shutdown() })
|
||||
|
||||
st := sc.GetMetrics()
|
||||
if st.MemoryCacheCapacity != 1000000 {
|
||||
t.Errorf("MemoryCacheCapacity=%d, want 1000000 (configured 1MB)", st.MemoryCacheCapacity)
|
||||
}
|
||||
if st.DiskCacheCapacity != 0 {
|
||||
t.Errorf("DiskCacheCapacity=%d, want 0 (no disk configured)", st.DiskCacheCapacity)
|
||||
}
|
||||
if st.DiskCacheFullRatio != 0 {
|
||||
t.Errorf("DiskCacheFullRatio=%v, want 0 (no disk)", st.DiskCacheFullRatio)
|
||||
}
|
||||
|
||||
rec := httptest.NewRecorder()
|
||||
metrics.WriteText(rec, sc.GetMetrics())
|
||||
body := rec.Body.String()
|
||||
if !strings.Contains(body, "memory_cache_capacity 1000000\n") {
|
||||
t.Errorf("WriteText missing memory_cache_capacity 1000000: %q", body)
|
||||
}
|
||||
if !strings.Contains(body, "disk_cache_capacity 0\n") {
|
||||
t.Errorf("WriteText missing disk_cache_capacity 0: %q", body)
|
||||
}
|
||||
if !strings.Contains(body, "# TYPE disk_cache_full_ratio gauge") {
|
||||
t.Errorf("WriteText missing # TYPE disk_cache_full_ratio gauge: %q", body)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("mixed pending attach reports capacity without Size", func(t *testing.T) {
|
||||
td := t.TempDir()
|
||||
diskPath := filepath.Join(td, "disk")
|
||||
if err := os.MkdirAll(diskPath, 0755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
hold := make(chan struct{})
|
||||
var holdOnce sync.Once
|
||||
closeHold := func() { holdOnce.Do(func() { close(hold) }) }
|
||||
disk.RegisterInitHold(diskPath, hold)
|
||||
t.Cleanup(func() {
|
||||
closeHold()
|
||||
disk.ClearInitHold(diskPath)
|
||||
})
|
||||
|
||||
sc, err := New("127.0.0.1:0", "1MB", "10MB", diskPath, "", "lru", "lru", 10, 1, "0", nil, "")
|
||||
if err != nil {
|
||||
t.Fatalf("New mixed: %v", err)
|
||||
}
|
||||
t.Cleanup(func() { sc.Shutdown() })
|
||||
t.Cleanup(closeHold) // before Shutdown: attach is blocked in Size() until the hold closes
|
||||
|
||||
// Pending window is held open; if GetMetrics called disk.Size() it would
|
||||
// block on the barrier, so a bounded wait proves non-blocking behavior.
|
||||
done := make(chan *metrics.Stats, 1)
|
||||
go func() { done <- sc.GetMetrics() }()
|
||||
select {
|
||||
case st := <-done:
|
||||
if got := st.DiskTierReady; got != 0 {
|
||||
t.Fatalf("immediate DiskTierReady=%d, want 0 (pending)", got)
|
||||
}
|
||||
if st.DiskCacheCapacity != 10000000 {
|
||||
t.Errorf("pending DiskCacheCapacity=%d, want 10000000 (configured 10MB)", st.DiskCacheCapacity)
|
||||
}
|
||||
if st.MemoryCacheCapacity != 1000000 {
|
||||
t.Errorf("pending MemoryCacheCapacity=%d, want 1000000 (configured 1MB)", st.MemoryCacheCapacity)
|
||||
}
|
||||
if st.DiskCacheFullRatio != 0 {
|
||||
t.Errorf("pending DiskCacheFullRatio=%v, want 0 (size not reported while pending)", st.DiskCacheFullRatio)
|
||||
}
|
||||
case <-time.After(2 * time.Second):
|
||||
t.Fatal("GetMetrics blocked during pending attach (must not call disk.Size())")
|
||||
}
|
||||
|
||||
closeHold()
|
||||
_ = sc.disk.Size()
|
||||
deadline := time.Now().Add(2 * time.Second)
|
||||
for time.Now().Before(deadline) {
|
||||
if sc.GetMetrics().DiskTierReady == 1 {
|
||||
break
|
||||
}
|
||||
time.Sleep(1 * time.Millisecond)
|
||||
}
|
||||
if got := sc.GetMetrics().DiskTierReady; got != 1 {
|
||||
t.Fatalf("DiskTierReady=%d after barrier, want 1 (ready)", got)
|
||||
}
|
||||
|
||||
// Post-attach writes prefer the slow (disk) tier, so a write produces a
|
||||
// non-zero disk size and hence a non-zero occupancy ratio.
|
||||
w, err := sc.vfs.Create("occupancy-key", 128)
|
||||
if err != nil {
|
||||
t.Fatalf("Create failed after attach: %v", err)
|
||||
}
|
||||
if _, err := w.Write(make([]byte, 128)); err != nil {
|
||||
t.Fatalf("Write failed: %v", err)
|
||||
}
|
||||
if err := w.Close(); err != nil {
|
||||
t.Fatalf("Close failed: %v", err)
|
||||
}
|
||||
|
||||
st := sc.GetMetrics()
|
||||
if st.DiskCacheCapacity != 10000000 {
|
||||
t.Errorf("post-attach DiskCacheCapacity=%d, want 10000000", st.DiskCacheCapacity)
|
||||
}
|
||||
if st.DiskCacheSize <= 0 {
|
||||
t.Errorf("post-attach DiskCacheSize=%d, want > 0 after a write", st.DiskCacheSize)
|
||||
}
|
||||
if st.DiskCacheFullRatio <= 0 || st.DiskCacheFullRatio > 1 {
|
||||
t.Errorf("post-attach DiskCacheFullRatio=%v, want in (0,1]", st.DiskCacheFullRatio)
|
||||
}
|
||||
|
||||
rec := httptest.NewRecorder()
|
||||
metrics.WriteText(rec, st)
|
||||
body := rec.Body.String()
|
||||
if !strings.Contains(body, "disk_cache_capacity 10000000\n") {
|
||||
t.Errorf("WriteText missing disk_cache_capacity 10000000: %q", body)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user