From dd72668c2d88ed2da400ba939c23fe56fbbbecec Mon Sep 17 00:00:00 2001 From: pike Date: Tue, 8 Sep 2026 19:53:58 +0000 Subject: [PATCH] metrics: Prometheus text exposition for /metrics /metrics was Prometheus-ish name/value text with a custom banner and text/plain Content-Type, so strict scrapers could fail. Emit Prometheus text 0.0.4 (# HELP, # TYPE, counter|gauge) with stable metric names, and set Content-Type to text/plain; version=0.0.4; charset=utf-8. --- README.md | 2 + steamcache/handler.go | 4 +- steamcache/metrics/metrics.go | 71 +++++++++++------- steamcache/metrics/metrics_test.go | 116 +++++++++++++++++++++++++++++ steamcache/negative_cache_test.go | 12 +++ steamcache/range_test.go | 12 +++ steamcache/steamcache_test.go | 12 +++ 7 files changed, 200 insertions(+), 29 deletions(-) diff --git a/README.md b/README.md index deae975..2ade58b 100644 --- a/README.md +++ b/README.md @@ -109,6 +109,8 @@ Heartbeat also returns `X-SteamCache-Disk-Tier: pending|ready|disabled` (`disabl These are the cache process's own `/metrics` and `/lancache-heartbeat` endpoints. There is no separate metrics daemon. +`/metrics` is Prometheus text exposition format 0.0.4 (`Content-Type: text/plain; version=0.0.4; charset=utf-8`) so Prometheus and compatible scrapers can pull it. Metric names in the table above are unchanged; each series is preceded by `# HELP` and `# TYPE`. + If you changed `listen_address`, point curl at that host:port instead. For a full SteamPrefill validation workflow (small caches, coalescing, GC), see [Validating Full Functionality](#validating-full-functionality-with-external-tools). ### Development Workflow diff --git a/steamcache/handler.go b/steamcache/handler.go index 3f0b2fb..b0e2fa0 100644 --- a/steamcache/handler.go +++ b/steamcache/handler.go @@ -76,9 +76,9 @@ func (sc *SteamCache) handleSpecialEndpoints(w http.ResponseWriter, r *http.Requ } if r.URL.String() == "/metrics" { - // Return metrics in a simple text format + // Prometheus text exposition format 0.0.4 stats := sc.GetMetrics() - w.Header().Set("Content-Type", "text/plain") + w.Header().Set("Content-Type", "text/plain; version=0.0.4; charset=utf-8") w.WriteHeader(http.StatusOK) metrics.WriteText(w, stats) return true diff --git a/steamcache/metrics/metrics.go b/steamcache/metrics/metrics.go index c224205..e5dc52e 100644 --- a/steamcache/metrics/metrics.go +++ b/steamcache/metrics/metrics.go @@ -344,42 +344,59 @@ type Stats struct { LastResetTime time.Time } -// WriteText emits the Prometheus-style text metrics to the ResponseWriter. -// Promoted from internal handler per Phase 3 for better package ownership. +// WriteText emits Prometheus text exposition format 0.0.4 to the ResponseWriter. +// Each metric family is # HELP, then # TYPE, then one or more sample lines. +// Metric names are stable; labeled series keep service=%q (Prometheus-valid quotes). // All fmt.Fprintf errors are intentionally discarded via _ = : this is a best-effort // read-only debug endpoint; client disconnects or write errors during metrics dump // are not actionable (do not affect cache correctness or require retries). func WriteText(w http.ResponseWriter, stats *Stats) { - _, _ = fmt.Fprintf(w, "# SteamCache2 Metrics\n") - _, _ = fmt.Fprintf(w, "total_requests %d\n", stats.TotalRequests) - _, _ = fmt.Fprintf(w, "cache_hits %d\n", stats.CacheHits) - _, _ = fmt.Fprintf(w, "cache_misses %d\n", stats.CacheMisses) - _, _ = fmt.Fprintf(w, "negative_cache_hits %d\n", stats.NegativeCacheHits) - _, _ = fmt.Fprintf(w, "cache_coalesced %d\n", stats.CacheCoalesced) - _, _ = fmt.Fprintf(w, "range_cache %d\n", stats.RangeCache) - _, _ = fmt.Fprintf(w, "range_upstream %d\n", stats.RangeUpstream) - _, _ = fmt.Fprintf(w, "errors %d\n", stats.Errors) - _, _ = fmt.Fprintf(w, "rate_limited %d\n", stats.RateLimited) - _, _ = fmt.Fprintf(w, "upstream_errors %d\n", stats.UpstreamErrors) - _, _ = fmt.Fprintf(w, "cache_write_failures %d\n", stats.CacheWriteFailures) - _, _ = fmt.Fprintf(w, "memory_cache_hits %d\n", stats.MemoryCacheHits) - _, _ = fmt.Fprintf(w, "disk_cache_hits %d\n", stats.DiskCacheHits) - _, _ = fmt.Fprintf(w, "promotions %d\n", stats.Promotions) - _, _ = fmt.Fprintf(w, "evictions %d\n", stats.Evictions) - _, _ = fmt.Fprintf(w, "capacity_pressure_events %d\n", stats.CapacityPressureEvents) + writeInt(w, "total_requests", "Total HTTP requests handled.", "counter", stats.TotalRequests) + writeInt(w, "cache_hits", "Requests served from cache.", "counter", stats.CacheHits) + writeInt(w, "cache_misses", "Requests not found in cache.", "counter", stats.CacheMisses) + writeInt(w, "negative_cache_hits", "404/410 served from a still-valid negative cache entry.", "counter", stats.NegativeCacheHits) + writeInt(w, "cache_coalesced", "Requests coalesced onto an in-flight upstream fetch.", "counter", stats.CacheCoalesced) + writeInt(w, "range_cache", "Range requests served as 206 from an already-cached object.", "counter", stats.RangeCache) + writeInt(w, "range_upstream", "Range requests that required an upstream fetch, served as 206.", "counter", stats.RangeUpstream) + writeInt(w, "errors", "Request errors.", "counter", stats.Errors) + writeInt(w, "rate_limited", "Requests rejected by rate limiting.", "counter", stats.RateLimited) + writeInt(w, "upstream_errors", "Errors talking to upstream.", "counter", stats.UpstreamErrors) + writeInt(w, "cache_write_failures", "Failures writing objects into cache.", "counter", stats.CacheWriteFailures) + writeInt(w, "memory_cache_hits", "Hits served from the memory tier.", "counter", stats.MemoryCacheHits) + writeInt(w, "disk_cache_hits", "Hits served from the disk tier.", "counter", stats.DiskCacheHits) + writeInt(w, "promotions", "Objects promoted from disk to memory.", "counter", stats.Promotions) + writeInt(w, "evictions", "Objects evicted from cache.", "counter", stats.Evictions) + writeInt(w, "capacity_pressure_events", "Soft eviction under the memory or disk cap, and/or disk ENOSPC.", "counter", stats.CapacityPressureEvents) + + writeHelpType(w, "service_errors", "Errors attributed to a named service.", "counter") for svc, cnt := range stats.ServiceErrors { _, _ = fmt.Fprintf(w, "service_errors{service=%q} %d\n", svc, cnt) } + writeHelpType(w, "service_requests", "Requests attributed to a named service.", "counter") for svc, cnt := range stats.ServiceRequests { _, _ = fmt.Fprintf(w, "service_requests{service=%q} %d\n", svc, cnt) } - _, _ = fmt.Fprintf(w, "hit_rate %.4f\n", stats.HitRate) - _, _ = fmt.Fprintf(w, "avg_response_time_ms %.2f\n", float64(stats.AvgResponseTime.Nanoseconds())/1e6) - _, _ = fmt.Fprintf(w, "total_bytes_served %d\n", stats.TotalBytesServed) - _, _ = fmt.Fprintf(w, "total_bytes_saved %d\n", stats.TotalBytesSaved) - _, _ = fmt.Fprintf(w, "memory_cache_size %d\n", stats.MemoryCacheSize) - _, _ = fmt.Fprintf(w, "disk_cache_size %d\n", stats.DiskCacheSize) - _, _ = fmt.Fprintf(w, "disk_tier_ready %d\n", stats.DiskTierReady) - _, _ = fmt.Fprintf(w, "uptime_seconds %.2f\n", stats.Uptime.Seconds()) + writeFloat(w, "hit_rate", "Cache hits divided by total requests.", "gauge", "%.4f", stats.HitRate) + writeFloat(w, "avg_response_time_ms", "Average response time in milliseconds.", "gauge", "%.2f", float64(stats.AvgResponseTime.Nanoseconds())/1e6) + writeInt(w, "total_bytes_served", "Total bytes sent to clients.", "counter", stats.TotalBytesServed) + writeInt(w, "total_bytes_saved", "Bytes served from cache instead of being re-downloaded from upstream.", "counter", stats.TotalBytesSaved) + writeInt(w, "memory_cache_size", "Current memory cache size in bytes.", "gauge", stats.MemoryCacheSize) + writeInt(w, "disk_cache_size", "Current disk cache size in bytes.", "gauge", stats.DiskCacheSize) + writeInt(w, "disk_tier_ready", "1 if the disk tier is attached or no disk is configured; 0 while attach is pending.", "gauge", stats.DiskTierReady) + writeFloat(w, "uptime_seconds", "Process uptime in seconds.", "gauge", "%.2f", stats.Uptime.Seconds()) +} + +func writeHelpType(w http.ResponseWriter, name, help, typ string) { + _, _ = fmt.Fprintf(w, "# HELP %s %s\n# TYPE %s %s\n", name, help, name, typ) +} + +func writeInt(w http.ResponseWriter, name, help, typ string, v int64) { + writeHelpType(w, name, help, typ) + _, _ = fmt.Fprintf(w, "%s %d\n", name, v) +} + +func writeFloat(w http.ResponseWriter, name, help, typ, valFmt string, v float64) { + writeHelpType(w, name, help, typ) + _, _ = fmt.Fprintf(w, "%s "+valFmt+"\n", name, v) } diff --git a/steamcache/metrics/metrics_test.go b/steamcache/metrics/metrics_test.go index 89ee3ab..058c9ac 100644 --- a/steamcache/metrics/metrics_test.go +++ b/steamcache/metrics/metrics_test.go @@ -4,7 +4,9 @@ import ( "bytes" "errors" "net/http/httptest" + "strings" "testing" + "time" ) func TestCapacityPressureEventsWriteTextAndReset(t *testing.T) { @@ -47,6 +49,15 @@ func TestCapacityPressureEventsWriteTextAndReset(t *testing.T) { if !bytes.Contains(body, []byte("evictions 2")) { t.Errorf("WriteText missing evictions 2: %q", rec.Body.String()) } + if !bytes.Contains(body, []byte("# HELP capacity_pressure_events")) { + t.Errorf("WriteText missing # HELP capacity_pressure_events: %q", rec.Body.String()) + } + if !bytes.Contains(body, []byte("# TYPE capacity_pressure_events counter")) { + t.Errorf("WriteText missing # TYPE capacity_pressure_events counter: %q", rec.Body.String()) + } + if !bytes.Contains(body, []byte("# TYPE evictions counter")) { + t.Errorf("WriteText missing # TYPE evictions counter: %q", rec.Body.String()) + } m.Reset() st = m.GetStats() @@ -61,3 +72,108 @@ func TestNoteSoftEvictionNilMetrics(t *testing.T) { NoteSoftEviction(nil, "memory", 10) NoteNoSpace(nil, errors.New("ENOSPC")) } + +func TestWriteTextPrometheusExposition(t *testing.T) { + t.Parallel() + st := &Stats{ + TotalRequests: 10, + CacheHits: 4, + CacheMisses: 6, + NegativeCacheHits: 1, + CacheCoalesced: 2, + RangeCache: 3, + RangeUpstream: 5, + Errors: 1, + RateLimited: 1, + UpstreamErrors: 1, + CacheWriteFailures: 1, + MemoryCacheHits: 2, + DiskCacheHits: 2, + Promotions: 1, + Evictions: 1, + CapacityPressureEvents: 1, + ServiceErrors: map[string]int64{"steam": 2}, + ServiceRequests: map[string]int64{"steam": 7}, + HitRate: 0.4, + AvgResponseTime: 2 * time.Millisecond, + TotalBytesServed: 100, + TotalBytesSaved: 50, + MemoryCacheSize: 8, + DiskCacheSize: 16, + DiskTierReady: 1, + Uptime: 3 * time.Second, + } + rec := httptest.NewRecorder() + WriteText(rec, st) + body := rec.Body.String() + + if strings.Contains(body, "# SteamCache2 Metrics") { + t.Error("non-standard # SteamCache2 Metrics banner must not be present") + } + + counters := []string{ + "total_requests", "cache_hits", "cache_misses", "negative_cache_hits", + "cache_coalesced", "range_cache", "range_upstream", "errors", "rate_limited", + "upstream_errors", "cache_write_failures", "memory_cache_hits", "disk_cache_hits", + "promotions", "evictions", "capacity_pressure_events", "service_errors", + "service_requests", "total_bytes_served", "total_bytes_saved", + } + gauges := []string{ + "hit_rate", "avg_response_time_ms", "memory_cache_size", "disk_cache_size", + "disk_tier_ready", "uptime_seconds", + } + for _, name := range counters { + assertHelpType(t, body, name, "counter") + } + for _, name := range gauges { + assertHelpType(t, body, name, "gauge") + } + + if !strings.Contains(body, `service_errors{service="steam"} 2`) { + t.Errorf("missing labeled service_errors sample: %q", body) + } + if !strings.Contains(body, `service_requests{service="steam"} 7`) { + t.Errorf("missing labeled service_requests sample: %q", body) + } + if !strings.Contains(body, "disk_tier_ready 1\n") { + t.Errorf("missing disk_tier_ready 1 sample: %q", body) + } + if !strings.Contains(body, "range_cache 3\n") { + t.Errorf("missing range_cache 3 sample: %q", body) + } + if !strings.Contains(body, "negative_cache_hits 1\n") { + t.Errorf("missing negative_cache_hits 1 sample: %q", body) + } +} + +func assertHelpType(t *testing.T, body, name, typ string) { + t.Helper() + help := "# HELP " + name + " " + typeLine := "# TYPE " + name + " " + typ + iHelp := strings.Index(body, help) + if iHelp < 0 { + t.Errorf("missing %q", help) + return + } + iType := strings.Index(body[iHelp:], typeLine) + if iType < 0 { + t.Errorf("missing %q after HELP for %s", typeLine, name) + return + } + afterType := body[iHelp+iType+len(typeLine):] + if !strings.HasPrefix(afterType, "\n") { + t.Errorf("# TYPE %s not followed by newline", name) + return + } + sample := afterType[1:] + if !strings.HasPrefix(sample, name+" ") && !strings.HasPrefix(sample, name+"{") { + t.Errorf("sample for %s does not follow TYPE; next line starts %q", name, firstLine(sample)) + } +} + +func firstLine(s string) string { + if i := strings.IndexByte(s, '\n'); i >= 0 { + return s[:i] + } + return s +} diff --git a/steamcache/negative_cache_test.go b/steamcache/negative_cache_test.go index 216e021..53dbbfd 100644 --- a/steamcache/negative_cache_test.go +++ b/steamcache/negative_cache_test.go @@ -61,6 +61,9 @@ func TestNegativeCache404(t *testing.T) { if !bytes.Contains(mrec.Body.Bytes(), []byte("negative_cache_hits")) { t.Errorf("/metrics missing negative_cache_hits:\n%s", mrec.Body.String()) } + if ct := mrec.Header().Get("Content-Type"); ct != "text/plain; version=0.0.4; charset=utf-8" { + t.Errorf("/metrics Content-Type=%q, want Prometheus text 0.0.4", ct) + } } func TestNegativeCache410(t *testing.T) { @@ -207,4 +210,13 @@ func TestWriteTextNegativeCacheHits(t *testing.T) { if !bytes.Contains(out, []byte("negative_cache_hits 0\n")) { t.Errorf("/metrics missing negative_cache_hits 0:\n%s", out) } + if ct := resp.Header.Get("Content-Type"); ct != "text/plain; version=0.0.4; charset=utf-8" { + t.Errorf("/metrics Content-Type=%q, want Prometheus text 0.0.4", ct) + } + if !bytes.Contains(out, []byte("# HELP negative_cache_hits ")) { + t.Errorf("/metrics missing # HELP negative_cache_hits:\n%s", out) + } + if !bytes.Contains(out, []byte("# TYPE negative_cache_hits counter")) { + t.Errorf("/metrics missing # TYPE negative_cache_hits counter:\n%s", out) + } } diff --git a/steamcache/range_test.go b/steamcache/range_test.go index 66b17e6..f29df7d 100644 --- a/steamcache/range_test.go +++ b/steamcache/range_test.go @@ -313,6 +313,18 @@ func TestRangeMetricsWriteText(t *testing.T) { if !strings.Contains(text, "range_upstream 1\n") { t.Errorf("/metrics missing 'range_upstream 1' line:\n%s", text) } + if ct := resp.Header.Get("Content-Type"); ct != "text/plain; version=0.0.4; charset=utf-8" { + t.Errorf("/metrics Content-Type=%q, want Prometheus text 0.0.4", ct) + } + if !strings.Contains(text, "# HELP range_cache ") { + t.Errorf("/metrics missing # HELP range_cache:\n%s", text) + } + if !strings.Contains(text, "# TYPE range_cache counter") { + t.Errorf("/metrics missing # TYPE range_cache counter:\n%s", text) + } + if !strings.Contains(text, "# TYPE range_upstream counter") { + t.Errorf("/metrics missing # TYPE range_upstream counter:\n%s", text) + } } // TestStreamCachedResponseRange206 is a focused unit test for streamCachedResponse: diff --git a/steamcache/steamcache_test.go b/steamcache/steamcache_test.go index b58e8ab..72c9338 100644 --- a/steamcache/steamcache_test.go +++ b/steamcache/steamcache_test.go @@ -562,6 +562,12 @@ func TestMetrics(t *testing.T) { if !bytes.Contains(rec.Body.Bytes(), []byte("negative_cache_hits")) { t.Error("WriteText output missing negative_cache_hits") } + if !bytes.Contains(rec.Body.Bytes(), []byte("# HELP total_requests")) { + t.Error("WriteText output missing # HELP total_requests") + } + if !bytes.Contains(rec.Body.Bytes(), []byte("# TYPE total_requests counter")) { + t.Error("WriteText output missing # TYPE total_requests counter") + } } // Removed old TestKeyGeneration - replaced with TestURLHashing that uses SHA256 @@ -1128,6 +1134,12 @@ func TestDiskOnlyDelayedAttach(t *testing.T) { if !bytes.Contains(rec.Body.Bytes(), []byte("capacity_pressure_events")) { t.Errorf("WriteText output missing capacity_pressure_events: %q", rec.Body.String()) } + if !bytes.Contains(rec.Body.Bytes(), []byte("# TYPE disk_tier_ready gauge")) { + t.Errorf("WriteText output missing # TYPE disk_tier_ready gauge: %q", rec.Body.String()) + } + if !bytes.Contains(rec.Body.Bytes(), []byte("# TYPE capacity_pressure_events counter")) { + t.Errorf("WriteText output missing # TYPE capacity_pressure_events counter: %q", rec.Body.String()) + } } // TestDiskTierSignalMemoryOnly covers memory-only mode: DiskTierReady=1 (N/A, not