Merge pull request 'metrics: Prometheus text exposition for /metrics' (#51) from metrics/prometheus-exposition into main

metrics: Prometheus text exposition for /metrics (#51)

Closes #49

README.md

@@ -109,6 +109,8 @@ Heartbeat also returns `X-SteamCache-Disk-Tier: pending|ready|disabled` (`disabl
 
 These are the cache process's own `/metrics` and `/lancache-heartbeat` endpoints. There is no separate metrics daemon.
 
+`/metrics` is Prometheus text exposition format 0.0.4 (`Content-Type: text/plain; version=0.0.4; charset=utf-8`) so Prometheus and compatible scrapers can pull it. Metric names in the table above are unchanged; each series is preceded by `# HELP` and `# TYPE`.
+
 If you changed `listen_address`, point curl at that host:port instead. For a full SteamPrefill validation workflow (small caches, coalescing, GC), see [Validating Full Functionality](#validating-full-functionality-with-external-tools).
 
 ### Development Workflow

steamcache/handler.go

@@ -76,9 +76,9 @@ func (sc *SteamCache) handleSpecialEndpoints(w http.ResponseWriter, r *http.Requ
 	}
 
 	if r.URL.String() == "/metrics" {
-		// Return metrics in a simple text format
+		// Prometheus text exposition format 0.0.4
 		stats := sc.GetMetrics()
-		w.Header().Set("Content-Type", "text/plain")
+		w.Header().Set("Content-Type", "text/plain; version=0.0.4; charset=utf-8")
 		w.WriteHeader(http.StatusOK)
 		metrics.WriteText(w, stats)
 		return true

steamcache/metrics/metrics.go

@@ -344,42 +344,59 @@ type Stats struct {
 	LastResetTime          time.Time
 }
 
-// WriteText emits the Prometheus-style text metrics to the ResponseWriter.
-// Promoted from internal handler per Phase 3 for better package ownership.
+// WriteText emits Prometheus text exposition format 0.0.4 to the ResponseWriter.
+// Each metric family is # HELP, then # TYPE, then one or more sample lines.
+// Metric names are stable; labeled series keep service=%q (Prometheus-valid quotes).
 // All fmt.Fprintf errors are intentionally discarded via _ = : this is a best-effort
 // read-only debug endpoint; client disconnects or write errors during metrics dump
 // are not actionable (do not affect cache correctness or require retries).
 func WriteText(w http.ResponseWriter, stats *Stats) {
-	_, _ = fmt.Fprintf(w, "# SteamCache2 Metrics\n")
-	_, _ = fmt.Fprintf(w, "total_requests %d\n", stats.TotalRequests)
-	_, _ = fmt.Fprintf(w, "cache_hits %d\n", stats.CacheHits)
-	_, _ = fmt.Fprintf(w, "cache_misses %d\n", stats.CacheMisses)
-	_, _ = fmt.Fprintf(w, "negative_cache_hits %d\n", stats.NegativeCacheHits)
-	_, _ = fmt.Fprintf(w, "cache_coalesced %d\n", stats.CacheCoalesced)
-	_, _ = fmt.Fprintf(w, "range_cache %d\n", stats.RangeCache)
-	_, _ = fmt.Fprintf(w, "range_upstream %d\n", stats.RangeUpstream)
-	_, _ = fmt.Fprintf(w, "errors %d\n", stats.Errors)
-	_, _ = fmt.Fprintf(w, "rate_limited %d\n", stats.RateLimited)
-	_, _ = fmt.Fprintf(w, "upstream_errors %d\n", stats.UpstreamErrors)
-	_, _ = fmt.Fprintf(w, "cache_write_failures %d\n", stats.CacheWriteFailures)
-	_, _ = fmt.Fprintf(w, "memory_cache_hits %d\n", stats.MemoryCacheHits)
-	_, _ = fmt.Fprintf(w, "disk_cache_hits %d\n", stats.DiskCacheHits)
-	_, _ = fmt.Fprintf(w, "promotions %d\n", stats.Promotions)
-	_, _ = fmt.Fprintf(w, "evictions %d\n", stats.Evictions)
-	_, _ = fmt.Fprintf(w, "capacity_pressure_events %d\n", stats.CapacityPressureEvents)
+	writeInt(w, "total_requests", "Total HTTP requests handled.", "counter", stats.TotalRequests)
+	writeInt(w, "cache_hits", "Requests served from cache.", "counter", stats.CacheHits)
+	writeInt(w, "cache_misses", "Requests not found in cache.", "counter", stats.CacheMisses)
+	writeInt(w, "negative_cache_hits", "404/410 served from a still-valid negative cache entry.", "counter", stats.NegativeCacheHits)
+	writeInt(w, "cache_coalesced", "Requests coalesced onto an in-flight upstream fetch.", "counter", stats.CacheCoalesced)
+	writeInt(w, "range_cache", "Range requests served as 206 from an already-cached object.", "counter", stats.RangeCache)
+	writeInt(w, "range_upstream", "Range requests that required an upstream fetch, served as 206.", "counter", stats.RangeUpstream)
+	writeInt(w, "errors", "Request errors.", "counter", stats.Errors)
+	writeInt(w, "rate_limited", "Requests rejected by rate limiting.", "counter", stats.RateLimited)
+	writeInt(w, "upstream_errors", "Errors talking to upstream.", "counter", stats.UpstreamErrors)
+	writeInt(w, "cache_write_failures", "Failures writing objects into cache.", "counter", stats.CacheWriteFailures)
+	writeInt(w, "memory_cache_hits", "Hits served from the memory tier.", "counter", stats.MemoryCacheHits)
+	writeInt(w, "disk_cache_hits", "Hits served from the disk tier.", "counter", stats.DiskCacheHits)
+	writeInt(w, "promotions", "Objects promoted from disk to memory.", "counter", stats.Promotions)
+	writeInt(w, "evictions", "Objects evicted from cache.", "counter", stats.Evictions)
+	writeInt(w, "capacity_pressure_events", "Soft eviction under the memory or disk cap, and/or disk ENOSPC.", "counter", stats.CapacityPressureEvents)
+
+	writeHelpType(w, "service_errors", "Errors attributed to a named service.", "counter")
 	for svc, cnt := range stats.ServiceErrors {
 		_, _ = fmt.Fprintf(w, "service_errors{service=%q} %d\n", svc, cnt)
 	}
+	writeHelpType(w, "service_requests", "Requests attributed to a named service.", "counter")
 	for svc, cnt := range stats.ServiceRequests {
 		_, _ = fmt.Fprintf(w, "service_requests{service=%q} %d\n", svc, cnt)
 	}
-	_, _ = fmt.Fprintf(w, "hit_rate %.4f\n", stats.HitRate)
-	_, _ = fmt.Fprintf(w, "avg_response_time_ms %.2f\n", float64(stats.AvgResponseTime.Nanoseconds())/1e6)
-	_, _ = fmt.Fprintf(w, "total_bytes_served %d\n", stats.TotalBytesServed)
-	_, _ = fmt.Fprintf(w, "total_bytes_saved %d\n", stats.TotalBytesSaved)
 
-	_, _ = fmt.Fprintf(w, "memory_cache_size %d\n", stats.MemoryCacheSize)
-	_, _ = fmt.Fprintf(w, "disk_cache_size %d\n", stats.DiskCacheSize)
-	_, _ = fmt.Fprintf(w, "disk_tier_ready %d\n", stats.DiskTierReady)
-	_, _ = fmt.Fprintf(w, "uptime_seconds %.2f\n", stats.Uptime.Seconds())
+	writeFloat(w, "hit_rate", "Cache hits divided by total requests.", "gauge", "%.4f", stats.HitRate)
+	writeFloat(w, "avg_response_time_ms", "Average response time in milliseconds.", "gauge", "%.2f", float64(stats.AvgResponseTime.Nanoseconds())/1e6)
+	writeInt(w, "total_bytes_served", "Total bytes sent to clients.", "counter", stats.TotalBytesServed)
+	writeInt(w, "total_bytes_saved", "Bytes served from cache instead of being re-downloaded from upstream.", "counter", stats.TotalBytesSaved)
+	writeInt(w, "memory_cache_size", "Current memory cache size in bytes.", "gauge", stats.MemoryCacheSize)
+	writeInt(w, "disk_cache_size", "Current disk cache size in bytes.", "gauge", stats.DiskCacheSize)
+	writeInt(w, "disk_tier_ready", "1 if the disk tier is attached or no disk is configured; 0 while attach is pending.", "gauge", stats.DiskTierReady)
+	writeFloat(w, "uptime_seconds", "Process uptime in seconds.", "gauge", "%.2f", stats.Uptime.Seconds())
+}
+
+func writeHelpType(w http.ResponseWriter, name, help, typ string) {
+	_, _ = fmt.Fprintf(w, "# HELP %s %s\n# TYPE %s %s\n", name, help, name, typ)
+}
+
+func writeInt(w http.ResponseWriter, name, help, typ string, v int64) {
+	writeHelpType(w, name, help, typ)
+	_, _ = fmt.Fprintf(w, "%s %d\n", name, v)
+}
+
+func writeFloat(w http.ResponseWriter, name, help, typ, valFmt string, v float64) {
+	writeHelpType(w, name, help, typ)
+	_, _ = fmt.Fprintf(w, "%s "+valFmt+"\n", name, v)
 }

steamcache/metrics/metrics_test.go

@@ -4,7 +4,9 @@ import (
 	"bytes"
 	"errors"
 	"net/http/httptest"
+	"strings"
 	"testing"
+	"time"
 )
 
 func TestCapacityPressureEventsWriteTextAndReset(t *testing.T) {
@@ -47,6 +49,15 @@ func TestCapacityPressureEventsWriteTextAndReset(t *testing.T) {
 	if !bytes.Contains(body, []byte("evictions 2")) {
 		t.Errorf("WriteText missing evictions 2: %q", rec.Body.String())
 	}
+	if !bytes.Contains(body, []byte("# HELP capacity_pressure_events")) {
+		t.Errorf("WriteText missing # HELP capacity_pressure_events: %q", rec.Body.String())
+	}
+	if !bytes.Contains(body, []byte("# TYPE capacity_pressure_events counter")) {
+		t.Errorf("WriteText missing # TYPE capacity_pressure_events counter: %q", rec.Body.String())
+	}
+	if !bytes.Contains(body, []byte("# TYPE evictions counter")) {
+		t.Errorf("WriteText missing # TYPE evictions counter: %q", rec.Body.String())
+	}
 
 	m.Reset()
 	st = m.GetStats()
@@ -61,3 +72,108 @@ func TestNoteSoftEvictionNilMetrics(t *testing.T) {
 	NoteSoftEviction(nil, "memory", 10)
 	NoteNoSpace(nil, errors.New("ENOSPC"))
 }
+
+func TestWriteTextPrometheusExposition(t *testing.T) {
+	t.Parallel()
+	st := &Stats{
+		TotalRequests:          10,
+		CacheHits:              4,
+		CacheMisses:            6,
+		NegativeCacheHits:      1,
+		CacheCoalesced:         2,
+		RangeCache:             3,
+		RangeUpstream:          5,
+		Errors:                 1,
+		RateLimited:            1,
+		UpstreamErrors:         1,
+		CacheWriteFailures:     1,
+		MemoryCacheHits:        2,
+		DiskCacheHits:          2,
+		Promotions:             1,
+		Evictions:              1,
+		CapacityPressureEvents: 1,
+		ServiceErrors:          map[string]int64{"steam": 2},
+		ServiceRequests:        map[string]int64{"steam": 7},
+		HitRate:                0.4,
+		AvgResponseTime:        2 * time.Millisecond,
+		TotalBytesServed:       100,
+		TotalBytesSaved:        50,
+		MemoryCacheSize:        8,
+		DiskCacheSize:          16,
+		DiskTierReady:          1,
+		Uptime:                 3 * time.Second,
+	}
+	rec := httptest.NewRecorder()
+	WriteText(rec, st)
+	body := rec.Body.String()
+
+	if strings.Contains(body, "# SteamCache2 Metrics") {
+		t.Error("non-standard # SteamCache2 Metrics banner must not be present")
+	}
+
+	counters := []string{
+		"total_requests", "cache_hits", "cache_misses", "negative_cache_hits",
+		"cache_coalesced", "range_cache", "range_upstream", "errors", "rate_limited",
+		"upstream_errors", "cache_write_failures", "memory_cache_hits", "disk_cache_hits",
+		"promotions", "evictions", "capacity_pressure_events", "service_errors",
+		"service_requests", "total_bytes_served", "total_bytes_saved",
+	}
+	gauges := []string{
+		"hit_rate", "avg_response_time_ms", "memory_cache_size", "disk_cache_size",
+		"disk_tier_ready", "uptime_seconds",
+	}
+	for _, name := range counters {
+		assertHelpType(t, body, name, "counter")
+	}
+	for _, name := range gauges {
+		assertHelpType(t, body, name, "gauge")
+	}
+
+	if !strings.Contains(body, `service_errors{service="steam"} 2`) {
+		t.Errorf("missing labeled service_errors sample: %q", body)
+	}
+	if !strings.Contains(body, `service_requests{service="steam"} 7`) {
+		t.Errorf("missing labeled service_requests sample: %q", body)
+	}
+	if !strings.Contains(body, "disk_tier_ready 1\n") {
+		t.Errorf("missing disk_tier_ready 1 sample: %q", body)
+	}
+	if !strings.Contains(body, "range_cache 3\n") {
+		t.Errorf("missing range_cache 3 sample: %q", body)
+	}
+	if !strings.Contains(body, "negative_cache_hits 1\n") {
+		t.Errorf("missing negative_cache_hits 1 sample: %q", body)
+	}
+}
+
+func assertHelpType(t *testing.T, body, name, typ string) {
+	t.Helper()
+	help := "# HELP " + name + " "
+	typeLine := "# TYPE " + name + " " + typ
+	iHelp := strings.Index(body, help)
+	if iHelp < 0 {
+		t.Errorf("missing %q", help)
+		return
+	}
+	iType := strings.Index(body[iHelp:], typeLine)
+	if iType < 0 {
+		t.Errorf("missing %q after HELP for %s", typeLine, name)
+		return
+	}
+	afterType := body[iHelp+iType+len(typeLine):]
+	if !strings.HasPrefix(afterType, "\n") {
+		t.Errorf("# TYPE %s not followed by newline", name)
+		return
+	}
+	sample := afterType[1:]
+	if !strings.HasPrefix(sample, name+" ") && !strings.HasPrefix(sample, name+"{") {
+		t.Errorf("sample for %s does not follow TYPE; next line starts %q", name, firstLine(sample))
+	}
+}
+
+func firstLine(s string) string {
+	if i := strings.IndexByte(s, '\n'); i >= 0 {
+		return s[:i]
+	}
+	return s
+}

steamcache/negative_cache_test.go

@@ -61,6 +61,9 @@ func TestNegativeCache404(t *testing.T) {
 	if !bytes.Contains(mrec.Body.Bytes(), []byte("negative_cache_hits")) {
 		t.Errorf("/metrics missing negative_cache_hits:\n%s", mrec.Body.String())
 	}
+	if ct := mrec.Header().Get("Content-Type"); ct != "text/plain; version=0.0.4; charset=utf-8" {
+		t.Errorf("/metrics Content-Type=%q, want Prometheus text 0.0.4", ct)
+	}
 }
 
 func TestNegativeCache410(t *testing.T) {
@@ -207,4 +210,13 @@ func TestWriteTextNegativeCacheHits(t *testing.T) {
 	if !bytes.Contains(out, []byte("negative_cache_hits 0\n")) {
 		t.Errorf("/metrics missing negative_cache_hits 0:\n%s", out)
 	}
+	if ct := resp.Header.Get("Content-Type"); ct != "text/plain; version=0.0.4; charset=utf-8" {
+		t.Errorf("/metrics Content-Type=%q, want Prometheus text 0.0.4", ct)
+	}
+	if !bytes.Contains(out, []byte("# HELP negative_cache_hits ")) {
+		t.Errorf("/metrics missing # HELP negative_cache_hits:\n%s", out)
+	}
+	if !bytes.Contains(out, []byte("# TYPE negative_cache_hits counter")) {
+		t.Errorf("/metrics missing # TYPE negative_cache_hits counter:\n%s", out)
+	}
 }

steamcache/range_test.go

@@ -313,6 +313,18 @@ func TestRangeMetricsWriteText(t *testing.T) {
 	if !strings.Contains(text, "range_upstream 1\n") {
 		t.Errorf("/metrics missing 'range_upstream 1' line:\n%s", text)
 	}
+	if ct := resp.Header.Get("Content-Type"); ct != "text/plain; version=0.0.4; charset=utf-8" {
+		t.Errorf("/metrics Content-Type=%q, want Prometheus text 0.0.4", ct)
+	}
+	if !strings.Contains(text, "# HELP range_cache ") {
+		t.Errorf("/metrics missing # HELP range_cache:\n%s", text)
+	}
+	if !strings.Contains(text, "# TYPE range_cache counter") {
+		t.Errorf("/metrics missing # TYPE range_cache counter:\n%s", text)
+	}
+	if !strings.Contains(text, "# TYPE range_upstream counter") {
+		t.Errorf("/metrics missing # TYPE range_upstream counter:\n%s", text)
+	}
 }
 
 // TestStreamCachedResponseRange206 is a focused unit test for streamCachedResponse:

steamcache/steamcache_test.go

@@ -562,6 +562,12 @@ func TestMetrics(t *testing.T) {
 	if !bytes.Contains(rec.Body.Bytes(), []byte("negative_cache_hits")) {
 		t.Error("WriteText output missing negative_cache_hits")
 	}
+	if !bytes.Contains(rec.Body.Bytes(), []byte("# HELP total_requests")) {
+		t.Error("WriteText output missing # HELP total_requests")
+	}
+	if !bytes.Contains(rec.Body.Bytes(), []byte("# TYPE total_requests counter")) {
+		t.Error("WriteText output missing # TYPE total_requests counter")
+	}
 }
 
 // Removed old TestKeyGeneration - replaced with TestURLHashing that uses SHA256
@@ -1128,6 +1134,12 @@ func TestDiskOnlyDelayedAttach(t *testing.T) {
 	if !bytes.Contains(rec.Body.Bytes(), []byte("capacity_pressure_events")) {
 		t.Errorf("WriteText output missing capacity_pressure_events: %q", rec.Body.String())
 	}
+	if !bytes.Contains(rec.Body.Bytes(), []byte("# TYPE disk_tier_ready gauge")) {
+		t.Errorf("WriteText output missing # TYPE disk_tier_ready gauge: %q", rec.Body.String())
+	}
+	if !bytes.Contains(rec.Body.Bytes(), []byte("# TYPE capacity_pressure_events counter")) {
+		t.Errorf("WriteText output missing # TYPE capacity_pressure_events counter: %q", rec.Body.String())
+	}
 }
 
 // TestDiskTierSignalMemoryOnly covers memory-only mode: DiskTierReady=1 (N/A, not