ops: Signal disk-full and eviction capacity pressure
When the disk (or memory) tier is at cap or the volume returns ENOSPC, ops currently look like random misses with no clear "we are dropping data." Count those events as capacity_pressure_events on /metrics and log tier plus reason so operators can tell capacity pressure from a cold cache, without changing the existing evictions counter. Link: #36
This commit is contained in:
@@ -85,6 +85,7 @@ curl -s -i http://localhost/lancache-heartbeat
|
|||||||
| `memory_cache_hits` / `disk_cache_hits` | Which tier served the hits |
|
| `memory_cache_hits` / `disk_cache_hits` | Which tier served the hits |
|
||||||
| `total_requests` / `errors` | Volume and failures |
|
| `total_requests` / `errors` | Volume and failures |
|
||||||
| `disk_tier_ready` | `0` while disk slow-tier attach pending; `1` when attached, or when no disk configured (N/A — not waiting) |
|
| `disk_tier_ready` | `0` while disk slow-tier attach pending; `1` when attached, or when no disk configured (N/A — not waiting) |
|
||||||
|
| `capacity_pressure_events` | Soft eviction under the memory or disk cap, and/or disk Create/Write/Mkdir hitting ENOSPC (volume full). Distinct from cold-cache misses and from the existing `evictions` counter. Logs `tier` (memory or disk) and `reason` (eviction or enospc). |
|
||||||
|
|
||||||
A first pass through new content is mostly misses (`hit_rate` near 0). Repeat the same content and `cache_hits` / `hit_rate` should rise.
|
A first pass through new content is mostly misses (`hit_rate` near 0). Repeat the same content and `cache_hits` / `hit_rate` should rise.
|
||||||
|
|
||||||
@@ -296,6 +297,7 @@ See `config.Validate()` and `steamcache.New` error paths. This ensures the LAN a
|
|||||||
- Startup logs: Info "Disk slow tier attach pending..." then later "Disk slow tier attached (...)" for disk-only and mixed modes.
|
- Startup logs: Info "Disk slow tier attach pending..." then later "Disk slow tier attached (...)" for disk-only and mixed modes.
|
||||||
- `/metrics` exposes `disk_tier_ready` 0/1 and stays responsive during attach (GetMetrics does not block on Size while pending).
|
- `/metrics` exposes `disk_tier_ready` 0/1 and stays responsive during attach (GetMetrics does not block on Size while pending).
|
||||||
- `/lancache-heartbeat` header `X-SteamCache-Disk-Tier` mirrors that state.
|
- `/lancache-heartbeat` header `X-SteamCache-Disk-Tier` mirrors that state.
|
||||||
|
- `/metrics` `capacity_pressure_events` counts times the cache dropped data under capacity pressure (soft eviction at the memory or disk cap, or disk Create/Write/Mkdir returning ENOSPC). Logs include `tier=memory|disk` and `reason=eviction|enospc` so operators can grep and tell this apart from a cold cache. The existing `evictions` counter is unchanged.
|
||||||
|
|
||||||
#### Garbage Collection Algorithms
|
#### Garbage Collection Algorithms
|
||||||
|
|
||||||
|
|||||||
@@ -7,6 +7,8 @@ import (
|
|||||||
"sync"
|
"sync"
|
||||||
"sync/atomic"
|
"sync/atomic"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
|
"s1d3sw1ped/steamcache2/steamcache/logger"
|
||||||
)
|
)
|
||||||
|
|
||||||
// Metrics tracks various performance and operational metrics
|
// Metrics tracks various performance and operational metrics
|
||||||
@@ -27,13 +29,14 @@ type Metrics struct {
|
|||||||
TotalBytesSaved int64 // bytes served from cache instead of being re-downloaded from upstream
|
TotalBytesSaved int64 // bytes served from cache instead of being re-downloaded from upstream
|
||||||
|
|
||||||
// Cache metrics
|
// Cache metrics
|
||||||
MemoryCacheSize int64
|
MemoryCacheSize int64
|
||||||
DiskCacheSize int64
|
DiskCacheSize int64
|
||||||
MemoryCacheHits int64
|
MemoryCacheHits int64
|
||||||
DiskCacheHits int64
|
DiskCacheHits int64
|
||||||
Promotions int64
|
Promotions int64
|
||||||
Evictions int64
|
Evictions int64
|
||||||
DiskTierReady int64 // 0=pending (or unset), 1=ready or no-disk (N/A)
|
CapacityPressureEvents int64 // soft eviction under cap and/or disk ENOSPC
|
||||||
|
DiskTierReady int64 // 0=pending (or unset), 1=ready or no-disk (N/A)
|
||||||
|
|
||||||
// Expanded observability (upstream breakdowns, cache write failures, per-service errors)
|
// Expanded observability (upstream breakdowns, cache write failures, per-service errors)
|
||||||
UpstreamErrors int64
|
UpstreamErrors int64
|
||||||
@@ -163,8 +166,39 @@ func (m *Metrics) GetServiceRequests(service string) int64 {
|
|||||||
return m.ServiceRequests[service]
|
return m.ServiceRequests[service]
|
||||||
}
|
}
|
||||||
|
|
||||||
func (m *Metrics) IncrementPromotions() { atomic.AddInt64(&m.Promotions, 1) }
|
func (m *Metrics) IncrementPromotions() { atomic.AddInt64(&m.Promotions, 1) }
|
||||||
func (m *Metrics) IncrementEvictions() { atomic.AddInt64(&m.Evictions, 1) }
|
func (m *Metrics) IncrementEvictions() { atomic.AddInt64(&m.Evictions, 1) }
|
||||||
|
func (m *Metrics) IncrementCapacityPressureEvents() { atomic.AddInt64(&m.CapacityPressureEvents, 1) }
|
||||||
|
|
||||||
|
// NoteSoftEviction records one cap-pressure eviction batch that freed bytes.
|
||||||
|
// Keeps the existing evictions counter and also increments capacity_pressure_events.
|
||||||
|
// Nil m is safe: the log still fires so ops can grep without metrics wired.
|
||||||
|
func NoteSoftEviction(m *Metrics, tier string, evicted uint) {
|
||||||
|
if evicted == 0 {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if m != nil {
|
||||||
|
m.IncrementEvictions()
|
||||||
|
m.IncrementCapacityPressureEvents()
|
||||||
|
}
|
||||||
|
logger.Logger.Info().
|
||||||
|
Str("tier", tier).
|
||||||
|
Str("reason", "eviction").
|
||||||
|
Uint("bytes_evicted", evicted).
|
||||||
|
Msg("cache capacity pressure")
|
||||||
|
}
|
||||||
|
|
||||||
|
// NoteNoSpace records a disk Create/Write/Mkdir ENOSPC (or equivalent) event.
|
||||||
|
func NoteNoSpace(m *Metrics, err error) {
|
||||||
|
if m != nil {
|
||||||
|
m.IncrementCapacityPressureEvents()
|
||||||
|
}
|
||||||
|
logger.Logger.Warn().
|
||||||
|
Str("tier", "disk").
|
||||||
|
Str("reason", "enospc").
|
||||||
|
Err(err).
|
||||||
|
Msg("cache capacity pressure")
|
||||||
|
}
|
||||||
|
|
||||||
// Additional observability counters
|
// Additional observability counters
|
||||||
func (m *Metrics) IncrementUpstreamErrors() { atomic.AddInt64(&m.UpstreamErrors, 1) }
|
func (m *Metrics) IncrementUpstreamErrors() { atomic.AddInt64(&m.UpstreamErrors, 1) }
|
||||||
@@ -209,31 +243,32 @@ func (m *Metrics) GetStats() *Stats {
|
|||||||
}
|
}
|
||||||
|
|
||||||
return &Stats{
|
return &Stats{
|
||||||
TotalRequests: totalRequests,
|
TotalRequests: totalRequests,
|
||||||
CacheHits: cacheHits,
|
CacheHits: cacheHits,
|
||||||
CacheMisses: cacheMisses,
|
CacheMisses: cacheMisses,
|
||||||
CacheCoalesced: atomic.LoadInt64(&m.CacheCoalesced),
|
CacheCoalesced: atomic.LoadInt64(&m.CacheCoalesced),
|
||||||
RangeCache: atomic.LoadInt64(&m.RangeCache),
|
RangeCache: atomic.LoadInt64(&m.RangeCache),
|
||||||
RangeUpstream: atomic.LoadInt64(&m.RangeUpstream),
|
RangeUpstream: atomic.LoadInt64(&m.RangeUpstream),
|
||||||
Errors: atomic.LoadInt64(&m.Errors),
|
Errors: atomic.LoadInt64(&m.Errors),
|
||||||
RateLimited: atomic.LoadInt64(&m.RateLimited),
|
RateLimited: atomic.LoadInt64(&m.RateLimited),
|
||||||
HitRate: hitRate,
|
HitRate: hitRate,
|
||||||
AvgResponseTime: avgResponseTime,
|
AvgResponseTime: avgResponseTime,
|
||||||
TotalBytesServed: atomic.LoadInt64(&m.TotalBytesServed),
|
TotalBytesServed: atomic.LoadInt64(&m.TotalBytesServed),
|
||||||
TotalBytesSaved: atomic.LoadInt64(&m.TotalBytesSaved),
|
TotalBytesSaved: atomic.LoadInt64(&m.TotalBytesSaved),
|
||||||
MemoryCacheSize: atomic.LoadInt64(&m.MemoryCacheSize),
|
MemoryCacheSize: atomic.LoadInt64(&m.MemoryCacheSize),
|
||||||
DiskCacheSize: atomic.LoadInt64(&m.DiskCacheSize),
|
DiskCacheSize: atomic.LoadInt64(&m.DiskCacheSize),
|
||||||
DiskTierReady: atomic.LoadInt64(&m.DiskTierReady),
|
DiskTierReady: atomic.LoadInt64(&m.DiskTierReady),
|
||||||
MemoryCacheHits: atomic.LoadInt64(&m.MemoryCacheHits),
|
MemoryCacheHits: atomic.LoadInt64(&m.MemoryCacheHits),
|
||||||
DiskCacheHits: atomic.LoadInt64(&m.DiskCacheHits),
|
DiskCacheHits: atomic.LoadInt64(&m.DiskCacheHits),
|
||||||
Promotions: atomic.LoadInt64(&m.Promotions),
|
Promotions: atomic.LoadInt64(&m.Promotions),
|
||||||
Evictions: atomic.LoadInt64(&m.Evictions),
|
Evictions: atomic.LoadInt64(&m.Evictions),
|
||||||
ServiceRequests: serviceRequests,
|
CapacityPressureEvents: atomic.LoadInt64(&m.CapacityPressureEvents),
|
||||||
UpstreamErrors: atomic.LoadInt64(&m.UpstreamErrors),
|
ServiceRequests: serviceRequests,
|
||||||
CacheWriteFailures: atomic.LoadInt64(&m.CacheWriteFailures),
|
UpstreamErrors: atomic.LoadInt64(&m.UpstreamErrors),
|
||||||
ServiceErrors: serviceErrors,
|
CacheWriteFailures: atomic.LoadInt64(&m.CacheWriteFailures),
|
||||||
Uptime: time.Since(m.StartTime),
|
ServiceErrors: serviceErrors,
|
||||||
LastResetTime: m.LastResetTime,
|
Uptime: time.Since(m.StartTime),
|
||||||
|
LastResetTime: m.LastResetTime,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -254,6 +289,7 @@ func (m *Metrics) Reset() {
|
|||||||
atomic.StoreInt64(&m.DiskCacheHits, 0)
|
atomic.StoreInt64(&m.DiskCacheHits, 0)
|
||||||
atomic.StoreInt64(&m.Promotions, 0)
|
atomic.StoreInt64(&m.Promotions, 0)
|
||||||
atomic.StoreInt64(&m.Evictions, 0)
|
atomic.StoreInt64(&m.Evictions, 0)
|
||||||
|
atomic.StoreInt64(&m.CapacityPressureEvents, 0)
|
||||||
atomic.StoreInt64(&m.UpstreamErrors, 0)
|
atomic.StoreInt64(&m.UpstreamErrors, 0)
|
||||||
atomic.StoreInt64(&m.CacheWriteFailures, 0)
|
atomic.StoreInt64(&m.CacheWriteFailures, 0)
|
||||||
|
|
||||||
@@ -284,18 +320,19 @@ type Stats struct {
|
|||||||
TotalBytesSaved int64
|
TotalBytesSaved int64
|
||||||
MemoryCacheSize int64
|
MemoryCacheSize int64
|
||||||
|
|
||||||
DiskCacheSize int64
|
DiskCacheSize int64
|
||||||
DiskTierReady int64
|
DiskTierReady int64
|
||||||
MemoryCacheHits int64
|
MemoryCacheHits int64
|
||||||
DiskCacheHits int64
|
DiskCacheHits int64
|
||||||
Promotions int64
|
Promotions int64
|
||||||
Evictions int64
|
Evictions int64
|
||||||
UpstreamErrors int64
|
CapacityPressureEvents int64
|
||||||
CacheWriteFailures int64
|
UpstreamErrors int64
|
||||||
ServiceErrors map[string]int64
|
CacheWriteFailures int64
|
||||||
ServiceRequests map[string]int64
|
ServiceErrors map[string]int64
|
||||||
Uptime time.Duration
|
ServiceRequests map[string]int64
|
||||||
LastResetTime time.Time
|
Uptime time.Duration
|
||||||
|
LastResetTime time.Time
|
||||||
}
|
}
|
||||||
|
|
||||||
// WriteText emits the Prometheus-style text metrics to the ResponseWriter.
|
// WriteText emits the Prometheus-style text metrics to the ResponseWriter.
|
||||||
@@ -319,6 +356,7 @@ func WriteText(w http.ResponseWriter, stats *Stats) {
|
|||||||
_, _ = fmt.Fprintf(w, "disk_cache_hits %d\n", stats.DiskCacheHits)
|
_, _ = fmt.Fprintf(w, "disk_cache_hits %d\n", stats.DiskCacheHits)
|
||||||
_, _ = fmt.Fprintf(w, "promotions %d\n", stats.Promotions)
|
_, _ = fmt.Fprintf(w, "promotions %d\n", stats.Promotions)
|
||||||
_, _ = fmt.Fprintf(w, "evictions %d\n", stats.Evictions)
|
_, _ = fmt.Fprintf(w, "evictions %d\n", stats.Evictions)
|
||||||
|
_, _ = fmt.Fprintf(w, "capacity_pressure_events %d\n", stats.CapacityPressureEvents)
|
||||||
for svc, cnt := range stats.ServiceErrors {
|
for svc, cnt := range stats.ServiceErrors {
|
||||||
_, _ = fmt.Fprintf(w, "service_errors{service=%q} %d\n", svc, cnt)
|
_, _ = fmt.Fprintf(w, "service_errors{service=%q} %d\n", svc, cnt)
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -0,0 +1,63 @@
|
|||||||
|
package metrics
|
||||||
|
|
||||||
|
import (
|
||||||
|
"bytes"
|
||||||
|
"errors"
|
||||||
|
"net/http/httptest"
|
||||||
|
"testing"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestCapacityPressureEventsWriteTextAndReset(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
m := NewMetrics()
|
||||||
|
if got := m.GetStats().CapacityPressureEvents; got != 0 {
|
||||||
|
t.Fatalf("initial CapacityPressureEvents=%d, want 0", got)
|
||||||
|
}
|
||||||
|
|
||||||
|
NoteSoftEviction(m, "memory", 0)
|
||||||
|
if got := m.GetStats().CapacityPressureEvents; got != 0 {
|
||||||
|
t.Fatalf("zero-byte eviction counted: %d", got)
|
||||||
|
}
|
||||||
|
|
||||||
|
NoteSoftEviction(m, "memory", 128)
|
||||||
|
st := m.GetStats()
|
||||||
|
if st.CapacityPressureEvents != 1 {
|
||||||
|
t.Fatalf("after memory eviction, CapacityPressureEvents=%d, want 1", st.CapacityPressureEvents)
|
||||||
|
}
|
||||||
|
if st.Evictions != 1 {
|
||||||
|
t.Fatalf("after memory eviction, Evictions=%d, want 1 (existing counter kept)", st.Evictions)
|
||||||
|
}
|
||||||
|
|
||||||
|
NoteSoftEviction(m, "disk", 64)
|
||||||
|
NoteNoSpace(m, errors.New("no space left on device"))
|
||||||
|
st = m.GetStats()
|
||||||
|
if st.CapacityPressureEvents != 3 {
|
||||||
|
t.Fatalf("after disk eviction + ENOSPC, CapacityPressureEvents=%d, want 3", st.CapacityPressureEvents)
|
||||||
|
}
|
||||||
|
if st.Evictions != 2 {
|
||||||
|
t.Fatalf("ENOSPC must not increment evictions; Evictions=%d, want 2", st.Evictions)
|
||||||
|
}
|
||||||
|
|
||||||
|
rec := httptest.NewRecorder()
|
||||||
|
WriteText(rec, st)
|
||||||
|
body := rec.Body.Bytes()
|
||||||
|
if !bytes.Contains(body, []byte("capacity_pressure_events 3")) {
|
||||||
|
t.Errorf("WriteText missing capacity_pressure_events 3: %q", rec.Body.String())
|
||||||
|
}
|
||||||
|
if !bytes.Contains(body, []byte("evictions 2")) {
|
||||||
|
t.Errorf("WriteText missing evictions 2: %q", rec.Body.String())
|
||||||
|
}
|
||||||
|
|
||||||
|
m.Reset()
|
||||||
|
st = m.GetStats()
|
||||||
|
if st.CapacityPressureEvents != 0 || st.Evictions != 0 {
|
||||||
|
t.Errorf("after Reset, CapacityPressureEvents=%d Evictions=%d, want 0", st.CapacityPressureEvents, st.Evictions)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestNoteSoftEvictionNilMetrics(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
// Must not panic when metrics are not wired (unit tests / early init).
|
||||||
|
NoteSoftEviction(nil, "memory", 10)
|
||||||
|
NoteNoSpace(nil, errors.New("ENOSPC"))
|
||||||
|
}
|
||||||
@@ -1101,6 +1101,9 @@ func TestDiskOnlyDelayedAttach(t *testing.T) {
|
|||||||
if !bytes.Contains(rec.Body.Bytes(), []byte("disk_tier_ready 1")) {
|
if !bytes.Contains(rec.Body.Bytes(), []byte("disk_tier_ready 1")) {
|
||||||
t.Errorf("WriteText output missing \"disk_tier_ready 1\": %q", rec.Body.String())
|
t.Errorf("WriteText output missing \"disk_tier_ready 1\": %q", rec.Body.String())
|
||||||
}
|
}
|
||||||
|
if !bytes.Contains(rec.Body.Bytes(), []byte("capacity_pressure_events")) {
|
||||||
|
t.Errorf("WriteText output missing capacity_pressure_events: %q", rec.Body.String())
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// TestDiskTierSignalMemoryOnly covers memory-only mode: DiskTierReady=1 (N/A, not
|
// TestDiskTierSignalMemoryOnly covers memory-only mode: DiskTierReady=1 (N/A, not
|
||||||
|
|||||||
+20
-16
@@ -403,11 +403,13 @@ func (d *DiskFS) Create(key string, size int64) (io.WriteCloser, error) {
|
|||||||
dir := filepath.Dir(path)
|
dir := filepath.Dir(path)
|
||||||
// 0700 (not 0755): per-shard cache dirs hold untrusted CDN content; restrict to owner only (G301 addressed).
|
// 0700 (not 0755): per-shard cache dirs hold untrusted CDN content; restrict to owner only (G301 addressed).
|
||||||
if err := os.MkdirAll(dir, 0700); err != nil {
|
if err := os.MkdirAll(dir, 0700); err != nil {
|
||||||
|
d.recordIfNoSpace(err)
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
|
|
||||||
file, err := os.Create(path) // #nosec G304 -- path built by pathForKey from sanitized (Clean, no ..) hash-derived key under trusted disk.root; no untrusted file inclusion
|
file, err := os.Create(path) // #nosec G304 -- path built by pathForKey from sanitized (Clean, no ..) hash-derived key under trusted disk.root; no untrusted file inclusion
|
||||||
if err != nil {
|
if err != nil {
|
||||||
|
d.recordIfNoSpace(err)
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -438,7 +440,19 @@ type diskWriteCloser struct {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func (dwc *diskWriteCloser) Write(p []byte) (n int, err error) {
|
func (dwc *diskWriteCloser) Write(p []byte) (n int, err error) {
|
||||||
return dwc.file.Write(p)
|
n, err = dwc.file.Write(p)
|
||||||
|
if err != nil {
|
||||||
|
dwc.disk.recordIfNoSpace(err)
|
||||||
|
}
|
||||||
|
return n, err
|
||||||
|
}
|
||||||
|
|
||||||
|
// recordIfNoSpace increments capacity_pressure_events and logs when err is ENOSPC (or Windows disk-full).
|
||||||
|
func (d *DiskFS) recordIfNoSpace(err error) {
|
||||||
|
if !isNoSpaceError(err) {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
metrics.NoteNoSpace(d.metrics, err)
|
||||||
}
|
}
|
||||||
|
|
||||||
func (dwc *diskWriteCloser) Close() error {
|
func (dwc *diskWriteCloser) Close() error {
|
||||||
@@ -722,9 +736,7 @@ func (d *DiskFS) EvictLRU(bytesNeeded uint) uint {
|
|||||||
}
|
}
|
||||||
d.mu.Unlock()
|
d.mu.Unlock()
|
||||||
|
|
||||||
if d.metrics != nil && evicted > 0 {
|
metrics.NoteSoftEviction(d.metrics, "disk", evicted)
|
||||||
d.metrics.IncrementEvictions()
|
|
||||||
}
|
|
||||||
return evicted
|
return evicted
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -775,9 +787,7 @@ func (d *DiskFS) EvictBySize(bytesNeeded uint, ascending bool) uint {
|
|||||||
}
|
}
|
||||||
d.mu.Unlock()
|
d.mu.Unlock()
|
||||||
|
|
||||||
if d.metrics != nil && evicted > 0 {
|
metrics.NoteSoftEviction(d.metrics, "disk", evicted)
|
||||||
d.metrics.IncrementEvictions()
|
|
||||||
}
|
|
||||||
return evicted
|
return evicted
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -826,9 +836,7 @@ func (d *DiskFS) EvictFIFO(bytesNeeded uint) uint {
|
|||||||
}
|
}
|
||||||
d.mu.Unlock()
|
d.mu.Unlock()
|
||||||
|
|
||||||
if d.metrics != nil && evicted > 0 {
|
metrics.NoteSoftEviction(d.metrics, "disk", evicted)
|
||||||
d.metrics.IncrementEvictions()
|
|
||||||
}
|
|
||||||
return evicted
|
return evicted
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -882,9 +890,7 @@ func (d *DiskFS) EvictLFU(bytesNeeded uint) uint {
|
|||||||
}
|
}
|
||||||
d.mu.Unlock()
|
d.mu.Unlock()
|
||||||
|
|
||||||
if d.metrics != nil && evicted > 0 {
|
metrics.NoteSoftEviction(d.metrics, "disk", evicted)
|
||||||
d.metrics.IncrementEvictions()
|
|
||||||
}
|
|
||||||
return evicted
|
return evicted
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -939,8 +945,6 @@ func (d *DiskFS) EvictHybrid(bytesNeeded uint) uint {
|
|||||||
}
|
}
|
||||||
d.mu.Unlock()
|
d.mu.Unlock()
|
||||||
|
|
||||||
if d.metrics != nil && evicted > 0 {
|
metrics.NoteSoftEviction(d.metrics, "disk", evicted)
|
||||||
d.metrics.IncrementEvictions()
|
|
||||||
}
|
|
||||||
return evicted
|
return evicted
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -11,6 +11,7 @@ import (
|
|||||||
"testing"
|
"testing"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
|
"s1d3sw1ped/steamcache2/steamcache/metrics"
|
||||||
"s1d3sw1ped/steamcache2/vfs"
|
"s1d3sw1ped/steamcache2/vfs"
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -122,6 +123,42 @@ func TestDiskFS_InitPopulatesIndexOnRestart(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestDiskFS_CapacityPressureOnEvict(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
td := t.TempDir()
|
||||||
|
d, err := New(td, 500, nil)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
_ = d.Size()
|
||||||
|
met := metrics.NewMetrics()
|
||||||
|
d.SetMetrics(met)
|
||||||
|
for i := 0; i < 3; i++ {
|
||||||
|
k := "f" + string(rune('0'+i))
|
||||||
|
w, cerr := d.Create(k, 200)
|
||||||
|
if cerr != nil {
|
||||||
|
t.Fatal(cerr)
|
||||||
|
}
|
||||||
|
if _, werr := w.Write(make([]byte, 200)); werr != nil {
|
||||||
|
t.Fatal(werr)
|
||||||
|
}
|
||||||
|
if cerr := w.Close(); cerr != nil {
|
||||||
|
t.Fatal(cerr)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
evicted := d.EvictLRU(100)
|
||||||
|
if evicted == 0 {
|
||||||
|
t.Fatalf("expected eviction under cap, size=%d cap=%d", d.Size(), d.Capacity())
|
||||||
|
}
|
||||||
|
st := met.GetStats()
|
||||||
|
if st.Evictions == 0 {
|
||||||
|
t.Error("evictions counter not incremented under disk cap pressure")
|
||||||
|
}
|
||||||
|
if st.CapacityPressureEvents == 0 {
|
||||||
|
t.Error("capacity_pressure_events not incremented under disk cap pressure")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestDiskFS_EvictAndLazyStat(t *testing.T) {
|
func TestDiskFS_EvictAndLazyStat(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
td := t.TempDir()
|
td := t.TempDir()
|
||||||
@@ -138,10 +175,21 @@ func TestDiskFS_EvictAndLazyStat(t *testing.T) {
|
|||||||
w.Write(make([]byte, 120))
|
w.Write(make([]byte, 120))
|
||||||
w.Close()
|
w.Close()
|
||||||
}
|
}
|
||||||
|
met := metrics.NewMetrics()
|
||||||
|
d.SetMetrics(met)
|
||||||
ev := d.EvictLRU(200)
|
ev := d.EvictLRU(200)
|
||||||
if ev == 0 {
|
if ev == 0 {
|
||||||
t.Log("no evict (size calc async or snapshot tolerance?)")
|
t.Log("no evict (size calc async or snapshot tolerance?)")
|
||||||
}
|
}
|
||||||
|
if ev > 0 {
|
||||||
|
st := met.GetStats()
|
||||||
|
if st.Evictions == 0 {
|
||||||
|
t.Error("evictions counter not incremented after disk EvictLRU freed bytes")
|
||||||
|
}
|
||||||
|
if st.CapacityPressureEvents == 0 {
|
||||||
|
t.Error("capacity_pressure_events not incremented after disk EvictLRU freed bytes")
|
||||||
|
}
|
||||||
|
}
|
||||||
// Explicit post-evict consistency checks: for any key no longer visible via Stat, its on-disk
|
// Explicit post-evict consistency checks: for any key no longer visible via Stat, its on-disk
|
||||||
// file must be absent (verifies coordinated unlink + no resurrection via lazy discovery).
|
// file must be absent (verifies coordinated unlink + no resurrection via lazy discovery).
|
||||||
// Keys still present after this small evict are allowed (accounting tolerance in raw DiskFS).
|
// Keys still present after this small evict are allowed (accounting tolerance in raw DiskFS).
|
||||||
|
|||||||
@@ -0,0 +1,14 @@
|
|||||||
|
//go:build !windows
|
||||||
|
|
||||||
|
package disk
|
||||||
|
|
||||||
|
import (
|
||||||
|
"errors"
|
||||||
|
|
||||||
|
"golang.org/x/sys/unix"
|
||||||
|
)
|
||||||
|
|
||||||
|
// isNoSpaceError reports whether err is ENOSPC (or wraps it).
|
||||||
|
func isNoSpaceError(err error) bool {
|
||||||
|
return err != nil && errors.Is(err, unix.ENOSPC)
|
||||||
|
}
|
||||||
@@ -0,0 +1,58 @@
|
|||||||
|
//go:build !windows
|
||||||
|
|
||||||
|
package disk
|
||||||
|
|
||||||
|
import (
|
||||||
|
"io"
|
||||||
|
"os"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"golang.org/x/sys/unix"
|
||||||
|
|
||||||
|
"s1d3sw1ped/steamcache2/steamcache/metrics"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestIsNoSpaceError(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
if isNoSpaceError(nil) {
|
||||||
|
t.Error("nil must not be ENOSPC")
|
||||||
|
}
|
||||||
|
if isNoSpaceError(io.EOF) {
|
||||||
|
t.Error("EOF must not be ENOSPC")
|
||||||
|
}
|
||||||
|
if !isNoSpaceError(unix.ENOSPC) {
|
||||||
|
t.Error("unix.ENOSPC should match")
|
||||||
|
}
|
||||||
|
wrapped := &os.PathError{Op: "write", Path: "x", Err: unix.ENOSPC}
|
||||||
|
if !isNoSpaceError(wrapped) {
|
||||||
|
t.Error("PathError wrapping ENOSPC should match")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestDiskFS_ENOSPCCapacityPressure(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
d, err := New(t.TempDir(), 1024, nil)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
met := metrics.NewMetrics()
|
||||||
|
d.SetMetrics(met)
|
||||||
|
|
||||||
|
d.recordIfNoSpace(io.EOF)
|
||||||
|
if got := met.GetStats().CapacityPressureEvents; got != 0 {
|
||||||
|
t.Fatalf("non-ENOSPC counted: %d", got)
|
||||||
|
}
|
||||||
|
|
||||||
|
d.recordIfNoSpace(unix.ENOSPC)
|
||||||
|
if got := met.GetStats().CapacityPressureEvents; got != 1 {
|
||||||
|
t.Fatalf("unix.ENOSPC: CapacityPressureEvents=%d, want 1", got)
|
||||||
|
}
|
||||||
|
if got := met.GetStats().Evictions; got != 0 {
|
||||||
|
t.Fatalf("ENOSPC must not increment evictions, got %d", got)
|
||||||
|
}
|
||||||
|
|
||||||
|
d.recordIfNoSpace(&os.PathError{Op: "write", Path: "p", Err: unix.ENOSPC})
|
||||||
|
if got := met.GetStats().CapacityPressureEvents; got != 2 {
|
||||||
|
t.Fatalf("wrapped ENOSPC: CapacityPressureEvents=%d, want 2", got)
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,17 @@
|
|||||||
|
//go:build windows
|
||||||
|
|
||||||
|
package disk
|
||||||
|
|
||||||
|
import (
|
||||||
|
"errors"
|
||||||
|
|
||||||
|
"golang.org/x/sys/windows"
|
||||||
|
)
|
||||||
|
|
||||||
|
// isNoSpaceError reports whether err is a Windows disk-full equivalent of ENOSPC.
|
||||||
|
func isNoSpaceError(err error) bool {
|
||||||
|
if err == nil {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
return errors.Is(err, windows.ERROR_DISK_FULL) || errors.Is(err, windows.ERROR_HANDLE_DISK_FULL)
|
||||||
|
}
|
||||||
@@ -0,0 +1,56 @@
|
|||||||
|
//go:build windows
|
||||||
|
|
||||||
|
package disk
|
||||||
|
|
||||||
|
import (
|
||||||
|
"io"
|
||||||
|
"os"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"golang.org/x/sys/windows"
|
||||||
|
|
||||||
|
"s1d3sw1ped/steamcache2/steamcache/metrics"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestIsNoSpaceError(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
if isNoSpaceError(nil) {
|
||||||
|
t.Error("nil must not be disk-full")
|
||||||
|
}
|
||||||
|
if isNoSpaceError(io.EOF) {
|
||||||
|
t.Error("EOF must not be disk-full")
|
||||||
|
}
|
||||||
|
if !isNoSpaceError(windows.ERROR_DISK_FULL) {
|
||||||
|
t.Error("ERROR_DISK_FULL should match")
|
||||||
|
}
|
||||||
|
if !isNoSpaceError(windows.ERROR_HANDLE_DISK_FULL) {
|
||||||
|
t.Error("ERROR_HANDLE_DISK_FULL should match")
|
||||||
|
}
|
||||||
|
wrapped := &os.PathError{Op: "write", Path: "x", Err: windows.ERROR_DISK_FULL}
|
||||||
|
if !isNoSpaceError(wrapped) {
|
||||||
|
t.Error("PathError wrapping ERROR_DISK_FULL should match")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestDiskFS_ENOSPCCapacityPressure(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
d, err := New(t.TempDir(), 1024, nil)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
met := metrics.NewMetrics()
|
||||||
|
d.SetMetrics(met)
|
||||||
|
|
||||||
|
d.recordIfNoSpace(io.EOF)
|
||||||
|
if got := met.GetStats().CapacityPressureEvents; got != 0 {
|
||||||
|
t.Fatalf("non-ENOSPC counted: %d", got)
|
||||||
|
}
|
||||||
|
|
||||||
|
d.recordIfNoSpace(windows.ERROR_DISK_FULL)
|
||||||
|
if got := met.GetStats().CapacityPressureEvents; got != 1 {
|
||||||
|
t.Fatalf("ERROR_DISK_FULL: CapacityPressureEvents=%d, want 1", got)
|
||||||
|
}
|
||||||
|
if got := met.GetStats().Evictions; got != 0 {
|
||||||
|
t.Fatalf("disk-full must not increment evictions, got %d", got)
|
||||||
|
}
|
||||||
|
}
|
||||||
+5
-15
@@ -358,9 +358,7 @@ func (m *MemoryFS) EvictLRU(bytesNeeded uint) uint {
|
|||||||
}
|
}
|
||||||
m.mu.Unlock()
|
m.mu.Unlock()
|
||||||
|
|
||||||
if m.metrics != nil && evicted > 0 {
|
metrics.NoteSoftEviction(m.metrics, "memory", evicted)
|
||||||
m.metrics.IncrementEvictions()
|
|
||||||
}
|
|
||||||
return evicted
|
return evicted
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -413,9 +411,7 @@ func (m *MemoryFS) EvictBySize(bytesNeeded uint, ascending bool) uint {
|
|||||||
}
|
}
|
||||||
m.mu.Unlock()
|
m.mu.Unlock()
|
||||||
|
|
||||||
if m.metrics != nil && evicted > 0 {
|
metrics.NoteSoftEviction(m.metrics, "memory", evicted)
|
||||||
m.metrics.IncrementEvictions()
|
|
||||||
}
|
|
||||||
return evicted
|
return evicted
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -464,9 +460,7 @@ func (m *MemoryFS) EvictFIFO(bytesNeeded uint) uint {
|
|||||||
}
|
}
|
||||||
m.mu.Unlock()
|
m.mu.Unlock()
|
||||||
|
|
||||||
if m.metrics != nil && evicted > 0 {
|
metrics.NoteSoftEviction(m.metrics, "memory", evicted)
|
||||||
m.metrics.IncrementEvictions()
|
|
||||||
}
|
|
||||||
return evicted
|
return evicted
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -520,9 +514,7 @@ func (m *MemoryFS) EvictLFU(bytesNeeded uint) uint {
|
|||||||
}
|
}
|
||||||
m.mu.Unlock()
|
m.mu.Unlock()
|
||||||
|
|
||||||
if m.metrics != nil && evicted > 0 {
|
metrics.NoteSoftEviction(m.metrics, "memory", evicted)
|
||||||
m.metrics.IncrementEvictions()
|
|
||||||
}
|
|
||||||
return evicted
|
return evicted
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -578,8 +570,6 @@ func (m *MemoryFS) EvictHybrid(bytesNeeded uint) uint {
|
|||||||
}
|
}
|
||||||
m.mu.Unlock()
|
m.mu.Unlock()
|
||||||
|
|
||||||
if m.metrics != nil && evicted > 0 {
|
metrics.NoteSoftEviction(m.metrics, "memory", evicted)
|
||||||
m.metrics.IncrementEvictions()
|
|
||||||
}
|
|
||||||
return evicted
|
return evicted
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -8,6 +8,8 @@ import (
|
|||||||
"sync/atomic"
|
"sync/atomic"
|
||||||
"testing"
|
"testing"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
|
"s1d3sw1ped/steamcache2/steamcache/metrics"
|
||||||
)
|
)
|
||||||
|
|
||||||
func TestMemoryFS_Basic(t *testing.T) {
|
func TestMemoryFS_Basic(t *testing.T) {
|
||||||
@@ -60,6 +62,8 @@ func TestMemoryFS_EvictUnderPressure(t *testing.T) {
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
|
met := metrics.NewMetrics()
|
||||||
|
m.SetMetrics(met)
|
||||||
// create 3x200 = 600 >500, should trigger internal? but direct evict call
|
// create 3x200 = 600 >500, should trigger internal? but direct evict call
|
||||||
for i := 0; i < 3; i++ {
|
for i := 0; i < 3; i++ {
|
||||||
w, _ := m.Create("f"+string(rune('0'+i)), 200)
|
w, _ := m.Create("f"+string(rune('0'+i)), 200)
|
||||||
@@ -71,6 +75,13 @@ func TestMemoryFS_EvictUnderPressure(t *testing.T) {
|
|||||||
if evicted == 0 || m.Size() > 500 {
|
if evicted == 0 || m.Size() > 500 {
|
||||||
t.Errorf("evict failed: evicted=%d size=%d", evicted, m.Size())
|
t.Errorf("evict failed: evicted=%d size=%d", evicted, m.Size())
|
||||||
}
|
}
|
||||||
|
st := met.GetStats()
|
||||||
|
if st.Evictions == 0 {
|
||||||
|
t.Error("evictions counter not incremented under memory cap pressure")
|
||||||
|
}
|
||||||
|
if st.CapacityPressureEvents == 0 {
|
||||||
|
t.Error("capacity_pressure_events not incremented under memory cap pressure")
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestMemoryFS_SizeNeverExceedsAfterEvict(t *testing.T) {
|
func TestMemoryFS_SizeNeverExceedsAfterEvict(t *testing.T) {
|
||||||
|
|||||||
Reference in New Issue
Block a user