ops: Signal disk-full and eviction capacity pressure

When the disk (or memory) tier is at cap or the volume returns ENOSPC,
ops currently look like random misses with no clear "we are dropping
data." Count those events as capacity_pressure_events on /metrics and
log tier plus reason so operators can tell capacity pressure from a
cold cache, without changing the existing evictions counter.

Link: #36
This commit is contained in:
ash
2026-09-07 19:23:55 +00:00
parent a0e929c525
commit de43a71929
12 changed files with 382 additions and 78 deletions
+20 -16
View File
@@ -403,11 +403,13 @@ func (d *DiskFS) Create(key string, size int64) (io.WriteCloser, error) {
dir := filepath.Dir(path)
// 0700 (not 0755): per-shard cache dirs hold untrusted CDN content; restrict to owner only (G301 addressed).
if err := os.MkdirAll(dir, 0700); err != nil {
d.recordIfNoSpace(err)
return nil, err
}
file, err := os.Create(path) // #nosec G304 -- path built by pathForKey from sanitized (Clean, no ..) hash-derived key under trusted disk.root; no untrusted file inclusion
if err != nil {
d.recordIfNoSpace(err)
return nil, err
}
@@ -438,7 +440,19 @@ type diskWriteCloser struct {
}
func (dwc *diskWriteCloser) Write(p []byte) (n int, err error) {
return dwc.file.Write(p)
n, err = dwc.file.Write(p)
if err != nil {
dwc.disk.recordIfNoSpace(err)
}
return n, err
}
// recordIfNoSpace increments capacity_pressure_events and logs when err is ENOSPC (or Windows disk-full).
func (d *DiskFS) recordIfNoSpace(err error) {
if !isNoSpaceError(err) {
return
}
metrics.NoteNoSpace(d.metrics, err)
}
func (dwc *diskWriteCloser) Close() error {
@@ -722,9 +736,7 @@ func (d *DiskFS) EvictLRU(bytesNeeded uint) uint {
}
d.mu.Unlock()
if d.metrics != nil && evicted > 0 {
d.metrics.IncrementEvictions()
}
metrics.NoteSoftEviction(d.metrics, "disk", evicted)
return evicted
}
@@ -775,9 +787,7 @@ func (d *DiskFS) EvictBySize(bytesNeeded uint, ascending bool) uint {
}
d.mu.Unlock()
if d.metrics != nil && evicted > 0 {
d.metrics.IncrementEvictions()
}
metrics.NoteSoftEviction(d.metrics, "disk", evicted)
return evicted
}
@@ -826,9 +836,7 @@ func (d *DiskFS) EvictFIFO(bytesNeeded uint) uint {
}
d.mu.Unlock()
if d.metrics != nil && evicted > 0 {
d.metrics.IncrementEvictions()
}
metrics.NoteSoftEviction(d.metrics, "disk", evicted)
return evicted
}
@@ -882,9 +890,7 @@ func (d *DiskFS) EvictLFU(bytesNeeded uint) uint {
}
d.mu.Unlock()
if d.metrics != nil && evicted > 0 {
d.metrics.IncrementEvictions()
}
metrics.NoteSoftEviction(d.metrics, "disk", evicted)
return evicted
}
@@ -939,8 +945,6 @@ func (d *DiskFS) EvictHybrid(bytesNeeded uint) uint {
}
d.mu.Unlock()
if d.metrics != nil && evicted > 0 {
d.metrics.IncrementEvictions()
}
metrics.NoteSoftEviction(d.metrics, "disk", evicted)
return evicted
}