feat(envd): add mem_cache metric to track page cache usage (#2269)

arkamar · github-actions[bot] · web-flow · commit 8a404ee35eb4 · 2026-03-31T17:57:14.000+02:00
* feat(envd): add mem_cache metric to track page cache usage across full pipeline

Expose the Linux page cache (Cached from /proc/meminfo) as a new mem_cache
metric. Previously only mem_used was reported, which excludes page cache and
can underreport actual memory pressure inside Firecracker VMs.

The metric flows through: envd -&gt; orchestrator OTel gauges -&gt; ClickHouse -&gt;
public and edge APIs. Version-gated to envd &gt;= 0.5.9 for backward compat.

* fix(api): make memCache a required field in SandboxMetric schemas

All other metric fields (memUsed, memTotal, diskUsed, etc.) are required in
the OpenAPI specs. Leaving memCache optional caused oapi-codegen to generate
it as a pointer (*int64) with omitempty, which is inconsistent with the rest
of the struct and would make it the only metric that could be absent from API
responses.

* chore: auto-commit generated changes

* fix(telemetry): add description and unit for ram cache gauge

---------

Co-authored-by: github-actions[bot] &lt;github-actions[bot]@users.noreply.github.com&gt;
diff --git a/packages/api/internal/api/api.gen.go b/packages/api/internal/api/api.gen.go
diff --git a/packages/api/internal/clusters/resources_local.go b/packages/api/internal/clusters/resources_local.go
@@ -78,6 +78,7 @@ func (l *LocalClusterResourceProvider) GetSandboxMetrics(ctx context.Context, te
 			CpuCount:      int32(m.CPUCount),
 			MemTotal:      int64(m.MemTotal),
 			MemUsed:       int64(m.MemUsed),
+			MemCache:      int64(m.MemCache),
 			DiskTotal:     int64(m.DiskTotal),
 			DiskUsed:      int64(m.DiskUsed),
 		}
@@ -105,6 +106,7 @@ func (l *LocalClusterResourceProvider) GetSandboxesMetrics(ctx context.Context,
 			CpuCount:      int32(m.CPUCount),
 			MemTotal:      int64(m.MemTotal),
 			MemUsed:       int64(m.MemUsed),
+			MemCache:      int64(m.MemCache),
 			DiskTotal:     int64(m.DiskTotal),
 			DiskUsed:      int64(m.DiskUsed),
 		}
diff --git a/packages/api/internal/clusters/resources_remote.go b/packages/api/internal/clusters/resources_remote.go
@@ -66,6 +66,7 @@ func (r *ClusterResourceProviderImpl) GetSandboxMetrics(ctx context.Context, tea
 			CpuCount:      m.CpuCount,
 			MemTotal:      m.MemTotal,
 			MemUsed:       m.MemUsed,
+			MemCache:      m.MemCache,
 			DiskTotal:     m.DiskTotal,
 			DiskUsed:      m.DiskUsed,
 		}
@@ -98,6 +99,7 @@ func (r *ClusterResourceProviderImpl) GetSandboxesMetrics(ctx context.Context, t
 			CpuCount:      v.CpuCount,
 			MemTotal:      v.MemTotal,
 			MemUsed:       v.MemUsed,
+			MemCache:      v.MemCache,
 			DiskTotal:     v.DiskTotal,
 			DiskUsed:      v.DiskUsed,
 		}
diff --git a/packages/clickhouse/pkg/sandbox.go b/packages/clickhouse/pkg/sandbox.go
@@ -19,6 +19,7 @@ type Metrics struct {
 	CPUUsedPercent float64   `ch:"cpu_used"`
 	MemTotal       float64   `ch:"ram_total"`
 	MemUsed        float64   `ch:"ram_used"`
+	MemCache       float64   `ch:"ram_cache"`
 	DiskTotal      float64   `ch:"disk_total"`
 	DiskUsed       float64   `ch:"disk_used"`
 }
@@ -30,6 +31,7 @@ SELECT sandbox_id,
        argMaxIf(value, timestamp, metric_name = '%s')  AS cpu_used,
        argMaxIf(value, timestamp, metric_name = '%s')  AS ram_total,
        argMaxIf(value, timestamp, metric_name = '%s')  AS ram_used,
+       argMaxIf(value, timestamp, metric_name = '%s')  AS ram_cache,
        argMaxIf(value, timestamp, metric_name = '%s')  AS disk_total,
        argMaxIf(value, timestamp, metric_name = '%s')  AS disk_used,
        -- All metrics are recorded at the same time, so we can use max(timestamp) to get the latest one
@@ -39,7 +41,7 @@ WHERE  sandbox_id IN ?
        AND team_id = ?
 GROUP  BY sandbox_id,
           team_id; 
-`, telemetry.SandboxCpuTotalGaugeName, telemetry.SandboxCpuUsedGaugeName, telemetry.SandboxRamTotalGaugeName, telemetry.SandboxRamUsedGaugeName, telemetry.SandboxDiskTotalGaugeName, telemetry.SandboxDiskUsedGaugeName)
+`, telemetry.SandboxCpuTotalGaugeName, telemetry.SandboxCpuUsedGaugeName, telemetry.SandboxRamTotalGaugeName, telemetry.SandboxRamUsedGaugeName, telemetry.SandboxRamCacheGaugeName, telemetry.SandboxDiskTotalGaugeName, telemetry.SandboxDiskUsedGaugeName)
 
 // QueryLatestMetrics returns rows ordered by timestamp, paged by limit.
 func (c *Client) QueryLatestMetrics(ctx context.Context, sandboxIDs []string, teamID string) ([]Metrics, error) {
@@ -82,6 +84,7 @@ SELECT   toStartOfInterval(timestamp, interval {step:UInt32} second) AS ts,
          maxIf(value, metric_name = '%s')				          	 AS cpu_used,
          maxIf(value, metric_name = '%s')         					 AS ram_total,
          maxIf(value, metric_name = '%s')          					 AS ram_used,
+         maxIf(value, metric_name = '%s')          					 AS ram_cache,
          maxIf(value, metric_name = '%s')        					 AS disk_total,
          maxIf(value, metric_name = '%s')         					 AS disk_used
 FROM     sandbox_metrics_gauge s
@@ -91,7 +94,7 @@ AND      timestamp >= {start_time:DateTime64}
 AND      timestamp <= {end_time:DateTime64}
 GROUP BY ts
 ORDER BY ts;
-`, telemetry.SandboxCpuTotalGaugeName, telemetry.SandboxCpuUsedGaugeName, telemetry.SandboxRamTotalGaugeName, telemetry.SandboxRamUsedGaugeName, telemetry.SandboxDiskTotalGaugeName, telemetry.SandboxDiskUsedGaugeName)
+`, telemetry.SandboxCpuTotalGaugeName, telemetry.SandboxCpuUsedGaugeName, telemetry.SandboxRamTotalGaugeName, telemetry.SandboxRamUsedGaugeName, telemetry.SandboxRamCacheGaugeName, telemetry.SandboxDiskTotalGaugeName, telemetry.SandboxDiskUsedGaugeName)
 
 func (c *Client) QuerySandboxTimeRange(ctx context.Context, sandboxID string, teamID string) (time.Time, time.Time, error) {
 	var start, end time.Time
diff --git a/packages/envd/internal/api/api.gen.go b/packages/envd/internal/api/api.gen.go
diff --git a/packages/envd/internal/host/metrics.go b/packages/envd/internal/host/metrics.go
@@ -25,6 +25,7 @@ type Metrics struct {
 
 	MemTotal uint64 `json:"mem_total"` // Total virtual memory in bytes
 	MemUsed  uint64 `json:"mem_used"`  // Used virtual memory in bytes
+	MemCache uint64 `json:"mem_cache"` // Cached memory (page cache) in bytes
 
 	DiskUsed  uint64 `json:"disk_used"`  // Used disk space in bytes
 	DiskTotal uint64 `json:"disk_total"` // Total disk space in bytes
@@ -68,6 +69,7 @@ func GetMetrics() (*Metrics, error) {
 		MemTotalMiB:    memTotalMiB,
 		MemTotal:       v.Total,
 		MemUsed:        v.Used,
+		MemCache:       v.Cached,
 		DiskUsed:       diskMetrics.Total - diskMetrics.Available,
 		DiskTotal:      diskMetrics.Total,
 	}, nil
diff --git a/packages/envd/pkg/version.go b/packages/envd/pkg/version.go
@@ -1,3 +1,3 @@
 package pkg
 
-const Version = "0.5.8"
+const Version = "0.5.9"
diff --git a/packages/envd/spec/envd.yaml b/packages/envd/spec/envd.yaml
@@ -349,6 +349,9 @@ components:
         mem_used:
           type: integer
           description: Used virtual memory in bytes
+        mem_cache:
+          type: integer
+          description: Cached memory (page cache) in bytes
         mem_total_mib:
           type: integer
           description: Total virtual memory in MiB
diff --git a/packages/orchestrator/pkg/metrics/sandboxes.go b/packages/orchestrator/pkg/metrics/sandboxes.go
@@ -33,6 +33,7 @@ const (
 	minEnvVersionForMetricsTimestamp = "0.1.3"
 	minEnvdVersionForMemoryPrecise   = "0.2.4"
 	minEnvdVersionForDiskMetrics     = "0.2.4"
+	minEnvdVersionForCacheMetrics    = "0.5.9"
 
 	timeoutGetMetrics         = 100 * time.Millisecond
 	metricsParallelismFactor  = 5 // Used to calculate number of concurrently sandbox metrics requests
@@ -57,6 +58,7 @@ type SandboxObserver struct {
 	cpuUsed     metric.Float64ObservableGauge
 	memoryTotal metric.Int64ObservableGauge
 	memoryUsed  metric.Int64ObservableGauge
+	memoryCache metric.Int64ObservableGauge
 	diskTotal   metric.Int64ObservableGauge
 	diskUsed    metric.Int64ObservableGauge
 }
@@ -108,6 +110,11 @@ func NewSandboxObserver(ctx context.Context, nodeID, serviceName, serviceCommit,
 		return nil, fmt.Errorf("failed to create memory used gauge: %w", err)
 	}
 
+	memoryCache, err := telemetry.GetGaugeInt(meter, telemetry.SandboxRamCacheGaugeName)
+	if err != nil {
+		return nil, fmt.Errorf("failed to create memory cache gauge: %w", err)
+	}
+
 	diskTotal, err := telemetry.GetGaugeInt(meter, telemetry.SandboxDiskTotalGaugeName)
 	if err != nil {
 		return nil, fmt.Errorf("failed to create disk total gauge: %w", err)
@@ -127,6 +134,7 @@ func NewSandboxObserver(ctx context.Context, nodeID, serviceName, serviceCommit,
 		cpuUsed:        cpuUsed,
 		memoryTotal:    memoryTotal,
 		memoryUsed:     memoryUsed,
+		memoryCache:    memoryCache,
 		diskTotal:      diskTotal,
 		diskUsed:       diskUsed,
 	}
@@ -229,6 +237,14 @@ func (so *SandboxObserver) startObserving() (metric.Registration, error) {
 					o.ObserveInt64(so.memoryTotal, memoryTotal, attributes)
 					o.ObserveInt64(so.memoryUsed, memoryUsed, attributes)
 
+					ok, err = utils.IsGTEVersion(sbx.Config.Envd.Version, minEnvdVersionForCacheMetrics)
+					if err != nil {
+						logger.L().Error(ctx, "Failed to check envd version for cache metrics", zap.Error(err), logger.WithSandboxID(sbx.Runtime.SandboxID))
+					}
+					if ok {
+						o.ObserveInt64(so.memoryCache, sbxMetrics.MemCache, attributes)
+					}
+
 					ok, err = utils.IsGTEVersion(sbx.Config.Envd.Version, minEnvdVersionForDiskMetrics)
 					if err != nil {
 						logger.L().Error(ctx, "Failed to check envd version for disk metrics", zap.Error(err), logger.WithSandboxID(sbx.Runtime.SandboxID))
@@ -266,7 +282,7 @@ func (so *SandboxObserver) startObserving() (metric.Registration, error) {
 			}
 
 			return nil
-		}, so.cpuTotal, so.cpuUsed, so.memoryTotal, so.memoryUsed, so.diskTotal, so.diskUsed)
+		}, so.cpuTotal, so.cpuUsed, so.memoryTotal, so.memoryUsed, so.memoryCache, so.diskTotal, so.diskUsed)
 	if err != nil {
 		return nil, err
 	}
diff --git a/packages/orchestrator/pkg/sandbox/envd/envd.gen.go b/packages/orchestrator/pkg/sandbox/envd/envd.gen.go
diff --git a/packages/orchestrator/pkg/sandbox/metrics.go b/packages/orchestrator/pkg/sandbox/metrics.go
@@ -18,6 +18,7 @@ type Metrics struct {
 
 	MemTotal int64 `json:"mem_total"` // Total virtual memory in bytes
 	MemUsed  int64 `json:"mem_used"`  // Used virtual memory in bytes
+	MemCache int64 `json:"mem_cache"` // Cached memory (page cache) in bytes
 
 	DiskUsed  int64 `json:"disk_used"`  // Used disk space in bytes
 	DiskTotal int64 `json:"disk_total"` // Total disk space in bytes
diff --git a/packages/shared/pkg/http/edge/generated.go b/packages/shared/pkg/http/edge/generated.go
diff --git a/packages/shared/pkg/logger/sandbox/sandbox_logger.go b/packages/shared/pkg/logger/sandbox/sandbox_logger.go
@@ -27,6 +27,7 @@ type SandboxMetricsFields struct {
 	CPUUsedPercent float32
 	MemTotalMiB    uint64
 	MemUsedMiB     uint64
+	MemCacheBytes  uint64
 }
 
 func (sl *SandboxLogger) Metrics(ctx context.Context, metrics SandboxMetricsFields) {
@@ -38,6 +39,7 @@ func (sl *SandboxLogger) Metrics(ctx context.Context, metrics SandboxMetricsFiel
 		zap.Uint32("cpuCount", metrics.CPUCount),
 		zap.Uint64("memTotalMiB", metrics.MemTotalMiB),
 		zap.Uint64("memUsedMiB", metrics.MemUsedMiB),
+		zap.Uint64("memCacheBytes", metrics.MemCacheBytes),
 	)
 }
 
diff --git a/packages/shared/pkg/telemetry/meters.go b/packages/shared/pkg/telemetry/meters.go
@@ -114,6 +114,7 @@ const (
 	// Sandbox metrics
 	SandboxRamUsedGaugeName   GaugeIntType = "e2b.sandbox.ram.used"
 	SandboxRamTotalGaugeName  GaugeIntType = "e2b.sandbox.ram.total"
+	SandboxRamCacheGaugeName  GaugeIntType = "e2b.sandbox.ram.cache"
 	SandboxCpuTotalGaugeName  GaugeIntType = "e2b.sandbox.cpu.total"
 	SandboxDiskUsedGaugeName  GaugeIntType = "e2b.sandbox.disk.used"
 	SandboxDiskTotalGaugeName GaugeIntType = "e2b.sandbox.disk.total"
@@ -217,6 +218,7 @@ var gaugeIntDesc = map[GaugeIntType]string{
 	ApiOrchestratorCountMeterName: "Counter of running orchestrators.",
 	SandboxRamUsedGaugeName:       "Amount of RAM used by the sandbox.",
 	SandboxRamTotalGaugeName:      "Amount of RAM available to the sandbox.",
+	SandboxRamCacheGaugeName:      "Amount of RAM used by the page cache in the sandbox.",
 	SandboxCpuTotalGaugeName:      "Amount of CPU available to the sandbox.",
 	SandboxDiskUsedGaugeName:      "Amount of disk space used by the sandbox.",
 	SandboxDiskTotalGaugeName:     "Amount of disk space available to the sandbox.",
@@ -228,6 +230,7 @@ var gaugeIntUnits = map[GaugeIntType]string{
 	ApiOrchestratorCountMeterName: "{orchestrator}",
 	SandboxRamUsedGaugeName:       "{By}",
 	SandboxRamTotalGaugeName:      "{By}",
+	SandboxRamCacheGaugeName:      "{By}",
 	SandboxCpuTotalGaugeName:      "{count}",
 	SandboxDiskUsedGaugeName:      "{By}",
 	SandboxDiskTotalGaugeName:     "{By}",
diff --git a/spec/openapi-edge.yml b/spec/openapi-edge.yml
@@ -254,6 +254,7 @@ components:
         - cpu_count
         - mem_total
         - mem_used
+        - mem_cache
         - disk_total
         - disk_used
       properties:
@@ -281,6 +282,10 @@ components:
           type: integer
           format: int64
           description: Used memory in bytes
+        mem_cache:
+          type: integer
+          format: int64
+          description: Cached memory (page cache) in bytes
         disk_total:
           type: integer
           format: int64
diff --git a/spec/openapi.yml b/spec/openapi.yml
@@ -388,6 +388,7 @@ components:
         - cpuUsedPct
         - memUsed
         - memTotal
+        - memCache
         - diskUsed
         - diskTotal
       properties:
@@ -416,6 +417,10 @@ components:
           type: integer
           format: int64
           description: Total memory in bytes
+        memCache:
+          type: integer
+          format: int64
+          description: Cached memory (page cache) in bytes
         diskUsed:
           type: integer
           format: int64
diff --git a/tests/integration/internal/api/generated.go b/tests/integration/internal/api/generated.go
diff --git a/tests/integration/internal/envd/generated.go b/tests/integration/internal/envd/generated.go

Original file line number	Diff line number	Diff line change
`@@ -1,3 +1,3 @@`
`1`	`1`	`package pkg`
`2`	`2`
`3`		`-const Version = "0.5.8"`
	`3`	`+const Version = "0.5.9"`