Add host-level memory pressure metrics to system memory check (#44808)

### What does this PR do?

Adds memory pressure metrics from `/proc/vmstat` to the Linux system memory check.

**New metrics:**

| Metric | Type | Tags | Description |
|--------|------|------|-------------|
| `system.mem.allocstall` | MonotonicCount | `zone` | Direct reclaim stalls per memory zone |
| `system.mem.pgscan_direct` | MonotonicCount | - | Pages scanned during direct (synchronous) reclaim |
| `system.mem.pgsteal_direct` | MonotonicCount | - | Pages reclaimed during direct reclaim |
| `system.mem.pgscan_kswapd` | MonotonicCount | - | Pages scanned by kswapd background daemon |
| `system.mem.pgsteal_kswapd` | MonotonicCount | - | Pages reclaimed by kswapd |

Similar to #1605 (commit_limit/committed_as) and #6708 (context_switches).

### Motivation

These counters expose kernel memory reclaim activity:

- **allocstall**: Increments when a process enters direct reclaim, blocking until memory is freed. Indicates memory pressure where kswapd cannot keep up with demand. Per-zone counters (`dma`, `dma32`, `normal`, `movable`) help distinguish zone-specific pressure from system-wide pressure. ([kernel docs](https://www.kernel.org/doc/html/latest/admin-guide/mm/concepts.html), [Red Hat on memory zones](https://www.redhat.com/en/blog/memory-zones-and-linux-driver-issue))

- **pgscan/pgsteal**: Track pages scanned vs reclaimed during memory reclaim. The ratio `pgsteal/pgscan` indicates reclaim efficiency—values below 30% suggest the kernel is scanning many pages to find few reclaimable ones. `_direct` variants measure synchronous reclaim in the allocation path; `_kswapd` variants measure background reclaim. ([kernel memory management docs](https://www.kernel.org/doc/gorman/html/understand/understand013.html))

### Describe how you validated your changes

- `go test -tags test ./pkg/collector/corechecks/system/memory/...` — all tests pass
- `TestMemoryCheckLinux` updated to verify new metrics with correct values and zone tags
- `TestSwapMemoryError` updated to handle vmstat unavailability path

### Additional Notes

- Linux-only (relies on `/proc/vmstat`)
- Respects `procfs_path` config for containerized environments
- Per-zone `allocstall_*` counters available since kernel 4.8; older kernels have a single global counter (not collected)
- No configuration required—metrics collected automatically with existing memory check
- This PR was created with the help of [Claude Code](https://github.com/anthropics/claude-code)


Co-authored-by: fabbing <fabbing@free.fr>
Co-authored-by: fabrice.buoro <fabrice.buoro@datadoghq.com>

Author: mrIncompetent

cmd/agent/dist/conf.d/memory.d/conf.yaml.default

@@ -1,2 +1,16 @@
+## This file is overwritten upon Agent upgrade.
+## To make modifications to the check configuration, copy this file
+## to `memory.yaml` and make your changes on that file.
+
+init_config:
+
 instances:
-  - {}
+  -
+
+    ## @param collect_memory_pressure - boolean - optional - default: false
+    ## Set to true to collect memory pressure metrics from /proc/vmstat
+    ## (allocstall, pgscan_direct, pgsteal_direct, pgscan_kswapd, pgsteal_kswapd).
+    ## These metrics help detect kernel memory pressure before OOM events occur.
+    ## Linux only.
+    #
+    # collect_memory_pressure: false

pkg/collector/corechecks/system/memory/memory_nix.go

@@ -8,24 +8,39 @@
 package memory
 
 import (
+	"bufio"
 	"errors"
+	"os"
 	"runtime"
+	"strconv"
+	"strings"
 
 	"github.com/shirou/gopsutil/v4/mem"
+	"gopkg.in/yaml.v2"
 
-	"github.com/DataDog/datadog-agent/pkg/util/log"
-
+	"github.com/DataDog/datadog-agent/comp/core/autodiscovery/integration"
+	"github.com/DataDog/datadog-agent/pkg/aggregator/sender"
 	core "github.com/DataDog/datadog-agent/pkg/collector/corechecks"
+	pkgconfigsetup "github.com/DataDog/datadog-agent/pkg/config/setup"
+	"github.com/DataDog/datadog-agent/pkg/util/log"
 )
 
 // For testing purpose
 var virtualMemory = mem.VirtualMemory
 var swapMemory = mem.SwapMemory
 var runtimeOS = runtime.GOOS
+var openProcFile = func(path string) (*os.File, error) {
+	return os.Open(path)
+}
+
+type memoryInstanceConfig struct {
+	CollectMemoryPressure bool `yaml:"collect_memory_pressure"`
+}
 
-// Check doesn't need additional fields
+// Check collects memory metrics
 type Check struct {
 	core.CheckBase
+	instanceConfig memoryInstanceConfig
 }
 
 const mbSize float64 = 1024 * 1024
@@ -96,6 +111,11 @@ func (c *Check) linuxSpecificVirtualMemoryCheck(v *mem.VirtualMemoryStat) error
 	sender.Gauge("system.mem.commit_limit", float64(v.CommitLimit)/mbSize, "", nil)
 	sender.Gauge("system.mem.committed_as", float64(v.CommittedAS)/mbSize, "", nil)
 	sender.Gauge("system.swap.cached", float64(v.SwapCached)/mbSize, "", nil)
+
+	if c.instanceConfig.CollectMemoryPressure {
+		c.collectVMStatPressureMetrics(sender)
+	}
+
 	return nil
 }
 
@@ -108,3 +128,78 @@ func (c *Check) freebsdSpecificVirtualMemoryCheck(v *mem.VirtualMemoryStat) erro
 	sender.Gauge("system.mem.cached", float64(v.Cached)/mbSize, "", nil)
 	return nil
 }
+
+// Configure configures the memory check
+func (c *Check) Configure(senderManager sender.SenderManager, _ uint64, rawInstance integration.Data, rawInitConfig integration.Data, source string) error {
+	err := c.CommonConfigure(senderManager, rawInitConfig, rawInstance, source)
+	if err != nil {
+		return err
+	}
+
+	s, err := c.GetSender()
+	if err != nil {
+		return err
+	}
+	s.FinalizeCheckServiceTag()
+
+	return yaml.Unmarshal(rawInstance, &c.instanceConfig)
+}
+
+func (c *Check) collectVMStatPressureMetrics(sender sender.Sender) {
+	procfsPath := "/proc"
+	if pkgconfigsetup.Datadog().IsSet("procfs_path") {
+		procfsPath = pkgconfigsetup.Datadog().GetString("procfs_path")
+	}
+
+	filePath := procfsPath + "/vmstat"
+	file, err := openProcFile(filePath)
+	if err != nil {
+		log.Debugf("memory.Check: could not open %s: %v", filePath, err)
+		return
+	}
+	defer file.Close()
+
+	allocstallByZone := make(map[string]uint64)
+	var pgscanDirect, pgstealDirect uint64
+	var pgscanKswapd, pgstealKswapd uint64
+
+	scanner := bufio.NewScanner(file)
+	for scanner.Scan() {
+		fields := strings.Fields(scanner.Text())
+		if len(fields) < 2 {
+			continue
+		}
+
+		value, err := strconv.ParseUint(fields[1], 10, 64)
+		if err != nil {
+			continue
+		}
+
+		switch {
+		case strings.HasPrefix(fields[0], "allocstall_"):
+			zoneName := strings.TrimPrefix(fields[0], "allocstall_")
+			allocstallByZone[zoneName] = value
+		case fields[0] == "pgscan_direct":
+			pgscanDirect = value
+		case fields[0] == "pgsteal_direct":
+			pgstealDirect = value
+		case fields[0] == "pgscan_kswapd":
+			pgscanKswapd = value
+		case fields[0] == "pgsteal_kswapd":
+			pgstealKswapd = value
+		}
+	}
+
+	if err := scanner.Err(); err != nil {
+		log.Debugf("memory.Check: error reading %s: %v", filePath, err)
+		return
+	}
+
+	for zoneName, value := range allocstallByZone {
+		sender.MonotonicCount("system.mem.allocstall", float64(value), "", []string{"zone:" + zoneName})
+	}
+	sender.MonotonicCount("system.mem.pgscan_direct", float64(pgscanDirect), "", nil)
+	sender.MonotonicCount("system.mem.pgsteal_direct", float64(pgstealDirect), "", nil)
+	sender.MonotonicCount("system.mem.pgscan_kswapd", float64(pgscanKswapd), "", nil)
+	sender.MonotonicCount("system.mem.pgsteal_kswapd", float64(pgstealKswapd), "", nil)
+}

pkg/collector/corechecks/system/memory/memory_nix_test.go

@@ -9,6 +9,8 @@ package memory
 
 import (
 	"errors"
+	"os"
+	"strings"
 	"testing"
 
 	"github.com/shirou/gopsutil/v4/mem"
@@ -60,6 +62,29 @@ func TestMemoryCheckLinux(t *testing.T) {
 	virtualMemory = VirtualMemory
 	swapMemory = SwapMemory
 	memCheck := new(Check)
+	memCheck.instanceConfig.CollectMemoryPressure = true
+
+	vmstatContent := `nr_free_pages 12345
+allocstall_dma 10
+allocstall_dma32 20
+allocstall_normal 30
+allocstall_movable 40
+pgscan_kswapd 1000
+pgsteal_kswapd 900
+pgscan_direct 500
+pgsteal_direct 450
+`
+	openProcFile = func(path string) (*os.File, error) {
+		if strings.HasSuffix(path, "/vmstat") {
+			r, w, _ := os.Pipe()
+			go func() {
+				w.WriteString(vmstatContent)
+				w.Close()
+			}()
+			return r, nil
+		}
+		return nil, errors.New("file not found")
+	}
 
 	mock := mocksender.NewMockSender(memCheck.ID())
 
@@ -85,14 +110,26 @@ func TestMemoryCheckLinux(t *testing.T) {
 	mock.On("Gauge", "system.swap.cached", 25000000000.0/mbSize, "", []string(nil)).Return().Times(1)
 	mock.On("Rate", "system.swap.swap_in", 21.0/mbSize, "", []string(nil)).Return().Times(1)
 	mock.On("Rate", "system.swap.swap_out", 22.0/mbSize, "", []string(nil)).Return().Times(1)
+
+	// vmstat pressure metrics with zone tags
+	mock.On("MonotonicCount", "system.mem.allocstall", float64(10), "", []string{"zone:dma"}).Return().Times(1)
+	mock.On("MonotonicCount", "system.mem.allocstall", float64(20), "", []string{"zone:dma32"}).Return().Times(1)
+	mock.On("MonotonicCount", "system.mem.allocstall", float64(30), "", []string{"zone:normal"}).Return().Times(1)
+	mock.On("MonotonicCount", "system.mem.allocstall", float64(40), "", []string{"zone:movable"}).Return().Times(1)
+	mock.On("MonotonicCount", "system.mem.pgscan_direct", float64(500), "", []string(nil)).Return().Times(1)
+	mock.On("MonotonicCount", "system.mem.pgsteal_direct", float64(450), "", []string(nil)).Return().Times(1)
+	mock.On("MonotonicCount", "system.mem.pgscan_kswapd", float64(1000), "", []string(nil)).Return().Times(1)
+	mock.On("MonotonicCount", "system.mem.pgsteal_kswapd", float64(900), "", []string(nil)).Return().Times(1)
+
 	mock.On("FinalizeCheckServiceTag").Return().Times(1)
 	mock.On("Commit").Return().Times(1)
 	memCheck.Configure(mock.GetSenderManager(), 0, nil, nil, "")
 	err := memCheck.Run()
 	require.Nil(t, err)
 
 	mock.AssertExpectations(t)
-	mock.AssertNumberOfCalls(t, "Gauge", 18)
+	mock.AssertNumberOfCalls(t, "Gauge", 18)         // Standard memory metrics
+	mock.AssertNumberOfCalls(t, "MonotonicCount", 8) // 4 allocstall zones + 4 other vmstat
 	mock.AssertNumberOfCalls(t, "Rate", 2)
 	mock.AssertNumberOfCalls(t, "Commit", 1)
 }
@@ -186,6 +223,11 @@ func TestSwapMemoryError(t *testing.T) {
 	swapMemory = func() (*mem.SwapMemoryStat, error) { return nil, errors.New("some error") }
 	memCheck := new(Check)
 
+	// Mock proc files to return errors (simulates non-Linux or unavailable)
+	openProcFile = func(_ string) (*os.File, error) {
+		return nil, errors.New("file not found")
+	}
+
 	mock := mocksender.NewMockSender(memCheck.ID())
 
 	runtimeOS = "linux"

releasenotes/notes/add-system-memory-pressure-metrics-d8beb319a56786f8.yaml

@@ -0,0 +1,16 @@
+# Each section from every release note are combined when the
+# CHANGELOG.rst is rendered. So the text needs to be worded so that
+# it does not depend on any information only available in another
+# section. This may mean repeating some details, but each section
+# must be readable independently of the other.
+#
+# Each section note must be formatted as reStructuredText.
+---
+features:
+  - |
+    The system memory check on Linux can now collect memory pressure metrics
+    from /proc/vmstat to help detect memory pressure before OOM events occur.
+    To enable, set ``collect_memory_pressure: true`` in the memory check configuration.
+    New metrics: ``system.mem.allocstall`` (with ``zone`` tag),
+    ``system.mem.pgscan_direct``, ``system.mem.pgsteal_direct``,
+    ``system.mem.pgscan_kswapd``, ``system.mem.pgsteal_kswapd``.