diff --git a/cgroup/cpu.go b/cgroup/cpu.go index 00a2ba7..833c25d 100644 --- a/cgroup/cpu.go +++ b/cgroup/cpu.go @@ -16,7 +16,7 @@ type CPUStat struct { LimitCores float64 } -func (cg Cgroup) CpuStat() *CPUStat { +func (cg *Cgroup) CpuStat() *CPUStat { cpu, cpuacct := cg.subsystems["cpu"], cg.subsystems["cpuacct"] if cpu == "" || cpuacct == "" { st, _ := cg.cpuStatV2() @@ -26,7 +26,7 @@ func (cg Cgroup) CpuStat() *CPUStat { return st } -func (cg Cgroup) cpuStatV1() (*CPUStat, error) { +func (cg *Cgroup) cpuStatV1() (*CPUStat, error) { if cg.subsystems["cpu"] == "" || cg.subsystems["cpuacct"] == "" { return nil, nil } @@ -56,7 +56,7 @@ func (cg Cgroup) cpuStatV1() (*CPUStat, error) { return res, nil } -func (cg Cgroup) cpuStatV2() (*CPUStat, error) { +func (cg *Cgroup) cpuStatV2() (*CPUStat, error) { if cg.subsystems[""] == "" { return nil, nil } diff --git a/cgroup/fixtures/cgroup/kubepods.slice/kubepods-besteffort.slice/kubepods-besteffort-pod8712f785_1a3e_41ec_a00b_e2dcc77431cb.slice/docker-73051af271105c07e1f493b34856a77e665e3b0b4fc72f76c807dfbffeb881bd.scope/cpu.pressure b/cgroup/fixtures/cgroup/kubepods.slice/kubepods-besteffort.slice/kubepods-besteffort-pod8712f785_1a3e_41ec_a00b_e2dcc77431cb.slice/docker-73051af271105c07e1f493b34856a77e665e3b0b4fc72f76c807dfbffeb881bd.scope/cpu.pressure new file mode 100644 index 0000000..989830a --- /dev/null +++ b/cgroup/fixtures/cgroup/kubepods.slice/kubepods-besteffort.slice/kubepods-besteffort-pod8712f785_1a3e_41ec_a00b_e2dcc77431cb.slice/docker-73051af271105c07e1f493b34856a77e665e3b0b4fc72f76c807dfbffeb881bd.scope/cpu.pressure @@ -0,0 +1,2 @@ +some avg10=0.00 avg60=0.00 avg300=0.00 total=465907442 +full avg10=0.00 avg60=0.00 avg300=0.00 total=463529433 diff --git a/cgroup/fixtures/cgroup/kubepods.slice/kubepods-besteffort.slice/kubepods-besteffort-pod8712f785_1a3e_41ec_a00b_e2dcc77431cb.slice/docker-73051af271105c07e1f493b34856a77e665e3b0b4fc72f76c807dfbffeb881bd.scope/io.pressure b/cgroup/fixtures/cgroup/kubepods.slice/kubepods-besteffort.slice/kubepods-besteffort-pod8712f785_1a3e_41ec_a00b_e2dcc77431cb.slice/docker-73051af271105c07e1f493b34856a77e665e3b0b4fc72f76c807dfbffeb881bd.scope/io.pressure new file mode 100644 index 0000000..23bb276 --- /dev/null +++ b/cgroup/fixtures/cgroup/kubepods.slice/kubepods-besteffort.slice/kubepods-besteffort-pod8712f785_1a3e_41ec_a00b_e2dcc77431cb.slice/docker-73051af271105c07e1f493b34856a77e665e3b0b4fc72f76c807dfbffeb881bd.scope/io.pressure @@ -0,0 +1,2 @@ +some avg10=0.00 avg60=0.00 avg300=0.05 total=17657662684 +full avg10=0.00 avg60=0.00 avg300=0.05 total=17636951020 diff --git a/cgroup/fixtures/cgroup/kubepods.slice/kubepods-besteffort.slice/kubepods-besteffort-pod8712f785_1a3e_41ec_a00b_e2dcc77431cb.slice/docker-73051af271105c07e1f493b34856a77e665e3b0b4fc72f76c807dfbffeb881bd.scope/memory.pressure b/cgroup/fixtures/cgroup/kubepods.slice/kubepods-besteffort.slice/kubepods-besteffort-pod8712f785_1a3e_41ec_a00b_e2dcc77431cb.slice/docker-73051af271105c07e1f493b34856a77e665e3b0b4fc72f76c807dfbffeb881bd.scope/memory.pressure new file mode 100644 index 0000000..fdf1ef4 --- /dev/null +++ b/cgroup/fixtures/cgroup/kubepods.slice/kubepods-besteffort.slice/kubepods-besteffort-pod8712f785_1a3e_41ec_a00b_e2dcc77431cb.slice/docker-73051af271105c07e1f493b34856a77e665e3b0b4fc72f76c807dfbffeb881bd.scope/memory.pressure @@ -0,0 +1,2 @@ +some avg10=0.00 avg60=0.00 avg300=0.00 total=6937313991 +full avg10=0.00 avg60=0.00 avg300=0.00 total=6934649214 diff --git a/cgroup/psi.go b/cgroup/psi.go new file mode 100644 index 0000000..c36d60f --- /dev/null +++ b/cgroup/psi.go @@ -0,0 +1,85 @@ +package cgroup + +import ( + "os" + "path" + "strconv" + "strings" + + "github.com/coroot/coroot-node-agent/common" + "k8s.io/klog/v2" +) + +type PSIStats struct { + CPUSecondsSome float64 + CPUSecondsFull float64 + MemorySecondsSome float64 + MemorySecondsFull float64 + IOSecondsSome float64 + IOSecondsFull float64 +} + +type PressureTotals struct { + SomeSecondsTotal float64 + FullSecondsTotal float64 +} + +func (cg *Cgroup) PSI() *PSIStats { + if cg.subsystems[""] == "" { + return nil + } + stats := &PSIStats{} + for _, controller := range []string{"cpu", "memory", "io"} { + p, err := cg.readPressure(controller) + if err != nil { + if !common.IsNotExist(err) { + klog.Warningln(err) + } + return nil + } + switch controller { + case "cpu": + stats.CPUSecondsSome = p.SomeSecondsTotal + stats.CPUSecondsFull = p.FullSecondsTotal + case "memory": + stats.MemorySecondsSome = p.SomeSecondsTotal + stats.MemorySecondsFull = p.FullSecondsTotal + case "io": + stats.IOSecondsSome = p.SomeSecondsTotal + stats.IOSecondsFull = p.FullSecondsTotal + } + } + return stats +} + +func (cg *Cgroup) readPressure(controller string) (*PressureTotals, error) { + data, err := os.ReadFile(path.Join(cg2Root, cg.subsystems[""], controller+".pressure")) + if err != nil { + return nil, err + } + pressure := &PressureTotals{} + for _, line := range strings.Split(strings.TrimSpace(string(data)), "\n") { + parts := strings.Fields(line) + if len(parts) == 0 { + continue + } + kind := parts[0] + for _, p := range parts[1:] { + if strings.HasPrefix(p, "total=") { + vStr := strings.TrimPrefix(p, "total=") + v, err := strconv.ParseUint(vStr, 10, 64) + if err != nil { + return nil, err + } + switch kind { + case "some": + pressure.SomeSecondsTotal = float64(v) / 1e6 // microseconds to seconds + case "full": + pressure.FullSecondsTotal = float64(v) / 1e6 + } + break + } + } + } + return pressure, nil +} diff --git a/cgroup/psi_test.go b/cgroup/psi_test.go new file mode 100644 index 0000000..51f627c --- /dev/null +++ b/cgroup/psi_test.go @@ -0,0 +1,27 @@ +package cgroup + +import ( + "path" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestCgroupPSI(t *testing.T) { + cgRoot = "fixtures/cgroup" + cg2Root = "fixtures/cgroup" + + cg, _ := NewFromProcessCgroupFile(path.Join("fixtures/proc/400/cgroup")) + stat := cg.PSI() + require.NotNil(t, stat) + assert.Equal(t, float64(465907442)/1e6, stat.CPUSecondsSome) + assert.Equal(t, float64(463529433)/1e6, stat.CPUSecondsFull) + assert.Equal(t, float64(6937313991)/1e6, stat.MemorySecondsSome) + assert.Equal(t, float64(6934649214)/1e6, stat.MemorySecondsFull) + assert.Equal(t, float64(17657662684)/1e6, stat.IOSecondsSome) + assert.Equal(t, float64(17636951020)/1e6, stat.IOSecondsFull) + + cg, _ = NewFromProcessCgroupFile(path.Join("fixtures/proc/100/cgroup")) + assert.Nil(t, cg.PSI()) +} diff --git a/containers/container.go b/containers/container.go index aa1a3d3..e0abf60 100644 --- a/containers/container.go +++ b/containers/container.go @@ -265,6 +265,15 @@ func (c *Container) Collect(ch chan<- prometheus.Metric) { } } + if psi := c.cgroup.PSI(); psi != nil { + ch <- counter(metrics.PsiCPU, psi.CPUSecondsSome, "some") + ch <- counter(metrics.PsiCPU, psi.CPUSecondsFull, "full") + ch <- counter(metrics.PsiMemory, psi.MemorySecondsSome, "some") + ch <- counter(metrics.PsiMemory, psi.MemorySecondsFull, "full") + ch <- counter(metrics.PsiIO, psi.IOSecondsSome, "some") + ch <- counter(metrics.PsiIO, psi.IOSecondsFull, "full") + } + if c.oomKills > 0 { ch <- counter(metrics.OOMKills, float64(c.oomKills)) } diff --git a/containers/metrics.go b/containers/metrics.go index 7647c98..4612314 100644 --- a/containers/metrics.go +++ b/containers/metrics.go @@ -19,6 +19,10 @@ var metrics = struct { MemoryCache *prometheus.Desc OOMKills *prometheus.Desc + PsiCPU *prometheus.Desc + PsiMemory *prometheus.Desc + PsiIO *prometheus.Desc + DiskDelay *prometheus.Desc DiskSize *prometheus.Desc DiskUsed *prometheus.Desc @@ -71,6 +75,10 @@ var metrics = struct { MemoryCache: metric("container_resources_memory_cache_bytes", "Amount of page cache memory allocated by the container"), OOMKills: metric("container_oom_kills_total", "Total number of times the container was terminated by the OOM killer"), + PsiCPU: metric("container_resources_cpu_pressure_waiting_seconds_total", "Total time in seconds tha the container were delayed due to CPU pressure", "kind"), + PsiMemory: metric("container_resources_memory_pressure_waiting_seconds_total", "Total time in seconds that the container were delayed due to memory pressure", "kind"), + PsiIO: metric("container_resources_io_pressure_waiting_seconds_total", "Total time in seconds that the container were delayed due to I/O pressure", "kind"), + DiskDelay: metric("container_resources_disk_delay_seconds_total", "Total time duration processes of the container have been waiting fot I/Os to complete"), DiskSize: metric("container_resources_disk_size_bytes", "Total capacity of the volume", "mount_point", "device", "volume"), DiskUsed: metric("container_resources_disk_used_bytes", "Used capacity of the volume", "mount_point", "device", "volume"),