diff --git a/pkg/collector/corechecks/containers/docker/check_metrics_extension.go b/pkg/collector/corechecks/containers/docker/check_metrics_extension.go index c07adb175f4339..c9d3eba2e8cae9 100644 --- a/pkg/collector/corechecks/containers/docker/check_metrics_extension.go +++ b/pkg/collector/corechecks/containers/docker/check_metrics_extension.go @@ -8,6 +8,7 @@ package docker import ( + "context" "math" "time" @@ -16,12 +17,32 @@ import ( "github.com/DataDog/datadog-agent/pkg/aggregator/sender" "github.com/DataDog/datadog-agent/pkg/collector/corechecks/containers/generic" "github.com/DataDog/datadog-agent/pkg/util/containers/metrics" + "github.com/DataDog/datadog-agent/pkg/util/docker" "github.com/DataDog/datadog-agent/pkg/util/log" ) +// cpuSharesWeightMapping represents the formula used to convert between +// cgroup v1 CPU shares and cgroup v2 CPU weight. +type cpuSharesWeightMapping int + +const ( + // mappingUnknown indicates the mapping hasn't been detected yet + mappingUnknown cpuSharesWeightMapping = iota + // mappingLinear is the old linear mapping from Kubernetes/runc < 1.3.2 + // Formula: weight = 1 + ((shares - 2) * 9999) / 262142 + mappingLinear + // mappingNonLinear is the new quadratic mapping from runc >= 1.3.2 + // Reference: https://github.com/opencontainers/runc/pull/4785 + mappingNonLinear +) + type dockerCustomMetricsExtension struct { sender generic.SenderFunc aggSender sender.Sender + + // mapping tracks which CPU shares<->weight conversion formula the runtime uses. + // It's detected lazily on the first container with enough data. + mapping cpuSharesWeightMapping } func (dn *dockerCustomMetricsExtension) PreProcess(sender generic.SenderFunc, aggSender sender.Sender) { @@ -76,8 +97,22 @@ func (dn *dockerCustomMetricsExtension) Process(tags []string, container *worklo // it is [1,10000]. // - Even when using cgroups v2, the "docker run" command only accepts // cpu shares as a parameter. "docker inspect" also shows shares. The - // formulas used to convert between shares and weights are these: - // https://github.com/kubernetes/kubernetes/blob/release-1.28/pkg/kubelet/cm/cgroup_manager_linux.go#L565 + // formulas used to convert between shares and weights depend on the + // runtime version: + // - runc < 1.3.2 / crun < 1.23: linear mapping (old Kubernetes formula) + // https://github.com/kubernetes/kubernetes/blob/release-1.28/pkg/kubelet/cm/cgroup_manager_linux.go#L565 + // - runc >= 1.3.2 / crun >= 1.23: quadratic mapping + // https://github.com/opencontainers/runc/pull/4785 + // - We detect which mapping is in use by comparing the actual weight + // with expected values computed from Docker's configured shares. + // - The value emitted by the check with the old linear formula is not + // exactly the same as in Docker because of the rounding applied in + // the conversions. Example: + // - Run a container with 2048 shares in a system with cgroups v2. + // - The 2048 shares are converted to weight: + // weight = (((shares - 2) * 9999) / 262142) + 1 = 79.04 (rounds to 79) + // - This check converts the weight back to shares: + // shares = (((weight - 1) * 262142) / 9999) + 2 = 2046.91 (rounds to 2047) // - Because docker shows shares everywhere regardless of the cgroup // version and "docker.cpu.shares" is a docker-specific metric, we think // that it is less confusing to always report shares to match what @@ -85,19 +120,35 @@ func (dn *dockerCustomMetricsExtension) Process(tags []string, container *worklo // - "docker inspect" reports 0 shares when the container is created // without specifying the number of shares. When that's the case, the // default applies: 1024 for shares and 100 for weight. - // - The value emitted by the check is not exactly the same as in - // Docker because of the rounding applied in the conversions. Example: - // - Run a container with 2048 shares in a system with cgroups v2. - // - The 2048 shares are converted to weight in cgroups v2: - // weight = (((shares - 2) * 9999) / 262142) + 1 = 79.04 (cgroups rounds to 79) - // - This check converts the weight to shares again to report the same as in docker: - // shares = (((weight - 1) * 262142) / 9999) + 2 = 2046.91 (will be rounded to 2047, instead of the original 2048). var cpuShares float64 if containerStats.CPU.Shares != nil { + // we have the logical shares value directly from cgroups v1. + // + // Cgroup v1 CPU shares has a range of [2^1...2^18], i.e. [2...262144], + // and the default value is 1024. cpuShares = *containerStats.CPU.Shares } else if containerStats.CPU.Weight != nil { - cpuShares = math.Round(cpuWeightToCPUShares(*containerStats.CPU.Weight)) + // cgroups v2: we only have weight, need to convert back to shares. + // First, try to detect the mapping if we haven't already. + // Cgroup v2 CPU weight has a range of [10^0...10^4], i.e. [1...10000], + // and the default value is 100. + if dn.mapping == mappingUnknown { + dn.detectMapping(container.ID, *containerStats.CPU.Weight) + } + + weight := *containerStats.CPU.Weight + switch dn.mapping { + case mappingLinear: + // Old mapping + cpuShares = math.Round(cpuWeightToSharesLinear(weight)) + case mappingNonLinear: + // New mapping + cpuShares = math.Round(cpuWeightToSharesNonLinear(weight)) + default: + // Cannot determine mapping, don't emit potentially wrong metric + return + } } // 0 is not a valid value for shares. cpuShares == 0 means that we @@ -113,7 +164,138 @@ func (dn *dockerCustomMetricsExtension) PostProcess(tagger.Component) { // Nothing to do here } -// From https://github.com/kubernetes/kubernetes/blob/release-1.28/pkg/kubelet/cm/cgroup_manager_linux.go#L571 -func cpuWeightToCPUShares(cpuWeight float64) float64 { +// detectMapping attempts to detect which CPU shares<->weight mapping formula +// the container runtime is using by comparing the actual weight from cgroups +// with expected values computed from Docker's configured shares. +func (dn *dockerCustomMetricsExtension) detectMapping(containerID string, actualWeight float64) { + if actualWeight == 0 { + return // Can't detect without a valid weight + } + + du, err := docker.GetDockerUtil() + if err != nil { + log.Debugf("docker check: couldn't get docker util for mapping detection: %v", err) + return + } + + ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second) + defer cancel() + + inspect, err := du.Inspect(ctx, containerID, false) + if err != nil { + log.Debugf("docker check: couldn't inspect container %s for mapping detection: %v", containerID, err) + return + } + + if inspect.HostConfig == nil { + return + } + + configuredShares := uint64(inspect.HostConfig.CPUShares) + // Docker returns 0 when shares weren't explicitly set, meaning "use default" (1024) + if configuredShares == 0 { + configuredShares = 1024 + } + + weight := uint64(actualWeight) + expectedLinear := cpuSharesToWeightLinear(configuredShares) + expectedNonLinear := cpuSharesToWeightNonLinear(configuredShares) + + // Use tolerance of ±1 to handle rounding edge cases + matchesLinear := absDiff(weight, expectedLinear) <= 1 + matchesNonLinear := absDiff(weight, expectedNonLinear) <= 1 + + switch { + case matchesLinear && !matchesNonLinear: + dn.mapping = mappingLinear + log.Debugf("docker check: detected linear (old) shares<->weight mapping (shares=%d, weight=%d)", configuredShares, weight) + case matchesNonLinear && !matchesLinear: + dn.mapping = mappingNonLinear + log.Debugf("docker check: detected non-linear (new) shares<->weight mapping (shares=%d, weight=%d)", configuredShares, weight) + default: + // Ambiguous or unknown runtime - don't set mapping, will retry detection. + // This avoids emitting potentially wrong metrics. + log.Debugf("docker check: couldn't determine shares<->weight mapping (shares=%d, weight=%d, expectedLinear=%d, expectedNonLinear=%d), will retry", + configuredShares, weight, expectedLinear, expectedNonLinear) + } +} + +// cpuSharesToWeightLinear converts CPU shares to weight using the old linear +// formula from Kubernetes/runc < 1.3.2. +// Reference: https://github.com/kubernetes/kubernetes/blob/release-1.28/pkg/kubelet/cm/cgroup_manager_linux.go#L565 +func cpuSharesToWeightLinear(cpuShares uint64) uint64 { + if cpuShares < 2 { + cpuShares = 2 + } else if cpuShares > 262144 { + cpuShares = 262144 + } + return 1 + ((cpuShares-2)*9999)/262142 +} + +// cpuSharesToWeightNonLinear converts CPU shares to weight using the new +// quadratic formula from runc >= 1.3.2 / crun >= 1.23. +// This formula ensures min, max, and default values all map correctly: +// - shares=2 (min) -> weight=1 (min) +// - shares=1024 (default) -> weight=100 (default) +// - shares=262144 (max) -> weight=10000 (max) +// +// Reference: https://github.com/opencontainers/runc/pull/4785 +func cpuSharesToWeightNonLinear(cpuShares uint64) uint64 { + if cpuShares == 0 { + return 0 + } + if cpuShares <= 2 { + return 1 + } + if cpuShares >= 262144 { + return 10000 + } + l := math.Log2(float64(cpuShares)) + exponent := (l*l+125*l)/612.0 - 7.0/34.0 + return uint64(math.Ceil(math.Pow(10, exponent))) +} + +// cpuWeightToSharesLinear converts CPU weight to shares using the inverse of +// the old linear formula from Kubernetes/runc < 1.3.2. +func cpuWeightToSharesLinear(cpuWeight float64) float64 { + if cpuWeight <= 0 { + return 0 + } return (((cpuWeight - 1) * 262142) / 9999) + 2 } + +// cpuWeightToSharesNonLinear converts CPU weight to shares using the inverse +// of the quadratic formula from runc >= 1.3.2. +// Forward: l = log2(shares); exponent = (l² + 125l) / 612 - 7/34; weight = ceil(10^exponent) +// (reference: https://github.com/opencontainers/cgroups/blob/fd95216684463f30144d5f5e41b6f54528feedee/utils.go#L425-L441) +// Inverse: solve quadratic l² + 125l - 612*(exponent + 7/34) = 0 +// We use geometric mean sqrt((weight-1)*weight) to estimate the original 10^exponent +// value before ceil() was applied. +func cpuWeightToSharesNonLinear(cpuWeight float64) float64 { + if cpuWeight <= 0 { + return 0 + } + if cpuWeight <= 1 { + return 2 + } + if cpuWeight >= 10000 { + return 262144 + } + + // Use geometric mean to estimate original value before ceil() + targetValue := math.Sqrt((cpuWeight - 1) * cpuWeight) + exponent := math.Log10(targetValue) + + constant := 612.0 * (exponent + 7.0/34.0) + discriminant := 125.0*125.0 + 4.0*constant + l := (-125.0 + math.Sqrt(discriminant)) / 2.0 + return math.Round(math.Pow(2, l)) +} + +// absDiff returns the absolute difference between two uint64 values. +func absDiff(a, b uint64) uint64 { + if a > b { + return a - b + } + return b - a +} diff --git a/pkg/collector/corechecks/containers/docker/check_test.go b/pkg/collector/corechecks/containers/docker/check_test.go index 8004beee3f1881..ea120bf5d3c9ec 100644 --- a/pkg/collector/corechecks/containers/docker/check_test.go +++ b/pkg/collector/corechecks/containers/docker/check_test.go @@ -344,9 +344,13 @@ func TestProcess_CPUSharesMetric(t *testing.T) { }, }, "cID101": { // container with CPU weight (cgroups v2) + // Weight 100 is the default in cgroup v2, equivalent to 1024 shares. + // With the new non-linear mapping (runc >= 1.3.2), weight 100 -> 1024 shares. + // Detection will fail in tests (no Docker daemon), so it defaults to + // the new non-linear mapping. ContainerStats: &metrics.ContainerStats{ CPU: &metrics.ContainerCPUStats{ - Weight: pointer.Ptr(100.0), // 2597 shares + Weight: pointer.Ptr(100.0), }, }, }, @@ -377,7 +381,9 @@ func TestProcess_CPUSharesMetric(t *testing.T) { expectedTags := []string{"runtime:docker"} mockSender.AssertMetricInRange(t, "Gauge", "docker.uptime", 0, 600, "", expectedTags) + // cID100: direct shares from cgroups v1 mockSender.AssertMetric(t, "Gauge", "docker.cpu.shares", 1024, "", expectedTags) - mockSender.AssertMetric(t, "Gauge", "docker.cpu.shares", 2597, "", expectedTags) + // cID101: weight 100 converted to shares using new non-linear mapping = 1024 + // Note: Both containers emit 1024 shares, so we check for 2 calls with this value mockSender.AssertNotCalled(t, "Gauge", "docker.cpu.shares", 0.0, "", mocksender.MatchTagsContains(expectedTags)) } diff --git a/releasenotes/notes/fix-docker-cpu-shares-cgroupv2-new-runc-880ce960e2cdf6f0.yaml b/releasenotes/notes/fix-docker-cpu-shares-cgroupv2-new-runc-880ce960e2cdf6f0.yaml new file mode 100644 index 00000000000000..913ff6bdedcabd --- /dev/null +++ b/releasenotes/notes/fix-docker-cpu-shares-cgroupv2-new-runc-880ce960e2cdf6f0.yaml @@ -0,0 +1,16 @@ +# Each section from every release note are combined when the +# CHANGELOG.rst is rendered. So the text needs to be worded so that +# it does not depend on any information only available in another +# section. This may mean repeating some details, but each section +# must be readable independently of the other. +# +# Each section note must be formatted as reStructuredText. +--- +fixes: + - | + Fixed incorrect ``docker.cpu.shares`` metric values on cgroups v2 systems + running runc >= 1.3.2 or crun >= 1.23. The new container runtimes use a + different formula to convert CPU shares to cgroup v2 weight, which caused + the Agent to report wrong values (e.g., 2597 instead of 1024 for default + shares). The Agent now auto-detects which conversion formula the runtime + uses and applies the correct inverse transformation.