Add Kubernetes based Resource Monitoring (#6748)

* Draft

* Draft

* Add ResourceQuotasProvider

* Fix public API surface, remove Quotas provider component

* Switch layering of the abstractions to support Kubernetes metadata

* Fix namespace

* Remove ClusterMetadata

* Add Linux changes

* Add abstractions project

* Add changes after API Review

* Add changes after API Review pt.2

* Fixes

* Add reading memory min and low for cgroupsv2

* Add unit tests

* Fix typos

* Address PR comments and fix lint issues

* Fix flaky test

* Fix tests verified files

* Fix tests, switch to experimental

* Fix errors

* Fix more errors

* Remove TestUtilities reference

Author: amadeuszl

src/Libraries/Microsoft.Extensions.Diagnostics.ResourceMonitoring.Kubernetes/KubernetesMetadata.cs

@@ -0,0 +1,57 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+
+using System;
+using System.Globalization;
+
+namespace Microsoft.Extensions.Diagnostics.ResourceMonitoring.Kubernetes;
+
+internal sealed class KubernetesMetadata
+{
+    /// <summary>
+    /// Gets or sets the resource memory limit the container is allowed to use in bytes.
+    /// </summary>
+    public ulong LimitsMemory { get; set; }
+
+    /// <summary>
+    /// Gets or sets the resource CPU limit the container is allowed to use in millicores.
+    /// </summary>
+    public ulong LimitsCpu { get; set; }
+
+    /// <summary>
+    /// Gets or sets the resource memory request the container is allowed to use in bytes.
+    /// </summary>
+    public ulong RequestsMemory { get; set; }
+
+    /// <summary>
+    /// Gets or sets the resource CPU request the container is allowed to use in millicores.
+    /// </summary>
+    public ulong RequestsCpu { get; set; }
+
+    public static KubernetesMetadata FromEnvironmentVariables(string environmentVariablePrefix)
+    {
+        return new KubernetesMetadata
+        {
+            LimitsMemory = GetEnvironmentVariableAsUInt64($"{environmentVariablePrefix}LIMITS_MEMORY"),
+            LimitsCpu = GetEnvironmentVariableAsUInt64($"{environmentVariablePrefix}LIMITS_CPU"),
+            RequestsMemory = GetEnvironmentVariableAsUInt64($"{environmentVariablePrefix}REQUESTS_MEMORY"),
+            RequestsCpu = GetEnvironmentVariableAsUInt64($"{environmentVariablePrefix}REQUESTS_CPU"),
+        };
+    }
+
+    private static ulong GetEnvironmentVariableAsUInt64(string variableName)
+    {
+        var value = Environment.GetEnvironmentVariable(variableName);
+        if (string.IsNullOrWhiteSpace(value))
+        {
+            return 0;
+        }
+
+        if (!ulong.TryParse(value, NumberStyles.None, CultureInfo.InvariantCulture, out ulong result))
+        {
+            throw new InvalidOperationException($"Environment variable '{variableName}' contains invalid value '{value}'. Expected a non-negative integer.");
+        }
+
+        return result;
+    }
+}

src/Libraries/Microsoft.Extensions.Diagnostics.ResourceMonitoring.Kubernetes/KubernetesResourceQuotaProvider.cs

@@ -0,0 +1,46 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+
+using Microsoft.Shared.Diagnostics;
+
+namespace Microsoft.Extensions.Diagnostics.ResourceMonitoring.Kubernetes;
+
+internal sealed class KubernetesResourceQuotaProvider : ResourceQuotaProvider
+{
+    private const double MillicoresPerCore = 1000.0;
+    private KubernetesMetadata _kubernetesMetadata;
+
+    public KubernetesResourceQuotaProvider(KubernetesMetadata kubernetesMetadata)
+    {
+        _ = Throw.IfNull(kubernetesMetadata);
+        _kubernetesMetadata = kubernetesMetadata;
+    }
+
+    public override ResourceQuota GetResourceQuota()
+    {
+        ResourceQuota quota = new()
+        {
+            BaselineCpuInCores = ConvertMillicoreToCpuUnit(_kubernetesMetadata.RequestsCpu),
+            MaxCpuInCores = ConvertMillicoreToCpuUnit(_kubernetesMetadata.LimitsCpu),
+            BaselineMemoryInBytes = _kubernetesMetadata.RequestsMemory,
+            MaxMemoryInBytes = _kubernetesMetadata.LimitsMemory,
+        };
+
+        if (quota.BaselineCpuInCores <= 0.0)
+        {
+            quota.BaselineCpuInCores = quota.MaxCpuInCores;
+        }
+
+        if (quota.BaselineMemoryInBytes == 0)
+        {
+            quota.BaselineMemoryInBytes = quota.MaxMemoryInBytes;
+        }
+
+        return quota;
+    }
+
+    private static double ConvertMillicoreToCpuUnit(ulong millicores)
+    {
+        return millicores / MillicoresPerCore;
+    }
+}

src/Libraries/Microsoft.Extensions.Diagnostics.ResourceMonitoring.Kubernetes/KubernetesResourceQuotaServiceCollectionExtensions.cs

@@ -0,0 +1,45 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+
+using Microsoft.Extensions.DependencyInjection.Extensions;
+using Microsoft.Extensions.Diagnostics.ResourceMonitoring;
+using Microsoft.Extensions.Diagnostics.ResourceMonitoring.Kubernetes;
+
+namespace Microsoft.Extensions.DependencyInjection;
+
+/// <summary>
+/// Lets you configure and register Kubernetes resource monitoring components.
+/// </summary>
+public static class KubernetesResourceQuotaServiceCollectionExtensions
+{
+    /// <summary>
+    /// Configures and adds an Kubernetes resource monitoring components to a service collection altogether with necessary basic resource monitoring components.
+    /// </summary>
+    /// <param name="services">The dependency injection container to add the Kubernetes resource monitoring to.</param>
+    /// <param name="environmentVariablePrefix">Optional value of prefix used to read environment variables in the container.</param>
+    /// <returns>The value of <paramref name="services" />.</returns>
+    /// <remarks>
+    /// <para>
+    /// If you have configured your Kubernetes container with Downward API to add environment variable <c>MYCLUSTER_LIMITS_CPU</c> with CPU limits,
+    /// then you should pass <c>MYCLUSTER_</c> to <paramref name="environmentVariablePrefix"/> parameter. Environment variables will be read during DI Container resolution.
+    /// </para>
+    /// <para>
+    /// <strong>Important:</strong> Do not call <see cref="ResourceMonitoringServiceCollectionExtensions.AddResourceMonitoring(IServiceCollection)"/> 
+    /// if you are using this method, as it already includes all necessary resource monitoring components and registers a Kubernetes-specific 
+    /// <see cref="ResourceQuotaProvider"/> implementation. Calling both methods may result in conflicting service registrations.
+    /// </para>
+    /// </remarks>
+    public static IServiceCollection AddKubernetesResourceMonitoring(
+        this IServiceCollection services,
+        string? environmentVariablePrefix = default)
+    {
+        services.TryAddSingleton<ResourceQuotaProvider>(sp =>
+        {
+            return new KubernetesResourceQuotaProvider(KubernetesMetadata.FromEnvironmentVariables(environmentVariablePrefix ?? string.Empty));
+        });
+
+        _ = services.AddResourceMonitoring();
+
+        return services;
+    }
+}

src/Libraries/Microsoft.Extensions.Diagnostics.ResourceMonitoring.Kubernetes/Microsoft.Extensions.Diagnostics.ResourceMonitoring.Kubernetes.csproj

@@ -0,0 +1,29 @@
+<Project Sdk="Microsoft.NET.Sdk">
+  <PropertyGroup>
+    <RootNamespace>Microsoft.Extensions.Diagnostics.ResourceMonitoring.Kubernetes</RootNamespace>
+    <Description>Provides Kubernetes data for measurements of processor and memory usage.</Description>
+    <Workstream>ResourceMonitoring</Workstream>
+    <NoWarn Condition="'$(TargetFramework)' == 'net462'">$(NoWarn);CS0436</NoWarn>
+  </PropertyGroup>
+
+  <PropertyGroup>
+    <InjectExperimentalAttributeOnLegacy>true</InjectExperimentalAttributeOnLegacy>
+    <InjectSharedInstruments>true</InjectSharedInstruments>
+  </PropertyGroup>
+
+  <PropertyGroup>
+    <Stage>dev</Stage>
+    <StageDevDiagnosticId>EXTEXP0016</StageDevDiagnosticId>
+    <MinCodeCoverage>99</MinCodeCoverage>
+    <MinMutationScore>90</MinMutationScore>
+  </PropertyGroup>
+
+  <ItemGroup>
+    <ProjectReference Include="..\Microsoft.Extensions.Diagnostics.ResourceMonitoring\Microsoft.Extensions.Diagnostics.ResourceMonitoring.csproj" />
+  </ItemGroup>
+
+  <ItemGroup>
+    <InternalsVisibleToDynamicProxyGenAssembly2 Include="*" />
+    <InternalsVisibleToTest Include="$(AssemblyName).Tests" />
+  </ItemGroup>
+</Project>

src/Libraries/Microsoft.Extensions.Diagnostics.ResourceMonitoring.Kubernetes/Microsoft.Extensions.Diagnostics.ResourceMonitoring.Kubernetes.json

@@ -0,0 +1,24 @@
+# Microsoft.Extensions.Diagnostics.ResourceMonitoring.Kubernetes
+
+Registers `ResourceQuota` implementation specific to Kubernetes.
+
+## Install the package
+
+From the command-line:
+
+```console
+dotnet add package Microsoft.Extensions.Diagnostics.ResourceMonitoring.Kubernetes
+```
+
+Or directly in the C# project file:
+
+```xml
+<ItemGroup>
+  <PackageReference Include="Microsoft.Extensions.Diagnostics.ResourceMonitoring.Kubernetes" Version="[CURRENTVERSION]" />
+</ItemGroup>
+```
+
+
+## Feedback & Contributing
+
+We welcome feedback and contributions in [our GitHub repo](https://github.com/dotnet/extensions).

src/Libraries/Microsoft.Extensions.Diagnostics.ResourceMonitoring.Kubernetes/README.md

@@ -95,4 +95,10 @@ internal interface ILinuxUtilizationParser
     /// </summary>
     /// <returns>The number of CPU periods.</returns>
     long GetCgroupPeriodsIntervalInMicroSecondsV2();
+
+    /// <summary>
+    /// For CgroupV2 only. Reads the file /sys/fs/cgroup/memory.min, if 0 reads the file /sys/fs/cgroup/memory.low.
+    /// </summary>
+    /// <returns>The minimum memory allocated in bytes, or 0 if not available.</returns>
+    ulong GetMinMemoryInBytes();
 }

src/Libraries/Microsoft.Extensions.Diagnostics.ResourceMonitoring/Linux/ILinuxUtilizationParser.cs

@@ -0,0 +1,44 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+
+using Microsoft.Extensions.Options;
+
+namespace Microsoft.Extensions.Diagnostics.ResourceMonitoring.Linux;
+
+internal sealed class LinuxResourceQuotaProvider : ResourceQuotaProvider
+{
+    private readonly ILinuxUtilizationParser _parser;
+    private bool _useLinuxCalculationV2;
+
+    public LinuxResourceQuotaProvider(ILinuxUtilizationParser parser, IOptions<ResourceMonitoringOptions> options)
+    {
+        _parser = parser;
+        _useLinuxCalculationV2 = options.Value.UseLinuxCalculationV2;
+    }
+
+    public override ResourceQuota GetResourceQuota()
+    {
+        var resourceQuota = new ResourceQuota();
+        if (_useLinuxCalculationV2)
+        {
+            resourceQuota.MaxCpuInCores = _parser.GetCgroupLimitV2();
+            resourceQuota.BaselineCpuInCores = _parser.GetCgroupRequestCpuV2();
+        }
+        else
+        {
+            resourceQuota.MaxCpuInCores = _parser.GetCgroupLimitedCpus();
+            resourceQuota.BaselineCpuInCores = _parser.GetCgroupRequestCpu();
+        }
+
+        resourceQuota.MaxMemoryInBytes = _parser.GetAvailableMemoryInBytes();
+        resourceQuota.BaselineMemoryInBytes = _parser.GetMinMemoryInBytes();
+
+        if (resourceQuota.BaselineMemoryInBytes == 0)
+        {
+            resourceQuota.BaselineMemoryInBytes = resourceQuota.MaxMemoryInBytes;
+        }
+
+        return resourceQuota;
+    }
+}
+

src/Libraries/Microsoft.Extensions.Diagnostics.ResourceMonitoring/Linux/LinuxResourceQuotaProvider.cs

@@ -368,6 +368,12 @@ static void ThrowException(ReadOnlySpan<char> content) =>
                 $"Could not parse '{_cpuSetCpus}'. Expected comma-separated list of integers, with dashes (\"-\") based ranges (\"0\", \"2-6,12\") but got '{new string(content)}'.");
     }
 
+    /// <summary>
+    /// In CgroupV1 there's no equivalent to memory.min or memory.low files.
+    /// </summary>
+    /// <returns>0.</returns>
+    public ulong GetMinMemoryInBytes() => 0;
+
     /// <remarks>
     /// The input must contain only number. If there is something more than whitespace before the number, it will return failure (-1).
     /// </remarks>

src/Libraries/Microsoft.Extensions.Diagnostics.ResourceMonitoring/Linux/LinuxUtilizationParserCgroupV1.cs

@@ -23,6 +23,8 @@ internal sealed class LinuxUtilizationParserCgroupV2 : ILinuxUtilizationParser
     private const string CpuStat = "cpu.stat"; // File containing CPU usage in nanoseconds.
     private const string CpuLimit = "cpu.max"; // File with amount of CPU time available to the group along with the accounting period in microseconds.
     private const string CpuRequest = "cpu.weight"; // CPU weights, also known as shares in cgroup v1, is used for resource allocation.
+    private const string MemoryMin = "memory.min"; // File contains min memory, set for QoS by K8s
+    private const string MemoryLow = "memory.low"; // File contains min memory, set if there was memory reservation for container
     private static readonly ObjectPool<BufferWriter<char>> _sharedBufferWriterPool = BufferWriterPool.CreateBufferWriterPool<char>();
 
     /// <remarks>
@@ -513,6 +515,50 @@ static void ThrowException(ReadOnlySpan<char> content) =>
                 $"Could not parse '{_cpuSetCpus}'. Expected comma-separated list of integers, with dashes (\"-\") based ranges (\"0\", \"2-6,12\") but got '{new string(content)}'.");
     }
 
+    public ulong GetMinMemoryInBytes()
+    {
+        FileInfo memoryMinFile = new(GetCgroupPath(MemoryMin));
+        if (_fileSystem.Exists(memoryMinFile))
+        {
+            using ReturnableBufferWriter<char> bufferWriter = new(_sharedBufferWriterPool);
+            _fileSystem.ReadAll(memoryMinFile, bufferWriter.Buffer);
+
+            ReadOnlySpan<char> memoryMinBuffer = bufferWriter.Buffer.WrittenSpan;
+
+            _ = GetNextNumber(memoryMinBuffer, out long memoryMin);
+
+            if (memoryMin == -1)
+            {
+                Throw.InvalidOperationException($"Could not parse '{memoryMinFile}' content. Expected to find memory minimum in bytes but got '{new string(memoryMinBuffer)}' instead.");
+            }
+
+            if (memoryMin != 0)
+            {
+                return (ulong)memoryMin;
+            }
+        }
+
+        FileInfo memoryLowFile = new(GetCgroupPath(MemoryLow));
+        if (_fileSystem.Exists(memoryLowFile))
+        {
+            using ReturnableBufferWriter<char> bufferWriter = new(_sharedBufferWriterPool);
+            _fileSystem.ReadAll(memoryLowFile, bufferWriter.Buffer);
+
+            ReadOnlySpan<char> memoryLowBuffer = bufferWriter.Buffer.WrittenSpan;
+
+            _ = GetNextNumber(memoryLowBuffer, out long memoryLow);
+
+            if (memoryLow == -1)
+            {
+                Throw.InvalidOperationException($"Could not parse '{memoryLowFile}' content. Expected to find memory low in bytes but got '{new string(memoryLowBuffer)}' instead.");
+            }
+
+            return (ulong)memoryLow;
+        }
+
+        return 0;
+    }
+
     private static (long cpuUsageNanoseconds, long nrPeriods) ParseCpuUsageFromFile(IFileSystem fileSystem, FileInfo cpuUsageFile)
     {
         // The values we are interested in start with these prefixes