4.9.300 Hotfix (#11004)

* Update version to 4.1039.300

* Clear release notes

* 4.9.300 Hotfix (#11001)

* Enhancement to memory metrics reporting (#10984)

* Improved memory metrics reporting using CGroup data for Linux consumption.

* Minor logging improvements.

* Removed OS check as it is not needed since this publisher will be enabled only for Linux.

* Log only when IsCGroupMemoryMetricsEnabled property value changes.

* Update LinuxContainerLegionMetricsPublisher to use CGroup memory metrics

* Reordering the location of property before method, to fix SA 1201 Stylecop warning.

* Fixes circular dependency when resolving LinuxContainerLegionMetricsPublisher (#10991)

* Avoid circular dependency when resolving LinuxContainerLegionMetricsPublisher.

* Revert launchsettings.json chane.

* Adding a placeholder mode host startup test for Linux consumption on legion

* Update test to verify SKU specific services are resolved from DI container.

* Resolve conflict in release notes

* Switch to Type check instead of type name check, in the test.

* release notes insanity fix.

---------

Co-authored-by: Azure Functions Release <[email protected]>
Co-authored-by: Shyju Krishnankutty <[email protected]>

Author: 3 people

release_notes.md

@@ -3,4 +3,5 @@
 <!-- Please add your release notes in the following format:
 - My change description (#PR)
 -->
-- Add JitTrace Files for v4.1039
+- Improved memory metrics reporting using CGroup data for Linux consumption (#10968)
+- Avoid circular dependency when resolving LinuxContainerLegionMetricsPublisher. (#10991)

src/Directory.Version.props

@@ -1,6 +1,6 @@
 <Project>
   <PropertyGroup>
-    <VersionPrefix>4.1039.200</VersionPrefix>
+    <VersionPrefix>4.1039.300</VersionPrefix>
     <UpdateBuildNumber>true</UpdateBuildNumber>
   </PropertyGroup>
 </Project>

src/WebJobs.Script.WebHost/Metrics/CgroupMemoryUsageHelper.cs

@@ -0,0 +1,51 @@
+// Copyright (c) .NET Foundation. All rights reserved.
+// Licensed under the MIT License. See License.txt in the project root for license information.
+
+using System;
+using System.IO;
+using Microsoft.Extensions.Logging;
+
+namespace Microsoft.Azure.WebJobs.Script.WebHost.Metrics
+{
+    internal static class CgroupMemoryUsageHelper
+    {
+        private const string CgroupPathV1 = "/sys/fs/cgroup/memory/memory.usage_in_bytes";
+        private const string CgroupPathV2 = "/sys/fs/cgroup/memory.current";
+
+        /// <summary>
+        /// Retrieves the memory usage of the control group in bytes, supporting both cgroup v1 and v2.
+        /// </summary>
+        /// <param name="logger">An <see cref="ILogger{TCategoryName}"/> instance for logging.</param>
+        /// <returns>The memory usage in bytes if available; otherwise, 0.</returns>
+        internal static long GetMemoryUsageInBytes(ILogger logger)
+        {
+            try
+            {
+                if (TryReadMemoryUsage(CgroupPathV2, out var usageInBytes) || TryReadMemoryUsage(CgroupPathV1, out usageInBytes))
+                {
+                    return usageInBytes;
+                }
+
+                logger.LogWarning("Memory usage not available from either control group v1 or v2.");
+                return 0;
+            }
+            catch (Exception ex)
+            {
+                logger.LogError(ex, "Error reading control group resource usage.");
+                return 0;
+            }
+        }
+
+        private static bool TryReadMemoryUsage(string path, out long memoryUsageInBytes)
+        {
+            memoryUsageInBytes = 0;
+
+            if (!File.Exists(path))
+            {
+                return false;
+            }
+
+            return long.TryParse(File.ReadAllText(path), out memoryUsageInBytes);
+        }
+    }
+}

src/WebJobs.Script.WebHost/Metrics/LinuxContainerLegionMetricsPublisher.cs

@@ -7,8 +7,10 @@
 using System.Threading;
 using System.Threading.Tasks;
 using Microsoft.Azure.Functions.Platform.Metrics.LinuxConsumption;
+using Microsoft.Azure.WebJobs.Script.Config;
 using Microsoft.Azure.WebJobs.Script.Diagnostics;
 using Microsoft.Azure.WebJobs.Script.WebHost.Configuration;
+using Microsoft.Extensions.DependencyInjection;
 using Microsoft.Extensions.Logging;
 using Microsoft.Extensions.Options;
 
@@ -21,29 +23,34 @@ public sealed class LinuxContainerLegionMetricsPublisher : IMetricsPublisher, ID
         private readonly TimeSpan _memorySnapshotInterval = TimeSpan.FromMilliseconds(1000);
         private readonly TimeSpan _timerStartDelay = TimeSpan.FromSeconds(2);
         private readonly IOptionsMonitor<StandbyOptions> _standbyOptions;
-        private readonly IDisposable _standbyOptionsOnChangeSubscription;
+        private readonly IDisposable _standbyOptionsOnChangeListener;
+        private readonly IDisposable _hostingConfigOptionsOnChangeListener;
         private readonly IEnvironment _environment;
         private readonly ILogger _logger;
-        private readonly IScriptHostManager _scriptHostManager;
+        private readonly IServiceProvider _serviceProvider;
         private readonly string _containerName;
+        private readonly IOptionsMonitor<FunctionsHostingConfigOptions> _hostingConfigOptions;
 
         private IMetricsLogger _metricsLogger;
         private TimeSpan _metricPublishInterval;
         private Process _process;
         private Timer _processMonitorTimer;
         private Timer _metricsPublisherTimer;
         private bool _initialized = false;
+        private bool _isCGroupMemoryMetricsEnabled = false;
 
         public LinuxContainerLegionMetricsPublisher(IEnvironment environment, IOptionsMonitor<StandbyOptions> standbyOptions,
             IOptions<LinuxConsumptionLegionMetricsPublisherOptions> options, ILogger<LinuxContainerLegionMetricsPublisher> logger,
-            IFileSystem fileSystem, ILinuxConsumptionMetricsTracker metricsTracker, IScriptHostManager scriptHostManager,
+            IFileSystem fileSystem, ILinuxConsumptionMetricsTracker metricsTracker, IServiceProvider serviceProvider,
+            IOptionsMonitor<FunctionsHostingConfigOptions> functionsHostingConfigOptions,
             int? metricsPublishIntervalMS = null)
         {
             _standbyOptions = standbyOptions ?? throw new ArgumentNullException(nameof(standbyOptions));
             _environment = environment ?? throw new ArgumentNullException(nameof(environment));
             _logger = logger ?? throw new ArgumentNullException(nameof(logger));
             _metricsTracker = metricsTracker ?? throw new ArgumentNullException(nameof(metricsTracker));
-            _scriptHostManager = scriptHostManager ?? throw new ArgumentNullException(nameof(scriptHostManager));
+            _serviceProvider = serviceProvider ?? throw new ArgumentNullException(nameof(serviceProvider));
+            _hostingConfigOptions = functionsHostingConfigOptions ?? throw new ArgumentNullException(nameof(functionsHostingConfigOptions));
             _containerName = options.Value.ContainerName;
 
             // Set this to 15 minutes worth of files
@@ -59,27 +66,24 @@ public LinuxContainerLegionMetricsPublisher(IEnvironment environment, IOptionsMo
 
             if (_standbyOptions.CurrentValue.InStandbyMode)
             {
-                _standbyOptionsOnChangeSubscription = _standbyOptions.OnChange(o => OnStandbyOptionsChange(o));
+                _standbyOptionsOnChangeListener = _standbyOptions.OnChange(o => OnStandbyOptionsChange(o));
             }
             else
             {
                 Start();
             }
+
+            _hostingConfigOptionsOnChangeListener = _hostingConfigOptions.OnChange(OnHostingConfigOptionsChanged);
         }
 
-        public IMetricsLogger MetricsLogger
+        private IMetricsLogger MetricsLogger => _metricsLogger ??= _serviceProvider.GetRequiredService<IMetricsLogger>();
+
+        private void OnHostingConfigOptionsChanged(FunctionsHostingConfigOptions newOptions)
         {
-            get
+            if (newOptions.IsCGroupMemoryMetricsEnabled != _isCGroupMemoryMetricsEnabled)
             {
-                if (_metricsLogger == null)
-                {
-                    if (!Utility.TryGetHostService<IMetricsLogger>(_scriptHostManager, out _metricsLogger))
-                    {
-                        throw new InvalidOperationException($"Unable to resolve {nameof(IMetricsLogger)} service.");
-                    }
-                }
-
-                return _metricsLogger;
+                _logger.LogInformation("CGroup memory metrics enabled: {Enabled}", newOptions.IsCGroupMemoryMetricsEnabled);
+                _isCGroupMemoryMetricsEnabled = newOptions.IsCGroupMemoryMetricsEnabled;
             }
         }
 
@@ -185,11 +189,20 @@ private void OnProcessMonitorTimer(object state)
         {
             try
             {
-                _process.Refresh();
-                var commitSizeBytes = _process.WorkingSet64;
-                if (commitSizeBytes != 0)
+                long memoryUsageInBytes;
+                if (_hostingConfigOptions.CurrentValue.IsCGroupMemoryMetricsEnabled)
+                {
+                    memoryUsageInBytes = CgroupMemoryUsageHelper.GetMemoryUsageInBytes(_logger);
+                }
+                else
+                {
+                    _process.Refresh();
+                    memoryUsageInBytes = _process.WorkingSet64;
+                }
+
+                if (memoryUsageInBytes != 0)
                 {
-                    AddMemoryActivity(DateTime.UtcNow, commitSizeBytes);
+                    AddMemoryActivity(DateTime.UtcNow, memoryUsageInBytes);
                 }
             }
             catch (Exception e)
@@ -244,6 +257,8 @@ public void Dispose()
             _metricsPublisherTimer = null;
 
             _metricsTracker.OnDiagnosticEvent -= OnMetricsDiagnosticEvent;
+            _standbyOptionsOnChangeListener?.Dispose();
+            _hostingConfigOptionsOnChangeListener?.Dispose();
         }
 
         internal class Metrics

src/WebJobs.Script.WebHost/Metrics/LinuxContainerMetricsPublisher.cs

@@ -4,6 +4,7 @@
 using System;
 using System.Collections.Concurrent;
 using System.Diagnostics;
+using System.IO;
 using System.Linq;
 using System.Net.Http;
 using System.Net.Http.Formatting;
@@ -19,7 +20,7 @@
 
 namespace Microsoft.Azure.WebJobs.Script.WebHost.Metrics
 {
-    public class LinuxContainerMetricsPublisher : IMetricsPublisher
+    public sealed class LinuxContainerMetricsPublisher : IMetricsPublisher, IDisposable
     {
         public const string PublishMemoryActivityPath = "/memoryactivity";
         public const string PublishFunctionActivityPath = "/functionactivity";
@@ -36,10 +37,11 @@ public class LinuxContainerMetricsPublisher : IMetricsPublisher
         private readonly TimeSpan _metricPublishInterval = TimeSpan.FromMilliseconds(30 * 1000);
         private readonly TimeSpan _timerStartDelay = TimeSpan.FromSeconds(2);
         private readonly IOptionsMonitor<StandbyOptions> _standbyOptions;
-        private readonly IDisposable _standbyOptionsOnChangeSubscription;
+        private readonly IDisposable _standbyOptionsOnChangeListener;
+        private readonly IDisposable _hostingConfigOptionsOnChangeListener;
         private readonly string _requestUri;
         private readonly IEnvironment _environment;
-        private readonly IOptions<FunctionsHostingConfigOptions> _hostingConfigOptions;
+        private readonly IOptionsMonitor<FunctionsHostingConfigOptions> _hostingConfigOptions;
 
         // Buffer for all memory activities for this container.
         private BlockingCollection<MemoryActivity> _memoryActivities;
@@ -64,8 +66,9 @@ public class LinuxContainerMetricsPublisher : IMetricsPublisher
         private int _errorCount = 0;
         private string _stampName;
         private bool _initialized = false;
+        private bool _isCGroupMemoryMetricsEnabled = false;
 
-        public LinuxContainerMetricsPublisher(IEnvironment environment, IOptionsMonitor<StandbyOptions> standbyOptions, ILogger<LinuxContainerMetricsPublisher> logger, HostNameProvider hostNameProvider, IOptions<FunctionsHostingConfigOptions> functionsHostingConfigOptions, HttpClient httpClient = null)
+        public LinuxContainerMetricsPublisher(IEnvironment environment, IOptionsMonitor<StandbyOptions> standbyOptions, ILogger<LinuxContainerMetricsPublisher> logger, HostNameProvider hostNameProvider, IOptionsMonitor<FunctionsHostingConfigOptions> functionsHostingConfigOptions, HttpClient httpClient = null)
         {
             _standbyOptions = standbyOptions ?? throw new ArgumentNullException(nameof(standbyOptions));
             _environment = environment ?? throw new ArgumentNullException(nameof(environment));
@@ -86,12 +89,23 @@ public LinuxContainerMetricsPublisher(IEnvironment environment, IOptionsMonitor<
             _httpClient = (httpClient != null) ? httpClient : CreateMetricsPublisherHttpClient();
             if (_standbyOptions.CurrentValue.InStandbyMode)
             {
-                _standbyOptionsOnChangeSubscription = _standbyOptions.OnChange(o => OnStandbyOptionsChange());
+                _standbyOptionsOnChangeListener = _standbyOptions.OnChange(o => OnStandbyOptionsChange());
             }
             else
             {
                 Start();
             }
+
+            _hostingConfigOptionsOnChangeListener = _hostingConfigOptions.OnChange(OnHostingConfigOptionsChanged);
+        }
+
+        private void OnHostingConfigOptionsChanged(FunctionsHostingConfigOptions newOptions)
+        {
+            if (newOptions.IsCGroupMemoryMetricsEnabled != _isCGroupMemoryMetricsEnabled)
+            {
+                _logger.LogInformation("CGroup memory metrics enabled: {Enabled}", newOptions.IsCGroupMemoryMetricsEnabled);
+                _isCGroupMemoryMetricsEnabled = newOptions.IsCGroupMemoryMetricsEnabled;
+            }
         }
 
         private void OnStandbyOptionsChange()
@@ -234,7 +248,6 @@ internal async Task SendRequest<T>(ConcurrentQueue<T> activitiesToPublish, strin
             try
             {
                 var request = BuildRequest(HttpMethod.Post, publishPath, activitiesToPublish.ToArray());
-
                 HttpResponseMessage response = await _httpClient.SendAsync(request);
                 response.EnsureSuccessStatusCode();
             }
@@ -265,11 +278,20 @@ private void OnProcessMonitorTimer(object state)
         {
             try
             {
-                _process.Refresh();
-                var commitSizeBytes = _process.WorkingSet64;
-                if (commitSizeBytes != 0)
+                long memoryUsageInBytes;
+                if (_hostingConfigOptions.CurrentValue.IsCGroupMemoryMetricsEnabled)
+                {
+                    memoryUsageInBytes = CgroupMemoryUsageHelper.GetMemoryUsageInBytes(_logger);
+                }
+                else
+                {
+                    _process.Refresh();
+                    memoryUsageInBytes = _process.WorkingSet64;
+                }
+
+                if (memoryUsageInBytes != 0)
                 {
-                    AddMemoryActivity(DateTime.UtcNow, commitSizeBytes);
+                    AddMemoryActivity(DateTime.UtcNow, memoryUsageInBytes);
                 }
             }
             catch (Exception e)
@@ -292,7 +314,7 @@ private HttpRequestMessage BuildRequest<TContent>(HttpMethod method, string path
             request.Headers.Add(HostNameHeader, _hostNameProvider.Value);
             request.Headers.Add(StampNameHeader, _stampName);
 
-            if (_hostingConfigOptions.Value.SwtIssuerEnabled)
+            if (_hostingConfigOptions.CurrentValue.SwtIssuerEnabled)
             {
                 string swtToken = SimpleWebTokenHelper.CreateToken(DateTime.UtcNow.AddMinutes(5));
                 request.Headers.Add(ScriptConstants.SiteRestrictedTokenHeaderName, swtToken);
@@ -313,5 +335,16 @@ public void OnFunctionCompleted(string functionName, string invocationId)
         {
             // nothing to do
         }
+
+        public void Dispose()
+        {
+            _standbyOptionsOnChangeListener?.Dispose();
+            _hostingConfigOptionsOnChangeListener?.Dispose();
+            _processMonitorTimer?.Dispose();
+            _metricsPublisherTimer?.Dispose();
+            _httpClient?.Dispose();
+            _memoryActivities?.Dispose();
+            _functionActivities?.Dispose();
+        }
     }
 }

src/WebJobs.Script.WebHost/WebHostServiceCollectionExtensions.cs

@@ -1,6 +1,7 @@
 // Copyright (c) .NET Foundation. All rights reserved.
 // Licensed under the MIT License. See License.txt in the project root for license information.
 
+using System;
 using System.IO.Abstractions;
 using System.Net.Http;
 using System.Runtime.InteropServices;
@@ -304,8 +305,9 @@ private static void AddLinuxContainerServices(this IServiceCollection services)
                     var logger = s.GetService<ILogger<LinuxContainerLegionMetricsPublisher>>();
                     var metricsTracker = s.GetService<ILinuxConsumptionMetricsTracker>();
                     var standbyOptions = s.GetService<IOptionsMonitor<StandbyOptions>>();
-                    var scriptHostManager = s.GetService<IScriptHostManager>();
-                    return new LinuxContainerLegionMetricsPublisher(environment, standbyOptions, options, logger, new FileSystem(), metricsTracker, scriptHostManager);
+                    var serviceProvider = s.GetService<IServiceProvider>();
+                    var hostingConfigOptions = s.GetService<IOptionsMonitor<FunctionsHostingConfigOptions>>();
+                    return new LinuxContainerLegionMetricsPublisher(environment, standbyOptions, options, logger, new FileSystem(), metricsTracker, serviceProvider, hostingConfigOptions);
                 }
                 else if (environment.IsFlexConsumptionSku())
                 {
@@ -320,7 +322,7 @@ private static void AddLinuxContainerServices(this IServiceCollection services)
                     var logger = s.GetService<ILogger<LinuxContainerMetricsPublisher>>();
                     var standbyOptions = s.GetService<IOptionsMonitor<StandbyOptions>>();
                     var hostNameProvider = s.GetService<HostNameProvider>();
-                    var hostingConfigOptions = s.GetService<IOptions<FunctionsHostingConfigOptions>>();
+                    var hostingConfigOptions = s.GetService<IOptionsMonitor<FunctionsHostingConfigOptions>>();
                     return new LinuxContainerMetricsPublisher(environment, standbyOptions, logger, hostNameProvider, hostingConfigOptions);
                 }
 

src/WebJobs.Script/Config/FunctionsHostingConfigOptions.cs

@@ -78,6 +78,22 @@ internal bool SwtIssuerEnabled
             }
         }
 
+        /// <summary>
+        /// Gets or sets a value indicating whether to use cgroup memory metrics for reporting memory usage.
+        /// </summary>
+        internal bool IsCGroupMemoryMetricsEnabled
+        {
+            get
+            {
+                return GetFeatureAsBooleanOrDefault(ScriptConstants.FeatureFlagEnableCGroupMemoryMetrics, false);
+            }
+
+            set
+            {
+                _features[ScriptConstants.FeatureFlagEnableCGroupMemoryMetrics] = value ? "1" : "0";
+            }
+        }
+
         /// <summary>
         /// Gets or sets a string delimited by '|' that contains a list of admin APIs that are allowed to
         /// be invoked internally by platform components.

src/WebJobs.Script/ScriptConstants.cs

@@ -142,6 +142,7 @@ public static class ScriptConstants
         public const string FeatureFlagDisableOrderedInvocationMessages = "DisableOrderedInvocationMessages";
         public const string FeatureFlagEnableAzureMonitorTimeIsoFormat = "EnableAzureMonitorTimeIsoFormat";
         public const string FeatureFlagEnableTestDataSuppression = "EnableTestDataSuppression";
+        public const string FeatureFlagEnableCGroupMemoryMetrics = "EnableCGroupMemoryMetrics";
         public const string HostingConfigDisableLinuxAppServiceDetailedExecutionEvents = "DisableLinuxExecutionDetails";
         public const string HostingConfigDisableLinuxAppServiceExecutionEventLogBackoff = "DisableLinuxLogBackoff";
         public const string FeatureFlagEnableLegacyDurableVersionCheck = "EnableLegacyDurableVersionCheck";