HostMetrics published every 30 seconds via flex metrics publisher (#11019)

Author: liliankasem

release_notes.md

@@ -12,4 +12,5 @@
 - Update Python Worker Version to [4.37.0](https://github.com/Azure/azure-functions-python-worker/releases/tag/4.37.0)
 - Add runtime and process metrics. (#11034)
 - Add `win-arm64` and `linux-arm64` to the list of PowerShell runtimes; added filter for `osx` RIDs (includes `osx-x64` and `osx-arm64`) (#11013)
-- Disable Diagnostic Events when Table Storage is not accessible (#10996)
+- Disable Diagnostic Events when Table Storage is not accessible (#10996)
+- Update flex metric publisher to publish every 30 seconds (regardless of actvity) (#11019)

src/WebJobs.Script.WebHost/Configuration/FlexConsumptionMetricsPublisherOptions.cs

@@ -14,12 +14,14 @@ public class FlexConsumptionMetricsPublisherOptions
         internal const int DefaultMetricsPublishIntervalMS = 5000;
         internal const int DefaultMinimumActivityIntervalMS = 1000;
         internal const int DefaultMetricsGranularityMS = 100;
+        internal const int DefaultKeepAliveIntervalMS = 30000;
 
         public FlexConsumptionMetricsPublisherOptions()
         {
             MetricsPublishIntervalMS = DefaultMetricsPublishIntervalMS;
             MinimumActivityIntervalMS = DefaultMinimumActivityIntervalMS;
             MetricsGranularityMS = DefaultMetricsGranularityMS;
+            KeepAliveIntervalMS = DefaultKeepAliveIntervalMS;
             InitialPublishDelayMS = Utility.ColdStartDelayMS;
 
             // Default this to 15 minutes worth of files
@@ -66,5 +68,14 @@ public FlexConsumptionMetricsPublisherOptions()
         /// When over this limit, the oldest files will be deleted to make room for new files.
         /// </remarks>
         public int MaxFileCount { get; set; }
+
+        /// <summary>
+        /// Gets or sets the maximum interval at which metrics are published.
+        /// </summary>
+        /// <remarks>
+        /// This is required to ensure that we publish something even if there are no
+        /// new metrics or activity to report so that we have a heartbeat.
+        /// </remarks>
+        public int KeepAliveIntervalMS { get; set; }
     }
 }

src/WebJobs.Script.WebHost/Metrics/FlexConsumptionMetricsPublisher.cs

@@ -36,6 +36,7 @@ public class FlexConsumptionMetricsPublisher : IMetricsPublisher, IDisposable
         private IDisposable _standbyOptionsOnChangeSubscription;
         private TimeSpan _metricPublishInterval;
         private TimeSpan _initialPublishDelay;
+        private DateTime _lastPublishTime = DateTime.UtcNow;
 
         public FlexConsumptionMetricsPublisher(IEnvironment environment, IOptionsMonitor<StandbyOptions> standbyOptions, IOptions<FlexConsumptionMetricsPublisherOptions> options,
             ILogger<FlexConsumptionMetricsPublisher> logger, IFileSystem fileSystem, IHostMetricsProvider metricsProvider)
@@ -105,9 +106,12 @@ internal async Task OnPublishMetrics(DateTime now)
                     }
                 }
 
-                if (FunctionExecutionCount == 0 && FunctionExecutionTimeMS == 0 && !IsAlwaysReady && !_metricsProvider.HasMetrics())
+                bool hasActivity = FunctionExecutionCount > 0 || FunctionExecutionTimeMS > 0 || _metricsProvider.HasMetrics();
+                bool shouldForcePublish = (now - _lastPublishTime) >= TimeSpan.FromMilliseconds(_options.KeepAliveIntervalMS);
+
+                if (!hasActivity && !shouldForcePublish && !IsAlwaysReady)
                 {
-                    // no activity to report
+                    // No activity and not time for keep-alive publish & not always ready
                     return;
                 }
 
@@ -135,6 +139,7 @@ internal async Task OnPublishMetrics(DateTime now)
                     }
 
                     FunctionExecutionTimeMS = FunctionExecutionCount = 0;
+                    _lastPublishTime = now;
                 }
 
                 await _metricsFileManager.PublishMetricsAsync(metrics);

test/WebJobs.Script.Tests/Metrics/FlexConsumptionMetricsPublisherTests.cs

@@ -111,6 +111,7 @@ public async Task OnPublishMetrics_WritesFileAndResetsCounts(bool isAlwaysReadyI
 
             FlexConsumptionMetricsPublisher.Metrics metrics = null;
             FileInfo metricsFile = null;
+
             if (!isAlwaysReadyInstance)
             {
                 // don't expect a file to be written when no activity
@@ -531,6 +532,35 @@ public async Task OnPublishMetrics_OutstandingActivityIsPublished()
             Assert.Equal(2000, metrics.ExecutionTimeMS);
         }
 
+        [Fact]
+        public async Task OnPublishMetrics_KeepAliveMetricsPublished_WhenNoActivity()
+        {
+            CleanupMetricsFiles();
+
+            var publisher = CreatePublisher();
+
+            DateTime now = DateTime.UtcNow;
+
+            // Should not publish because there is no activity and keepalive internal has not passed
+            await publisher.OnPublishMetrics(now);
+            var files = GetMetricsFilesSafe(_metricsFilePath);
+            Assert.Equal(0, files.Length);
+
+            // Simulate 31 seconds passing with no function activity
+            now = now.AddSeconds(31);
+
+            // Should publish as keepalive interval has passed
+            await publisher.OnPublishMetrics(now);
+            files = GetMetricsFilesSafe(_metricsFilePath);
+            Assert.Equal(1, files.Length);
+
+            var metrics = await ReadMetricsAsync(files[0].FullName, deleteFile: true);
+
+            // We expect all activity values to be zero as there has been no activity
+            Assert.Equal(0, metrics.ExecutionCount);
+            Assert.Equal(0, metrics.ExecutionTimeMS);
+        }
+
         [Fact]
         public void OnFunctionCompleted_NoOutstandingInvocations_IgnoresEvent()
         {