Delay Adaptive Sampling (#3097)

Introducing delay in configuring Adaptive Sampling

Author: RohitRanjanMS

src/Microsoft.Azure.WebJobs.Logging.ApplicationInsights/ApplicationInsightsLoggerOptions.cs

@@ -155,6 +155,18 @@ public TimeSpan QuickPulseInitializationDelay
         /// </summary>
         public bool EnableMetricsCustomDimensionOptimization { get; set; } = false;
 
+        /// <summary>
+        /// Gets or sets the flag that enables Adaptive Sampling delay.
+        /// Enabled by default.
+        /// </summary>
+        public bool EnableAdaptiveSamplingDelay { get; set; } = true;
+
+        /// <summary>
+        /// Specifies the delay time for initializing Adaptive Sampling, allowing more initialization logs to be sent to Application Insights.
+        /// The default value is 15 seconds.
+        /// </summary>
+        public TimeSpan AdaptiveSamplingInitializationDelay { get; set; } = TimeSpan.FromSeconds(15);
+
         public string Format()
         {
             JObject sampling = null;
@@ -246,6 +258,8 @@ public string Format()
                 { nameof(DiagnosticsEventListenerLogLevel), DiagnosticsEventListenerLogLevel?.ToString() },
                 { nameof(EnableAutocollectedMetricsExtractor), EnableAutocollectedMetricsExtractor },
                 { nameof(EnableMetricsCustomDimensionOptimization), EnableMetricsCustomDimensionOptimization },
+                { nameof(EnableAdaptiveSamplingDelay), EnableAdaptiveSamplingDelay },
+                { nameof(AdaptiveSamplingInitializationDelay), AdaptiveSamplingInitializationDelay },
             };
 
             return options.ToString(Formatting.Indented);

src/Microsoft.Azure.WebJobs.Logging.ApplicationInsights/Extensions/ApplicationInsightsServiceCollectionExtensions.cs

@@ -388,16 +388,14 @@ private static void SetupTelemetryConfiguration(
             {
                 configuration.TelemetryProcessorChainBuilder.Use((next) =>
                 {
-                    var processor = new AdaptiveSamplingTelemetryProcessor(options.SamplingSettings, null, next);
-                    if (options.SamplingExcludedTypes != null)
+                    if (options.EnableAdaptiveSamplingDelay)
                     {
-                        processor.ExcludedTypes = options.SamplingExcludedTypes;
+                        return new DelayedSamplingProcessor(next, options);
                     }
-                    if (options.SamplingIncludedTypes != null)
+                    else
                     {
-                        processor.IncludedTypes = options.SamplingIncludedTypes;
+                        return TelemetryProcessorFactory.CreateAdaptiveSamplingProcessor(options, next);
                     }
-                    return processor;
                 });
             }
 

src/Microsoft.Azure.WebJobs.Logging.ApplicationInsights/Processors/DelayedSamplingProcessor.cs

@@ -0,0 +1,45 @@
+// Copyright (c) .NET Foundation. All rights reserved.
+// Licensed under the MIT License. See License.txt in the project root for license information.
+
+using Microsoft.ApplicationInsights.Channel;
+using Microsoft.ApplicationInsights.Extensibility;
+using Microsoft.ApplicationInsights.WindowsServer.TelemetryChannel;
+using System.Threading.Tasks;
+
+namespace Microsoft.Azure.WebJobs.Logging.ApplicationInsights
+{
+    internal class DelayedSamplingProcessor : ITelemetryProcessor
+    {
+        private readonly AdaptiveSamplingTelemetryProcessor _samplingProcessor;
+        private ITelemetryProcessor _next;
+        private bool _isSamplingEnabled = false;        
+
+        public DelayedSamplingProcessor(ITelemetryProcessor next, ApplicationInsightsLoggerOptions options)
+        {
+            _next = next;            
+            _samplingProcessor = TelemetryProcessorFactory.CreateAdaptiveSamplingProcessor(options, next);
+
+            // Start a timer to enable sampling after a delay        
+            Task.Delay(options.AdaptiveSamplingInitializationDelay).ContinueWith(t => EnableSampling());
+        }
+
+        public void Process(ITelemetry item)
+        {
+            if (_isSamplingEnabled)
+            {
+                // Forward to Adaptive Sampling processor
+                _samplingProcessor.Process(item);
+            }
+            else
+            {
+                // Bypass sampling
+                _next.Process(item);
+            }
+        }
+
+        private void EnableSampling()
+        {
+            _isSamplingEnabled = true;
+        }
+    }
+}

src/Microsoft.Azure.WebJobs.Logging.ApplicationInsights/Processors/TelemetryProcessorExtensions.cs

@@ -0,0 +1,27 @@
+// Copyright (c) .NET Foundation. All rights reserved.
+// Licensed under the MIT License. See License.txt in the project root for license information.
+
+using Microsoft.ApplicationInsights.Extensibility;
+using Microsoft.ApplicationInsights.WindowsServer.TelemetryChannel;
+
+namespace Microsoft.Azure.WebJobs.Logging.ApplicationInsights
+{
+    internal static class TelemetryProcessorFactory
+    {
+        internal static AdaptiveSamplingTelemetryProcessor CreateAdaptiveSamplingProcessor(ApplicationInsightsLoggerOptions options, ITelemetryProcessor next = null)
+        {
+            // Create the sampling processor
+            var samplingProcessor = new AdaptiveSamplingTelemetryProcessor(options.SamplingSettings, null, next);
+
+            if (options.SamplingExcludedTypes != null)
+            {
+                samplingProcessor.ExcludedTypes = options.SamplingExcludedTypes;
+            }
+            if (options.SamplingIncludedTypes != null)
+            {
+                samplingProcessor.IncludedTypes = options.SamplingIncludedTypes;
+            }
+            return samplingProcessor;
+        }
+    }
+}

src/Microsoft.Azure.WebJobs.Logging.ApplicationInsights/WebJobs.Logging.ApplicationInsights.csproj

@@ -3,7 +3,7 @@
   <Import Project="..\..\build\common.props" />
 
   <PropertyGroup>
-    <Version>3.0.41$(VersionSuffix)</Version>
+    <Version>3.0.42$(VersionSuffix)</Version>
     <InformationalVersion>$(Version) Commit hash: $(CommitHash)</InformationalVersion>
     <TargetFramework>netstandard2.0</TargetFramework>
     <PackageId>Microsoft.Azure.WebJobs.Logging.ApplicationInsights</PackageId>

test/Microsoft.Azure.WebJobs.Host.UnitTests/Loggers/ApplicationInsightsConfigurationTests.cs

@@ -225,6 +225,7 @@ public void DependencyInjectionConfiguration_ConfiguresSampling()
                         o.SamplingSettings = samplingSettings;
                         o.SamplingExcludedTypes = samplingExcludedTypes;
                         o.SamplingIncludedTypes = samplingIncludedTypes;
+                        o.EnableAdaptiveSamplingDelay = false;
                     });
                 })
                 .Build())
@@ -242,7 +243,43 @@ public void DependencyInjectionConfiguration_ConfiguresSampling()
             }
         }
 
-        
+        [Fact]
+        public void DependencyInjectionConfiguration_ConfiguresDelayedSampling()
+        {
+            var samplingSettings = new SamplingPercentageEstimatorSettings { MaxTelemetryItemsPerSecond = 1 };
+            var samplingExcludedTypes = "PageView;Request";
+            var samplingIncludedTypes = "Trace";
+            using (var host = new HostBuilder()
+                .ConfigureLogging(b =>
+                {
+                    b.AddApplicationInsightsWebJobs(o =>
+                    {
+                        o.InstrumentationKey = "some key";
+                        o.SamplingSettings = samplingSettings;
+                        o.SamplingExcludedTypes = samplingExcludedTypes;
+                        o.SamplingIncludedTypes = samplingIncludedTypes;
+                    });
+                })
+                .Build())
+            {
+                var config = host.Services.GetService<TelemetryConfiguration>();
+                Assert.Equal(5, config.TelemetryProcessors.Count);
+                Assert.IsType<OperationFilteringTelemetryProcessor>(config.TelemetryProcessors[0]);
+                Assert.IsType<QuickPulseTelemetryProcessor>(config.TelemetryProcessors[1]);
+                Assert.IsType<FilteringTelemetryProcessor>(config.TelemetryProcessors[2]);
+                Assert.IsType<DelayedSamplingProcessor>(config.TelemetryProcessors[3]);
+
+                // Use reflection to access the private field "_samplingProcessor"
+                var samplingProcessorField = typeof(DelayedSamplingProcessor)
+                    .GetField("_samplingProcessor", BindingFlags.NonPublic | BindingFlags.Instance);
+                var samplingProcessorInstance = (AdaptiveSamplingTelemetryProcessor)samplingProcessorField.GetValue(config.TelemetryProcessors[3]);
+
+                Assert.Equal(samplingSettings.MaxTelemetryItemsPerSecond, samplingProcessorInstance.MaxTelemetryItemsPerSecond);
+                Assert.Equal(samplingExcludedTypes, samplingProcessorInstance.ExcludedTypes);
+                Assert.Equal(samplingIncludedTypes, samplingProcessorInstance.IncludedTypes);
+            }
+        }
+
         [Fact]
         public void DependencyInjectionConfiguration_EnableLiveMetricsFilters()
         {
@@ -589,6 +626,7 @@ public void DependencyInjectionConfiguration_NoFilterConfiguresSampling()
                     {
                         o.InstrumentationKey = "some key";
                         o.SamplingSettings = samplingSettings;
+                        o.EnableAdaptiveSamplingDelay = false;
                     });
                 }).Build())
             {