Add metadata provider timeout host config (#10526)

Author: liliankasem

release_notes.md

@@ -8,3 +8,6 @@
 - Improving console log handling during specialization (#10345)
 - Update Node.js Worker Version to [3.10.1](https://github.com/Azure/azure-functions-nodejs-worker/releases/tag/v3.10.1)
 - Remove packages `Microsoft.Azure.Cosmos.Table` and `Microsoft.Azure.DocumentDB.Core` (#10503)
+- Implement host configuration property to allow configuration of the metadata provider timeout period (#10526)
+  - The value can be set via `metadataProviderTimeout` in host.json and defaults to "00:00:30" (30 seconds).
+  - For logic apps, unless configured via the host.json, the timeout is disabled by default.

src/WebJobs.Script/Config/ConfigurationSectionNames.cs

@@ -25,5 +25,6 @@ public static class ConfigurationSectionNames
         public const string SequentialJobHostRestart = JobHost + ":sequentialRestart";
         public const string SendCanceledInvocationsToWorker = "sendCanceledInvocationsToWorker";
         public const string TelemetryMode = "telemetryMode";
+        public const string MetadataProviderTimeout = "metadataProviderTimeout";
     }
 }

src/WebJobs.Script/Config/HostJsonFileConfigurationSource.cs

@@ -51,7 +51,8 @@ public class HostJsonFileConfigurationProvider : ConfigurationProvider
             {
                 "version", "functionTimeout", "retry", "functions", "http", "watchDirectories", "watchFiles", "queues", "serviceBus",
                 "eventHub", "singleton", "logging", "aggregator", "healthMonitor", "extensionBundle", "managedDependencies",
-                "customHandler", "httpWorker", "extensions", "concurrency", ConfigurationSectionNames.SendCanceledInvocationsToWorker
+                "customHandler", "httpWorker", "extensions", "concurrency", ConfigurationSectionNames.SendCanceledInvocationsToWorker,
+                ConfigurationSectionNames.MetadataProviderTimeout
             };
 
             private readonly HostJsonFileConfigurationSource _configurationSource;

src/WebJobs.Script/Config/ScriptJobHostOptions.cs

@@ -135,5 +135,10 @@ public string RootScriptPath
         /// Gets or sets the telemetry mode.
         /// </summary>
         internal TelemetryMode TelemetryMode { get; set; } = TelemetryMode.ApplicationInsights;
+
+        /// <summary>
+        /// Gets or sets a value indicating the timeout duration for the function metadata provider.
+        /// </summary>
+        public TimeSpan MetadataProviderTimeout { get; set; } = TimeSpan.Zero;
     }
 }

src/WebJobs.Script/Config/ScriptJobHostOptionsSetup.cs

@@ -2,6 +2,7 @@
 // Licensed under the MIT License. See License.txt in the project root for license information.
 
 using System;
+using System.Threading;
 using Microsoft.Azure.WebJobs.Script.Diagnostics.OpenTelemetry;
 using Microsoft.Extensions.Configuration;
 using Microsoft.Extensions.Options;
@@ -18,6 +19,7 @@ internal class ScriptJobHostOptionsSetup : IConfigureOptions<ScriptJobHostOption
         internal static readonly TimeSpan DefaultConsumptionFunctionTimeout = TimeSpan.FromMinutes(5);
         internal static readonly TimeSpan MaxFunctionTimeoutDynamic = TimeSpan.FromMinutes(10);
         internal static readonly TimeSpan DefaultFunctionTimeout = TimeSpan.FromMinutes(30);
+        internal static readonly TimeSpan DefaultMetadataProviderTimeout = TimeSpan.FromSeconds(30);
 
         public ScriptJobHostOptionsSetup(IConfiguration configuration, IEnvironment environment, IOptions<ScriptApplicationHostOptions> applicationHostOptions)
         {
@@ -59,6 +61,12 @@ public void Configure(ScriptJobHostOptions options)
             // FunctionTimeout
             ConfigureFunctionTimeout(options);
 
+            if (options.MetadataProviderTimeout == TimeSpan.Zero)
+            {
+                // If the timeout value is not configured and we are running in a logic app, we want to disable the timeout
+                options.MetadataProviderTimeout = _environment.IsLogicApp() ? Timeout.InfiniteTimeSpan : DefaultMetadataProviderTimeout;
+            }
+
             // If we have a read only file system, override any configuration and
             // disable file watching
             if (_applicationHostOptions.Value.IsFileSystemReadOnly)
@@ -88,7 +96,7 @@ public void Configure(ScriptJobHostOptions options)
 
         private void ConfigureFunctionTimeout(ScriptJobHostOptions options)
         {
-            if (options.FunctionTimeout == null)
+            if (options.FunctionTimeout is null)
             {
                 options.FunctionTimeout = (_environment.IsConsumptionSku() && !_environment.IsFlexConsumptionSku()) ? DefaultConsumptionFunctionTimeout : DefaultFunctionTimeout;
             }

src/WebJobs.Script/Host/FunctionMetadataManager.cs

@@ -6,6 +6,7 @@
 using System.Collections.Generic;
 using System.Collections.Immutable;
 using System.Linq;
+using System.Threading;
 using System.Threading.Tasks;
 using Microsoft.Azure.WebJobs.Logging;
 using Microsoft.Azure.WebJobs.Script.Description;
@@ -21,9 +22,8 @@ namespace Microsoft.Azure.WebJobs.Script
 {
     public class FunctionMetadataManager : IFunctionMetadataManager
     {
-        private const string FunctionConfigurationErrorMessage = "Unable to determine the primary function script.Make sure atleast one script file is present.Try renaming your entry point script to 'run' or alternatively you can specify the name of the entry point script explicitly by adding a 'scriptFile' property to your function metadata.";
+        private const string FunctionConfigurationErrorMessage = "Unable to determine the primary function script. Make sure at least one script file is present. Try renaming your entry point script to 'run' or alternatively you can specify the name of the entry point script explicitly by adding a 'scriptFile' property to your function metadata.";
         private const string MetadataProviderName = "Custom";
-        private const int DefaultMetadataProviderTimeoutInSeconds = 30;
         private readonly IServiceProvider _serviceProvider;
         private IFunctionMetadataProvider _functionMetadataProvider;
         private bool _isHttpWorker;
@@ -57,9 +57,6 @@ public FunctionMetadataManager(IOptions<ScriptJobHostOptions> scriptOptions, IFu
             };
         }
 
-        // Property is settable for testing purposes.
-        internal int MetadataProviderTimeoutInSeconds { get; set; } = DefaultMetadataProviderTimeoutInSeconds;
-
         public ImmutableDictionary<string, ImmutableArray<string>> Errors { get; private set; }
 
         public bool TryGetFunctionMetadata(string functionName, out FunctionMetadata functionMetadata, bool forceRefresh)
@@ -221,11 +218,12 @@ private void AddMetadataFromCustomProviders(IEnumerable<IFunctionProvider> funct
             _logger.ReadingFunctionMetadataFromProvider(MetadataProviderName);
 
             var functionProviderTasks = new List<Task<ImmutableArray<FunctionMetadata>>>();
+            var metadataProviderTimeout = _scriptOptions.Value.MetadataProviderTimeout;
 
             foreach (var functionProvider in functionProviders)
             {
                 var getFunctionMetadataFromProviderTask = functionProvider.GetFunctionMetadataAsync();
-                var delayTask = Task.Delay(TimeSpan.FromSeconds(MetadataProviderTimeoutInSeconds));
+                var delayTask = Task.Delay(metadataProviderTimeout);
 
                 var completedTask = Task.WhenAny(getFunctionMetadataFromProviderTask, delayTask).ContinueWith(t =>
                 {
@@ -235,7 +233,7 @@ private void AddMetadataFromCustomProviders(IEnumerable<IFunctionProvider> funct
                     }
 
                     // Timeout case.
-                    throw new TimeoutException($"Timeout occurred while retrieving metadata from provider '{functionProvider.GetType().FullName}'. The operation exceeded the configured timeout of {MetadataProviderTimeoutInSeconds} seconds.");
+                    throw new TimeoutException($"Timeout occurred while retrieving metadata from provider '{functionProvider.GetType().FullName}'. The operation exceeded the configured timeout of {metadataProviderTimeout.TotalSeconds} seconds.");
                 });
 
                 functionProviderTasks.Add(completedTask);

test/WebJobs.Script.Tests/Configuration/ScriptJobHostOptionsSetupTests.cs

@@ -334,6 +334,38 @@ public void Configure_WithConfiguration_AppliesTelemetryMode(string setting, str
             Assert.Equal(expectedMode, options.TelemetryMode.ToString());
         }
 
+        [Fact]
+        public void Configure_MetadataProviderTimeout_ValidateDefaultTimeoutValue()
+        {
+            var options = GetConfiguredOptions(new Dictionary<string, string>());
+
+            Assert.Equal(TimeSpan.FromSeconds(30), options.MetadataProviderTimeout);
+        }
+
+        [Fact]
+        public void Configure_MetadataProviderTimeout_IsLogicApp_SetTimeoutToInfinite()
+        {
+            var environment = new TestEnvironment();
+            environment.SetEnvironmentVariable(EnvironmentSettingNames.AppKind, "workflowapp");
+
+            var options = GetConfiguredOptions(new Dictionary<string, string>(), environment);
+
+            Assert.Equal(TimeSpan.FromMilliseconds(-1), options.MetadataProviderTimeout);
+        }
+
+        [Fact]
+        public void Configure_MetadataProviderTimeout_AppliesConfiguredTimeoutValue()
+        {
+            var settings = new Dictionary<string, string>
+            {
+                { ConfigurationPath.Combine(ConfigurationSectionNames.JobHost, "metadataProviderTimeout"), "00:00:43" }
+            };
+
+            var options = GetConfiguredOptions(settings);
+
+            Assert.Equal(TimeSpan.FromSeconds(43), options.MetadataProviderTimeout);
+        }
+
         private ScriptJobHostOptions GetConfiguredOptions(Dictionary<string, string> settings, IEnvironment environment = null)
         {
             ScriptJobHostOptionsSetup setup = CreateSetupWithConfiguration(settings, environment);

test/WebJobs.Script.Tests/FunctionMetadataManagerTests.cs

@@ -17,13 +17,12 @@
 using Microsoft.WebJobs.Script.Tests;
 using Moq;
 using Xunit;
-using static Microsoft.Azure.AppService.Proxy.Common.Constants.WellKnownHttpHeaders;
 
 namespace Microsoft.Azure.WebJobs.Script.Tests
 {
     public class FunctionMetadataManagerTests
     {
-        private const string _expectedErrorMessage = "Unable to determine the primary function script.Make sure atleast one script file is present.Try renaming your entry point script to 'run' or alternatively you can specify the name of the entry point script explicitly by adding a 'scriptFile' property to your function metadata.";
+        private const string _expectedErrorMessage = "Unable to determine the primary function script. Make sure at least one script file is present. Try renaming your entry point script to 'run' or alternatively you can specify the name of the entry point script explicitly by adding a 'scriptFile' property to your function metadata.";
         private ScriptJobHostOptions _scriptJobHostOptions = new ScriptJobHostOptions();
         private Mock<IFunctionMetadataProvider> _mockFunctionMetadataProvider;
         private FunctionMetadataManager _testFunctionMetadataManager;
@@ -256,7 +255,7 @@ public void FunctionMetadataManager_LoadFunctionMetadata_Throws_WhenFunctionProv
                 mockFunctionMetadataProvider.Object, new List<IFunctionProvider>() { goodFunctionMetadataProvider.Object, badFunctionMetadataProvider.Object }, new OptionsWrapper<HttpWorkerOptions>(_defaultHttpWorkerOptions), loggerFactory, new TestOptionsMonitor<LanguageWorkerOptions>(TestHelpers.GetTestLanguageWorkerOptions()));
 
             // Set the timeout to 1 second for the test.
-            testFunctionMetadataManager.MetadataProviderTimeoutInSeconds = 1;
+            _scriptJobHostOptions.MetadataProviderTimeout = TimeSpan.FromSeconds(1);
 
             var exception = Assert.Throws<TimeoutException>(() => testFunctionMetadataManager.LoadFunctionMetadata());
             Assert.Contains($"Timeout occurred while retrieving metadata from provider '{badFunctionMetadataProvider.Object.GetType().FullName}'. The operation exceeded the configured timeout of 1 seconds.", exception.Message);