Introduce host configuration profile feature (#11324)

* Introduce host configuration profile feature

* http:routePrefix -> extensions:http:routePrefix

* Fix tests, update exception

* Add unit tests for HostConfigurationProfile

* Address unit tests

* Fix test

* Remove unused usings

* mcp -> mcp-customer-handler

* Fix test

* Address PR comments

* Fix mcp-custom-handler typo

* Add release notes

* Fix merge

Author: jviau

release_notes.md

@@ -9,6 +9,8 @@
 - Add JitTrace Files for v4.1043 (#11281)
 - Update `Microsoft.Azure.WebJobs` reference to `3.0.42` (#11309)
 - Adding Activity wrapper to create a function-level request telemetry when none exists (#11311)
+- Introduce 'configurationProfile' functionality (#11324)
+- Remove request size limit for Host <--> Worker communication (#11295)
 - Setting current activity status for failed invocations (#11313)
 - Adding test coverage for `Utility.IsAzureMonitorLoggingEnabled` (#11322)
 - Reduce allocations in `Utility.IsAzureMonitorLoggingEnabled` (#11323)

src/WebJobs.Script/Config/ConfigurationSectionNames.cs

@@ -1,4 +1,4 @@
-// Copyright (c) .NET Foundation. All rights reserved.
+// Copyright (c) .NET Foundation. All rights reserved.
 // Licensed under the MIT License. See License.txt in the project root for license information.
 
 namespace Microsoft.Azure.WebJobs.Script.Configuration

src/WebJobs.Script/Config/HostConfigurationProfile.cs

@@ -0,0 +1,59 @@
+// Copyright (c) .NET Foundation. All rights reserved.
+// Licensed under the MIT License. See License.txt in the project root for license information.
+
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using Microsoft.Extensions.Configuration;
+
+namespace Microsoft.Azure.WebJobs.Script.Configuration
+{
+    public sealed class HostConfigurationProfile
+    {
+        public const string SectionKey = "configurationProfile";
+
+        // note: profile name consts are intentionally private.
+        // This ensures tests will fail if these values are changed without updating the test also.
+        private const string DefaultProfile = "default";
+
+        private const string McpCustomerHandlerProfile = "mcp-custom-handler";
+
+        // Make sure to update this as new profiles are added.
+        private const string SupportedValues = $"'', '{DefaultProfile}', '{McpCustomerHandlerProfile}'";
+
+        public static readonly HostConfigurationProfile Default = new(DefaultProfile, []);
+
+        public static readonly HostConfigurationProfile McpCustomHandler = new(
+            McpCustomerHandlerProfile,
+            [
+                KeyValuePair.Create(ConfigurationPath.Combine(
+                    ConfigurationSectionNames.CustomHandler, "enableHttpProxyingRequest"), "true"),
+                KeyValuePair.Create(ConfigurationPath.Combine(
+                    ConfigurationSectionNames.Http, "routePrefix"), string.Empty),
+            ]);
+
+        private HostConfigurationProfile(
+            string name,
+            IEnumerable<KeyValuePair<string, string>> configuration)
+        {
+            Name = name;
+            Configuration = [.. configuration, KeyValuePair.Create(SectionKey, name)];
+        }
+
+        public string Name { get; }
+
+        public IEnumerable<KeyValuePair<string, string>> Configuration { get; }
+
+        public static HostConfigurationProfile Get(string name)
+        {
+            ArgumentNullException.ThrowIfNull(name);
+            return name.ToLowerInvariant() switch
+            {
+                McpCustomerHandlerProfile => McpCustomHandler,
+                "" or DefaultProfile => Default,
+                _ => throw new NotSupportedException(
+                        $"Configuration profile '{name}' is not supported. Supported values: {SupportedValues}."),
+            };
+        }
+    }
+}

src/WebJobs.Script/Config/HostJsonFileConfigurationOptions.cs

@@ -0,0 +1,59 @@
+// Copyright (c) .NET Foundation. All rights reserved.
+// Licensed under the MIT License. See License.txt in the project root for license information.
+
+#nullable enable
+
+using System;
+using Newtonsoft.Json.Linq;
+
+namespace Microsoft.Azure.WebJobs.Script.Configuration
+{
+    public sealed class HostJsonFileConfigurationOptions
+    {
+        private const string ConfigProfileKey = "configurationProfile";
+        private const string ConfigProfileEnvKey = $"{ConfigurationSectionNames.JobHost}__{ConfigProfileKey}";
+
+        public HostJsonFileConfigurationOptions(ScriptApplicationHostOptions hostOptions)
+        {
+            ArgumentNullException.ThrowIfNull(hostOptions);
+            Host = hostOptions;
+        }
+
+        public HostJsonFileConfigurationOptions(
+            IEnvironment environment, ScriptApplicationHostOptions hostOptions)
+            : this(hostOptions)
+        {
+            ArgumentNullException.ThrowIfNull(environment);
+            ArgumentNullException.ThrowIfNull(hostOptions);
+
+            // Right now we explicitly read config profile from environment variable only.
+            // At the time of this commit there was 0 config sources already loaded. Environment
+            // vars are added to IConfiguration later in the pipeline. If we do eventually have
+            // config sources earlier we will need to consider if we want to read from those as well
+            // here.
+            ConfigProfile = environment.GetEnvironmentVariable(ConfigProfileEnvKey);
+            WorkerRuntime = environment.GetFunctionsWorkerRuntime();
+            IsLogicApp = environment.IsLogicApp();
+        }
+
+        public string? ConfigProfile { get; init; }
+
+        public string WorkerRuntime { get; init; } = string.Empty;
+
+        public bool IsLogicApp { get; init; }
+
+        public ScriptApplicationHostOptions Host { get; }
+
+        public HostConfigurationProfile GetConfigProfile(JObject hostFile)
+        {
+            ArgumentNullException.ThrowIfNull(hostFile);
+
+            // Right now this is ONLY set via env variable, which will always take precedence over host.json.
+            // If in the future we allow this to be set via other means (e.g. CLI arg), we may need to revisit precedence.
+            // If config profile is not set via env, check host.json for the value.
+            string profile = ConfigProfile ?? hostFile.GetValue(ConfigProfileKey)?.Value<string>() ?? string.Empty;
+
+            return HostConfigurationProfile.Get(profile);
+        }
+    }
+}

src/WebJobs.Script/Config/HostJsonFileConfigurationSource.cs

@@ -1,4 +1,4 @@
-// Copyright (c) .NET Foundation. All rights reserved.
+// Copyright (c) .NET Foundation. All rights reserved.
 // Licensed under the MIT License. See License.txt in the project root for license information.
 
 using System;
@@ -10,7 +10,6 @@
 using Microsoft.Azure.WebJobs.Logging;
 using Microsoft.Azure.WebJobs.Script.Diagnostics;
 using Microsoft.Azure.WebJobs.Script.Diagnostics.Extensions;
-using Microsoft.Azure.WebJobs.Script.Workers.Rpc;
 using Microsoft.Extensions.Configuration;
 using Microsoft.Extensions.Logging;
 using Newtonsoft.Json;
@@ -23,22 +22,23 @@ public class HostJsonFileConfigurationSource : IConfigurationSource
         private readonly ILogger _logger;
         private readonly IMetricsLogger _metricsLogger;
 
-        public HostJsonFileConfigurationSource(ScriptApplicationHostOptions applicationHostOptions, IEnvironment environment, ILoggerFactory loggerFactory, IMetricsLogger metricsLogger)
+        public HostJsonFileConfigurationSource(
+            HostJsonFileConfigurationOptions options,
+            ILoggerFactory loggerFactory,
+            IMetricsLogger metricsLogger)
         {
-            if (loggerFactory == null)
-            {
-                throw new ArgumentNullException(nameof(loggerFactory));
-            }
+            ArgumentNullException.ThrowIfNull(options);
+            ArgumentNullException.ThrowIfNull(loggerFactory);
+            ArgumentNullException.ThrowIfNull(metricsLogger);
 
-            HostOptions = applicationHostOptions;
-            Environment = environment;
-            _metricsLogger = metricsLogger;
+            Options = options;
             _logger = loggerFactory.CreateLogger(LogCategories.Startup);
+            _metricsLogger = metricsLogger;
         }
 
-        public ScriptApplicationHostOptions HostOptions { get; }
+        private HostJsonFileConfigurationOptions Options { get; }
 
-        public IEnvironment Environment { get; }
+        private ScriptApplicationHostOptions HostOptions => Options.Host;
 
         public IConfigurationProvider Build(IConfigurationBuilder builder)
         {
@@ -47,30 +47,41 @@ public IConfigurationProvider Build(IConfigurationBuilder builder)
 
         public class HostJsonFileConfigurationProvider : ConfigurationProvider
         {
-            private static readonly string[] WellKnownHostJsonProperties = new[]
-            {
+            private static readonly string[] WellKnownHostJsonProperties =
+            [
                 "version", "functionTimeout", "retry", "functions", "http", "watchDirectories", "watchFiles", "queues", "serviceBus",
                 "eventHub", "singleton", "logging", "aggregator", "healthMonitor", "extensionBundle", "managedDependencies",
                 "customHandler", "httpWorker", "extensions", "concurrency", "telemetryMode", ConfigurationSectionNames.SendCanceledInvocationsToWorker,
                 ConfigurationSectionNames.MetadataProviderTimeout, "isDefaultHostConfig"
-            };
+            ];
 
             private readonly HostJsonFileConfigurationSource _configurationSource;
-            private readonly Stack<string> _path;
+            private readonly Stack<string> _path = new();
             private readonly ILogger _logger;
             private readonly IMetricsLogger _metricsLogger;
 
-            public HostJsonFileConfigurationProvider(HostJsonFileConfigurationSource configurationSource, ILogger logger, IMetricsLogger metricsLogger)
+            public HostJsonFileConfigurationProvider(
+                HostJsonFileConfigurationSource configurationSource, ILogger logger, IMetricsLogger metricsLogger)
             {
                 _configurationSource = configurationSource;
-                _path = new Stack<string>();
                 _logger = logger;
                 _metricsLogger = metricsLogger;
             }
 
+            private HostJsonFileConfigurationOptions Options => _configurationSource.Options;
+
             public override void Load()
             {
                 JObject hostJson = LoadHostConfigurationFile();
+
+                // Apply profile settings first, so that they can be overridden by host.json settings.
+                HostConfigurationProfile profile = GetConfigProfile(hostJson);
+                _logger.LogDebug("Loading host configuration profile '{profileName}'.", profile.Name);
+                foreach ((string key, string value) in profile.Configuration)
+                {
+                    Data[ConfigurationPath.Combine(ConfigurationSectionNames.JobHost, key)] = value;
+                }
+
                 ProcessObject(hostJson);
             }
 
@@ -141,9 +152,7 @@ private JObject LoadHostConfigurationFile()
                     // to the startup logger until we've read configuration settings and can create the real logger.
                     // The "startup" logger is used in this class for startup related logs. The public logger is used
                     // for all other logging after startup.
-
-                    ScriptApplicationHostOptions options = _configurationSource.HostOptions;
-                    string hostFilePath = Path.Combine(options.ScriptPath, ScriptConstants.HostMetadataFileName);
+                    string hostFilePath = Path.Combine(Options.Host.ScriptPath, ScriptConstants.HostMetadataFileName);
                     JObject hostConfigObject = LoadHostConfig(hostFilePath);
                     hostConfigObject = InitializeHostConfig(hostFilePath, hostConfigObject);
 
@@ -192,10 +201,11 @@ private JObject InitializeHostConfig(string hostJsonPath, JObject hostConfigObje
                         throw new HostConfigurationException(errorMsg.ToString());
                     }
                 }
+
                 return hostConfigObject;
             }
 
-            internal JObject LoadHostConfig(string configFilePath)
+            private JObject LoadHostConfig(string configFilePath)
             {
                 using (_metricsLogger.LatencyEvent(MetricEventNames.LoadHostConfiguration))
                 {
@@ -227,11 +237,11 @@ internal JObject LoadHostConfig(string configFilePath)
                         // So a newly created function app from the portal would have no host.json. In that case we need to
                         // create a new function app with host.json that includes a matching extension bundle based on the app kind.
                         hostConfigObject = GetDefaultHostConfigObject();
-                        string bundleId = _configurationSource.Environment.IsLogicApp() ?
+                        string bundleId = Options.IsLogicApp ?
                                             ScriptConstants.WorkFlowExtensionBundleId :
                                             ScriptConstants.DefaultExtensionBundleId;
 
-                        string bundleVersion = _configurationSource.Environment.IsLogicApp() ?
+                        string bundleVersion = Options.IsLogicApp ?
                                             ScriptConstants.LogicAppDefaultExtensionBundleVersion :
                                             ScriptConstants.DefaultExtensionBundleVersion;
 
@@ -248,15 +258,29 @@ private JObject GetDefaultHostConfigObject()
             {
                 // isDefaultHostConfig is used to determine if the host.json file was created by the system
                 var hostJsonJObj = JObject.Parse("{'version': '2.0', 'isDefaultHostConfig': true}");
-                if (string.Equals(_configurationSource.Environment.GetEnvironmentVariable(RpcWorkerConstants.FunctionWorkerRuntimeSettingName), "powershell", StringComparison.InvariantCultureIgnoreCase)
-                    && !_configurationSource.HostOptions.IsFileSystemReadOnly)
+                if (string.Equals(Options.WorkerRuntime, "powershell", StringComparison.InvariantCultureIgnoreCase)
+                    && !Options.Host.IsFileSystemReadOnly)
                 {
                     hostJsonJObj.Add("managedDependency", JToken.Parse("{'Enabled': true}"));
                 }
 
                 return hostJsonJObj;
             }
 
+            private HostConfigurationProfile GetConfigProfile(JObject hostFile)
+            {
+                ArgumentNullException.ThrowIfNull(hostFile);
+
+                try
+                {
+                    return Options.GetConfigProfile(hostFile);
+                }
+                catch (NotSupportedException ex)
+                {
+                    throw new HostConfigurationException(ex.Message, ex);
+                }
+            }
+
             private void TryWriteHostJson(string filePath, JObject content)
             {
                 if (!_configurationSource.HostOptions.IsFileSystemReadOnly)
@@ -278,7 +302,7 @@ private void TryWriteHostJson(string filePath, JObject content)
 
             private JObject TryAddBundleConfiguration(JObject content, string bundleId, string bundleVersion)
             {
-                if (!_configurationSource.HostOptions.IsFileSystemReadOnly)
+                if (!Options.Host.IsFileSystemReadOnly)
                 {
                     string bundleConfiguration = "{ 'id': '" + bundleId + "', 'version': '" + bundleVersion + "'}";
                     content.Add("extensionBundle", JToken.Parse(bundleConfiguration));

src/WebJobs.Script/ScriptHostBuilderExtensions.cs

@@ -80,7 +80,7 @@ public static IHostBuilder AddScriptHost(this IHostBuilder builder,
                                                  IMetricsLogger metricsLogger,
                                                  Action<IWebJobsBuilder> configureWebJobs = null)
         {
-            loggerFactory = loggerFactory ?? NullLoggerFactory.Instance;
+            loggerFactory ??= NullLoggerFactory.Instance;
 
             builder.SetAzureFunctionsConfigurationRoot();
             // Host configuration
@@ -103,7 +103,8 @@ public static IHostBuilder AddScriptHost(this IHostBuilder builder,
             {
                 if (!context.Properties.ContainsKey(ScriptConstants.SkipHostJsonConfigurationKey))
                 {
-                    configBuilder.Add(new HostJsonFileConfigurationSource(applicationOptions, SystemEnvironment.Instance, loggerFactory, metricsLogger));
+                    HostJsonFileConfigurationOptions hostJsonConfigOptions = new(SystemEnvironment.Instance, applicationOptions);
+                    configBuilder.Add(new HostJsonFileConfigurationSource(hostJsonConfigOptions, loggerFactory, metricsLogger));
                 }
                 // Adding hosting config into job host configuration
                 configBuilder.Add(new FunctionsHostingConfigSource(SystemEnvironment.Instance));
@@ -128,18 +129,24 @@ public static IHostBuilder AddScriptHost(this IHostBuilder builder,
             builder.ConfigureAppConfiguration((context, configBuilder) =>
             {
                 // Pre-build configuration here to load bundles and to store for later validation.
-                var config = configBuilder.Build();
-                var extensionBundleOptions = GetExtensionBundleOptions(config);
-                FunctionsHostingConfigOptions configOption = new FunctionsHostingConfigOptions();
-                var optionsSetup = new FunctionsHostingConfigOptionsSetup(config);
+                IConfigurationRoot config = configBuilder.Build();
+                ExtensionBundleOptions extensionBundleOptions = GetExtensionBundleOptions(config);
+                FunctionsHostingConfigOptions configOption = new();
+                FunctionsHostingConfigOptionsSetup optionsSetup = new(config);
                 optionsSetup.Configure(configOption);
 
                 var extensionRequirementOptions = applicationOptions.RootServiceProvider.GetService<IOptions<ExtensionRequirementOptions>>();
 
-                var bundleManager = new ExtensionBundleManager(extensionBundleOptions, SystemEnvironment.Instance, loggerFactory, configOption);
+                ExtensionBundleManager bundleManager = new(extensionBundleOptions, SystemEnvironment.Instance, loggerFactory, configOption);
                 var metadataServiceManager = applicationOptions.RootServiceProvider.GetService<IFunctionMetadataManager>();
 
-                var locator = new ScriptStartupTypeLocator(applicationOptions.ScriptPath, loggerFactory.CreateLogger<ScriptStartupTypeLocator>(), bundleManager, metadataServiceManager, metricsLogger, extensionRequirementOptions);
+                ScriptStartupTypeLocator locator = new(
+                    applicationOptions.ScriptPath,
+                    loggerFactory.CreateLogger<ScriptStartupTypeLocator>(),
+                    bundleManager,
+                    metadataServiceManager,
+                    metricsLogger,
+                    extensionRequirementOptions);
 
                 // The locator (and thus the bundle manager) need to be created now in order to configure app configuration.
                 // Store them so they do not need to be re-created later when configuring services.

test/WebJobs.Script.Tests.Shared/TestEnvironment.cs

@@ -5,7 +5,6 @@
 using System.Collections;
 using System.Collections.Generic;
 using System.Linq;
-using System.Text;
 
 namespace Microsoft.Azure.WebJobs.Script.Tests
 {
@@ -31,6 +30,12 @@ public TestEnvironment(IDictionary<string, string> variables, bool is64BitProces
 
         public bool Is64BitProcess => _is64BitProcess;
 
+        public string this[string key]
+        {
+            get => GetEnvironmentVariable(key);
+            set => SetEnvironmentVariable(key, value);
+        }
+
         public void Clear()
         {
             _variables.Clear();