Implementing a resolver that resolves worker configurations from specified probing paths

release_notes.md

@@ -5,3 +5,4 @@
 -->
 - Update Python Worker Version to [4.40.1](https://github.com/Azure/azure-functions-python-worker/releases/tag/4.40.1)
 - Add JitTrace Files for v4.1044
+- Implementing a resolver that resolves worker configurations from specified probing paths (#11258)

src/WebJobs.Script.WebHost/WebHostServiceCollectionExtensions.cs

@@ -239,7 +239,18 @@ public static void AddWebJobsScriptHost(this IServiceCollection services, IConfi
             services.ConfigureOptionsWithChangeTokenSource<WorkerConfigurationResolverOptions, WorkerConfigurationResolverOptionsSetup, HostBuiltChangeTokenSource<WorkerConfigurationResolverOptions>>();
             services.ConfigureOptionsWithChangeTokenSource<LanguageWorkerOptions, LanguageWorkerOptionsSetup, HostBuiltChangeTokenSource<LanguageWorkerOptions>>();
 
-            services.AddSingleton<IWorkerConfigurationResolver, DefaultWorkerConfigurationResolver>();
+            services.AddSingleton<IWorkerConfigurationResolver>(p =>
+            {
+                var workerConfigurationResolverOptions = p.GetService<IOptionsMonitor<WorkerConfigurationResolverOptions>>();
+                var workerProfileManager = p.GetService<IWorkerProfileManager>();
+                var loggerFactory = p.GetService<ILoggerFactory>();
+                var metricsLogger = p.GetService<IMetricsLogger>();
+
+                return workerConfigurationResolverOptions?.CurrentValue?.IsDynamicWorkerResolutionEnabled is true ?
+                            new DynamicWorkerConfigurationResolver(loggerFactory, metricsLogger, FileUtility.Instance, workerProfileManager, SystemRuntimeInformation.Instance, workerConfigurationResolverOptions) :
+                            new DefaultWorkerConfigurationResolver(loggerFactory, metricsLogger, FileUtility.Instance, workerProfileManager, SystemRuntimeInformation.Instance, workerConfigurationResolverOptions);
+            });
+
             services.TryAddSingleton<IDependencyValidator, DependencyValidator>();
             services.TryAddSingleton<IJobHostMiddlewarePipeline>(s => DefaultMiddlewarePipeline.Empty);
 

src/WebJobs.Script/Config/FunctionsHostingConfigOptions.cs

@@ -70,6 +70,29 @@ internal string WorkerIndexingDisabledApps
             }
         }
 
+        /// <summary>
+        /// Gets a string delimited by '|' that contains the names of the language workers available through probing paths outside of the Host.
+        /// </summary>
+        internal string WorkersAvailableForDynamicResolution
+        {
+            get
+            {
+                return GetFeature(RpcWorkerConstants.WorkersAvailableForDynamicResolution) ?? string.Empty;
+            }
+        }
+
+        /// <summary>
+        /// Gets a string delimited by '|' that contains the versions of language workers to be ignored during probing outside of the Host.
+        /// Example value: "Worker1Name:Version1|Worker1Name:Version2|Worker2Name:Version1|Worker3Name:Version1".
+        /// </summary>
+        internal string IgnoredWorkerVersions
+        {
+            get
+            {
+                return GetFeature(RpcWorkerConstants.IgnoredWorkerVersions) ?? string.Empty;
+            }
+        }
+
         /// <summary>
         /// Gets a value indicating whether Linux Log Backoff is disabled in the hosting config.
         /// </summary>

src/WebJobs.Script/Diagnostics/Extensions/LoggerExtension.cs

@@ -216,6 +216,11 @@ internal static class LoggerExtension
            new EventId(343, nameof(DefaultWorkersDirectoryPath)),
            "Workers Directory set to: {workersDirPath}");
 
+        private static readonly Action<ILogger, string, Exception> _workerProbingPaths =
+           LoggerMessage.Define<string>(LogLevel.Debug,
+           new EventId(344, nameof(WorkerProbingPaths)),
+           "Worker probing paths set to: {workerProbingPaths}");
+
         public static void PublishingMetrics(this ILogger logger, string metrics)
         {
             _publishingMetrics(logger, metrics, null);
@@ -423,6 +428,11 @@ public static void DefaultWorkersDirectoryPath(this ILogger logger, string worke
             _defaultWorkersDirectoryPath(logger, workersDirPath, null);
         }
 
+        public static void WorkerProbingPaths(this ILogger logger, string workerProbingPaths)
+        {
+            _workerProbingPaths(logger, workerProbingPaths, null);
+        }
+
         public static void OutdatedExtensionBundle(this ILogger logger, string currentVersion)
         {
             _outdatedExtensionBundle(logger, currentVersion, null);

src/WebJobs.Script/Environment/EnvironmentExtensions.cs

@@ -278,6 +278,16 @@ public static bool IsWindowsElasticPremium(this IEnvironment environment)
             return string.Equals(value, ScriptConstants.ElasticPremiumSku, StringComparison.OrdinalIgnoreCase);
         }
 
+        /// <summary>
+        /// Gets a value indicating whether the application is running in a Windows App Service environment.
+        /// </summary>
+        /// <param name="environment">The environment to verify.</param>
+        /// <returns><see cref="true"/> if running in a Windows App Service app; otherwise, false.</returns>
+        public static bool IsHostedWindowsEnvironment(this IEnvironment environment)
+        {
+            return environment.IsWindowsAzureManagedHosting() || environment.IsWindowsConsumption() || environment.IsWindowsElasticPremium();
+        }
+
         public static bool IsDynamicSku(this IEnvironment environment)
         {
             return environment.IsConsumptionSku() || environment.IsWindowsElasticPremium();
@@ -687,6 +697,16 @@ public static bool IsInProc(this IEnvironment environment, string workerRuntime
             return string.IsNullOrEmpty(workerRuntime) || string.Equals(workerRuntime, RpcWorkerConstants.DotNetLanguageWorkerName, StringComparison.OrdinalIgnoreCase);
         }
 
+        /// <summary>
+        /// Returns the Antares platform release channel specified by the environment variable.
+        /// Value of this setting could be "LATEST", "STANDARD" or "EXTENDED".
+        /// If the environment variable is not set, the method returns the default value "LATEST".
+        /// </summary>
+        public static string GetPlatformReleaseChannel(this IEnvironment environment)
+        {
+            return environment.GetEnvironmentVariable(AntaresPlatformReleaseChannel) ?? ScriptConstants.LatestPlatformChannelNameUpper;
+        }
+
         public static bool IsApplicationInsightsAgentEnabled(this IEnvironment environment)
         {
             // cache the value of the environment variable

src/WebJobs.Script/Environment/EnvironmentSettingNames.cs

@@ -1,4 +1,4 @@
-// Copyright (c) .NET Foundation. All rights reserved.
+// Copyright (c) .NET Foundation. All rights reserved.
 // Licensed under the MIT License. See License.txt in the project root for license information.
 
 namespace Microsoft.Azure.WebJobs.Script
@@ -35,6 +35,7 @@ public static class EnvironmentSettingNames
         public const string AppInsightsAgent = "APPLICATIONINSIGHTS_ENABLE_AGENT";
         public const string FunctionsExtensionVersion = "FUNCTIONS_EXTENSION_VERSION";
         public const string FunctionWorkerRuntime = "FUNCTIONS_WORKER_RUNTIME";
+        public const string WorkerProbingPaths = "WORKER_PROBING_PATHS";
         public const string ContainerName = "CONTAINER_NAME";
         public const string WebsitePodName = "WEBSITE_POD_NAME";
         public const string LegionServiceHost = "LEGION_SERVICE_HOST";

src/WebJobs.Script/JsonSerializerOptionsProvider.cs

@@ -19,6 +19,12 @@ public static class JsonSerializerOptionsProvider
         /// </summary>
         public static readonly JsonSerializerOptions Options = CreateJsonOptions();
 
+        /// <summary>
+        /// Shared Json serializer with the following setting:
+        /// - PropertyNameCaseInsensitive: true.
+        /// </summary>
+        public static readonly JsonSerializerOptions WorkerConfigJsonSerializerOptions = CreateWorkerConfigJsonOptions();
+
         private static JsonSerializerOptions CreateJsonOptions()
         {
             var options = new JsonSerializerOptions
@@ -32,5 +38,10 @@ private static JsonSerializerOptions CreateJsonOptions()
 
             return options;
         }
+
+        private static JsonSerializerOptions CreateWorkerConfigJsonOptions()
+        {
+            return new JsonSerializerOptions { PropertyNameCaseInsensitive = true };
+        }
     }
 }

src/WebJobs.Script/ScriptConstants.cs

@@ -133,6 +133,7 @@ public static class ScriptConstants
         public const string FeatureFlagDisableWebHostLogForwarding = "DisableWebHostLogForwarding";
         public const string FeatureFlagDisableMergedWebHostScriptHostConfiguration = "DisableMergedConfiguration";
         public const string FeatureFlagEnableWorkerIndexing = "EnableWorkerIndexing";
+        public const string FeatureFlagDisableDynamicWorkerResolution = "DisableDynamicWorkerResolution";
         public const string FeatureFlagEnableDebugTracing = "EnableDebugTracing";
         public const string FeatureFlagEnableProxies = "EnableProxies";
         public const string FeatureFlagStrictHISModeEnabled = "StrictHISModeEnabled";
@@ -248,6 +249,7 @@ public static class ScriptConstants
         public static readonly long DefaultMaxRequestBodySize = 104857600;
 
         public static readonly ImmutableArray<string> SystemLogCategoryPrefixes = ImmutableArray.Create("Microsoft.Azure.WebJobs.", "Function.", "Worker.", "Host.");
+        public static readonly ImmutableHashSet<string> HostCapabilities = ImmutableHashSet.Create<string>(StringComparer.OrdinalIgnoreCase);
 
         public static readonly string FunctionMetadataDirectTypeKey = "DirectType";
         public static readonly string LiveLogsSessionAIKey = "#AzFuncLiveLogsSessionId";

src/WebJobs.Script/Workers/Rpc/Configuration/DefaultWorkerConfigurationResolver.cs

@@ -1,10 +1,13 @@
-// Copyright (c) .NET Foundation. All rights reserved.
+// Copyright (c) .NET Foundation. All rights reserved.
 // Licensed under the MIT License. See License.txt in the project root for license information.
 
 using System;
 using System.Collections.Generic;
+using System.IO;
 using System.IO.Abstractions;
+using Microsoft.Azure.WebJobs.Script.Diagnostics;
 using Microsoft.Azure.WebJobs.Script.Diagnostics.Extensions;
+using Microsoft.Azure.WebJobs.Script.Workers.Profiles;
 using Microsoft.Extensions.Logging;
 using Microsoft.Extensions.Options;
 
@@ -15,36 +18,75 @@ internal sealed class DefaultWorkerConfigurationResolver : IWorkerConfigurationR
     {
         private readonly ILogger _logger;
         private readonly IOptionsMonitor<WorkerConfigurationResolverOptions> _workerConfigurationResolverOptions;
+        private readonly IMetricsLogger _metricsLogger;
+        private readonly IWorkerProfileManager _profileManager;
         private readonly IFileSystem _fileSystem;
+        private readonly ISystemRuntimeInformation _systemRuntimeInformation;
 
         public DefaultWorkerConfigurationResolver(ILoggerFactory loggerFactory,
+                                                    IMetricsLogger metricsLogger,
                                                     IFileSystem fileSystem,
+                                                    IWorkerProfileManager workerProfileManager,
+                                                    ISystemRuntimeInformation systemRuntimeInformation,
                                                     IOptionsMonitor<WorkerConfigurationResolverOptions> workerConfigurationResolverOptions)
         {
             ArgumentNullException.ThrowIfNull(loggerFactory);
             _logger = loggerFactory.CreateLogger(ScriptConstants.LogCategoryWorkerConfig);
             _workerConfigurationResolverOptions = workerConfigurationResolverOptions ?? throw new ArgumentNullException(nameof(workerConfigurationResolverOptions));
+            _metricsLogger = metricsLogger ?? throw new ArgumentNullException(nameof(metricsLogger));
             _fileSystem = fileSystem ?? throw new ArgumentNullException(nameof(fileSystem));
+            _profileManager = workerProfileManager ?? throw new ArgumentNullException(nameof(workerProfileManager));
+            _systemRuntimeInformation = systemRuntimeInformation ?? throw new ArgumentNullException(nameof(systemRuntimeInformation));
+            ArgumentNullException.ThrowIfNull(_workerConfigurationResolverOptions.CurrentValue);
         }
 
-        public WorkerConfigurationInfo GetConfigurationInfo()
+        public Dictionary<string, RpcWorkerConfig> GetWorkerConfigs()
         {
-            var workersRootDirPath = _workerConfigurationResolverOptions.CurrentValue.WorkersRootDirPath;
-            _logger.DefaultWorkersDirectoryPath(workersRootDirPath);
+            _logger.DefaultWorkersDirectoryPath(_workerConfigurationResolverOptions.CurrentValue.WorkersRootDirPath);
 
-            var workerConfigPaths = new List<string>();
+            return ResolveWorkerConfigsFromWithinHost(_workerConfigurationResolverOptions.CurrentValue,
+                                                            _logger,
+                                                            _fileSystem,
+                                                            _metricsLogger,
+                                                            _systemRuntimeInformation,
+                                                            _profileManager);
+        }
+
+        internal static Dictionary<string, RpcWorkerConfig> ResolveWorkerConfigsFromWithinHost(WorkerConfigurationResolverOptions resolverOptions,
+                                                                                    ILogger logger,
+                                                                                    IFileSystem fileSystem,
+                                                                                    IMetricsLogger metricsLogger,
+                                                                                    ISystemRuntimeInformation systemRuntimeInformation,
+                                                                                    IWorkerProfileManager profileManager,
+                                                                                    Dictionary<string, RpcWorkerConfig> availableRuntimeToConfigMap = null)
+        {
+            // `availableRuntimeToConfigMap` could be partially filled by DynamicWorkerConfigurationResolver, which searches for worker configs in probing paths.
+            // This applies to scenarios such as multi-language worker environment and placeholder mode where some worker configs are found in probing paths, while remaining configs will be loaded from the default path within the Host.
+            availableRuntimeToConfigMap = availableRuntimeToConfigMap ?? new Dictionary<string, RpcWorkerConfig>(StringComparer.OrdinalIgnoreCase);
 
-            foreach (var workerDir in _fileSystem.Directory.EnumerateDirectories(workersRootDirPath))
+            foreach (var workerPath in fileSystem.Directory.EnumerateDirectories(resolverOptions.WorkersRootDirPath))
             {
-                string workerConfigPath = _fileSystem.Path.Combine(workerDir, RpcWorkerConstants.WorkerConfigFileName);
+                var workerDir = Path.GetFileName(workerPath);
+
+                if (availableRuntimeToConfigMap.ContainsKey(workerDir) || WorkerConfigurationHelper.ShouldSkipWorkerDirectory(resolverOptions.WorkerRuntime, workerDir, resolverOptions.IsMultiLanguageWorkerEnvironment, resolverOptions.IsPlaceholderModeEnabled))
+                {
+                    continue;
+                }
+
+                (var workerDescription, var workerConfigJson) = WorkerConfigurationHelper.GetWorkerDescriptionAndConfig(workerPath, profileManager, resolverOptions.WorkerDescriptionOverrides, logger);
+                if (workerDescription is null || WorkerConfigurationHelper.IsWorkerDescriptionDisabled(workerDescription, logger))
+                {
+                    continue;
+                }
 
-                if (_fileSystem.File.Exists(workerConfigPath))
+                var workerConfig = WorkerConfigurationHelper.BuildWorkerConfig(resolverOptions, workerPath, workerConfigJson, workerDescription, metricsLogger, logger, systemRuntimeInformation);
+                if (workerConfig is not null)
                 {
-                    workerConfigPaths.Add(workerDir);
+                    availableRuntimeToConfigMap[workerDir] = workerConfig;
                 }
             }
 
-            return new WorkerConfigurationInfo(_workerConfigurationResolverOptions.CurrentValue.WorkersRootDirPath, workerConfigPaths);
+            return availableRuntimeToConfigMap;
         }
     }
 }