From 987fad0819b563e2d1ff415727d9dc8fc0a5f7b6 Mon Sep 17 00:00:00 2001 From: Surbhi Gupta Date: Sun, 25 May 2025 18:02:36 -0500 Subject: [PATCH 01/55] Changes for Decoupling workers implementation --- .../Config/FunctionsHostingConfigOptions.cs | 11 + .../Environment/EnvironmentExtensions.cs | 11 + .../Environment/EnvironmentSettingNames.cs | 1 + src/WebJobs.Script/ScriptConstants.cs | 2 + src/WebJobs.Script/Utility.cs | 5 + .../Workers/Http/HttpWorkerDescription.cs | 2 +- .../ProcessManagement/WorkerDescription.cs | 2 +- .../LanguageWorkerOptionsSetup.cs | 72 ++++- .../Configuration/RpcWorkerConfigFactory.cs | 205 +++++++------ .../WorkerConfigurationHelper.cs | 130 ++++++++ .../WorkerConfigurationResolver.cs | 278 ++++++++++++++++++ .../Rpc/IWorkerConfigurationResolver.cs | 12 + .../Workers/Rpc/RpcWorkerConstants.cs | 2 + .../Workers/Rpc/RpcWorkerDescription.cs | 8 +- .../LanguageWorkerOptionsSetupTests.cs | 9 +- .../Rpc/RpcWorkerConfigFactoryTests.cs | 167 ++++++++++- .../Workers/Rpc/RpcWorkerConfigTests.cs | 32 +- .../Rpc/WorkerConfigurationResolverTests.cs | 172 +++++++++++ 18 files changed, 979 insertions(+), 142 deletions(-) create mode 100644 src/WebJobs.Script/Workers/Rpc/Configuration/WorkerConfigurationHelper.cs create mode 100644 src/WebJobs.Script/Workers/Rpc/Configuration/WorkerConfigurationResolver.cs create mode 100644 src/WebJobs.Script/Workers/Rpc/IWorkerConfigurationResolver.cs create mode 100644 test/WebJobs.Script.Tests/Workers/Rpc/WorkerConfigurationResolverTests.cs diff --git a/src/WebJobs.Script/Config/FunctionsHostingConfigOptions.cs b/src/WebJobs.Script/Config/FunctionsHostingConfigOptions.cs index ad1382f7f2..64e9515506 100644 --- a/src/WebJobs.Script/Config/FunctionsHostingConfigOptions.cs +++ b/src/WebJobs.Script/Config/FunctionsHostingConfigOptions.cs @@ -123,6 +123,17 @@ internal string WorkerIndexingDisabledApps } } + /// + /// Gets a string delimited by '|' that contains the name of the apps with worker indexing disabled. + /// + internal string EnableProbingPathsForWorkers + { + get + { + return GetFeature(RpcWorkerConstants.EnableProbingPathsForWorkers) ?? string.Empty; + } + } + /// /// Gets a value indicating whether Linux Log Backoff is disabled in the hosting config. /// diff --git a/src/WebJobs.Script/Environment/EnvironmentExtensions.cs b/src/WebJobs.Script/Environment/EnvironmentExtensions.cs index fb41d1be38..9c8b13af30 100644 --- a/src/WebJobs.Script/Environment/EnvironmentExtensions.cs +++ b/src/WebJobs.Script/Environment/EnvironmentExtensions.cs @@ -283,6 +283,17 @@ public static bool IsWindowsElasticPremium(this IEnvironment environment) return string.Equals(value, ScriptConstants.ElasticPremiumSku, StringComparison.OrdinalIgnoreCase); } + + /// + /// Gets a value indicating whether the application is running in a Windows App Service environment. + /// + /// The environment to verify. + /// if running in a Windows App Service app; otherwise, false. + public static bool IsAnyWindows(this IEnvironment environment) + { + return environment.IsWindowsAzureManagedHosting() || environment.IsWindowsConsumption() || environment.IsWindowsElasticPremium(); + } + public static bool IsDynamicSku(this IEnvironment environment) { return environment.IsConsumptionSku() || environment.IsWindowsElasticPremium(); diff --git a/src/WebJobs.Script/Environment/EnvironmentSettingNames.cs b/src/WebJobs.Script/Environment/EnvironmentSettingNames.cs index cea779be92..80795be2a9 100644 --- a/src/WebJobs.Script/Environment/EnvironmentSettingNames.cs +++ b/src/WebJobs.Script/Environment/EnvironmentSettingNames.cs @@ -35,6 +35,7 @@ public static class EnvironmentSettingNames public const string AppInsightsAgent = "APPLICATIONINSIGHTS_ENABLE_AGENT"; public const string FunctionsExtensionVersion = "FUNCTIONS_EXTENSION_VERSION"; public const string FunctionWorkerRuntime = "FUNCTIONS_WORKER_RUNTIME"; + public const string WorkerProbingPaths = "WORKER_PROBING_PATHS"; public const string ContainerName = "CONTAINER_NAME"; public const string WebsitePodName = "WEBSITE_POD_NAME"; public const string LegionServiceHost = "LEGION_SERVICE_HOST"; diff --git a/src/WebJobs.Script/ScriptConstants.cs b/src/WebJobs.Script/ScriptConstants.cs index 838707abd0..a55d1c7a50 100644 --- a/src/WebJobs.Script/ScriptConstants.cs +++ b/src/WebJobs.Script/ScriptConstants.cs @@ -133,6 +133,8 @@ public static class ScriptConstants public const string FeatureFlagDisableWebHostLogForwarding = "DisableWebHostLogForwarding"; public const string FeatureFlagDisableMergedWebHostScriptHostConfiguration = "DisableMergedConfiguration"; public const string FeatureFlagEnableWorkerIndexing = "EnableWorkerIndexing"; + public const string FeatureFlagDisableWorkerProbingPaths = "DisableWorkerProbingPaths"; + public const string FeatureFlagEnableWorkerProbingPaths = "EnableWorkerProbingPaths"; public const string FeatureFlagEnableDebugTracing = "EnableDebugTracing"; public const string FeatureFlagEnableProxies = "EnableProxies"; public const string FeatureFlagStrictHISModeEnabled = "StrictHISModeEnabled"; diff --git a/src/WebJobs.Script/Utility.cs b/src/WebJobs.Script/Utility.cs index 12b79f65b9..020bdf9e62 100644 --- a/src/WebJobs.Script/Utility.cs +++ b/src/WebJobs.Script/Utility.cs @@ -1127,6 +1127,11 @@ public static FunctionAppContentEditingState GetFunctionAppContentEditingState(I } } + public static string GetPlatformReleaseChannel(IEnvironment environment) + { + return environment.GetEnvironmentVariable(EnvironmentSettingNames.AntaresPlatformReleaseChannel) ?? ScriptConstants.LatestPlatformChannelNameUpper; + } + public static bool TryReadAsBool(IDictionary properties, string propertyKey, out bool result) { if (properties.TryGetValue(propertyKey, out object valueObject)) diff --git a/src/WebJobs.Script/Workers/Http/HttpWorkerDescription.cs b/src/WebJobs.Script/Workers/Http/HttpWorkerDescription.cs index 190433eea6..db59eea2db 100644 --- a/src/WebJobs.Script/Workers/Http/HttpWorkerDescription.cs +++ b/src/WebJobs.Script/Workers/Http/HttpWorkerDescription.cs @@ -19,7 +19,7 @@ public class HttpWorkerDescription : WorkerDescription public override bool UseStdErrorStreamForErrorsOnly { get; set; } = true; - public override void ApplyDefaultsAndValidate(string inputWorkerDirectory, ILogger logger) + public override void ApplyDefaultsAndValidate(string inputWorkerDirectory, ILogger logger, bool probingPathEnabled = false) { if (inputWorkerDirectory == null) { diff --git a/src/WebJobs.Script/Workers/ProcessManagement/WorkerDescription.cs b/src/WebJobs.Script/Workers/ProcessManagement/WorkerDescription.cs index b950e33c3c..abae2c9278 100644 --- a/src/WebJobs.Script/Workers/ProcessManagement/WorkerDescription.cs +++ b/src/WebJobs.Script/Workers/ProcessManagement/WorkerDescription.cs @@ -57,7 +57,7 @@ public abstract class WorkerDescription /// public bool? IsDisabled { get; set; } - public abstract void ApplyDefaultsAndValidate(string workerDirectory, ILogger logger); + public abstract void ApplyDefaultsAndValidate(string workerDirectory, ILogger logger, bool probingPathEnabled = false); internal void ThrowIfFileNotExists(string inputFile, string paramName) { diff --git a/src/WebJobs.Script/Workers/Rpc/Configuration/LanguageWorkerOptionsSetup.cs b/src/WebJobs.Script/Workers/Rpc/Configuration/LanguageWorkerOptionsSetup.cs index 92bd078295..c070913d0b 100644 --- a/src/WebJobs.Script/Workers/Rpc/Configuration/LanguageWorkerOptionsSetup.cs +++ b/src/WebJobs.Script/Workers/Rpc/Configuration/LanguageWorkerOptionsSetup.cs @@ -4,8 +4,14 @@ using System; using System.Collections.Generic; using System.Diagnostics; +using System.IO; +using System.Linq; +using System.Text; +using System.Text.Json; +using Microsoft.Azure.WebJobs.Script.Config; using Microsoft.Azure.WebJobs.Script.Diagnostics; using Microsoft.Azure.WebJobs.Script.Workers.Profiles; +using Microsoft.Azure.WebJobs.Script.Workers.Rpc.Configuration; using Microsoft.Extensions.Configuration; using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Logging; @@ -21,13 +27,15 @@ internal class LanguageWorkerOptionsSetup : IConfigureOptions _functionsHostingConfigOptions; public LanguageWorkerOptionsSetup(IConfiguration configuration, ILoggerFactory loggerFactory, IEnvironment environment, IMetricsLogger metricsLogger, IWorkerProfileManager workerProfileManager, - IScriptHostManager scriptHostManager) + IScriptHostManager scriptHostManager, + IOptions functionsHostingConfigOptions) { if (loggerFactory is null) { @@ -39,6 +47,7 @@ public LanguageWorkerOptionsSetup(IConfiguration configuration, _environment = environment ?? throw new ArgumentNullException(nameof(environment)); _metricsLogger = metricsLogger ?? throw new ArgumentNullException(nameof(metricsLogger)); _workerProfileManager = workerProfileManager ?? throw new ArgumentNullException(nameof(workerProfileManager)); + _functionsHostingConfigOptions = functionsHostingConfigOptions ?? throw new ArgumentNullException(nameof(functionsHostingConfigOptions)); _logger = loggerFactory.CreateLogger("Host.LanguageWorkerConfig"); } @@ -72,9 +81,66 @@ public void Configure(LanguageWorkerOptions options) } } - var configFactory = new RpcWorkerConfigFactory(configuration, _logger, SystemRuntimeInformation.Instance, _environment, _metricsLogger, _workerProfileManager); + string jsonString = GetWorkerProbingPaths(); + + if (!string.IsNullOrEmpty(jsonString)) + { + using var jsonStream = new MemoryStream(Encoding.UTF8.GetBytes(jsonString)); + + configuration = new ConfigurationBuilder() + .AddConfiguration(configuration) + .AddJsonStream(jsonStream) + .Build(); + } + + var workerConfigurationResolver = new WorkerConfigurationResolver(configuration, _logger, _environment, _workerProfileManager, _functionsHostingConfigOptions); + var configFactory = new RpcWorkerConfigFactory(configuration, _logger, SystemRuntimeInformation.Instance, _environment, _metricsLogger, _workerProfileManager, workerConfigurationResolver, _functionsHostingConfigOptions); options.WorkerConfigs = configFactory.GetConfigs(); } + + public string GetWorkerProbingPaths() + { + var probingPaths = new List(); + string output = string.Empty; + + // If Env variable is available, read from there (works for linux) + var probingPathsEnvValue = _environment.GetEnvironmentVariableOrDefault(EnvironmentSettingNames.WorkerProbingPaths, null); + + if (!string.IsNullOrEmpty(probingPathsEnvValue)) + { + probingPaths = probingPathsEnvValue.Split(';', StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries).ToList() ?? new List(); + } + else + { + if (_environment.IsAnyWindows()) + { + // Harcoded site extensions path ("c:\\home\\SiteExtensions\\workers") for Windows until Antares starts setting this as Environment variable. + // probingPaths.Add("c:\\testData\\workers"); + +#pragma warning disable SYSLIB0012 // Type or member is obsolete + string assemblyLocalPath = Path.GetDirectoryName(new Uri(typeof(LanguageWorkerOptionsSetup).Assembly.CodeBase).LocalPath); +#pragma warning restore SYSLIB0012 // Type or member is obsolete + string workersDirPath = Path.Combine(assemblyLocalPath, "ProbingPaths"); + probingPaths.Add(workersDirPath); + _logger.LogInformation($"ProbingPaths setup via options: {workersDirPath}"); + } + } + + if (probingPaths.Any()) + { + var jsonObj = new + { + languageWorkers = new + { + probingPaths + } + }; + + output = JsonSerializer.Serialize(jsonObj, new JsonSerializerOptions { WriteIndented = true }); + } + + return output; + } } /// @@ -100,4 +166,4 @@ public void PostConfigure(string name, LanguageWorkerOptions options) logger.LogInformation(message); } } -} +} \ No newline at end of file diff --git a/src/WebJobs.Script/Workers/Rpc/Configuration/RpcWorkerConfigFactory.cs b/src/WebJobs.Script/Workers/Rpc/Configuration/RpcWorkerConfigFactory.cs index 3519a449af..ea9ce1dd68 100644 --- a/src/WebJobs.Script/Workers/Rpc/Configuration/RpcWorkerConfigFactory.cs +++ b/src/WebJobs.Script/Workers/Rpc/Configuration/RpcWorkerConfigFactory.cs @@ -9,10 +9,13 @@ using System.Text.Json; using System.Text.RegularExpressions; using System.Threading.Tasks; +using Microsoft.Azure.WebJobs.Script.Config; using Microsoft.Azure.WebJobs.Script.Diagnostics; using Microsoft.Azure.WebJobs.Script.Workers.Profiles; +using Microsoft.Azure.WebJobs.Script.Workers.Rpc.Configuration; using Microsoft.Extensions.Configuration; using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; namespace Microsoft.Azure.WebJobs.Script.Workers.Rpc { @@ -26,6 +29,8 @@ internal class RpcWorkerConfigFactory private readonly IMetricsLogger _metricsLogger; private readonly string _workerRuntime; private readonly IEnvironment _environment; + private readonly IWorkerConfigurationResolver _workerConfigurationResolver; + private readonly FunctionsHostingConfigOptions _functionsHostingConfigOptions; private readonly JsonSerializerOptions _jsonSerializerOptions = new() { PropertyNameCaseInsensitive = true @@ -38,7 +43,9 @@ public RpcWorkerConfigFactory(IConfiguration config, ISystemRuntimeInformation systemRuntimeInfo, IEnvironment environment, IMetricsLogger metricsLogger, - IWorkerProfileManager workerProfileManager) + IWorkerProfileManager workerProfileManager, + IWorkerConfigurationResolver workerConfigurationResolver, + IOptions functionsHostingConfigOptions) { _config = config ?? throw new ArgumentNullException(nameof(config)); _logger = logger ?? throw new ArgumentNullException(nameof(logger)); @@ -47,10 +54,14 @@ public RpcWorkerConfigFactory(IConfiguration config, _metricsLogger = metricsLogger ?? throw new ArgumentNullException(nameof(metricsLogger)); _profileManager = workerProfileManager ?? throw new ArgumentNullException(nameof(workerProfileManager)); _workerRuntime = _environment.GetEnvironmentVariable(RpcWorkerConstants.FunctionWorkerRuntimeSettingName); + _workerConfigurationResolver = workerConfigurationResolver ?? throw new ArgumentNullException(nameof(workerConfigurationResolver)); + _functionsHostingConfigOptions = functionsHostingConfigOptions.Value; WorkersDirPath = GetDefaultWorkersDirectory(Directory.Exists); var workersDirectorySection = _config.GetSection($"{RpcWorkerConstants.LanguageWorkersSectionName}:{WorkerConstants.WorkersDirectorySectionName}"); + ProbingPathsEnabled = AreProbingPathsEnabled(); + if (!string.IsNullOrEmpty(workersDirectorySection.Value)) { WorkersDirPath = workersDirectorySection.Value; @@ -59,6 +70,8 @@ public RpcWorkerConfigFactory(IConfiguration config, public string WorkersDirPath { get; } + internal bool ProbingPathsEnabled { get; } + public IList GetConfigs() { using (_metricsLogger.LatencyEvent(MetricEventNames.GetConfigs)) @@ -90,14 +103,30 @@ internal void BuildWorkerProviderDictionary() internal void AddProviders() { - _logger.LogDebug("Workers Directory set to: {WorkersDirPath}", WorkersDirPath); + if (ProbingPathsEnabled) + { + List probingPaths = GetWorkerProbingPaths(_config); - foreach (var workerDir in Directory.EnumerateDirectories(WorkersDirPath)) + _logger.LogDebug("Workers Directory set to: probingPaths = {probingPaths} and fallback path = {WorkersDirPath}", string.Join(", ", probingPaths), WorkersDirPath); + + List workerConfigs = _workerConfigurationResolver.GetWorkerConfigs(probingPaths, WorkersDirPath); + + foreach (var workerConfig in workerConfigs) + { + AddProvider(workerConfig); + } + } + else { - string workerConfigPath = Path.Combine(workerDir, RpcWorkerConstants.WorkerConfigFileName); - if (File.Exists(workerConfigPath)) + _logger.LogDebug("Workers Directory set to: {WorkersDirPath}", WorkersDirPath); + + foreach (var workerDir in Directory.EnumerateDirectories(WorkersDirPath)) { - AddProvider(workerDir); + string workerConfigPath = Path.Combine(workerDir, RpcWorkerConstants.WorkerConfigFileName); + if (File.Exists(workerConfigPath)) + { + AddProvider(workerDir); + } } } } @@ -144,30 +173,16 @@ internal void AddProvider(string workerDir) _logger.LogDebug("Found worker config: {workerConfigPath}", workerConfigPath); - var workerConfig = GetWorkerConfigJsonElement(workerConfigPath); - var workerDescriptionElement = workerConfig.GetProperty(WorkerConstants.WorkerDescription); - var workerDescription = workerDescriptionElement.Deserialize(_jsonSerializerOptions); - workerDescription.WorkerDirectory = workerDir; + var workerConfig = WorkerConfigurationHelper.GetWorkerConfigJsonElement(workerConfigPath); - // Read the profiles from worker description and load the profile for which the conditions match - if (workerConfig.TryGetProperty(WorkerConstants.WorkerDescriptionProfiles, out var profiles)) - { - List workerDescriptionProfiles = ReadWorkerDescriptionProfiles(profiles); - if (workerDescriptionProfiles.Count > 0) - { - _profileManager.SetWorkerDescriptionProfiles(workerDescriptionProfiles, workerDescription.Language); - _profileManager.LoadWorkerDescriptionFromProfiles(workerDescription, out workerDescription); - } - } - - // Check if any app settings are provided for that language - var languageSection = _config.GetSection($"{RpcWorkerConstants.LanguageWorkersSectionName}:{workerDescription.Language}"); - workerDescription.Arguments ??= new List(); - GetWorkerDescriptionFromAppSettings(workerDescription, languageSection); - AddArgumentsFromAppSettings(workerDescription, languageSection); - - // Validate workerDescription - workerDescription.ApplyDefaultsAndValidate(Directory.GetCurrentDirectory(), _logger); + RpcWorkerDescription workerDescription = WorkerConfigurationHelper.GetWorkerDescription( + workerConfig, + _jsonSerializerOptions, + workerDir, + ProbingPathsEnabled, + _profileManager, + _config, + _logger); if (workerDescription.IsDisabled == true) { @@ -203,7 +218,7 @@ internal void AddProvider(string workerDir) _workerDescriptionDictionary[workerDescription.Language] = rpcWorkerConfig; ReadLanguageWorkerFile(arguments.WorkerPath); - _logger.LogDebug("Added WorkerConfig for language: {language}", workerDescription.Language); + _logger.LogDebug("Added WorkerConfig for language: {language} from path: {path}", workerDescription.Language, workerDescription.DefaultWorkerPath); } } catch (Exception ex) when (!ex.IsFatal()) @@ -213,61 +228,6 @@ internal void AddProvider(string workerDir) } } - private static JsonElement GetWorkerConfigJsonElement(string workerConfigPath) - { - ReadOnlySpan jsonSpan = File.ReadAllBytes(workerConfigPath); - - if (jsonSpan.StartsWith([0xEF, 0xBB, 0xBF])) - { - jsonSpan = jsonSpan[3..]; // Skip UTF-8 Byte Order Mark (BOM) if present at the beginning of the file. - } - - var reader = new Utf8JsonReader(jsonSpan, isFinalBlock: true, state: default); - using var doc = JsonDocument.ParseValue(ref reader); - - return doc.RootElement.Clone(); - } - - private List ReadWorkerDescriptionProfiles(JsonElement profilesElement) - { - var profiles = profilesElement.Deserialize>(_jsonSerializerOptions); - - if (profiles == null || profiles.Count <= 0) - { - return new List(0); - } - - var descriptionProfiles = new List(profiles.Count); - - try - { - foreach (var profile in profiles) - { - var profileConditions = new List(profile.Conditions.Count); - - foreach (var descriptor in profile.Conditions) - { - if (!_profileManager.TryCreateWorkerProfileCondition(descriptor, out IWorkerProfileCondition condition)) - { - // Failed to resolve condition. This profile will be disabled using a mock false condition - _logger.LogInformation("Profile {name} is disabled. Cannot resolve the profile condition {condition}", profile.ProfileName, descriptor.Type); - condition = new FalseCondition(); - } - - profileConditions.Add(condition); - } - - descriptionProfiles.Add(new(profile.ProfileName, profileConditions, profile.Description)); - } - } - catch (Exception) - { - throw new FormatException("Failed to parse profiles in worker config."); - } - - return descriptionProfiles; - } - internal WorkerProcessCountOptions GetWorkerProcessCount(JsonElement workerConfig) { WorkerProcessCountOptions workerProcessCount = null; @@ -310,24 +270,6 @@ internal WorkerProcessCountOptions GetWorkerProcessCount(JsonElement workerConfi return workerProcessCount; } - private static void GetWorkerDescriptionFromAppSettings(RpcWorkerDescription workerDescription, IConfigurationSection languageSection) - { - var defaultExecutablePathSetting = languageSection.GetSection($"{WorkerConstants.WorkerDescriptionDefaultExecutablePath}"); - workerDescription.DefaultExecutablePath = defaultExecutablePathSetting.Value != null ? defaultExecutablePathSetting.Value : workerDescription.DefaultExecutablePath; - - var defaultRuntimeVersionAppSetting = languageSection.GetSection($"{WorkerConstants.WorkerDescriptionDefaultRuntimeVersion}"); - workerDescription.DefaultRuntimeVersion = defaultRuntimeVersionAppSetting.Value != null ? defaultRuntimeVersionAppSetting.Value : workerDescription.DefaultRuntimeVersion; - } - - internal static void AddArgumentsFromAppSettings(RpcWorkerDescription workerDescription, IConfigurationSection languageSection) - { - var argumentsSection = languageSection.GetSection($"{WorkerConstants.WorkerDescriptionArguments}"); - if (argumentsSection.Value != null) - { - ((List)workerDescription.Arguments).AddRange(Regex.Split(argumentsSection.Value, @"\s+")); - } - } - internal bool ShouldAddWorkerConfig(string workerDescriptionLanguage) { if (_environment.IsPlaceholderModeEnabled()) @@ -357,6 +299,61 @@ internal bool ShouldAddWorkerConfig(string workerDescriptionLanguage) return true; } + internal List GetWorkerProbingPaths(IConfiguration config) + { + List probingPaths = new List(); + + IConfigurationSection probingPathsSection = config.GetSection($"{RpcWorkerConstants.LanguageWorkersSectionName}") + ?.GetSection($"{RpcWorkerConstants.WorkerProbingPathsSectionName}"); + + var probingPathsList = probingPathsSection?.AsEnumerable(); + + if (probingPathsList is null || probingPathsSection is null) + { + return probingPaths; + } + + for (int i = 0; i < probingPathsList.Count(); i++) + { + var path = probingPathsSection.GetSection($"{i}").Value; + if (path is not null) + { + probingPaths.Add(path); + } + } + + return probingPaths; + } + + internal bool AreProbingPathsEnabled() + { + bool isFeatureFlagDisabled = FeatureFlags.IsEnabled(ScriptConstants.FeatureFlagDisableWorkerProbingPaths, _environment); + + if (isFeatureFlagDisabled) + { + return false; + } + + bool isFeatureFlagEnabled = FeatureFlags.IsEnabled(ScriptConstants.FeatureFlagEnableWorkerProbingPaths, _environment); + + if (isFeatureFlagEnabled) + { + return true; + } + + HashSet probingPathsEnabledWorkersViaHostingConfig = _functionsHostingConfigOptions.EnableProbingPathsForWorkers.ToLowerInvariant().Split("|", StringSplitOptions.RemoveEmptyEntries).ToHashSet(); + + if (!_environment.IsMultiLanguageRuntimeEnvironment()) + { + if (!string.IsNullOrWhiteSpace(_workerRuntime)) + { + return probingPathsEnabledWorkersViaHostingConfig.Contains(_workerRuntime); + } + } + + return probingPathsEnabledWorkersViaHostingConfig.Any() ? true : false; + } + private void ReadLanguageWorkerFile(string workerPath) { if (!_environment.IsPlaceholderModeEnabled() @@ -398,4 +395,4 @@ private void ReadLanguageWorkerFile(string workerPath) }); } } -} +} \ No newline at end of file diff --git a/src/WebJobs.Script/Workers/Rpc/Configuration/WorkerConfigurationHelper.cs b/src/WebJobs.Script/Workers/Rpc/Configuration/WorkerConfigurationHelper.cs new file mode 100644 index 0000000000..60f7c1b87a --- /dev/null +++ b/src/WebJobs.Script/Workers/Rpc/Configuration/WorkerConfigurationHelper.cs @@ -0,0 +1,130 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the MIT License. See License.txt in the project root for license information. + +using System; +using System.Collections.Generic; +using System.IO; +using System.Text.Json; +using System.Text.RegularExpressions; +using Microsoft.Azure.WebJobs.Script.Workers.Profiles; +using Microsoft.Extensions.Configuration; +using Microsoft.Extensions.Logging; + +namespace Microsoft.Azure.WebJobs.Script.Workers.Rpc.Configuration +{ + internal static class WorkerConfigurationHelper + { + internal static RpcWorkerDescription GetWorkerDescription( + JsonElement workerConfig, + JsonSerializerOptions jsonSerializerOptions, + string workerDir, + bool probingPathsEnabled, + IWorkerProfileManager profileManager, + IConfiguration config, + ILogger logger) + { + var workerDescriptionElement = workerConfig.GetProperty(WorkerConstants.WorkerDescription); + var workerDescription = workerDescriptionElement.Deserialize(jsonSerializerOptions); + workerDescription.WorkerDirectory = workerDir; + + // Read the profiles from worker description and load the profile for which the conditions match + if (workerConfig.TryGetProperty(WorkerConstants.WorkerDescriptionProfiles, out var profiles)) + { + List workerDescriptionProfiles = ReadWorkerDescriptionProfiles(profiles, jsonSerializerOptions, profileManager, logger); + if (workerDescriptionProfiles.Count > 0) + { + profileManager.SetWorkerDescriptionProfiles(workerDescriptionProfiles, workerDescription.Language); + profileManager.LoadWorkerDescriptionFromProfiles(workerDescription, out workerDescription); + } + } + + // Check if any app settings are provided for that language + var languageSection = config.GetSection($"{RpcWorkerConstants.LanguageWorkersSectionName}:{workerDescription.Language}"); + workerDescription.Arguments ??= new List(); + GetWorkerDescriptionFromAppSettings(workerDescription, languageSection); + AddArgumentsFromAppSettings(workerDescription, languageSection); + + // Validate workerDescription + workerDescription.ApplyDefaultsAndValidate(Directory.GetCurrentDirectory(), logger, probingPathsEnabled); + + return workerDescription; + } + + internal static JsonElement GetWorkerConfigJsonElement(string workerConfigPath) + { + ReadOnlySpan jsonSpan = File.ReadAllBytes(workerConfigPath); + + if (jsonSpan.StartsWith([0xEF, 0xBB, 0xBF])) + { + jsonSpan = jsonSpan[3..]; // Skip UTF-8 Byte Order Mark (BOM) if present at the beginning of the file. + } + + var reader = new Utf8JsonReader(jsonSpan, isFinalBlock: true, state: default); + using var doc = JsonDocument.ParseValue(ref reader); + + return doc.RootElement.Clone(); + } + + private static List ReadWorkerDescriptionProfiles( + JsonElement profilesElement, + JsonSerializerOptions jsonSerializerOptions, + IWorkerProfileManager profileManager, + ILogger logger) + { + var profiles = profilesElement.Deserialize>(jsonSerializerOptions); + + if (profiles == null || profiles.Count <= 0) + { + return new List(0); + } + + var descriptionProfiles = new List(profiles.Count); + + try + { + foreach (var profile in profiles) + { + var profileConditions = new List(profile.Conditions.Count); + + foreach (var descriptor in profile.Conditions) + { + if (!profileManager.TryCreateWorkerProfileCondition(descriptor, out IWorkerProfileCondition condition)) + { + // Failed to resolve condition. This profile will be disabled using a mock false condition + logger.LogInformation("Profile {name} is disabled. Cannot resolve the profile condition {condition}", profile.ProfileName, descriptor.Type); + condition = new FalseCondition(); + } + + profileConditions.Add(condition); + } + + descriptionProfiles.Add(new(profile.ProfileName, profileConditions, profile.Description)); + } + } + catch (Exception) + { + throw new FormatException("Failed to parse profiles in worker config."); + } + + return descriptionProfiles; + } + + private static void GetWorkerDescriptionFromAppSettings(RpcWorkerDescription workerDescription, IConfigurationSection languageSection) + { + var defaultExecutablePathSetting = languageSection?.GetSection($"{WorkerConstants.WorkerDescriptionDefaultExecutablePath}"); + workerDescription.DefaultExecutablePath = defaultExecutablePathSetting?.Value != null ? defaultExecutablePathSetting.Value : workerDescription.DefaultExecutablePath; + + var defaultRuntimeVersionAppSetting = languageSection?.GetSection($"{WorkerConstants.WorkerDescriptionDefaultRuntimeVersion}"); + workerDescription.DefaultRuntimeVersion = defaultRuntimeVersionAppSetting?.Value != null ? defaultRuntimeVersionAppSetting.Value : workerDescription.DefaultRuntimeVersion; + } + + internal static void AddArgumentsFromAppSettings(RpcWorkerDescription workerDescription, IConfigurationSection languageSection) + { + var argumentsSection = languageSection?.GetSection($"{WorkerConstants.WorkerDescriptionArguments}"); + if (argumentsSection?.Value != null) + { + ((List)workerDescription.Arguments).AddRange(Regex.Split(argumentsSection.Value, @"\s+")); + } + } + } +} \ No newline at end of file diff --git a/src/WebJobs.Script/Workers/Rpc/Configuration/WorkerConfigurationResolver.cs b/src/WebJobs.Script/Workers/Rpc/Configuration/WorkerConfigurationResolver.cs new file mode 100644 index 0000000000..2d6f6f1f81 --- /dev/null +++ b/src/WebJobs.Script/Workers/Rpc/Configuration/WorkerConfigurationResolver.cs @@ -0,0 +1,278 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the MIT License. See License.txt in the project root for license information. + +using System; +using System.Collections.Concurrent; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Text.Json; +using Microsoft.Azure.AppService.Proxy.Common.Extensions; +using Microsoft.Azure.WebJobs.Script.Config; +using Microsoft.Azure.WebJobs.Script.Workers.Profiles; +using Microsoft.Extensions.Configuration; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; + +namespace Microsoft.Azure.WebJobs.Script.Workers.Rpc.Configuration +{ + internal sealed class WorkerConfigurationResolver : IWorkerConfigurationResolver + { + private readonly IConfiguration _config; + private readonly ILogger _logger; + private readonly IWorkerProfileManager _profileManager; + private readonly IEnvironment _environment; + private readonly FunctionsHostingConfigOptions _functionsHostingConfigOptions; + private readonly JsonSerializerOptions _jsonSerializerOptions = new() + { + PropertyNameCaseInsensitive = true + }; + + private HashSet _probingPathsEnabledWorkersViaHostingConfig; + + public WorkerConfigurationResolver(IConfiguration config, + ILogger logger, + IEnvironment environment, + IWorkerProfileManager workerProfileManager, + IOptions functionsHostingConfigOptions) + { + _config = config ?? throw new ArgumentNullException(nameof(config)); + _logger = logger ?? throw new ArgumentNullException(nameof(logger)); + _environment = environment ?? throw new ArgumentNullException(nameof(environment)); + _profileManager = workerProfileManager ?? throw new ArgumentNullException(nameof(workerProfileManager)); + _functionsHostingConfigOptions = functionsHostingConfigOptions?.Value ?? throw new ArgumentNullException(nameof(functionsHostingConfigOptions)); + + _probingPathsEnabledWorkersViaHostingConfig = _functionsHostingConfigOptions.EnableProbingPathsForWorkers.ToLowerInvariant().Split("|").ToHashSet(); + } + + public List GetWorkerConfigs(List probingPaths, string fallbackPath) + { + var workerRuntime = _environment.GetEnvironmentVariable(EnvironmentSettingNames.FunctionWorkerRuntime); + + // Dictionary of language-name : workerConfig + ConcurrentDictionary outputDict = new ConcurrentDictionary(); + + // check worker release channel + string releaseChannel = Utility.GetPlatformReleaseChannel(_environment); + + if (!probingPaths.IsNullOrEmpty()) + { + foreach (var probingPath in probingPaths) + { + if (!string.IsNullOrEmpty(probingPath) && Directory.Exists(probingPath)) + { + foreach (var languageWorkerPath in Directory.EnumerateDirectories(probingPath)) + { + string languageWorkerFolder = Path.GetFileName(languageWorkerPath); + + _logger.LogInformation("Probing for language worker in path: {LanguageWorkerPath}", languageWorkerPath); + + // Only skip worker directories that don't match the current runtime. + // Do not skip non-worker directories like the function app payload directory + // && languageWorkerPath.StartsWith(fallbackPath)) + if (//!_probingPathsEnabledWorkersViaHostingConfig.Contains(languageWorkerFolder) || + (!_environment.IsMultiLanguageRuntimeEnvironment() && + workerRuntime is not null && + !workerRuntime.Equals(languageWorkerFolder, StringComparison.OrdinalIgnoreCase))) + { + continue; + } + + IEnumerable workerVersions = Directory.EnumerateDirectories(languageWorkerPath); + var versionsList = ParseWorkerVersions(workerVersions); + var versions = versionsList.OrderDescending(); + + if (outputDict.ContainsKey(languageWorkerFolder)) + { + continue; + } + + GetWorkerConfigsFromProbingPaths(versions, languageWorkerPath, languageWorkerFolder, releaseChannel, outputDict); + + _logger.LogInformation("Found worker config for {LanguageWorkerFolder} at {LanguageWorkerPath}", languageWorkerFolder, outputDict.GetValueOrDefault(languageWorkerFolder)); + } + } + } + } + + if (!_environment.IsMultiLanguageRuntimeEnvironment() && + workerRuntime is not null && + outputDict.ContainsKey(workerRuntime)) + { + return outputDict.Values.ToList(); + } + + // fallback path + GetWorkerConfigsFromWithinHost(fallbackPath, workerRuntime, outputDict); + + return outputDict.Values.ToList(); + } + + private void GetWorkerConfigsFromProbingPaths(IEnumerable versions, string languageWorkerPath, string languageWorkerFolder, string releaseChannel, ConcurrentDictionary outputDict) + { + int found = 0; + + // language worker version + foreach (Version versionFolder in versions) + { + string languageWorkerVersionPath = Path.Combine(languageWorkerPath, versionFolder.ToString()); + if (IsCompatibleWithHost(languageWorkerVersionPath)) + { + found++; + outputDict[languageWorkerFolder] = languageWorkerVersionPath; + + if (string.IsNullOrEmpty(releaseChannel) || !releaseChannel.Equals(ScriptConstants.StandardPlatformChannelNameUpper)) + { + // latest version is the default + break; + } + + if (found > 1 && releaseChannel.Equals(ScriptConstants.StandardPlatformChannelNameUpper)) + { + outputDict[languageWorkerFolder] = languageWorkerVersionPath; + break; + } + } + } + } + + private void GetWorkerConfigsFromWithinHost(string fallbackPath, string workerRuntime, ConcurrentDictionary outputDict) + { + _logger.LogInformation("Searching for worker configs in fallback path: {FallbackPath}", fallbackPath); + + // fallback path + if (Directory.Exists(fallbackPath)) + { + foreach (var workerDir in Directory.EnumerateDirectories(fallbackPath)) + { + string workerFolder = Path.GetFileName(workerDir); + + if (outputDict.ContainsKey(workerFolder) || + (!_environment.IsMultiLanguageRuntimeEnvironment() && + workerRuntime is not null && + !workerRuntime.Equals(workerFolder, StringComparison.OrdinalIgnoreCase))) + { + continue; + } + + string workerConfigPath = Path.Combine(workerDir, RpcWorkerConstants.WorkerConfigFileName); + if (File.Exists(workerConfigPath)) + { + outputDict[workerFolder] = workerDir; + _logger.LogInformation("Found worker in fallback path workerfolder = {workerFolder} and dir = {workerDir}", workerFolder, workerDir); + } + + if (!_environment.IsMultiLanguageRuntimeEnvironment() && + workerRuntime is not null && + outputDict.ContainsKey(workerRuntime)) + { + break; + } + } + } + } + + private List ParseWorkerVersions(IEnumerable workerVersions) + { + var versions = new List(); + + foreach (var workerVersion in workerVersions) + { + string versionFolder = Path.GetFileName(workerVersion); + + if (versionFolder.Length == 1) + { + versionFolder = versionFolder + ".0"; // Handle single digit versions like '1' as '1.0' + } + + if (Version.TryParse(versionFolder, out Version version)) + { + versions.Add(version); + } + else + { + Console.WriteLine($"Failed to parse version: '{versionFolder}'"); + } + } + + return versions; + } + + private HashSet GetHostCapabilities() + { + HashSet hostCapabilites = ["test-capability-1", "test-capability-2"]; + + return hostCapabilites; + } + + private bool IsCompatibleWithHost(string workerDir) + { + string workerConfigPath = Path.Combine(workerDir, RpcWorkerConstants.WorkerConfigFileName); + if (!File.Exists(workerConfigPath)) + { + return false; + } + + JsonElement workerConfig = WorkerConfigurationHelper.GetWorkerConfigJsonElement(workerConfigPath); + + // static capability resolution + bool doesHostRequirementMeet = DoesHostRequirementMeet(workerConfig); + + if (!doesHostRequirementMeet) + { + return false; + } + + // profiles evaluation + RpcWorkerDescription workerDescription = WorkerConfigurationHelper.GetWorkerDescription( + workerConfig, + _jsonSerializerOptions, + workerDir, + true, + _profileManager, + _config, + _logger); + + if (workerDescription.IsDisabled == true) + { + return false; + } + + return true; + } + + private HashSet GetHostRequirements(JsonElement workerConfig) + { + HashSet hostRequirements = new HashSet(); + + if (workerConfig.TryGetProperty("hostRequirements", out JsonElement configSection)) + { + var requirements = configSection.EnumerateArray(); + + foreach (var requirement in requirements) + { + hostRequirements.Add(requirement.GetString()); + } + } + + return hostRequirements; + } + + private bool DoesHostRequirementMeet(JsonElement workerConfig) + { + HashSet hostRequirements = GetHostRequirements(workerConfig); + + HashSet hostCapabilities = GetHostCapabilities(); + + foreach (var hostRequirement in hostRequirements) + { + if (!hostCapabilities.Contains(hostRequirement)) + { + return false; + } + } + + return true; + } + } +} \ No newline at end of file diff --git a/src/WebJobs.Script/Workers/Rpc/IWorkerConfigurationResolver.cs b/src/WebJobs.Script/Workers/Rpc/IWorkerConfigurationResolver.cs new file mode 100644 index 0000000000..53b3a4f019 --- /dev/null +++ b/src/WebJobs.Script/Workers/Rpc/IWorkerConfigurationResolver.cs @@ -0,0 +1,12 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the MIT License. See License.txt in the project root for license information. + +using System.Collections.Generic; + +namespace Microsoft.Azure.WebJobs.Script.Workers.Rpc.Configuration +{ + public interface IWorkerConfigurationResolver + { + internal List GetWorkerConfigs(List probingPaths, string fallbackPath); + } +} \ No newline at end of file diff --git a/src/WebJobs.Script/Workers/Rpc/RpcWorkerConstants.cs b/src/WebJobs.Script/Workers/Rpc/RpcWorkerConstants.cs index b7a41b980b..18d56c348e 100644 --- a/src/WebJobs.Script/Workers/Rpc/RpcWorkerConstants.cs +++ b/src/WebJobs.Script/Workers/Rpc/RpcWorkerConstants.cs @@ -26,6 +26,7 @@ public static class RpcWorkerConstants // Section names in host.json or AppSettings public const string LanguageWorkersSectionName = "languageWorkers"; + public const string WorkerProbingPathsSectionName = "probingPaths"; // Worker description constants public const string WorkerDescriptionLanguage = "language"; @@ -90,6 +91,7 @@ public static class RpcWorkerConstants public const string WorkerIndexingEnabled = "WORKER_INDEXING_ENABLED"; public const string WorkerIndexingDisabledApps = "WORKER_INDEXING_DISABLED_APPS"; + public const string EnableProbingPathsForWorkers = "ENABLE_PROBING_PATHS_FOR_WORKERS"; public const string RevertWorkerShutdownBehavior = "REVERT_WORKER_SHUTDOWN_BEHAVIOR"; public const string ShutdownWebhostWorkerChannelsOnHostShutdown = "ShutdownWebhostWorkerChannelsOnHostShutdown"; public const string ThrowOnMissingFunctionsWorkerRuntime = "THROW_ON_MISSING_FUNCTIONS_WORKER_RUNTIME"; diff --git a/src/WebJobs.Script/Workers/Rpc/RpcWorkerDescription.cs b/src/WebJobs.Script/Workers/Rpc/RpcWorkerDescription.cs index 570497343b..57d03131c3 100644 --- a/src/WebJobs.Script/Workers/Rpc/RpcWorkerDescription.cs +++ b/src/WebJobs.Script/Workers/Rpc/RpcWorkerDescription.cs @@ -87,7 +87,7 @@ public List Extensions public override bool UseStdErrorStreamForErrorsOnly { get; set; } = false; - public override void ApplyDefaultsAndValidate(string workerDirectory, ILogger logger) + public override void ApplyDefaultsAndValidate(string workerDirectory, ILogger logger, bool probingPathsEnabled = false) { if (workerDirectory == null) { @@ -97,7 +97,11 @@ public override void ApplyDefaultsAndValidate(string workerDirectory, ILogger lo WorkerDirectory = WorkerDirectory ?? workerDirectory; if (!string.IsNullOrEmpty(DefaultWorkerPath) && !Path.IsPathRooted(DefaultWorkerPath)) { - DefaultWorkerPath = Path.Combine(WorkerDirectory, DefaultWorkerPath); + if (probingPathsEnabled && DefaultWorkerPath.Contains(RpcWorkerConstants.RuntimeVersionPlaceholder)) + { + var versionDir = Path.GetFileName(WorkerDirectory); + WorkerDirectory = WorkerDirectory.Replace(versionDir, string.Empty); + } } if (string.IsNullOrEmpty(Language)) { diff --git a/test/WebJobs.Script.Tests/Configuration/LanguageWorkerOptionsSetupTests.cs b/test/WebJobs.Script.Tests/Configuration/LanguageWorkerOptionsSetupTests.cs index d3f2dea6c8..070ec14245 100644 --- a/test/WebJobs.Script.Tests/Configuration/LanguageWorkerOptionsSetupTests.cs +++ b/test/WebJobs.Script.Tests/Configuration/LanguageWorkerOptionsSetupTests.cs @@ -2,10 +2,15 @@ // Licensed under the MIT License. See License.txt in the project root for license information. using System; +using Microsoft.Azure.WebJobs.Script.Config; +using Microsoft.Azure.WebJobs.Script.Workers; using Microsoft.Azure.WebJobs.Script.Workers.Profiles; using Microsoft.Azure.WebJobs.Script.Workers.Rpc; +using Microsoft.Azure.WebJobs.Script.Workers.Rpc.Configuration; using Microsoft.Extensions.Configuration; +using Microsoft.Extensions.Logging; using Microsoft.Extensions.Logging.Abstractions; +using Microsoft.Extensions.Options; using Moq; using Xunit; @@ -53,7 +58,7 @@ public void LanguageWorkerOptions_Expected_ListOfConfigs(string workerRuntime) } }); - LanguageWorkerOptionsSetup setup = new LanguageWorkerOptionsSetup(configuration, NullLoggerFactory.Instance, testEnvironment, testMetricLogger, testProfileManager.Object, testScriptHostManager.Object); + LanguageWorkerOptionsSetup setup = new LanguageWorkerOptionsSetup(configuration, NullLoggerFactory.Instance, testEnvironment, testMetricLogger, testProfileManager.Object, testScriptHostManager.Object, new OptionsWrapper(new FunctionsHostingConfigOptions())); LanguageWorkerOptions options = new LanguageWorkerOptions(); setup.Configure(options); @@ -72,4 +77,4 @@ public void LanguageWorkerOptions_Expected_ListOfConfigs(string workerRuntime) } } } -} +} \ No newline at end of file diff --git a/test/WebJobs.Script.Tests/Workers/Rpc/RpcWorkerConfigFactoryTests.cs b/test/WebJobs.Script.Tests/Workers/Rpc/RpcWorkerConfigFactoryTests.cs index 62c135d6a2..41f03cdd9f 100644 --- a/test/WebJobs.Script.Tests/Workers/Rpc/RpcWorkerConfigFactoryTests.cs +++ b/test/WebJobs.Script.Tests/Workers/Rpc/RpcWorkerConfigFactoryTests.cs @@ -10,7 +10,10 @@ using Microsoft.Azure.WebJobs.Script.Workers; using Microsoft.Azure.WebJobs.Script.Workers.Profiles; using Microsoft.Azure.WebJobs.Script.Workers.Rpc; +using Microsoft.Azure.WebJobs.Script.Workers.Rpc.Configuration; using Microsoft.Extensions.Configuration; +using Microsoft.Extensions.Options; +using Moq; using Xunit; namespace Microsoft.Azure.WebJobs.Script.Tests.Workers.Rpc @@ -39,7 +42,9 @@ public void DefaultLanguageWorkersDir() var expectedWorkersDir = Path.Combine(Path.GetDirectoryName(new Uri(typeof(RpcWorkerConfigFactory).Assembly.Location).LocalPath), RpcWorkerConstants.DefaultWorkersDirectoryName); var config = new ConfigurationBuilder().Build(); var testLogger = new TestLogger("test"); - var configFactory = new RpcWorkerConfigFactory(config, testLogger, _testSysRuntimeInfo, _testEnvironment, new TestMetricsLogger(), _testWorkerProfileManager); + var hostingOptions = new OptionsWrapper(new FunctionsHostingConfigOptions()); + var workerConfigurationResolver = new WorkerConfigurationResolver(config, testLogger, _testEnvironment, _testWorkerProfileManager, hostingOptions); + var configFactory = new RpcWorkerConfigFactory(config, testLogger, _testSysRuntimeInfo, _testEnvironment, new TestMetricsLogger(), _testWorkerProfileManager, workerConfigurationResolver, hostingOptions); Assert.Equal(expectedWorkersDir, configFactory.WorkersDirPath); } @@ -72,7 +77,9 @@ public void LanguageWorker_WorkersDir_Set() }) .Build(); var testLogger = new TestLogger("test"); - var configFactory = new RpcWorkerConfigFactory(config, testLogger, _testSysRuntimeInfo, _testEnvironment, new TestMetricsLogger(), _testWorkerProfileManager); + var hostingOptions = new OptionsWrapper(new FunctionsHostingConfigOptions()); + var workerConfigurationResolver = new WorkerConfigurationResolver(config, testLogger, _testEnvironment, _testWorkerProfileManager, hostingOptions); + var configFactory = new RpcWorkerConfigFactory(config, testLogger, _testSysRuntimeInfo, _testEnvironment, new TestMetricsLogger(), _testWorkerProfileManager, workerConfigurationResolver, hostingOptions); Assert.Equal(expectedWorkersDir, configFactory.WorkersDirPath); } @@ -88,7 +95,9 @@ public void LanguageWorker_WorkersDir_NotSet() var config = configBuilder.Build(); var scriptSettingsManager = new ScriptSettingsManager(config); var testLogger = new TestLogger("test"); - var configFactory = new RpcWorkerConfigFactory(config, testLogger, _testSysRuntimeInfo, _testEnvironment, new TestMetricsLogger(), _testWorkerProfileManager); + var hostingOptions = new OptionsWrapper(new FunctionsHostingConfigOptions()); + var workerConfigurationResolver = new WorkerConfigurationResolver(config, testLogger, _testEnvironment, _testWorkerProfileManager, hostingOptions); + var configFactory = new RpcWorkerConfigFactory(config, testLogger, _testSysRuntimeInfo, _testEnvironment, new TestMetricsLogger(), _testWorkerProfileManager, workerConfigurationResolver, hostingOptions); Assert.Equal(expectedWorkersDir, configFactory.WorkersDirPath); } @@ -112,7 +121,9 @@ public void WorkerDescription_Skipped_When_Profile_Disables_Worker() var scriptSettingsManager = new ScriptSettingsManager(config); var testLogger = new TestLogger("test"); _testEnvironment.SetEnvironmentVariable("ENV_VAR_BAR", "True"); - var configFactory = new RpcWorkerConfigFactory(config, testLogger, _testSysRuntimeInfo, _testEnvironment, new TestMetricsLogger(), _testWorkerProfileManager); + var hostingOptions = new OptionsWrapper(new FunctionsHostingConfigOptions()); + var workerConfigurationResolver = new WorkerConfigurationResolver(config, testLogger, _testEnvironment, _testWorkerProfileManager, hostingOptions); + var configFactory = new RpcWorkerConfigFactory(config, testLogger, _testSysRuntimeInfo, _testEnvironment, new TestMetricsLogger(), _testWorkerProfileManager, workerConfigurationResolver, hostingOptions); var workerConfigs = configFactory.GetConfigs(); var errors = testLogger.GetLogMessages().Where(m => m.Exception != null).ToList(); @@ -134,13 +145,51 @@ public void JavaPath_FromEnvVars() var config = configBuilder.Build(); var scriptSettingsManager = new ScriptSettingsManager(config); var testLogger = new TestLogger("test"); - var configFactory = new RpcWorkerConfigFactory(config, testLogger, _testSysRuntimeInfo, _testEnvironment, new TestMetricsLogger(), _testWorkerProfileManager); + var hostingOptions = new OptionsWrapper(new FunctionsHostingConfigOptions()); + var workerConfigurationResolver = new WorkerConfigurationResolver(config, testLogger, _testEnvironment, _testWorkerProfileManager, hostingOptions); + var configFactory = new RpcWorkerConfigFactory(config, testLogger, _testSysRuntimeInfo, _testEnvironment, new TestMetricsLogger(), _testWorkerProfileManager, workerConfigurationResolver, hostingOptions); var workerConfigs = configFactory.GetConfigs(); var javaPath = workerConfigs.FirstOrDefault(c => c.Description.Language.Equals("java", StringComparison.OrdinalIgnoreCase)).Description.DefaultExecutablePath; Assert.DoesNotContain(@"%JAVA_HOME%", javaPath); Assert.Contains(@"/bin/java", javaPath); } + [Fact] + public void GetConfigs_ProbingPaths() + { + string probingPath1 = Path.GetFullPath("..\\..\\..\\..\\test\\TestWorkers\\ProbingPaths\\workers\\"); + string fallbackPath = Path.GetFullPath("..\\..\\..\\..\\test\\TestWorkers\\FallbackPath\\workers\\"); + + var mockEnvironment = new Mock(); + //mockEnvironment.Setup(p => p.GetEnvironmentVariable(EnvironmentSettingNames.AzureWebJobsFeatureFlags)).Returns(ScriptConstants.FeatureFlagDisableWorkerProbingPaths); + mockEnvironment.Setup(p => p.GetEnvironmentVariable(EnvironmentSettingNames.FunctionWorkerRuntime)).Returns("java"); + + var inMemorySettings = new Dictionary + { + ["languageWorkers:probingPaths:0"] = probingPath1, + }; + + IConfiguration config = new ConfigurationBuilder() + .AddInMemoryCollection(inMemorySettings) + .Build(); + + var scriptSettingsManager = new ScriptSettingsManager(config); + var testLogger = new TestLogger("test"); + + var hostingOptionsValue = new FunctionsHostingConfigOptions(); + hostingOptionsValue.Features.Add(RpcWorkerConstants.EnableProbingPathsForWorkers, "java"); + var hostingOptions = new OptionsWrapper(hostingOptionsValue); + + var workerConfigurationResolver = new WorkerConfigurationResolver(config, testLogger, mockEnvironment.Object, _testWorkerProfileManager, hostingOptions); + var configFactory = new RpcWorkerConfigFactory(config, testLogger, _testSysRuntimeInfo, mockEnvironment.Object, new TestMetricsLogger(), _testWorkerProfileManager, workerConfigurationResolver, hostingOptions); + var workerConfigs = configFactory.GetConfigs(); + + // check log messages + var logs = testLogger.GetLogMessages(); + Assert.Equal(1, workerConfigs.Count); + Assert.True(logs.Any(p => p.FormattedMessage.Contains("Workers Directory set to: probingPaths = " + probingPath1))); + } + [Fact] public void DefaultWorkerConfigs_Overrides_DefaultWorkerRuntimeVersion_AppSetting() { @@ -156,7 +205,9 @@ public void DefaultWorkerConfigs_Overrides_DefaultWorkerRuntimeVersion_AppSettin _testEnvironment.SetEnvironmentVariable(EnvironmentSettingNames.AzureWebsitePlaceholderMode, "1"); using var variables = new TestScopedSettings(scriptSettingsManager, testEnvVariables); - var configFactory = new RpcWorkerConfigFactory(config, testLogger, _testSysRuntimeInfo, _testEnvironment, new TestMetricsLogger(), _testWorkerProfileManager); + var hostingOptions = new OptionsWrapper(new FunctionsHostingConfigOptions()); + var workerConfigurationResolver = new WorkerConfigurationResolver(config, testLogger, _testEnvironment, _testWorkerProfileManager, hostingOptions); + var configFactory = new RpcWorkerConfigFactory(config, testLogger, _testSysRuntimeInfo, _testEnvironment, new TestMetricsLogger(), _testWorkerProfileManager, workerConfigurationResolver, hostingOptions); var workerConfigs = configFactory.GetConfigs(); var pythonWorkerConfig = workerConfigs.FirstOrDefault(w => w.Description.Language.Equals("python", StringComparison.OrdinalIgnoreCase)); var powershellWorkerConfig = workerConfigs.FirstOrDefault(w => w.Description.Language.Equals("powershell", StringComparison.OrdinalIgnoreCase)); @@ -167,22 +218,46 @@ public void DefaultWorkerConfigs_Overrides_DefaultWorkerRuntimeVersion_AppSettin Assert.Equal("7.4", powershellWorkerConfig.Description.DefaultRuntimeVersion); } - [Fact] - public void DefaultWorkerConfigs_Overrides_VersionAppSetting() + [Theory] + [InlineData("7.4", "7.4", null, null)] + [InlineData(null, "7.4", null, null)] + [InlineData("7.4", "7.4", "..\\..\\..\\..\\test\\TestWorkers\\ProbingPaths\\workers\\", "EnableWorkerProbingPaths")] + [InlineData(null, "7.4", "..\\..\\..\\..\\test\\TestWorkers\\ProbingPaths\\workers\\", "EnableWorkerProbingPaths")] + [InlineData("7.2", "7.2", null, null)] + [InlineData("7.2", "7.2", "..\\..\\..\\..\\test\\TestWorkers\\ProbingPaths\\workers\\", "EnableWorkerProbingPaths")] + [InlineData("7", "7", null, null)] + [InlineData("7", "7", "..\\..\\..\\..\\test\\TestWorkers\\ProbingPaths\\workers\\", "EnableWorkerProbingPaths")] + public void DefaultWorkerConfigs_Overrides_VersionAppSetting(string runtimeSettingVersion, string outputVersion, string probingPathValue, string enableProbingPath) { + var probingPath = string.IsNullOrEmpty(probingPathValue) ? probingPathValue : Path.GetFullPath(probingPathValue); + var testEnvironment = new TestEnvironment(); - testEnvironment.SetEnvironmentVariable("FUNCTIONS_WORKER_RUNTIME_VERSION", "7.4"); - testEnvironment.SetEnvironmentVariable("FUNCTIONS_WORKER_RUNTIME", "powerShell"); + testEnvironment.SetEnvironmentVariable("FUNCTIONS_WORKER_RUNTIME_VERSION", runtimeSettingVersion); + testEnvironment.SetEnvironmentVariable("FUNCTIONS_WORKER_RUNTIME", "powershell"); + testEnvironment.SetEnvironmentVariable(EnvironmentSettingNames.WorkerProbingPaths, $"{probingPath}"); + testEnvironment.SetEnvironmentVariable(EnvironmentSettingNames.AzureWebJobsFeatureFlags, enableProbingPath); + var configBuilder = ScriptSettingsManager.CreateDefaultConfigurationBuilder(); + configBuilder.AddInMemoryCollection(new Dictionary + { + { $"{RpcWorkerConstants.LanguageWorkersSectionName}:probingPaths:0", probingPath } + }); var config = configBuilder.Build(); var scriptSettingsManager = new ScriptSettingsManager(config); var testLogger = new TestLogger("test"); - var configFactory = new RpcWorkerConfigFactory(config, testLogger, _testSysRuntimeInfo, testEnvironment, new TestMetricsLogger(), _testWorkerProfileManager); + var hostingOptions = new OptionsWrapper(new FunctionsHostingConfigOptions()); + var workerConfigurationResolver = new WorkerConfigurationResolver(config, testLogger, testEnvironment, _testWorkerProfileManager, hostingOptions); + var configFactory = new RpcWorkerConfigFactory(config, testLogger, _testSysRuntimeInfo, testEnvironment, new TestMetricsLogger(), _testWorkerProfileManager, workerConfigurationResolver, hostingOptions); var workerConfigs = configFactory.GetConfigs(); var powershellWorkerConfig = workerConfigs.FirstOrDefault(w => w.Description.Language.Equals("powershell", StringComparison.OrdinalIgnoreCase)); Assert.Equal(1, workerConfigs.Count); Assert.NotNull(powershellWorkerConfig); - Assert.Equal("7.4", powershellWorkerConfig.Description.DefaultRuntimeVersion); + Assert.Equal(outputVersion, powershellWorkerConfig.Description.DefaultRuntimeVersion); + + if (probingPath is not null) + { + Assert.True(powershellWorkerConfig.Arguments.WorkerPath.Contains(probingPath)); + } } [Theory] @@ -204,7 +279,9 @@ public void ShouldAddProvider_Returns_Expected(string workerLanguage, string wor } var config = new ConfigurationBuilder().Build(); var testLogger = new TestLogger("test"); - RpcWorkerConfigFactory rpcWorkerConfigFactory = new RpcWorkerConfigFactory(config, testLogger, _testSysRuntimeInfo, _testEnvironment, new TestMetricsLogger(), _testWorkerProfileManager); + var hostingOptions = new OptionsWrapper(new FunctionsHostingConfigOptions()); + var workerConfigurationResolver = new WorkerConfigurationResolver(config, testLogger, _testEnvironment, _testWorkerProfileManager, hostingOptions); + RpcWorkerConfigFactory rpcWorkerConfigFactory = new RpcWorkerConfigFactory(config, testLogger, _testSysRuntimeInfo, _testEnvironment, new TestMetricsLogger(), _testWorkerProfileManager, workerConfigurationResolver, hostingOptions); _testEnvironment.SetEnvironmentVariable(RpcWorkerConstants.FunctionWorkerRuntimeSettingName, workerRuntime); Assert.Equal(expectedResult, rpcWorkerConfigFactory.ShouldAddWorkerConfig(workerLanguage)); } @@ -248,7 +325,9 @@ public void GetWorkerProcessCount_Tests(bool defaultWorkerConfig, bool setProces var config = new ConfigurationBuilder().Build(); var testLogger = new TestLogger("test"); - RpcWorkerConfigFactory rpcWorkerConfigFactory = new RpcWorkerConfigFactory(config, testLogger, _testSysRuntimeInfo, _testEnvironment, new TestMetricsLogger(), _testWorkerProfileManager); + var hostingOptions = new OptionsWrapper(new FunctionsHostingConfigOptions()); + var workerConfigurationResolver = new WorkerConfigurationResolver(config, testLogger, _testEnvironment, _testWorkerProfileManager, hostingOptions); + RpcWorkerConfigFactory rpcWorkerConfigFactory = new RpcWorkerConfigFactory(config, testLogger, _testSysRuntimeInfo, _testEnvironment, new TestMetricsLogger(), _testWorkerProfileManager, workerConfigurationResolver, hostingOptions); var result = rpcWorkerConfigFactory.GetWorkerProcessCount(workerConfig); if (defaultWorkerConfig) @@ -288,7 +367,9 @@ public void GetWorkerProcessCount_ThrowsException_Tests() var config = new ConfigurationBuilder().Build(); var testLogger = new TestLogger("test"); - RpcWorkerConfigFactory rpcWorkerConfigFactory = new RpcWorkerConfigFactory(config, testLogger, _testSysRuntimeInfo, _testEnvironment, new TestMetricsLogger(), _testWorkerProfileManager); + var hostingOptions = new OptionsWrapper(new FunctionsHostingConfigOptions()); + var workerConfigurationResolver = new WorkerConfigurationResolver(config, testLogger, _testEnvironment, _testWorkerProfileManager, hostingOptions); + RpcWorkerConfigFactory rpcWorkerConfigFactory = new RpcWorkerConfigFactory(config, testLogger, _testSysRuntimeInfo, _testEnvironment, new TestMetricsLogger(), _testWorkerProfileManager, workerConfigurationResolver, hostingOptions); var resultEx1 = Assert.Throws(() => rpcWorkerConfigFactory.GetWorkerProcessCount(workerConfig)); Assert.Contains("ProcessCount must be greater than 0", resultEx1.Message); @@ -301,6 +382,62 @@ public void GetWorkerProcessCount_ThrowsException_Tests() Assert.Contains("value could not be converted to System.TimeSpan", resultEx3.Message); } + [Fact] + public void GetWorkerProbingPaths_ReturnsExpectedPaths() + { + // Arrange + var inMemorySettings = new Dictionary + { + ["languageWorkers:probingPaths:0"] = @"C:\workers\path1", + ["languageWorkers:probingPaths:1"] = @"C:\workers\path2" + }; + + IConfiguration config = new ConfigurationBuilder() + .AddInMemoryCollection(inMemorySettings) + .Build(); + + var testLogger = new TestLogger("test"); + var hostingOptions = new OptionsWrapper(new FunctionsHostingConfigOptions()); + var workerConfigurationResolver = new WorkerConfigurationResolver(config, testLogger, _testEnvironment, _testWorkerProfileManager, hostingOptions); + RpcWorkerConfigFactory rpcWorkerConfigFactory = new RpcWorkerConfigFactory(config, testLogger, _testSysRuntimeInfo, _testEnvironment, new TestMetricsLogger(), _testWorkerProfileManager, workerConfigurationResolver, hostingOptions); + + // Act + var result = rpcWorkerConfigFactory.GetWorkerProbingPaths(config); + + // Assert + Assert.Equal(2, result.Count); + Assert.Contains(@"C:\workers\path1", result); + Assert.Contains(@"C:\workers\path2", result); + } + + [Theory] + [InlineData("languageWorkers:probingPaths:0", "", 1)] + [InlineData("languageWorkers:probingPaths", "C:\\workers\\path1", 0)] + [InlineData("languageWorkers", "C:\\workers\\path1", 0)] + public void GetWorkerProbingPaths_InvalidProbingPaths_ReturnsEmptyList(string key, string value, int count) + { + // Arrange + var inMemorySettings = new Dictionary + { + [key] = value, + }; + + IConfiguration config = new ConfigurationBuilder() + .AddInMemoryCollection(inMemorySettings) + .Build(); + + var testLogger = new TestLogger("test"); + var hostingOptions = new OptionsWrapper(new FunctionsHostingConfigOptions()); + var workerConfigurationResolver = new WorkerConfigurationResolver(config, testLogger, _testEnvironment, _testWorkerProfileManager, hostingOptions); + RpcWorkerConfigFactory rpcWorkerConfigFactory = new RpcWorkerConfigFactory(config, testLogger, _testSysRuntimeInfo, _testEnvironment, new TestMetricsLogger(), _testWorkerProfileManager, workerConfigurationResolver, hostingOptions); + + // Act + var result = rpcWorkerConfigFactory.GetWorkerProbingPaths(config); + + // Assert + Assert.Equal(count, result.Count); + } + private static JsonElement CreateWorkerConfig(int processCount, int maxProcessCount, string processStartupInterval, bool setProcessCountToCores) { using var stream = new MemoryStream(); diff --git a/test/WebJobs.Script.Tests/Workers/Rpc/RpcWorkerConfigTests.cs b/test/WebJobs.Script.Tests/Workers/Rpc/RpcWorkerConfigTests.cs index 1f696ad16b..a0d98f4d6f 100644 --- a/test/WebJobs.Script.Tests/Workers/Rpc/RpcWorkerConfigTests.cs +++ b/test/WebJobs.Script.Tests/Workers/Rpc/RpcWorkerConfigTests.cs @@ -12,8 +12,10 @@ using Microsoft.Azure.WebJobs.Script.Workers; using Microsoft.Azure.WebJobs.Script.Workers.Profiles; using Microsoft.Azure.WebJobs.Script.Workers.Rpc; +using Microsoft.Azure.WebJobs.Script.Workers.Rpc.Configuration; using Microsoft.Extensions.Configuration; using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; using Moq; using Xunit; @@ -282,10 +284,10 @@ public void ValidateWorkerDescription_ResolvesDotNetDefaultWorkerExecutablePath_ Extensions = new List(), DefaultExecutablePath = defaultExecutablePath, FileExists = path => - { - Assert.Equal(expectedExecutablePath, path); - return true; - } + { + Assert.Equal(expectedExecutablePath, path); + return true; + } }; workerDescription.ApplyDefaultsAndValidate(Directory.GetCurrentDirectory(), testLogger); @@ -313,10 +315,10 @@ public void ValidateWorkerDescription_DoesNotModifyDefaultWorkerExecutablePathAn Extensions = new List(), DefaultExecutablePath = defaultExecutablePath, FileExists = path => - { - Assert.Equal(expectedExecutablePath, path); - return false; - } + { + Assert.Equal(expectedExecutablePath, path); + return false; + } }; workerDescription.ApplyDefaultsAndValidate(Directory.GetCurrentDirectory(), testLogger); @@ -341,10 +343,10 @@ public void ValidateWorkerDescription_DoesNotModifyDefaultWorkerExecutablePath_W Extensions = new List(), DefaultExecutablePath = defaultExecutablePath, FileExists = path => - { - Assert.True(false, "FileExists should not be called"); - return false; - } + { + Assert.True(false, "FileExists should not be called"); + return false; + } }; workerDescription.ApplyDefaultsAndValidate(Directory.GetCurrentDirectory(), testLogger); @@ -683,7 +685,9 @@ private IEnumerable TestReadWorkerProviderFromConfig(IEnumerabl var scriptHostOptions = new ScriptJobHostOptions(); var scriptSettingsManager = new ScriptSettingsManager(config); var workerProfileManager = new Mock(); - var configFactory = new RpcWorkerConfigFactory(config, testLogger, _testSysRuntimeInfo, _testEnvironment, testMetricsLogger, workerProfileManager.Object); + var hostingOptions = new OptionsWrapper(new FunctionsHostingConfigOptions()); + var workerConfigurationResolver = new WorkerConfigurationResolver(config, testLogger, _testEnvironment, workerProfileManager.Object, hostingOptions); + var configFactory = new RpcWorkerConfigFactory(config, testLogger, _testSysRuntimeInfo, _testEnvironment, testMetricsLogger, workerProfileManager.Object, workerConfigurationResolver, hostingOptions); if (appSvcEnv) { @@ -757,4 +761,4 @@ private bool AreRequiredMetricsEmitted(TestMetricsLogger metricsLogger) return hasBegun && hasEnded && (metricsLogger.EventsBegan.Contains(MetricEventNames.GetConfigs) && metricsLogger.EventsEnded.Contains(MetricEventNames.GetConfigs)); } } -} +} \ No newline at end of file diff --git a/test/WebJobs.Script.Tests/Workers/Rpc/WorkerConfigurationResolverTests.cs b/test/WebJobs.Script.Tests/Workers/Rpc/WorkerConfigurationResolverTests.cs new file mode 100644 index 0000000000..8ca279c01e --- /dev/null +++ b/test/WebJobs.Script.Tests/Workers/Rpc/WorkerConfigurationResolverTests.cs @@ -0,0 +1,172 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the MIT License. See License.txt in the project root for license information. + +using System.Collections.Generic; +using System.IO; +using System.Linq; +using Microsoft.Azure.WebJobs.Script.Config; +using Microsoft.Azure.WebJobs.Script.Workers.Profiles; +using Microsoft.Azure.WebJobs.Script.Workers.Rpc; +using Microsoft.Azure.WebJobs.Script.Workers.Rpc.Configuration; +using Microsoft.Extensions.Configuration; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; +using Moq; +using Xunit; + +namespace Microsoft.Azure.WebJobs.Script.Tests.Workers.Rpc +{ + public class WorkerConfigurationResolverTests + { + private readonly Mock _mockProfileManager; + private readonly Mock _mockConfig; + private readonly Mock _mockLogger; + private readonly string _probingPath1 = Path.GetFullPath("..\\..\\..\\..\\test\\TestWorkers\\ProbingPaths\\workers\\"); + private readonly string _fallbackPath = Path.GetFullPath("workers"); + private List _probingPaths; + + public WorkerConfigurationResolverTests() + { + _mockProfileManager = new Mock(); + _mockConfig = new Mock(); + _mockLogger = new Mock(); + + _probingPaths = new List { _probingPath1, string.Empty, null, "path-not-exists" }; + } + + [Theory] + [InlineData("java", "LATEST", "2.19.0")] + [InlineData("java", "STANDARD", "2.18.0")] + [InlineData("node", "STANDARD", "3.10.1")] + public void GetWorkerConfigs_ReturnsExpectedConfigs(string languageWorker, string releaseChannel, string version) + { + // Arrange + var mockEnvironment = new Mock(); + mockEnvironment.Setup(p => p.GetEnvironmentVariable(EnvironmentSettingNames.FunctionWorkerRuntime)).Returns(languageWorker); + mockEnvironment.Setup(p => p.GetEnvironmentVariable(EnvironmentSettingNames.AntaresPlatformReleaseChannel)).Returns(releaseChannel); + mockEnvironment.Setup(p => p.GetEnvironmentVariable(EnvironmentSettingNames.AppKind)).Returns((string)null); + + // Act + var hostingOptionsValue = new FunctionsHostingConfigOptions(); + hostingOptionsValue.Features.Add(RpcWorkerConstants.EnableProbingPathsForWorkers, languageWorker); + var hostingOptions = new OptionsWrapper(hostingOptionsValue); + + var workerConfigurationResolver = new WorkerConfigurationResolver(_mockConfig.Object, _mockLogger.Object, mockEnvironment.Object, _mockProfileManager.Object, hostingOptions); + + var result = workerConfigurationResolver.GetWorkerConfigs(_probingPaths, _fallbackPath); + + // Assert + Assert.Single(result); + Assert.Contains(_probingPath1 + languageWorker + "\\" + version, result); + } + + [Theory] + [InlineData("LATEST", "java\\2.19.0", "node\\3.10.1", "powershell\\7.4", "dotnet-isolated", "python")] + [InlineData("STANDARD", "java\\2.18.0", "node\\3.10.1", "powershell\\7.2", "dotnet-isolated", "python")] + public void GetWorkerConfigs_MultiLanguageWorker_ReturnsExpectedConfigs(string releaseChannel, string java, string node, string powershell, string dotnetIsolated, string python) + { + // Arrange + var mockEnvironment = new Mock(); + mockEnvironment.Setup(p => p.GetEnvironmentVariable(EnvironmentSettingNames.AntaresPlatformReleaseChannel)).Returns(releaseChannel); + mockEnvironment.Setup(p => p.GetEnvironmentVariable(EnvironmentSettingNames.AppKind)).Returns(ScriptConstants.WorkFlowAppKind); + mockEnvironment.Setup(p => p.GetEnvironmentVariable(EnvironmentSettingNames.FunctionWorkerRuntime)).Returns((string)null); + + // Act + var hostingOptionsValue = new FunctionsHostingConfigOptions(); + hostingOptionsValue.Features.Add(RpcWorkerConstants.EnableProbingPathsForWorkers, "java|node|powershell"); + var hostingOptions = new OptionsWrapper(hostingOptionsValue); + + var workerConfigurationResolver = new WorkerConfigurationResolver(_mockConfig.Object, _mockLogger.Object, mockEnvironment.Object, _mockProfileManager.Object, hostingOptions); + + var result = workerConfigurationResolver.GetWorkerConfigs(_probingPaths, _fallbackPath); + + // Assert + Assert.Equal(result.Count, 5); + Assert.True(result.Any(r => r.Contains(_probingPath1 + java))); + Assert.True(result.Any(r => r.Contains(_probingPath1 + node))); + Assert.True(result.Any(r => r.Contains(_probingPath1 + powershell))); + Assert.True(result.Any(r => r.Contains(_fallbackPath + "\\" + dotnetIsolated))); + Assert.True(result.Any(r => r.Contains(_fallbackPath + "\\" + python))); + } + + [Theory] + [InlineData(null, "LATEST")] + [InlineData(null, "STANDARD")] + [InlineData("Empty", "LATEST")] + [InlineData("Empty", "STANDARD")] + public void GetWorkerConfigs_MultiLanguageWorker_NullOREmptyProbingPath_ReturnsExpectedConfigs(string probingPathValue, string releaseChannel) + { + // Arrange + var mockEnvironment = new Mock(); + mockEnvironment.Setup(p => p.GetEnvironmentVariable(EnvironmentSettingNames.AntaresPlatformReleaseChannel)).Returns(releaseChannel); + mockEnvironment.Setup(p => p.GetEnvironmentVariable(EnvironmentSettingNames.AppKind)).Returns(ScriptConstants.WorkFlowAppKind); + mockEnvironment.Setup(p => p.GetEnvironmentVariable(EnvironmentSettingNames.FunctionWorkerRuntime)).Returns((string)null); + + List probingPaths = null; + + if (probingPathValue == "Empty") + { + probingPaths = new List(); + } + + // Act + var hostingOptionsValue = new FunctionsHostingConfigOptions(); + hostingOptionsValue.Features.Add(RpcWorkerConstants.EnableProbingPathsForWorkers, "java|node|powershell"); + var hostingOptions = new OptionsWrapper(hostingOptionsValue); + + var workerConfigurationResolver = new WorkerConfigurationResolver(_mockConfig.Object, _mockLogger.Object, mockEnvironment.Object, _mockProfileManager.Object, hostingOptions); + + var result = workerConfigurationResolver.GetWorkerConfigs(probingPaths, _fallbackPath); + + // Assert + Assert.Equal(result.Count, 5); + Assert.True(result.Any(r => r.Contains(_fallbackPath + "\\java"))); + Assert.True(result.Any(r => r.Contains(_fallbackPath + "\\node"))); + Assert.True(result.Any(r => r.Contains(_fallbackPath + "\\powershell"))); + Assert.True(result.Any(r => r.Contains(_fallbackPath + "\\dotnet-isolated"))); + Assert.True(result.Any(r => r.Contains(_fallbackPath + "\\python"))); + } + + [Theory] + [InlineData(null, "LATEST", "java")] + [InlineData(null, "STANDARD", "java")] + [InlineData("Empty", "LATEST", "java")] + [InlineData("Empty", "STANDARD", "java")] + [InlineData(null, "STANDARD", "node")] + [InlineData("Empty", "LATEST", "node")] + [InlineData(null, "STANDARD", "powershell")] + [InlineData("Empty", "LATEST", "powershell")] + [InlineData(null, "LATEST", "dotnet-isolated")] + [InlineData(null, "STANDARD", "dotnet-isolated")] + [InlineData("Empty", "LATEST", "dotnet-isolated")] + [InlineData("Empty", "STANDARD", "dotnet-isolated")] + public void GetWorkerConfigs_NullOREmptyProbingPath_ReturnsExpectedConfigs(string probingPathValue, string releaseChannel, string languageWorker) + { + // Arrange + var mockEnvironment = new Mock(); + mockEnvironment.Setup(p => p.GetEnvironmentVariable(EnvironmentSettingNames.FunctionWorkerRuntime)).Returns(languageWorker); + mockEnvironment.Setup(p => p.GetEnvironmentVariable(EnvironmentSettingNames.AntaresPlatformReleaseChannel)).Returns(releaseChannel); + mockEnvironment.Setup(p => p.GetEnvironmentVariable(EnvironmentSettingNames.AppKind)).Returns((string)null); + + List probingPaths = null; + + if (probingPathValue == "Empty") + { + probingPaths = new List(); + } + + // Act + var hostingOptionsValue = new FunctionsHostingConfigOptions(); + hostingOptionsValue.Features.Add(RpcWorkerConstants.EnableProbingPathsForWorkers, "java|node|powershell"); + var hostingOptions = new OptionsWrapper(hostingOptionsValue); + + var workerConfigurationResolver = new WorkerConfigurationResolver(_mockConfig.Object, _mockLogger.Object, mockEnvironment.Object, _mockProfileManager.Object, hostingOptions); + + var result = workerConfigurationResolver.GetWorkerConfigs(probingPaths, _fallbackPath); + + // Assert + Assert.Equal(result.Count, 1); + Assert.True(result.Any(r => r.Contains(_fallbackPath + "\\" + languageWorker))); + } + } +} \ No newline at end of file From 734b02961c3a777c02a066b2eb7d5c6c3dc93d3e Mon Sep 17 00:00:00 2001 From: Surbhi Gupta Date: Sun, 25 May 2025 23:27:52 -0500 Subject: [PATCH 02/55] Added test data for unit tests --- .../Workers/Rpc/RpcWorkerDescription.cs | 4 ++++ .../workers/java/2.18.0/worker.config.json | 10 ++++++++++ .../workers/java/2.18.1/worker.config.json | 14 ++++++++++++++ .../workers/java/2.18.2/worker.config.json | 12 ++++++++++++ .../java/2.19.0/azure-functions-java-worker.jar | 1 + .../workers/java/2.19.0/worker.config.json | 11 +++++++++++ .../workers/node/3.10.1/worker.config.json | 14 ++++++++++++++ .../Microsoft.Azure.Functions.PowerShellWorker.dll | 1 + .../workers/powershell/7.2/worker.config.json | 11 +++++++++++ .../Microsoft.Azure.Functions.PowerShellWorker.dll | 1 + .../workers/powershell/7.4/worker.config.json | 11 +++++++++++ .../Microsoft.Azure.Functions.PowerShellWorker.dll | 1 + .../workers/powershell/7/worker.config.json | 11 +++++++++++ 13 files changed, 102 insertions(+) create mode 100644 test/TestWorkers/Probingpaths/workers/java/2.18.0/worker.config.json create mode 100644 test/TestWorkers/Probingpaths/workers/java/2.18.1/worker.config.json create mode 100644 test/TestWorkers/Probingpaths/workers/java/2.18.2/worker.config.json create mode 100644 test/TestWorkers/Probingpaths/workers/java/2.19.0/azure-functions-java-worker.jar create mode 100644 test/TestWorkers/Probingpaths/workers/java/2.19.0/worker.config.json create mode 100644 test/TestWorkers/Probingpaths/workers/node/3.10.1/worker.config.json create mode 100644 test/TestWorkers/Probingpaths/workers/powershell/7.2/Microsoft.Azure.Functions.PowerShellWorker.dll create mode 100644 test/TestWorkers/Probingpaths/workers/powershell/7.2/worker.config.json create mode 100644 test/TestWorkers/Probingpaths/workers/powershell/7.4/Microsoft.Azure.Functions.PowerShellWorker.dll create mode 100644 test/TestWorkers/Probingpaths/workers/powershell/7.4/worker.config.json create mode 100644 test/TestWorkers/Probingpaths/workers/powershell/7/Microsoft.Azure.Functions.PowerShellWorker.dll create mode 100644 test/TestWorkers/Probingpaths/workers/powershell/7/worker.config.json diff --git a/src/WebJobs.Script/Workers/Rpc/RpcWorkerDescription.cs b/src/WebJobs.Script/Workers/Rpc/RpcWorkerDescription.cs index 57d03131c3..9402bb293d 100644 --- a/src/WebJobs.Script/Workers/Rpc/RpcWorkerDescription.cs +++ b/src/WebJobs.Script/Workers/Rpc/RpcWorkerDescription.cs @@ -95,6 +95,7 @@ public override void ApplyDefaultsAndValidate(string workerDirectory, ILogger lo } Arguments = Arguments ?? new List(); WorkerDirectory = WorkerDirectory ?? workerDirectory; + if (!string.IsNullOrEmpty(DefaultWorkerPath) && !Path.IsPathRooted(DefaultWorkerPath)) { if (probingPathsEnabled && DefaultWorkerPath.Contains(RpcWorkerConstants.RuntimeVersionPlaceholder)) @@ -102,7 +103,10 @@ public override void ApplyDefaultsAndValidate(string workerDirectory, ILogger lo var versionDir = Path.GetFileName(WorkerDirectory); WorkerDirectory = WorkerDirectory.Replace(versionDir, string.Empty); } + + DefaultWorkerPath = Path.Combine(WorkerDirectory, DefaultWorkerPath); } + if (string.IsNullOrEmpty(Language)) { throw new ValidationException($"WorkerDescription {nameof(Language)} cannot be empty"); diff --git a/test/TestWorkers/Probingpaths/workers/java/2.18.0/worker.config.json b/test/TestWorkers/Probingpaths/workers/java/2.18.0/worker.config.json new file mode 100644 index 0000000000..12cca5b1d7 --- /dev/null +++ b/test/TestWorkers/Probingpaths/workers/java/2.18.0/worker.config.json @@ -0,0 +1,10 @@ +{ + "description": { + "language": "java", + "extensions": [".jar"], + "defaultExecutablePath": "%JAVA_HOME%/bin/java", + "defaultWorkerPath": "azure-functions-java-worker.jar", + "arguments": ["-XX:+TieredCompilation -XX:TieredStopAtLevel=1 -noverify -Djava.net.preferIPv4Stack=true -jar", "%JAVA_OPTS%", "%AZURE_FUNCTIONS_MESH_JAVA_OPTS%"] + }, + "profiles": [] +} \ No newline at end of file diff --git a/test/TestWorkers/Probingpaths/workers/java/2.18.1/worker.config.json b/test/TestWorkers/Probingpaths/workers/java/2.18.1/worker.config.json new file mode 100644 index 0000000000..89d426268c --- /dev/null +++ b/test/TestWorkers/Probingpaths/workers/java/2.18.1/worker.config.json @@ -0,0 +1,14 @@ +{ + "description": { + "language": "java", + "extensions": [".jar"], + "defaultExecutablePath": "%JAVA_HOME%/bin/java", + "defaultWorkerPath": "azure-functions-java-worker.jar", + "arguments": ["-XX:+TieredCompilation -XX:TieredStopAtLevel=1 -noverify -Djava.net.preferIPv4Stack=true -jar", "%JAVA_OPTS%", "%AZURE_FUNCTIONS_MESH_JAVA_OPTS%"] + }, + "hostRequirements": [ + "test-capability1", + "test-capability2" + ], + "profiles": [] +} \ No newline at end of file diff --git a/test/TestWorkers/Probingpaths/workers/java/2.18.2/worker.config.json b/test/TestWorkers/Probingpaths/workers/java/2.18.2/worker.config.json new file mode 100644 index 0000000000..8923070c36 --- /dev/null +++ b/test/TestWorkers/Probingpaths/workers/java/2.18.2/worker.config.json @@ -0,0 +1,12 @@ +{ + "description": { + "isDisabled": true, + "language": "java", + "extensions": [".jar"], + "defaultExecutablePath": "%JAVA_HOME%/bin/java", + "defaultWorkerPath": "azure-functions-java-worker.jar", + "arguments": ["-XX:+TieredCompilation -XX:TieredStopAtLevel=1 -noverify -Djava.net.preferIPv4Stack=true -jar", "%JAVA_OPTS%", "%AZURE_FUNCTIONS_MESH_JAVA_OPTS%"] + }, + "hostRequirements": [], + "profiles": [] +} \ No newline at end of file diff --git a/test/TestWorkers/Probingpaths/workers/java/2.19.0/azure-functions-java-worker.jar b/test/TestWorkers/Probingpaths/workers/java/2.19.0/azure-functions-java-worker.jar new file mode 100644 index 0000000000..07b06b15b8 --- /dev/null +++ b/test/TestWorkers/Probingpaths/workers/java/2.19.0/azure-functions-java-worker.jar @@ -0,0 +1 @@ +This is empty file needed to run the unit tests for RPCWorkerConfigFactory. It checks if the path mentions in DefaultWorkerPath exists or not before it loads the workerconfig. \ No newline at end of file diff --git a/test/TestWorkers/Probingpaths/workers/java/2.19.0/worker.config.json b/test/TestWorkers/Probingpaths/workers/java/2.19.0/worker.config.json new file mode 100644 index 0000000000..421dd1233a --- /dev/null +++ b/test/TestWorkers/Probingpaths/workers/java/2.19.0/worker.config.json @@ -0,0 +1,11 @@ +{ + "description": { + "language": "java", + "extensions": [".jar"], + "defaultExecutablePath": "%JAVA_HOME%/bin/java", + "defaultWorkerPath": "azure-functions-java-worker.jar", + "arguments": ["-XX:+TieredCompilation -XX:TieredStopAtLevel=1 -noverify -Djava.net.preferIPv4Stack=true -jar", "%JAVA_OPTS%", "%AZURE_FUNCTIONS_MESH_JAVA_OPTS%"] + }, + "hostRequirements": [], + "profiles": [] +} \ No newline at end of file diff --git a/test/TestWorkers/Probingpaths/workers/node/3.10.1/worker.config.json b/test/TestWorkers/Probingpaths/workers/node/3.10.1/worker.config.json new file mode 100644 index 0000000000..c20e0b4e45 --- /dev/null +++ b/test/TestWorkers/Probingpaths/workers/node/3.10.1/worker.config.json @@ -0,0 +1,14 @@ +{ + "description": { + "language": "node", + "extensions": [".js", ".mjs", ".cjs"], + "defaultExecutablePath": "node", + "defaultWorkerPath": "dist/src/nodejsWorker.js", + "workerIndexing": "true" + }, + "hostRequirements": ["test-capability-1", "test-capability-2"], + "processOptions": { + "initializationTimeout": "00:02:00", + "environmentReloadTimeout": "00:02:00" + } +} diff --git a/test/TestWorkers/Probingpaths/workers/powershell/7.2/Microsoft.Azure.Functions.PowerShellWorker.dll b/test/TestWorkers/Probingpaths/workers/powershell/7.2/Microsoft.Azure.Functions.PowerShellWorker.dll new file mode 100644 index 0000000000..07b06b15b8 --- /dev/null +++ b/test/TestWorkers/Probingpaths/workers/powershell/7.2/Microsoft.Azure.Functions.PowerShellWorker.dll @@ -0,0 +1 @@ +This is empty file needed to run the unit tests for RPCWorkerConfigFactory. It checks if the path mentions in DefaultWorkerPath exists or not before it loads the workerconfig. \ No newline at end of file diff --git a/test/TestWorkers/Probingpaths/workers/powershell/7.2/worker.config.json b/test/TestWorkers/Probingpaths/workers/powershell/7.2/worker.config.json new file mode 100644 index 0000000000..52cef391ed --- /dev/null +++ b/test/TestWorkers/Probingpaths/workers/powershell/7.2/worker.config.json @@ -0,0 +1,11 @@ +{ + "description":{ + "language":"powershell", + "extensions":[".ps1", ".psm1"], + "defaultExecutablePath":"dotnet", + "defaultWorkerPath":"%FUNCTIONS_WORKER_RUNTIME_VERSION%/Microsoft.Azure.Functions.PowerShellWorker.dll", + "supportedRuntimeVersions":["7", "7.2"], + "defaultRuntimeVersion": "7.2", + "sanitizeRuntimeVersionRegex":"\\d+\\.?\\d*" + } +} diff --git a/test/TestWorkers/Probingpaths/workers/powershell/7.4/Microsoft.Azure.Functions.PowerShellWorker.dll b/test/TestWorkers/Probingpaths/workers/powershell/7.4/Microsoft.Azure.Functions.PowerShellWorker.dll new file mode 100644 index 0000000000..07b06b15b8 --- /dev/null +++ b/test/TestWorkers/Probingpaths/workers/powershell/7.4/Microsoft.Azure.Functions.PowerShellWorker.dll @@ -0,0 +1 @@ +This is empty file needed to run the unit tests for RPCWorkerConfigFactory. It checks if the path mentions in DefaultWorkerPath exists or not before it loads the workerconfig. \ No newline at end of file diff --git a/test/TestWorkers/Probingpaths/workers/powershell/7.4/worker.config.json b/test/TestWorkers/Probingpaths/workers/powershell/7.4/worker.config.json new file mode 100644 index 0000000000..02fab11d36 --- /dev/null +++ b/test/TestWorkers/Probingpaths/workers/powershell/7.4/worker.config.json @@ -0,0 +1,11 @@ +{ + "description":{ + "language":"powershell", + "extensions":[".ps1", ".psm1"], + "defaultExecutablePath":"dotnet", + "defaultWorkerPath":"%FUNCTIONS_WORKER_RUNTIME_VERSION%/Microsoft.Azure.Functions.PowerShellWorker.dll", + "supportedRuntimeVersions":["7", "7.2", "7.4"], + "defaultRuntimeVersion": "7.4", + "sanitizeRuntimeVersionRegex":"\\d+\\.?\\d*" + } +} diff --git a/test/TestWorkers/Probingpaths/workers/powershell/7/Microsoft.Azure.Functions.PowerShellWorker.dll b/test/TestWorkers/Probingpaths/workers/powershell/7/Microsoft.Azure.Functions.PowerShellWorker.dll new file mode 100644 index 0000000000..07b06b15b8 --- /dev/null +++ b/test/TestWorkers/Probingpaths/workers/powershell/7/Microsoft.Azure.Functions.PowerShellWorker.dll @@ -0,0 +1 @@ +This is empty file needed to run the unit tests for RPCWorkerConfigFactory. It checks if the path mentions in DefaultWorkerPath exists or not before it loads the workerconfig. \ No newline at end of file diff --git a/test/TestWorkers/Probingpaths/workers/powershell/7/worker.config.json b/test/TestWorkers/Probingpaths/workers/powershell/7/worker.config.json new file mode 100644 index 0000000000..4a871e741b --- /dev/null +++ b/test/TestWorkers/Probingpaths/workers/powershell/7/worker.config.json @@ -0,0 +1,11 @@ +{ + "description":{ + "language":"powershell", + "extensions":[".ps1", ".psm1"], + "defaultExecutablePath":"dotnet", + "defaultWorkerPath":"%FUNCTIONS_WORKER_RUNTIME_VERSION%/Microsoft.Azure.Functions.PowerShellWorker.dll", + "supportedRuntimeVersions":["7", "7.2"], + "defaultRuntimeVersion":"7", + "sanitizeRuntimeVersionRegex":"\\d+\\.?\\d*" + } +} From cefdb92117169998d04cea6a385e287d37b8b924 Mon Sep 17 00:00:00 2001 From: Surbhi Gupta Date: Mon, 26 May 2025 02:02:17 -0500 Subject: [PATCH 03/55] Code cleanup and adding comments --- .../Config/FunctionsHostingConfigOptions.cs | 2 +- .../Environment/EnvironmentExtensions.cs | 1 - src/WebJobs.Script/Utility.cs | 60 +++++++++++++++---- .../LanguageWorkerOptionsSetup.cs | 60 +++++++++---------- .../Configuration/RpcWorkerConfigFactory.cs | 31 +--------- .../Rpc/IWorkerConfigurationResolver.cs | 6 ++ .../Workers/Rpc/RpcWorkerDescription.cs | 2 + .../FunctionsHostingConfigOptionsTest.cs | 3 + 8 files changed, 88 insertions(+), 77 deletions(-) diff --git a/src/WebJobs.Script/Config/FunctionsHostingConfigOptions.cs b/src/WebJobs.Script/Config/FunctionsHostingConfigOptions.cs index 64e9515506..5d2bbb9b11 100644 --- a/src/WebJobs.Script/Config/FunctionsHostingConfigOptions.cs +++ b/src/WebJobs.Script/Config/FunctionsHostingConfigOptions.cs @@ -124,7 +124,7 @@ internal string WorkerIndexingDisabledApps } /// - /// Gets a string delimited by '|' that contains the name of the apps with worker indexing disabled. + /// Gets a string delimited by '|' that contains the names of the language workers available through Probing paths outside of the Host. /// internal string EnableProbingPathsForWorkers { diff --git a/src/WebJobs.Script/Environment/EnvironmentExtensions.cs b/src/WebJobs.Script/Environment/EnvironmentExtensions.cs index 9c8b13af30..3deb7124da 100644 --- a/src/WebJobs.Script/Environment/EnvironmentExtensions.cs +++ b/src/WebJobs.Script/Environment/EnvironmentExtensions.cs @@ -283,7 +283,6 @@ public static bool IsWindowsElasticPremium(this IEnvironment environment) return string.Equals(value, ScriptConstants.ElasticPremiumSku, StringComparison.OrdinalIgnoreCase); } - /// /// Gets a value indicating whether the application is running in a Windows App Service environment. /// diff --git a/src/WebJobs.Script/Utility.cs b/src/WebJobs.Script/Utility.cs index 020bdf9e62..1c662ee499 100644 --- a/src/WebJobs.Script/Utility.cs +++ b/src/WebJobs.Script/Utility.cs @@ -1,6 +1,20 @@ // Copyright (c) .NET Foundation. All rights reserved. // Licensed under the MIT License. See License.txt in the project root for license information. +using Microsoft.AspNetCore.Http; +using Microsoft.Azure.WebJobs.Logging; +using Microsoft.Azure.WebJobs.Script.Config; +using Microsoft.Azure.WebJobs.Script.Description; +using Microsoft.Azure.WebJobs.Script.Diagnostics.Extensions; +using Microsoft.Azure.WebJobs.Script.Models; +using Microsoft.Azure.WebJobs.Script.Workers.Rpc; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Hosting; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; +using Newtonsoft.Json; +using Newtonsoft.Json.Converters; +using Newtonsoft.Json.Linq; using System; using System.Collections.Generic; using System.Collections.Specialized; @@ -16,19 +30,6 @@ using System.Threading; using System.Threading.Tasks; using System.Web; -using Microsoft.AspNetCore.Http; -using Microsoft.Azure.WebJobs.Logging; -using Microsoft.Azure.WebJobs.Script.Config; -using Microsoft.Azure.WebJobs.Script.Description; -using Microsoft.Azure.WebJobs.Script.Diagnostics.Extensions; -using Microsoft.Azure.WebJobs.Script.Models; -using Microsoft.Azure.WebJobs.Script.Workers.Rpc; -using Microsoft.Extensions.DependencyInjection; -using Microsoft.Extensions.Logging; -using Microsoft.Extensions.Options; -using Newtonsoft.Json; -using Newtonsoft.Json.Converters; -using Newtonsoft.Json.Linq; using FunctionMetadata = Microsoft.Azure.WebJobs.Script.Description.FunctionMetadata; namespace Microsoft.Azure.WebJobs.Script @@ -1065,6 +1066,39 @@ public static bool CanWorkerIndex(IEnumerable workerConfigs, IE return workerIndexingEnabled && workerIndexingAvailable; } + // Worker probing paths can be enabled or disabled via feature flags or hosting config options. Feature flags take precedence over hosting config options. + // Users can enable or disable probing paths via setting the appropriate feature flags. + // Probing paths can also be enabled for specific workers at stamp level via the hosting config options. + public static bool AreWorkerProbingPathsEnabled(IEnvironment environment, FunctionsHostingConfigOptions functionsHostingConfigOptions, string workerRuntime) + { + bool areProbingPathsDisabled = FeatureFlags.IsEnabled(ScriptConstants.FeatureFlagDisableWorkerProbingPaths, environment); + + if (areProbingPathsDisabled) + { + return false; + } + + bool areProbingPathsEnabled = FeatureFlags.IsEnabled(ScriptConstants.FeatureFlagEnableWorkerProbingPaths, environment); + + if (areProbingPathsEnabled) + { + return true; + } + + HashSet probingPathsEnabledWorkersViaHostingConfig = functionsHostingConfigOptions + .EnableProbingPathsForWorkers + ?.ToLowerInvariant() + ?.Split("|", StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries) + ?.ToHashSet(); + + if (!environment.IsMultiLanguageRuntimeEnvironment() && !string.IsNullOrWhiteSpace(workerRuntime)) + { + return probingPathsEnabledWorkersViaHostingConfig.Contains(workerRuntime); + } + + return probingPathsEnabledWorkersViaHostingConfig is not null ? probingPathsEnabledWorkersViaHostingConfig.Any() : false; + } + public static void LogAutorestGeneratedJsonIfExists(string rootScriptPath, ILogger logger) { string autorestGeneratedJsonPath = Path.Combine(rootScriptPath, ScriptConstants.AutorestGeenratedMetadataFileName); diff --git a/src/WebJobs.Script/Workers/Rpc/Configuration/LanguageWorkerOptionsSetup.cs b/src/WebJobs.Script/Workers/Rpc/Configuration/LanguageWorkerOptionsSetup.cs index c070913d0b..48a58daef3 100644 --- a/src/WebJobs.Script/Workers/Rpc/Configuration/LanguageWorkerOptionsSetup.cs +++ b/src/WebJobs.Script/Workers/Rpc/Configuration/LanguageWorkerOptionsSetup.cs @@ -21,6 +21,7 @@ namespace Microsoft.Azure.WebJobs.Script.Workers.Rpc { internal class LanguageWorkerOptionsSetup : IConfigureOptions { + private const string _windowsWorkerProbingPath = "c:\\home\\SiteExtensions\\workers"; // Harcoded site extensions path for Windows until Antares starts setting this as Environment variable. private readonly IConfiguration _configuration; private readonly ILogger _logger; private readonly IEnvironment _environment; @@ -81,16 +82,10 @@ public void Configure(LanguageWorkerOptions options) } } - string jsonString = GetWorkerProbingPaths(); - - if (!string.IsNullOrEmpty(jsonString)) + if (Utility.AreWorkerProbingPathsEnabled(_environment, _functionsHostingConfigOptions.Value, workerRuntime)) { - using var jsonStream = new MemoryStream(Encoding.UTF8.GetBytes(jsonString)); - - configuration = new ConfigurationBuilder() - .AddConfiguration(configuration) - .AddJsonStream(jsonStream) - .Build(); + string probingPathsString = GetWorkerProbingPaths(); + configuration = AddProbingPathsToConfiguration(configuration, probingPathsString); } var workerConfigurationResolver = new WorkerConfigurationResolver(configuration, _logger, _environment, _workerProfileManager, _functionsHostingConfigOptions); @@ -98,48 +93,49 @@ public void Configure(LanguageWorkerOptions options) options.WorkerConfigs = configFactory.GetConfigs(); } - public string GetWorkerProbingPaths() + internal string GetWorkerProbingPaths() { var probingPaths = new List(); - string output = string.Empty; + string probingPathsString = string.Empty; - // If Env variable is available, read from there (works for linux) - var probingPathsEnvValue = _environment.GetEnvironmentVariableOrDefault(EnvironmentSettingNames.WorkerProbingPaths, null); + // If Environment variable is set, read probing paths (works for linux) + string probingPathsEnvValue = _environment.GetEnvironmentVariableOrDefault(EnvironmentSettingNames.WorkerProbingPaths, null); if (!string.IsNullOrEmpty(probingPathsEnvValue)) { - probingPaths = probingPathsEnvValue.Split(';', StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries).ToList() ?? new List(); + probingPaths = probingPathsEnvValue.Split(';', StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries).ToList(); } else { if (_environment.IsAnyWindows()) { - // Harcoded site extensions path ("c:\\home\\SiteExtensions\\workers") for Windows until Antares starts setting this as Environment variable. - // probingPaths.Add("c:\\testData\\workers"); - -#pragma warning disable SYSLIB0012 // Type or member is obsolete - string assemblyLocalPath = Path.GetDirectoryName(new Uri(typeof(LanguageWorkerOptionsSetup).Assembly.CodeBase).LocalPath); -#pragma warning restore SYSLIB0012 // Type or member is obsolete - string workersDirPath = Path.Combine(assemblyLocalPath, "ProbingPaths"); - probingPaths.Add(workersDirPath); - _logger.LogInformation($"ProbingPaths setup via options: {workersDirPath}"); + probingPaths.Add(_windowsWorkerProbingPath); } } if (probingPaths.Any()) { - var jsonObj = new - { - languageWorkers = new - { - probingPaths - } - }; + probingPathsString = $"{{ {RpcWorkerConstants.LanguageWorkersSectionName}: " + + $"{{ {RpcWorkerConstants.WorkerProbingPathsSectionName} : " + + $"{probingPaths} }} }}"; + } - output = JsonSerializer.Serialize(jsonObj, new JsonSerializerOptions { WriteIndented = true }); + return probingPathsString; + } + + internal IConfiguration AddProbingPathsToConfiguration(IConfiguration configuration, string probingPathsString) + { + if (!string.IsNullOrEmpty(probingPathsString)) + { + using var jsonStream = new MemoryStream(Encoding.UTF8.GetBytes(probingPathsString)); + + configuration = new ConfigurationBuilder() + .AddConfiguration(configuration) + .AddJsonStream(jsonStream) + .Build(); } - return output; + return configuration; } } diff --git a/src/WebJobs.Script/Workers/Rpc/Configuration/RpcWorkerConfigFactory.cs b/src/WebJobs.Script/Workers/Rpc/Configuration/RpcWorkerConfigFactory.cs index ea9ce1dd68..675f6eda17 100644 --- a/src/WebJobs.Script/Workers/Rpc/Configuration/RpcWorkerConfigFactory.cs +++ b/src/WebJobs.Script/Workers/Rpc/Configuration/RpcWorkerConfigFactory.cs @@ -60,7 +60,7 @@ public RpcWorkerConfigFactory(IConfiguration config, WorkersDirPath = GetDefaultWorkersDirectory(Directory.Exists); var workersDirectorySection = _config.GetSection($"{RpcWorkerConstants.LanguageWorkersSectionName}:{WorkerConstants.WorkersDirectorySectionName}"); - ProbingPathsEnabled = AreProbingPathsEnabled(); + ProbingPathsEnabled = Utility.AreWorkerProbingPathsEnabled(_environment, _functionsHostingConfigOptions, _workerRuntime); if (!string.IsNullOrEmpty(workersDirectorySection.Value)) { @@ -325,35 +325,6 @@ internal List GetWorkerProbingPaths(IConfiguration config) return probingPaths; } - internal bool AreProbingPathsEnabled() - { - bool isFeatureFlagDisabled = FeatureFlags.IsEnabled(ScriptConstants.FeatureFlagDisableWorkerProbingPaths, _environment); - - if (isFeatureFlagDisabled) - { - return false; - } - - bool isFeatureFlagEnabled = FeatureFlags.IsEnabled(ScriptConstants.FeatureFlagEnableWorkerProbingPaths, _environment); - - if (isFeatureFlagEnabled) - { - return true; - } - - HashSet probingPathsEnabledWorkersViaHostingConfig = _functionsHostingConfigOptions.EnableProbingPathsForWorkers.ToLowerInvariant().Split("|", StringSplitOptions.RemoveEmptyEntries).ToHashSet(); - - if (!_environment.IsMultiLanguageRuntimeEnvironment()) - { - if (!string.IsNullOrWhiteSpace(_workerRuntime)) - { - return probingPathsEnabledWorkersViaHostingConfig.Contains(_workerRuntime); - } - } - - return probingPathsEnabledWorkersViaHostingConfig.Any() ? true : false; - } - private void ReadLanguageWorkerFile(string workerPath) { if (!_environment.IsPlaceholderModeEnabled() diff --git a/src/WebJobs.Script/Workers/Rpc/IWorkerConfigurationResolver.cs b/src/WebJobs.Script/Workers/Rpc/IWorkerConfigurationResolver.cs index 53b3a4f019..baaf4e2ad8 100644 --- a/src/WebJobs.Script/Workers/Rpc/IWorkerConfigurationResolver.cs +++ b/src/WebJobs.Script/Workers/Rpc/IWorkerConfigurationResolver.cs @@ -5,6 +5,12 @@ namespace Microsoft.Azure.WebJobs.Script.Workers.Rpc.Configuration { + /// + /// Interface to resolve WorkerConfigs using configured probing paths and fallback path. + /// + /// List of probing paths where workers are located. + /// A fallback path to check when workers cannot be found in the probing paths. + /// A list of paths to worker configuration files. public interface IWorkerConfigurationResolver { internal List GetWorkerConfigs(List probingPaths, string fallbackPath); diff --git a/src/WebJobs.Script/Workers/Rpc/RpcWorkerDescription.cs b/src/WebJobs.Script/Workers/Rpc/RpcWorkerDescription.cs index 9402bb293d..53adf78c1f 100644 --- a/src/WebJobs.Script/Workers/Rpc/RpcWorkerDescription.cs +++ b/src/WebJobs.Script/Workers/Rpc/RpcWorkerDescription.cs @@ -98,6 +98,8 @@ public override void ApplyDefaultsAndValidate(string workerDirectory, ILogger lo if (!string.IsNullOrEmpty(DefaultWorkerPath) && !Path.IsPathRooted(DefaultWorkerPath)) { + // If probing paths are enabled and DefaultWorkerPath contains FunctionWorkerRuntimeVersionSettingName + // then version becomes redundant in the path. Replacing version with an empty string to avoid deuplication. if (probingPathsEnabled && DefaultWorkerPath.Contains(RpcWorkerConstants.RuntimeVersionPlaceholder)) { var versionDir = Path.GetFileName(WorkerDirectory); diff --git a/test/WebJobs.Script.Tests/Configuration/FunctionsHostingConfigOptionsTest.cs b/test/WebJobs.Script.Tests/Configuration/FunctionsHostingConfigOptionsTest.cs index daebf4dfc2..b54a868089 100644 --- a/test/WebJobs.Script.Tests/Configuration/FunctionsHostingConfigOptionsTest.cs +++ b/test/WebJobs.Script.Tests/Configuration/FunctionsHostingConfigOptionsTest.cs @@ -75,6 +75,9 @@ public static IEnumerable PropertyValues yield return [nameof(FunctionsHostingConfigOptions.IsDotNetInProcDisabled), "DotNetInProcDisabled=1", true]; yield return [nameof(FunctionsHostingConfigOptions.IsDotNetInProcDisabled), "DotNetInProcDisabled=0", false]; + yield return [nameof(FunctionsHostingConfigOptions.EnableProbingPathsForWorkers), "EnableProbingPathsForWorkers=java|node", "java|node"]; + yield return [nameof(FunctionsHostingConfigOptions.EnableProbingPathsForWorkers), "EnableProbingPathsForWorkers=java", "java"]; + yield return [nameof(FunctionsHostingConfigOptions.IsTestDataSuppressionEnabled), "EnableTestDataSuppression=1", true]; yield return [nameof(FunctionsHostingConfigOptions.IsCGroupMemoryMetricsEnabled), "EnableCGroupMemoryMetrics=1", true]; From 021ac3ee8301f4cce3263ad785b7e10ba61b3f10 Mon Sep 17 00:00:00 2001 From: Surbhi Gupta Date: Mon, 26 May 2025 16:04:24 -0500 Subject: [PATCH 04/55] Code refactoring to check probing paths enabled only once --- src/WebJobs.Script/Utility.cs | 36 ++++----- .../LanguageWorkerOptionsSetup.cs | 19 +++-- .../Configuration/RpcWorkerConfigFactory.cs | 21 +++--- .../WorkerConfigurationResolver.cs | 14 +--- .../workers/java/2.18.0/worker.config.json | 2 +- .../workers/java/2.18.1/worker.config.json | 2 +- .../workers/java/2.18.2/worker.config.json | 2 +- .../workers/java/2.19.0/worker.config.json | 2 +- .../Rpc/RpcWorkerConfigFactoryTests.cs | 73 +++++++------------ .../Workers/Rpc/RpcWorkerConfigTests.cs | 5 +- .../Rpc/WorkerConfigurationResolverTests.cs | 23 +----- 11 files changed, 81 insertions(+), 118 deletions(-) diff --git a/src/WebJobs.Script/Utility.cs b/src/WebJobs.Script/Utility.cs index 1c662ee499..18c9d935fe 100644 --- a/src/WebJobs.Script/Utility.cs +++ b/src/WebJobs.Script/Utility.cs @@ -1,20 +1,6 @@ // Copyright (c) .NET Foundation. All rights reserved. // Licensed under the MIT License. See License.txt in the project root for license information. -using Microsoft.AspNetCore.Http; -using Microsoft.Azure.WebJobs.Logging; -using Microsoft.Azure.WebJobs.Script.Config; -using Microsoft.Azure.WebJobs.Script.Description; -using Microsoft.Azure.WebJobs.Script.Diagnostics.Extensions; -using Microsoft.Azure.WebJobs.Script.Models; -using Microsoft.Azure.WebJobs.Script.Workers.Rpc; -using Microsoft.Extensions.DependencyInjection; -using Microsoft.Extensions.Hosting; -using Microsoft.Extensions.Logging; -using Microsoft.Extensions.Options; -using Newtonsoft.Json; -using Newtonsoft.Json.Converters; -using Newtonsoft.Json.Linq; using System; using System.Collections.Generic; using System.Collections.Specialized; @@ -30,6 +16,20 @@ using System.Threading; using System.Threading.Tasks; using System.Web; +using Microsoft.AspNetCore.Http; +using Microsoft.Azure.WebJobs.Logging; +using Microsoft.Azure.WebJobs.Script.Config; +using Microsoft.Azure.WebJobs.Script.Description; +using Microsoft.Azure.WebJobs.Script.Diagnostics.Extensions; +using Microsoft.Azure.WebJobs.Script.Models; +using Microsoft.Azure.WebJobs.Script.Workers.Rpc; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Hosting; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; +using Newtonsoft.Json; +using Newtonsoft.Json.Converters; +using Newtonsoft.Json.Linq; using FunctionMetadata = Microsoft.Azure.WebJobs.Script.Description.FunctionMetadata; namespace Microsoft.Azure.WebJobs.Script @@ -1069,7 +1069,7 @@ public static bool CanWorkerIndex(IEnumerable workerConfigs, IE // Worker probing paths can be enabled or disabled via feature flags or hosting config options. Feature flags take precedence over hosting config options. // Users can enable or disable probing paths via setting the appropriate feature flags. // Probing paths can also be enabled for specific workers at stamp level via the hosting config options. - public static bool AreWorkerProbingPathsEnabled(IEnvironment environment, FunctionsHostingConfigOptions functionsHostingConfigOptions, string workerRuntime) + public static bool AreWorkerProbingPathsEnabled(IEnvironment environment, HashSet probingPathsEnabledWorkersViaHostingConfig, string workerRuntime) { bool areProbingPathsDisabled = FeatureFlags.IsEnabled(ScriptConstants.FeatureFlagDisableWorkerProbingPaths, environment); @@ -1085,12 +1085,6 @@ public static bool AreWorkerProbingPathsEnabled(IEnvironment environment, Functi return true; } - HashSet probingPathsEnabledWorkersViaHostingConfig = functionsHostingConfigOptions - .EnableProbingPathsForWorkers - ?.ToLowerInvariant() - ?.Split("|", StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries) - ?.ToHashSet(); - if (!environment.IsMultiLanguageRuntimeEnvironment() && !string.IsNullOrWhiteSpace(workerRuntime)) { return probingPathsEnabledWorkersViaHostingConfig.Contains(workerRuntime); diff --git a/src/WebJobs.Script/Workers/Rpc/Configuration/LanguageWorkerOptionsSetup.cs b/src/WebJobs.Script/Workers/Rpc/Configuration/LanguageWorkerOptionsSetup.cs index 48a58daef3..2b20d0c05b 100644 --- a/src/WebJobs.Script/Workers/Rpc/Configuration/LanguageWorkerOptionsSetup.cs +++ b/src/WebJobs.Script/Workers/Rpc/Configuration/LanguageWorkerOptionsSetup.cs @@ -82,14 +82,21 @@ public void Configure(LanguageWorkerOptions options) } } - if (Utility.AreWorkerProbingPathsEnabled(_environment, _functionsHostingConfigOptions.Value, workerRuntime)) + HashSet probingPathsEnabledWorkersViaHostingConfig = _functionsHostingConfigOptions.Value + ?.EnableProbingPathsForWorkers + ?.ToLowerInvariant() + ?.Split("|", StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries) + ?.ToHashSet(); + + bool probingPathsEnabled = Utility.AreWorkerProbingPathsEnabled(_environment, probingPathsEnabledWorkersViaHostingConfig, workerRuntime); + if (probingPathsEnabled) { string probingPathsString = GetWorkerProbingPaths(); configuration = AddProbingPathsToConfiguration(configuration, probingPathsString); } - var workerConfigurationResolver = new WorkerConfigurationResolver(configuration, _logger, _environment, _workerProfileManager, _functionsHostingConfigOptions); - var configFactory = new RpcWorkerConfigFactory(configuration, _logger, SystemRuntimeInformation.Instance, _environment, _metricsLogger, _workerProfileManager, workerConfigurationResolver, _functionsHostingConfigOptions); + var workerConfigurationResolver = new WorkerConfigurationResolver(configuration, _logger, _environment, _workerProfileManager, probingPathsEnabledWorkersViaHostingConfig); + var configFactory = new RpcWorkerConfigFactory(configuration, _logger, SystemRuntimeInformation.Instance, _environment, _metricsLogger, _workerProfileManager, workerConfigurationResolver, probingPathsEnabled); options.WorkerConfigs = configFactory.GetConfigs(); } @@ -115,9 +122,9 @@ internal string GetWorkerProbingPaths() if (probingPaths.Any()) { - probingPathsString = $"{{ {RpcWorkerConstants.LanguageWorkersSectionName}: " + - $"{{ {RpcWorkerConstants.WorkerProbingPathsSectionName} : " + - $"{probingPaths} }} }}"; + probingPathsString = $"{{ \"{RpcWorkerConstants.LanguageWorkersSectionName}\": " + + $"{{ \"{RpcWorkerConstants.WorkerProbingPathsSectionName}\": " + + $"\"{probingPaths}\" }} }}"; } return probingPathsString; diff --git a/src/WebJobs.Script/Workers/Rpc/Configuration/RpcWorkerConfigFactory.cs b/src/WebJobs.Script/Workers/Rpc/Configuration/RpcWorkerConfigFactory.cs index 675f6eda17..0543f09816 100644 --- a/src/WebJobs.Script/Workers/Rpc/Configuration/RpcWorkerConfigFactory.cs +++ b/src/WebJobs.Script/Workers/Rpc/Configuration/RpcWorkerConfigFactory.cs @@ -30,7 +30,6 @@ internal class RpcWorkerConfigFactory private readonly string _workerRuntime; private readonly IEnvironment _environment; private readonly IWorkerConfigurationResolver _workerConfigurationResolver; - private readonly FunctionsHostingConfigOptions _functionsHostingConfigOptions; private readonly JsonSerializerOptions _jsonSerializerOptions = new() { PropertyNameCaseInsensitive = true @@ -45,7 +44,7 @@ public RpcWorkerConfigFactory(IConfiguration config, IMetricsLogger metricsLogger, IWorkerProfileManager workerProfileManager, IWorkerConfigurationResolver workerConfigurationResolver, - IOptions functionsHostingConfigOptions) + bool probingPathsEnabled = false) { _config = config ?? throw new ArgumentNullException(nameof(config)); _logger = logger ?? throw new ArgumentNullException(nameof(logger)); @@ -55,12 +54,11 @@ public RpcWorkerConfigFactory(IConfiguration config, _profileManager = workerProfileManager ?? throw new ArgumentNullException(nameof(workerProfileManager)); _workerRuntime = _environment.GetEnvironmentVariable(RpcWorkerConstants.FunctionWorkerRuntimeSettingName); _workerConfigurationResolver = workerConfigurationResolver ?? throw new ArgumentNullException(nameof(workerConfigurationResolver)); - _functionsHostingConfigOptions = functionsHostingConfigOptions.Value; WorkersDirPath = GetDefaultWorkersDirectory(Directory.Exists); var workersDirectorySection = _config.GetSection($"{RpcWorkerConstants.LanguageWorkersSectionName}:{WorkerConstants.WorkersDirectorySectionName}"); - ProbingPathsEnabled = Utility.AreWorkerProbingPathsEnabled(_environment, _functionsHostingConfigOptions, _workerRuntime); + ProbingPathsEnabled = probingPathsEnabled; if (!string.IsNullOrEmpty(workersDirectorySection.Value)) { @@ -107,7 +105,7 @@ internal void AddProviders() { List probingPaths = GetWorkerProbingPaths(_config); - _logger.LogDebug("Workers Directory set to: probingPaths = {probingPaths} and fallback path = {WorkersDirPath}", string.Join(", ", probingPaths), WorkersDirPath); + _logger.LogDebug("Probing paths set to: {probingPaths} and Workers Directory set as fallback path: {WorkersDirPath}", string.Join(", ", probingPaths), WorkersDirPath); List workerConfigs = _workerConfigurationResolver.GetWorkerConfigs(probingPaths, WorkersDirPath); @@ -152,7 +150,10 @@ internal void AddProvider(string workerDir) try { // After specialization, load worker config only for the specified runtime unless it's a multi-language app. - if (!string.IsNullOrWhiteSpace(_workerRuntime) && !_environment.IsPlaceholderModeEnabled() && !_environment.IsMultiLanguageRuntimeEnvironment()) + if (!string.IsNullOrWhiteSpace(_workerRuntime) && + !_environment.IsPlaceholderModeEnabled() && + !_environment.IsMultiLanguageRuntimeEnvironment() && + !ProbingPathsEnabled) { string workerRuntime = Path.GetFileName(workerDir); // Only skip worker directories that don't match the current runtime. @@ -218,7 +219,7 @@ internal void AddProvider(string workerDir) _workerDescriptionDictionary[workerDescription.Language] = rpcWorkerConfig; ReadLanguageWorkerFile(arguments.WorkerPath); - _logger.LogDebug("Added WorkerConfig for language: {language} from path: {path}", workerDescription.Language, workerDescription.DefaultWorkerPath); + _logger.LogDebug("Added WorkerConfig for language: {language} with DefaultWorkerPath: {path}", workerDescription.Language, workerDescription.DefaultWorkerPath); } } catch (Exception ex) when (!ex.IsFatal()) @@ -303,7 +304,7 @@ internal List GetWorkerProbingPaths(IConfiguration config) { List probingPaths = new List(); - IConfigurationSection probingPathsSection = config.GetSection($"{RpcWorkerConstants.LanguageWorkersSectionName}") + IConfigurationSection probingPathsSection = config?.GetSection($"{RpcWorkerConstants.LanguageWorkersSectionName}") ?.GetSection($"{RpcWorkerConstants.WorkerProbingPathsSectionName}"); var probingPathsList = probingPathsSection?.AsEnumerable(); @@ -313,9 +314,9 @@ internal List GetWorkerProbingPaths(IConfiguration config) return probingPaths; } - for (int i = 0; i < probingPathsList.Count(); i++) + for (int item = 0; item < probingPathsList.Count(); item++) { - var path = probingPathsSection.GetSection($"{i}").Value; + var path = probingPathsSection.GetSection($"{item}")?.Value; if (path is not null) { probingPaths.Add(path); diff --git a/src/WebJobs.Script/Workers/Rpc/Configuration/WorkerConfigurationResolver.cs b/src/WebJobs.Script/Workers/Rpc/Configuration/WorkerConfigurationResolver.cs index 2d6f6f1f81..063d2066bf 100644 --- a/src/WebJobs.Script/Workers/Rpc/Configuration/WorkerConfigurationResolver.cs +++ b/src/WebJobs.Script/Workers/Rpc/Configuration/WorkerConfigurationResolver.cs @@ -8,11 +8,9 @@ using System.Linq; using System.Text.Json; using Microsoft.Azure.AppService.Proxy.Common.Extensions; -using Microsoft.Azure.WebJobs.Script.Config; using Microsoft.Azure.WebJobs.Script.Workers.Profiles; using Microsoft.Extensions.Configuration; using Microsoft.Extensions.Logging; -using Microsoft.Extensions.Options; namespace Microsoft.Azure.WebJobs.Script.Workers.Rpc.Configuration { @@ -22,27 +20,23 @@ internal sealed class WorkerConfigurationResolver : IWorkerConfigurationResolver private readonly ILogger _logger; private readonly IWorkerProfileManager _profileManager; private readonly IEnvironment _environment; - private readonly FunctionsHostingConfigOptions _functionsHostingConfigOptions; + private readonly HashSet _probingPathsEnabledWorkersViaHostingConfig; private readonly JsonSerializerOptions _jsonSerializerOptions = new() { PropertyNameCaseInsensitive = true }; - private HashSet _probingPathsEnabledWorkersViaHostingConfig; - public WorkerConfigurationResolver(IConfiguration config, ILogger logger, IEnvironment environment, IWorkerProfileManager workerProfileManager, - IOptions functionsHostingConfigOptions) + HashSet probingPathsEnabledWorkersViaHostingConfig) { _config = config ?? throw new ArgumentNullException(nameof(config)); _logger = logger ?? throw new ArgumentNullException(nameof(logger)); _environment = environment ?? throw new ArgumentNullException(nameof(environment)); _profileManager = workerProfileManager ?? throw new ArgumentNullException(nameof(workerProfileManager)); - _functionsHostingConfigOptions = functionsHostingConfigOptions?.Value ?? throw new ArgumentNullException(nameof(functionsHostingConfigOptions)); - - _probingPathsEnabledWorkersViaHostingConfig = _functionsHostingConfigOptions.EnableProbingPathsForWorkers.ToLowerInvariant().Split("|").ToHashSet(); + _probingPathsEnabledWorkersViaHostingConfig = probingPathsEnabledWorkersViaHostingConfig; } public List GetWorkerConfigs(List probingPaths, string fallbackPath) @@ -70,7 +64,7 @@ public List GetWorkerConfigs(List probingPaths, string fallbackP // Only skip worker directories that don't match the current runtime. // Do not skip non-worker directories like the function app payload directory // && languageWorkerPath.StartsWith(fallbackPath)) - if (//!_probingPathsEnabledWorkersViaHostingConfig.Contains(languageWorkerFolder) || + if (!_probingPathsEnabledWorkersViaHostingConfig.Contains(languageWorkerFolder) || (!_environment.IsMultiLanguageRuntimeEnvironment() && workerRuntime is not null && !workerRuntime.Equals(languageWorkerFolder, StringComparison.OrdinalIgnoreCase))) diff --git a/test/TestWorkers/Probingpaths/workers/java/2.18.0/worker.config.json b/test/TestWorkers/Probingpaths/workers/java/2.18.0/worker.config.json index 12cca5b1d7..5e65e09614 100644 --- a/test/TestWorkers/Probingpaths/workers/java/2.18.0/worker.config.json +++ b/test/TestWorkers/Probingpaths/workers/java/2.18.0/worker.config.json @@ -4,7 +4,7 @@ "extensions": [".jar"], "defaultExecutablePath": "%JAVA_HOME%/bin/java", "defaultWorkerPath": "azure-functions-java-worker.jar", - "arguments": ["-XX:+TieredCompilation -XX:TieredStopAtLevel=1 -noverify -Djava.net.preferIPv4Stack=true -jar", "%JAVA_OPTS%", "%AZURE_FUNCTIONS_MESH_JAVA_OPTS%"] + "arguments": [] }, "profiles": [] } \ No newline at end of file diff --git a/test/TestWorkers/Probingpaths/workers/java/2.18.1/worker.config.json b/test/TestWorkers/Probingpaths/workers/java/2.18.1/worker.config.json index 89d426268c..13ddd8acf5 100644 --- a/test/TestWorkers/Probingpaths/workers/java/2.18.1/worker.config.json +++ b/test/TestWorkers/Probingpaths/workers/java/2.18.1/worker.config.json @@ -4,7 +4,7 @@ "extensions": [".jar"], "defaultExecutablePath": "%JAVA_HOME%/bin/java", "defaultWorkerPath": "azure-functions-java-worker.jar", - "arguments": ["-XX:+TieredCompilation -XX:TieredStopAtLevel=1 -noverify -Djava.net.preferIPv4Stack=true -jar", "%JAVA_OPTS%", "%AZURE_FUNCTIONS_MESH_JAVA_OPTS%"] + "arguments": [] }, "hostRequirements": [ "test-capability1", diff --git a/test/TestWorkers/Probingpaths/workers/java/2.18.2/worker.config.json b/test/TestWorkers/Probingpaths/workers/java/2.18.2/worker.config.json index 8923070c36..a35215ccf2 100644 --- a/test/TestWorkers/Probingpaths/workers/java/2.18.2/worker.config.json +++ b/test/TestWorkers/Probingpaths/workers/java/2.18.2/worker.config.json @@ -5,7 +5,7 @@ "extensions": [".jar"], "defaultExecutablePath": "%JAVA_HOME%/bin/java", "defaultWorkerPath": "azure-functions-java-worker.jar", - "arguments": ["-XX:+TieredCompilation -XX:TieredStopAtLevel=1 -noverify -Djava.net.preferIPv4Stack=true -jar", "%JAVA_OPTS%", "%AZURE_FUNCTIONS_MESH_JAVA_OPTS%"] + "arguments": [] }, "hostRequirements": [], "profiles": [] diff --git a/test/TestWorkers/Probingpaths/workers/java/2.19.0/worker.config.json b/test/TestWorkers/Probingpaths/workers/java/2.19.0/worker.config.json index 421dd1233a..d801782347 100644 --- a/test/TestWorkers/Probingpaths/workers/java/2.19.0/worker.config.json +++ b/test/TestWorkers/Probingpaths/workers/java/2.19.0/worker.config.json @@ -4,7 +4,7 @@ "extensions": [".jar"], "defaultExecutablePath": "%JAVA_HOME%/bin/java", "defaultWorkerPath": "azure-functions-java-worker.jar", - "arguments": ["-XX:+TieredCompilation -XX:TieredStopAtLevel=1 -noverify -Djava.net.preferIPv4Stack=true -jar", "%JAVA_OPTS%", "%AZURE_FUNCTIONS_MESH_JAVA_OPTS%"] + "arguments": [] }, "hostRequirements": [], "profiles": [] diff --git a/test/WebJobs.Script.Tests/Workers/Rpc/RpcWorkerConfigFactoryTests.cs b/test/WebJobs.Script.Tests/Workers/Rpc/RpcWorkerConfigFactoryTests.cs index 41f03cdd9f..b431d92e17 100644 --- a/test/WebJobs.Script.Tests/Workers/Rpc/RpcWorkerConfigFactoryTests.cs +++ b/test/WebJobs.Script.Tests/Workers/Rpc/RpcWorkerConfigFactoryTests.cs @@ -42,9 +42,8 @@ public void DefaultLanguageWorkersDir() var expectedWorkersDir = Path.Combine(Path.GetDirectoryName(new Uri(typeof(RpcWorkerConfigFactory).Assembly.Location).LocalPath), RpcWorkerConstants.DefaultWorkersDirectoryName); var config = new ConfigurationBuilder().Build(); var testLogger = new TestLogger("test"); - var hostingOptions = new OptionsWrapper(new FunctionsHostingConfigOptions()); - var workerConfigurationResolver = new WorkerConfigurationResolver(config, testLogger, _testEnvironment, _testWorkerProfileManager, hostingOptions); - var configFactory = new RpcWorkerConfigFactory(config, testLogger, _testSysRuntimeInfo, _testEnvironment, new TestMetricsLogger(), _testWorkerProfileManager, workerConfigurationResolver, hostingOptions); + var workerConfigurationResolver = new WorkerConfigurationResolver(config, testLogger, _testEnvironment, _testWorkerProfileManager, new HashSet()); + var configFactory = new RpcWorkerConfigFactory(config, testLogger, _testSysRuntimeInfo, _testEnvironment, new TestMetricsLogger(), _testWorkerProfileManager, workerConfigurationResolver); Assert.Equal(expectedWorkersDir, configFactory.WorkersDirPath); } @@ -77,9 +76,8 @@ public void LanguageWorker_WorkersDir_Set() }) .Build(); var testLogger = new TestLogger("test"); - var hostingOptions = new OptionsWrapper(new FunctionsHostingConfigOptions()); - var workerConfigurationResolver = new WorkerConfigurationResolver(config, testLogger, _testEnvironment, _testWorkerProfileManager, hostingOptions); - var configFactory = new RpcWorkerConfigFactory(config, testLogger, _testSysRuntimeInfo, _testEnvironment, new TestMetricsLogger(), _testWorkerProfileManager, workerConfigurationResolver, hostingOptions); + var workerConfigurationResolver = new WorkerConfigurationResolver(config, testLogger, _testEnvironment, _testWorkerProfileManager, new HashSet()); + var configFactory = new RpcWorkerConfigFactory(config, testLogger, _testSysRuntimeInfo, _testEnvironment, new TestMetricsLogger(), _testWorkerProfileManager, workerConfigurationResolver); Assert.Equal(expectedWorkersDir, configFactory.WorkersDirPath); } @@ -95,9 +93,8 @@ public void LanguageWorker_WorkersDir_NotSet() var config = configBuilder.Build(); var scriptSettingsManager = new ScriptSettingsManager(config); var testLogger = new TestLogger("test"); - var hostingOptions = new OptionsWrapper(new FunctionsHostingConfigOptions()); - var workerConfigurationResolver = new WorkerConfigurationResolver(config, testLogger, _testEnvironment, _testWorkerProfileManager, hostingOptions); - var configFactory = new RpcWorkerConfigFactory(config, testLogger, _testSysRuntimeInfo, _testEnvironment, new TestMetricsLogger(), _testWorkerProfileManager, workerConfigurationResolver, hostingOptions); + var workerConfigurationResolver = new WorkerConfigurationResolver(config, testLogger, _testEnvironment, _testWorkerProfileManager, new HashSet()); + var configFactory = new RpcWorkerConfigFactory(config, testLogger, _testSysRuntimeInfo, _testEnvironment, new TestMetricsLogger(), _testWorkerProfileManager, workerConfigurationResolver); Assert.Equal(expectedWorkersDir, configFactory.WorkersDirPath); } @@ -121,9 +118,8 @@ public void WorkerDescription_Skipped_When_Profile_Disables_Worker() var scriptSettingsManager = new ScriptSettingsManager(config); var testLogger = new TestLogger("test"); _testEnvironment.SetEnvironmentVariable("ENV_VAR_BAR", "True"); - var hostingOptions = new OptionsWrapper(new FunctionsHostingConfigOptions()); - var workerConfigurationResolver = new WorkerConfigurationResolver(config, testLogger, _testEnvironment, _testWorkerProfileManager, hostingOptions); - var configFactory = new RpcWorkerConfigFactory(config, testLogger, _testSysRuntimeInfo, _testEnvironment, new TestMetricsLogger(), _testWorkerProfileManager, workerConfigurationResolver, hostingOptions); + var workerConfigurationResolver = new WorkerConfigurationResolver(config, testLogger, _testEnvironment, _testWorkerProfileManager, new HashSet()); + var configFactory = new RpcWorkerConfigFactory(config, testLogger, _testSysRuntimeInfo, _testEnvironment, new TestMetricsLogger(), _testWorkerProfileManager, workerConfigurationResolver); var workerConfigs = configFactory.GetConfigs(); var errors = testLogger.GetLogMessages().Where(m => m.Exception != null).ToList(); @@ -145,9 +141,8 @@ public void JavaPath_FromEnvVars() var config = configBuilder.Build(); var scriptSettingsManager = new ScriptSettingsManager(config); var testLogger = new TestLogger("test"); - var hostingOptions = new OptionsWrapper(new FunctionsHostingConfigOptions()); - var workerConfigurationResolver = new WorkerConfigurationResolver(config, testLogger, _testEnvironment, _testWorkerProfileManager, hostingOptions); - var configFactory = new RpcWorkerConfigFactory(config, testLogger, _testSysRuntimeInfo, _testEnvironment, new TestMetricsLogger(), _testWorkerProfileManager, workerConfigurationResolver, hostingOptions); + var workerConfigurationResolver = new WorkerConfigurationResolver(config, testLogger, _testEnvironment, _testWorkerProfileManager, new HashSet()); + var configFactory = new RpcWorkerConfigFactory(config, testLogger, _testSysRuntimeInfo, _testEnvironment, new TestMetricsLogger(), _testWorkerProfileManager, workerConfigurationResolver); var workerConfigs = configFactory.GetConfigs(); var javaPath = workerConfigs.FirstOrDefault(c => c.Description.Language.Equals("java", StringComparison.OrdinalIgnoreCase)).Description.DefaultExecutablePath; Assert.DoesNotContain(@"%JAVA_HOME%", javaPath); @@ -161,7 +156,6 @@ public void GetConfigs_ProbingPaths() string fallbackPath = Path.GetFullPath("..\\..\\..\\..\\test\\TestWorkers\\FallbackPath\\workers\\"); var mockEnvironment = new Mock(); - //mockEnvironment.Setup(p => p.GetEnvironmentVariable(EnvironmentSettingNames.AzureWebJobsFeatureFlags)).Returns(ScriptConstants.FeatureFlagDisableWorkerProbingPaths); mockEnvironment.Setup(p => p.GetEnvironmentVariable(EnvironmentSettingNames.FunctionWorkerRuntime)).Returns("java"); var inMemorySettings = new Dictionary @@ -176,18 +170,14 @@ public void GetConfigs_ProbingPaths() var scriptSettingsManager = new ScriptSettingsManager(config); var testLogger = new TestLogger("test"); - var hostingOptionsValue = new FunctionsHostingConfigOptions(); - hostingOptionsValue.Features.Add(RpcWorkerConstants.EnableProbingPathsForWorkers, "java"); - var hostingOptions = new OptionsWrapper(hostingOptionsValue); - - var workerConfigurationResolver = new WorkerConfigurationResolver(config, testLogger, mockEnvironment.Object, _testWorkerProfileManager, hostingOptions); - var configFactory = new RpcWorkerConfigFactory(config, testLogger, _testSysRuntimeInfo, mockEnvironment.Object, new TestMetricsLogger(), _testWorkerProfileManager, workerConfigurationResolver, hostingOptions); + var workerConfigurationResolver = new WorkerConfigurationResolver(config, testLogger, mockEnvironment.Object, _testWorkerProfileManager, new HashSet() { "java" }); + var configFactory = new RpcWorkerConfigFactory(config, testLogger, _testSysRuntimeInfo, mockEnvironment.Object, new TestMetricsLogger(), _testWorkerProfileManager, workerConfigurationResolver, true); var workerConfigs = configFactory.GetConfigs(); // check log messages var logs = testLogger.GetLogMessages(); Assert.Equal(1, workerConfigs.Count); - Assert.True(logs.Any(p => p.FormattedMessage.Contains("Workers Directory set to: probingPaths = " + probingPath1))); + Assert.True(logs.Any(p => p.FormattedMessage.Contains("Probing paths set to: " + probingPath1))); } [Fact] @@ -205,9 +195,8 @@ public void DefaultWorkerConfigs_Overrides_DefaultWorkerRuntimeVersion_AppSettin _testEnvironment.SetEnvironmentVariable(EnvironmentSettingNames.AzureWebsitePlaceholderMode, "1"); using var variables = new TestScopedSettings(scriptSettingsManager, testEnvVariables); - var hostingOptions = new OptionsWrapper(new FunctionsHostingConfigOptions()); - var workerConfigurationResolver = new WorkerConfigurationResolver(config, testLogger, _testEnvironment, _testWorkerProfileManager, hostingOptions); - var configFactory = new RpcWorkerConfigFactory(config, testLogger, _testSysRuntimeInfo, _testEnvironment, new TestMetricsLogger(), _testWorkerProfileManager, workerConfigurationResolver, hostingOptions); + var workerConfigurationResolver = new WorkerConfigurationResolver(config, testLogger, _testEnvironment, _testWorkerProfileManager, new HashSet()); + var configFactory = new RpcWorkerConfigFactory(config, testLogger, _testSysRuntimeInfo, _testEnvironment, new TestMetricsLogger(), _testWorkerProfileManager, workerConfigurationResolver); var workerConfigs = configFactory.GetConfigs(); var pythonWorkerConfig = workerConfigs.FirstOrDefault(w => w.Description.Language.Equals("python", StringComparison.OrdinalIgnoreCase)); var powershellWorkerConfig = workerConfigs.FirstOrDefault(w => w.Description.Language.Equals("powershell", StringComparison.OrdinalIgnoreCase)); @@ -235,7 +224,7 @@ public void DefaultWorkerConfigs_Overrides_VersionAppSetting(string runtimeSetti testEnvironment.SetEnvironmentVariable("FUNCTIONS_WORKER_RUNTIME_VERSION", runtimeSettingVersion); testEnvironment.SetEnvironmentVariable("FUNCTIONS_WORKER_RUNTIME", "powershell"); testEnvironment.SetEnvironmentVariable(EnvironmentSettingNames.WorkerProbingPaths, $"{probingPath}"); - testEnvironment.SetEnvironmentVariable(EnvironmentSettingNames.AzureWebJobsFeatureFlags, enableProbingPath); + // testEnvironment.SetEnvironmentVariable(EnvironmentSettingNames.AzureWebJobsFeatureFlags, enableProbingPath); var configBuilder = ScriptSettingsManager.CreateDefaultConfigurationBuilder(); configBuilder.AddInMemoryCollection(new Dictionary @@ -245,9 +234,8 @@ public void DefaultWorkerConfigs_Overrides_VersionAppSetting(string runtimeSetti var config = configBuilder.Build(); var scriptSettingsManager = new ScriptSettingsManager(config); var testLogger = new TestLogger("test"); - var hostingOptions = new OptionsWrapper(new FunctionsHostingConfigOptions()); - var workerConfigurationResolver = new WorkerConfigurationResolver(config, testLogger, testEnvironment, _testWorkerProfileManager, hostingOptions); - var configFactory = new RpcWorkerConfigFactory(config, testLogger, _testSysRuntimeInfo, testEnvironment, new TestMetricsLogger(), _testWorkerProfileManager, workerConfigurationResolver, hostingOptions); + var workerConfigurationResolver = new WorkerConfigurationResolver(config, testLogger, testEnvironment, _testWorkerProfileManager, new HashSet() { "powershell" }); + var configFactory = new RpcWorkerConfigFactory(config, testLogger, _testSysRuntimeInfo, testEnvironment, new TestMetricsLogger(), _testWorkerProfileManager, workerConfigurationResolver, enableProbingPath is null ? false : true); var workerConfigs = configFactory.GetConfigs(); var powershellWorkerConfig = workerConfigs.FirstOrDefault(w => w.Description.Language.Equals("powershell", StringComparison.OrdinalIgnoreCase)); Assert.Equal(1, workerConfigs.Count); @@ -279,9 +267,8 @@ public void ShouldAddProvider_Returns_Expected(string workerLanguage, string wor } var config = new ConfigurationBuilder().Build(); var testLogger = new TestLogger("test"); - var hostingOptions = new OptionsWrapper(new FunctionsHostingConfigOptions()); - var workerConfigurationResolver = new WorkerConfigurationResolver(config, testLogger, _testEnvironment, _testWorkerProfileManager, hostingOptions); - RpcWorkerConfigFactory rpcWorkerConfigFactory = new RpcWorkerConfigFactory(config, testLogger, _testSysRuntimeInfo, _testEnvironment, new TestMetricsLogger(), _testWorkerProfileManager, workerConfigurationResolver, hostingOptions); + var workerConfigurationResolver = new WorkerConfigurationResolver(config, testLogger, _testEnvironment, _testWorkerProfileManager, new HashSet()); + var rpcWorkerConfigFactory = new RpcWorkerConfigFactory(config, testLogger, _testSysRuntimeInfo, _testEnvironment, new TestMetricsLogger(), _testWorkerProfileManager, workerConfigurationResolver); _testEnvironment.SetEnvironmentVariable(RpcWorkerConstants.FunctionWorkerRuntimeSettingName, workerRuntime); Assert.Equal(expectedResult, rpcWorkerConfigFactory.ShouldAddWorkerConfig(workerLanguage)); } @@ -325,9 +312,8 @@ public void GetWorkerProcessCount_Tests(bool defaultWorkerConfig, bool setProces var config = new ConfigurationBuilder().Build(); var testLogger = new TestLogger("test"); - var hostingOptions = new OptionsWrapper(new FunctionsHostingConfigOptions()); - var workerConfigurationResolver = new WorkerConfigurationResolver(config, testLogger, _testEnvironment, _testWorkerProfileManager, hostingOptions); - RpcWorkerConfigFactory rpcWorkerConfigFactory = new RpcWorkerConfigFactory(config, testLogger, _testSysRuntimeInfo, _testEnvironment, new TestMetricsLogger(), _testWorkerProfileManager, workerConfigurationResolver, hostingOptions); + var workerConfigurationResolver = new WorkerConfigurationResolver(config, testLogger, _testEnvironment, _testWorkerProfileManager, new HashSet()); + var rpcWorkerConfigFactory = new RpcWorkerConfigFactory(config, testLogger, _testSysRuntimeInfo, _testEnvironment, new TestMetricsLogger(), _testWorkerProfileManager, workerConfigurationResolver); var result = rpcWorkerConfigFactory.GetWorkerProcessCount(workerConfig); if (defaultWorkerConfig) @@ -367,9 +353,8 @@ public void GetWorkerProcessCount_ThrowsException_Tests() var config = new ConfigurationBuilder().Build(); var testLogger = new TestLogger("test"); - var hostingOptions = new OptionsWrapper(new FunctionsHostingConfigOptions()); - var workerConfigurationResolver = new WorkerConfigurationResolver(config, testLogger, _testEnvironment, _testWorkerProfileManager, hostingOptions); - RpcWorkerConfigFactory rpcWorkerConfigFactory = new RpcWorkerConfigFactory(config, testLogger, _testSysRuntimeInfo, _testEnvironment, new TestMetricsLogger(), _testWorkerProfileManager, workerConfigurationResolver, hostingOptions); + var workerConfigurationResolver = new WorkerConfigurationResolver(config, testLogger, _testEnvironment, _testWorkerProfileManager, new HashSet()); + var rpcWorkerConfigFactory = new RpcWorkerConfigFactory(config, testLogger, _testSysRuntimeInfo, _testEnvironment, new TestMetricsLogger(), _testWorkerProfileManager, workerConfigurationResolver); var resultEx1 = Assert.Throws(() => rpcWorkerConfigFactory.GetWorkerProcessCount(workerConfig)); Assert.Contains("ProcessCount must be greater than 0", resultEx1.Message); @@ -397,9 +382,8 @@ public void GetWorkerProbingPaths_ReturnsExpectedPaths() .Build(); var testLogger = new TestLogger("test"); - var hostingOptions = new OptionsWrapper(new FunctionsHostingConfigOptions()); - var workerConfigurationResolver = new WorkerConfigurationResolver(config, testLogger, _testEnvironment, _testWorkerProfileManager, hostingOptions); - RpcWorkerConfigFactory rpcWorkerConfigFactory = new RpcWorkerConfigFactory(config, testLogger, _testSysRuntimeInfo, _testEnvironment, new TestMetricsLogger(), _testWorkerProfileManager, workerConfigurationResolver, hostingOptions); + var workerConfigurationResolver = new WorkerConfigurationResolver(config, testLogger, _testEnvironment, _testWorkerProfileManager, new HashSet()); + var rpcWorkerConfigFactory = new RpcWorkerConfigFactory(config, testLogger, _testSysRuntimeInfo, _testEnvironment, new TestMetricsLogger(), _testWorkerProfileManager, workerConfigurationResolver); // Act var result = rpcWorkerConfigFactory.GetWorkerProbingPaths(config); @@ -427,9 +411,8 @@ public void GetWorkerProbingPaths_InvalidProbingPaths_ReturnsEmptyList(string ke .Build(); var testLogger = new TestLogger("test"); - var hostingOptions = new OptionsWrapper(new FunctionsHostingConfigOptions()); - var workerConfigurationResolver = new WorkerConfigurationResolver(config, testLogger, _testEnvironment, _testWorkerProfileManager, hostingOptions); - RpcWorkerConfigFactory rpcWorkerConfigFactory = new RpcWorkerConfigFactory(config, testLogger, _testSysRuntimeInfo, _testEnvironment, new TestMetricsLogger(), _testWorkerProfileManager, workerConfigurationResolver, hostingOptions); + var workerConfigurationResolver = new WorkerConfigurationResolver(config, testLogger, _testEnvironment, _testWorkerProfileManager, new HashSet()); + var rpcWorkerConfigFactory = new RpcWorkerConfigFactory(config, testLogger, _testSysRuntimeInfo, _testEnvironment, new TestMetricsLogger(), _testWorkerProfileManager, workerConfigurationResolver); // Act var result = rpcWorkerConfigFactory.GetWorkerProbingPaths(config); diff --git a/test/WebJobs.Script.Tests/Workers/Rpc/RpcWorkerConfigTests.cs b/test/WebJobs.Script.Tests/Workers/Rpc/RpcWorkerConfigTests.cs index a0d98f4d6f..1f987caff0 100644 --- a/test/WebJobs.Script.Tests/Workers/Rpc/RpcWorkerConfigTests.cs +++ b/test/WebJobs.Script.Tests/Workers/Rpc/RpcWorkerConfigTests.cs @@ -685,9 +685,8 @@ private IEnumerable TestReadWorkerProviderFromConfig(IEnumerabl var scriptHostOptions = new ScriptJobHostOptions(); var scriptSettingsManager = new ScriptSettingsManager(config); var workerProfileManager = new Mock(); - var hostingOptions = new OptionsWrapper(new FunctionsHostingConfigOptions()); - var workerConfigurationResolver = new WorkerConfigurationResolver(config, testLogger, _testEnvironment, workerProfileManager.Object, hostingOptions); - var configFactory = new RpcWorkerConfigFactory(config, testLogger, _testSysRuntimeInfo, _testEnvironment, testMetricsLogger, workerProfileManager.Object, workerConfigurationResolver, hostingOptions); + var workerConfigurationResolver = new WorkerConfigurationResolver(config, testLogger, _testEnvironment, workerProfileManager.Object, new HashSet()); + var configFactory = new RpcWorkerConfigFactory(config, testLogger, _testSysRuntimeInfo, _testEnvironment, new TestMetricsLogger(), workerProfileManager.Object, workerConfigurationResolver); if (appSvcEnv) { diff --git a/test/WebJobs.Script.Tests/Workers/Rpc/WorkerConfigurationResolverTests.cs b/test/WebJobs.Script.Tests/Workers/Rpc/WorkerConfigurationResolverTests.cs index 8ca279c01e..4070ab201c 100644 --- a/test/WebJobs.Script.Tests/Workers/Rpc/WorkerConfigurationResolverTests.cs +++ b/test/WebJobs.Script.Tests/Workers/Rpc/WorkerConfigurationResolverTests.cs @@ -49,9 +49,7 @@ public void GetWorkerConfigs_ReturnsExpectedConfigs(string languageWorker, strin // Act var hostingOptionsValue = new FunctionsHostingConfigOptions(); hostingOptionsValue.Features.Add(RpcWorkerConstants.EnableProbingPathsForWorkers, languageWorker); - var hostingOptions = new OptionsWrapper(hostingOptionsValue); - - var workerConfigurationResolver = new WorkerConfigurationResolver(_mockConfig.Object, _mockLogger.Object, mockEnvironment.Object, _mockProfileManager.Object, hostingOptions); + var workerConfigurationResolver = new WorkerConfigurationResolver(_mockConfig.Object, _mockLogger.Object, mockEnvironment.Object, _mockProfileManager.Object, new HashSet() { languageWorker }); var result = workerConfigurationResolver.GetWorkerConfigs(_probingPaths, _fallbackPath); @@ -71,12 +69,7 @@ public void GetWorkerConfigs_MultiLanguageWorker_ReturnsExpectedConfigs(string r mockEnvironment.Setup(p => p.GetEnvironmentVariable(EnvironmentSettingNames.AppKind)).Returns(ScriptConstants.WorkFlowAppKind); mockEnvironment.Setup(p => p.GetEnvironmentVariable(EnvironmentSettingNames.FunctionWorkerRuntime)).Returns((string)null); - // Act - var hostingOptionsValue = new FunctionsHostingConfigOptions(); - hostingOptionsValue.Features.Add(RpcWorkerConstants.EnableProbingPathsForWorkers, "java|node|powershell"); - var hostingOptions = new OptionsWrapper(hostingOptionsValue); - - var workerConfigurationResolver = new WorkerConfigurationResolver(_mockConfig.Object, _mockLogger.Object, mockEnvironment.Object, _mockProfileManager.Object, hostingOptions); + var workerConfigurationResolver = new WorkerConfigurationResolver(_mockConfig.Object, _mockLogger.Object, mockEnvironment.Object, _mockProfileManager.Object, new HashSet() { "java", "node", "powershell" }); var result = workerConfigurationResolver.GetWorkerConfigs(_probingPaths, _fallbackPath); @@ -110,11 +103,7 @@ public void GetWorkerConfigs_MultiLanguageWorker_NullOREmptyProbingPath_ReturnsE } // Act - var hostingOptionsValue = new FunctionsHostingConfigOptions(); - hostingOptionsValue.Features.Add(RpcWorkerConstants.EnableProbingPathsForWorkers, "java|node|powershell"); - var hostingOptions = new OptionsWrapper(hostingOptionsValue); - - var workerConfigurationResolver = new WorkerConfigurationResolver(_mockConfig.Object, _mockLogger.Object, mockEnvironment.Object, _mockProfileManager.Object, hostingOptions); + var workerConfigurationResolver = new WorkerConfigurationResolver(_mockConfig.Object, _mockLogger.Object, mockEnvironment.Object, _mockProfileManager.Object, new HashSet() { "java", "node", "powershell" }); var result = workerConfigurationResolver.GetWorkerConfigs(probingPaths, _fallbackPath); @@ -156,11 +145,7 @@ public void GetWorkerConfigs_NullOREmptyProbingPath_ReturnsExpectedConfigs(strin } // Act - var hostingOptionsValue = new FunctionsHostingConfigOptions(); - hostingOptionsValue.Features.Add(RpcWorkerConstants.EnableProbingPathsForWorkers, "java|node|powershell"); - var hostingOptions = new OptionsWrapper(hostingOptionsValue); - - var workerConfigurationResolver = new WorkerConfigurationResolver(_mockConfig.Object, _mockLogger.Object, mockEnvironment.Object, _mockProfileManager.Object, hostingOptions); + var workerConfigurationResolver = new WorkerConfigurationResolver(_mockConfig.Object, _mockLogger.Object, mockEnvironment.Object, _mockProfileManager.Object, new HashSet() { "java", "node", "powershell" }); var result = workerConfigurationResolver.GetWorkerConfigs(probingPaths, _fallbackPath); From 36abb61406dbaa6676dd06565b3570109dbd117f Mon Sep 17 00:00:00 2001 From: Surbhi Gupta Date: Mon, 26 May 2025 16:40:21 -0500 Subject: [PATCH 05/55] Code refactoring --- .../LanguageWorkerOptionsSetup.cs | 49 ++++++----------- .../Configuration/RpcWorkerConfigFactory.cs | 40 +++----------- .../LanguageWorkerOptionsSetupTests.cs | 3 - .../Rpc/RpcWorkerConfigFactoryTests.cs | 55 ------------------- .../Workers/Rpc/RpcWorkerConfigTests.cs | 24 ++++---- 5 files changed, 36 insertions(+), 135 deletions(-) diff --git a/src/WebJobs.Script/Workers/Rpc/Configuration/LanguageWorkerOptionsSetup.cs b/src/WebJobs.Script/Workers/Rpc/Configuration/LanguageWorkerOptionsSetup.cs index 2b20d0c05b..a145b59c18 100644 --- a/src/WebJobs.Script/Workers/Rpc/Configuration/LanguageWorkerOptionsSetup.cs +++ b/src/WebJobs.Script/Workers/Rpc/Configuration/LanguageWorkerOptionsSetup.cs @@ -82,25 +82,30 @@ public void Configure(LanguageWorkerOptions options) } } - HashSet probingPathsEnabledWorkersViaHostingConfig = _functionsHostingConfigOptions.Value - ?.EnableProbingPathsForWorkers - ?.ToLowerInvariant() - ?.Split("|", StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries) - ?.ToHashSet(); - + HashSet probingPathsEnabledWorkersViaHostingConfig = GetWorkersEnabledViaHostingConfig(); bool probingPathsEnabled = Utility.AreWorkerProbingPathsEnabled(_environment, probingPathsEnabledWorkersViaHostingConfig, workerRuntime); + List probingPaths = null; + if (probingPathsEnabled) { - string probingPathsString = GetWorkerProbingPaths(); - configuration = AddProbingPathsToConfiguration(configuration, probingPathsString); + probingPaths = GetWorkerProbingPaths(); } var workerConfigurationResolver = new WorkerConfigurationResolver(configuration, _logger, _environment, _workerProfileManager, probingPathsEnabledWorkersViaHostingConfig); - var configFactory = new RpcWorkerConfigFactory(configuration, _logger, SystemRuntimeInformation.Instance, _environment, _metricsLogger, _workerProfileManager, workerConfigurationResolver, probingPathsEnabled); + var configFactory = new RpcWorkerConfigFactory(configuration, _logger, SystemRuntimeInformation.Instance, _environment, _metricsLogger, _workerProfileManager, workerConfigurationResolver, probingPathsEnabled, probingPaths); options.WorkerConfigs = configFactory.GetConfigs(); } - internal string GetWorkerProbingPaths() + internal HashSet GetWorkersEnabledViaHostingConfig() + { + return _functionsHostingConfigOptions.Value + ?.EnableProbingPathsForWorkers + ?.ToLowerInvariant() + ?.Split("|", StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries) + ?.ToHashSet(); + } + + internal List GetWorkerProbingPaths() { var probingPaths = new List(); string probingPathsString = string.Empty; @@ -120,29 +125,7 @@ internal string GetWorkerProbingPaths() } } - if (probingPaths.Any()) - { - probingPathsString = $"{{ \"{RpcWorkerConstants.LanguageWorkersSectionName}\": " + - $"{{ \"{RpcWorkerConstants.WorkerProbingPathsSectionName}\": " + - $"\"{probingPaths}\" }} }}"; - } - - return probingPathsString; - } - - internal IConfiguration AddProbingPathsToConfiguration(IConfiguration configuration, string probingPathsString) - { - if (!string.IsNullOrEmpty(probingPathsString)) - { - using var jsonStream = new MemoryStream(Encoding.UTF8.GetBytes(probingPathsString)); - - configuration = new ConfigurationBuilder() - .AddConfiguration(configuration) - .AddJsonStream(jsonStream) - .Build(); - } - - return configuration; + return probingPaths; } } diff --git a/src/WebJobs.Script/Workers/Rpc/Configuration/RpcWorkerConfigFactory.cs b/src/WebJobs.Script/Workers/Rpc/Configuration/RpcWorkerConfigFactory.cs index 0543f09816..4baa89966b 100644 --- a/src/WebJobs.Script/Workers/Rpc/Configuration/RpcWorkerConfigFactory.cs +++ b/src/WebJobs.Script/Workers/Rpc/Configuration/RpcWorkerConfigFactory.cs @@ -44,7 +44,8 @@ public RpcWorkerConfigFactory(IConfiguration config, IMetricsLogger metricsLogger, IWorkerProfileManager workerProfileManager, IWorkerConfigurationResolver workerConfigurationResolver, - bool probingPathsEnabled = false) + bool probingPathsEnabled = false, + List probingPaths = null) { _config = config ?? throw new ArgumentNullException(nameof(config)); _logger = logger ?? throw new ArgumentNullException(nameof(logger)); @@ -59,6 +60,7 @@ public RpcWorkerConfigFactory(IConfiguration config, var workersDirectorySection = _config.GetSection($"{RpcWorkerConstants.LanguageWorkersSectionName}:{WorkerConstants.WorkersDirectorySectionName}"); ProbingPathsEnabled = probingPathsEnabled; + WorkerProbingPaths = probingPaths; if (!string.IsNullOrEmpty(workersDirectorySection.Value)) { @@ -70,6 +72,8 @@ public RpcWorkerConfigFactory(IConfiguration config, internal bool ProbingPathsEnabled { get; } + internal List WorkerProbingPaths { get; } + public IList GetConfigs() { using (_metricsLogger.LatencyEvent(MetricEventNames.GetConfigs)) @@ -101,13 +105,11 @@ internal void BuildWorkerProviderDictionary() internal void AddProviders() { - if (ProbingPathsEnabled) + if (ProbingPathsEnabled && WorkerProbingPaths is not null) { - List probingPaths = GetWorkerProbingPaths(_config); - - _logger.LogDebug("Probing paths set to: {probingPaths} and Workers Directory set as fallback path: {WorkersDirPath}", string.Join(", ", probingPaths), WorkersDirPath); + _logger.LogDebug("Probing paths set to: {probingPaths} and Workers Directory set as fallback path: {WorkersDirPath}", string.Join(", ", WorkerProbingPaths), WorkersDirPath); - List workerConfigs = _workerConfigurationResolver.GetWorkerConfigs(probingPaths, WorkersDirPath); + List workerConfigs = _workerConfigurationResolver.GetWorkerConfigs(WorkerProbingPaths, WorkersDirPath); foreach (var workerConfig in workerConfigs) { @@ -300,32 +302,6 @@ internal bool ShouldAddWorkerConfig(string workerDescriptionLanguage) return true; } - internal List GetWorkerProbingPaths(IConfiguration config) - { - List probingPaths = new List(); - - IConfigurationSection probingPathsSection = config?.GetSection($"{RpcWorkerConstants.LanguageWorkersSectionName}") - ?.GetSection($"{RpcWorkerConstants.WorkerProbingPathsSectionName}"); - - var probingPathsList = probingPathsSection?.AsEnumerable(); - - if (probingPathsList is null || probingPathsSection is null) - { - return probingPaths; - } - - for (int item = 0; item < probingPathsList.Count(); item++) - { - var path = probingPathsSection.GetSection($"{item}")?.Value; - if (path is not null) - { - probingPaths.Add(path); - } - } - - return probingPaths; - } - private void ReadLanguageWorkerFile(string workerPath) { if (!_environment.IsPlaceholderModeEnabled() diff --git a/test/WebJobs.Script.Tests/Configuration/LanguageWorkerOptionsSetupTests.cs b/test/WebJobs.Script.Tests/Configuration/LanguageWorkerOptionsSetupTests.cs index 070ec14245..cde90f93df 100644 --- a/test/WebJobs.Script.Tests/Configuration/LanguageWorkerOptionsSetupTests.cs +++ b/test/WebJobs.Script.Tests/Configuration/LanguageWorkerOptionsSetupTests.cs @@ -3,12 +3,9 @@ using System; using Microsoft.Azure.WebJobs.Script.Config; -using Microsoft.Azure.WebJobs.Script.Workers; using Microsoft.Azure.WebJobs.Script.Workers.Profiles; using Microsoft.Azure.WebJobs.Script.Workers.Rpc; -using Microsoft.Azure.WebJobs.Script.Workers.Rpc.Configuration; using Microsoft.Extensions.Configuration; -using Microsoft.Extensions.Logging; using Microsoft.Extensions.Logging.Abstractions; using Microsoft.Extensions.Options; using Moq; diff --git a/test/WebJobs.Script.Tests/Workers/Rpc/RpcWorkerConfigFactoryTests.cs b/test/WebJobs.Script.Tests/Workers/Rpc/RpcWorkerConfigFactoryTests.cs index b431d92e17..a15352f7ab 100644 --- a/test/WebJobs.Script.Tests/Workers/Rpc/RpcWorkerConfigFactoryTests.cs +++ b/test/WebJobs.Script.Tests/Workers/Rpc/RpcWorkerConfigFactoryTests.cs @@ -12,7 +12,6 @@ using Microsoft.Azure.WebJobs.Script.Workers.Rpc; using Microsoft.Azure.WebJobs.Script.Workers.Rpc.Configuration; using Microsoft.Extensions.Configuration; -using Microsoft.Extensions.Options; using Moq; using Xunit; @@ -367,60 +366,6 @@ public void GetWorkerProcessCount_ThrowsException_Tests() Assert.Contains("value could not be converted to System.TimeSpan", resultEx3.Message); } - [Fact] - public void GetWorkerProbingPaths_ReturnsExpectedPaths() - { - // Arrange - var inMemorySettings = new Dictionary - { - ["languageWorkers:probingPaths:0"] = @"C:\workers\path1", - ["languageWorkers:probingPaths:1"] = @"C:\workers\path2" - }; - - IConfiguration config = new ConfigurationBuilder() - .AddInMemoryCollection(inMemorySettings) - .Build(); - - var testLogger = new TestLogger("test"); - var workerConfigurationResolver = new WorkerConfigurationResolver(config, testLogger, _testEnvironment, _testWorkerProfileManager, new HashSet()); - var rpcWorkerConfigFactory = new RpcWorkerConfigFactory(config, testLogger, _testSysRuntimeInfo, _testEnvironment, new TestMetricsLogger(), _testWorkerProfileManager, workerConfigurationResolver); - - // Act - var result = rpcWorkerConfigFactory.GetWorkerProbingPaths(config); - - // Assert - Assert.Equal(2, result.Count); - Assert.Contains(@"C:\workers\path1", result); - Assert.Contains(@"C:\workers\path2", result); - } - - [Theory] - [InlineData("languageWorkers:probingPaths:0", "", 1)] - [InlineData("languageWorkers:probingPaths", "C:\\workers\\path1", 0)] - [InlineData("languageWorkers", "C:\\workers\\path1", 0)] - public void GetWorkerProbingPaths_InvalidProbingPaths_ReturnsEmptyList(string key, string value, int count) - { - // Arrange - var inMemorySettings = new Dictionary - { - [key] = value, - }; - - IConfiguration config = new ConfigurationBuilder() - .AddInMemoryCollection(inMemorySettings) - .Build(); - - var testLogger = new TestLogger("test"); - var workerConfigurationResolver = new WorkerConfigurationResolver(config, testLogger, _testEnvironment, _testWorkerProfileManager, new HashSet()); - var rpcWorkerConfigFactory = new RpcWorkerConfigFactory(config, testLogger, _testSysRuntimeInfo, _testEnvironment, new TestMetricsLogger(), _testWorkerProfileManager, workerConfigurationResolver); - - // Act - var result = rpcWorkerConfigFactory.GetWorkerProbingPaths(config); - - // Assert - Assert.Equal(count, result.Count); - } - private static JsonElement CreateWorkerConfig(int processCount, int maxProcessCount, string processStartupInterval, bool setProcessCountToCores) { using var stream = new MemoryStream(); diff --git a/test/WebJobs.Script.Tests/Workers/Rpc/RpcWorkerConfigTests.cs b/test/WebJobs.Script.Tests/Workers/Rpc/RpcWorkerConfigTests.cs index 1f987caff0..51b2e97572 100644 --- a/test/WebJobs.Script.Tests/Workers/Rpc/RpcWorkerConfigTests.cs +++ b/test/WebJobs.Script.Tests/Workers/Rpc/RpcWorkerConfigTests.cs @@ -284,10 +284,10 @@ public void ValidateWorkerDescription_ResolvesDotNetDefaultWorkerExecutablePath_ Extensions = new List(), DefaultExecutablePath = defaultExecutablePath, FileExists = path => - { - Assert.Equal(expectedExecutablePath, path); - return true; - } + { + Assert.Equal(expectedExecutablePath, path); + return true; + } }; workerDescription.ApplyDefaultsAndValidate(Directory.GetCurrentDirectory(), testLogger); @@ -315,10 +315,10 @@ public void ValidateWorkerDescription_DoesNotModifyDefaultWorkerExecutablePathAn Extensions = new List(), DefaultExecutablePath = defaultExecutablePath, FileExists = path => - { - Assert.Equal(expectedExecutablePath, path); - return false; - } + { + Assert.Equal(expectedExecutablePath, path); + return false; + } }; workerDescription.ApplyDefaultsAndValidate(Directory.GetCurrentDirectory(), testLogger); @@ -343,10 +343,10 @@ public void ValidateWorkerDescription_DoesNotModifyDefaultWorkerExecutablePath_W Extensions = new List(), DefaultExecutablePath = defaultExecutablePath, FileExists = path => - { - Assert.True(false, "FileExists should not be called"); - return false; - } + { + Assert.True(false, "FileExists should not be called"); + return false; + } }; workerDescription.ApplyDefaultsAndValidate(Directory.GetCurrentDirectory(), testLogger); From 8929697155256bcaf9b2ad5db4f3cf6f66080150 Mon Sep 17 00:00:00 2001 From: Surbhi Gupta Date: Mon, 26 May 2025 17:02:36 -0500 Subject: [PATCH 06/55] Tests cleanup --- .../LanguageWorkerOptionsSetup.cs | 4 +- .../LanguageWorkerOptionsSetupTests.cs | 59 +++++++++++++++++++ .../Rpc/RpcWorkerConfigFactoryTests.cs | 28 ++------- 3 files changed, 67 insertions(+), 24 deletions(-) diff --git a/src/WebJobs.Script/Workers/Rpc/Configuration/LanguageWorkerOptionsSetup.cs b/src/WebJobs.Script/Workers/Rpc/Configuration/LanguageWorkerOptionsSetup.cs index a145b59c18..75554e16f4 100644 --- a/src/WebJobs.Script/Workers/Rpc/Configuration/LanguageWorkerOptionsSetup.cs +++ b/src/WebJobs.Script/Workers/Rpc/Configuration/LanguageWorkerOptionsSetup.cs @@ -82,7 +82,7 @@ public void Configure(LanguageWorkerOptions options) } } - HashSet probingPathsEnabledWorkersViaHostingConfig = GetWorkersEnabledViaHostingConfig(); + HashSet probingPathsEnabledWorkersViaHostingConfig = GetProbingPathsEnabledWorkersFromHostingConfig(); bool probingPathsEnabled = Utility.AreWorkerProbingPathsEnabled(_environment, probingPathsEnabledWorkersViaHostingConfig, workerRuntime); List probingPaths = null; @@ -96,7 +96,7 @@ public void Configure(LanguageWorkerOptions options) options.WorkerConfigs = configFactory.GetConfigs(); } - internal HashSet GetWorkersEnabledViaHostingConfig() + internal HashSet GetProbingPathsEnabledWorkersFromHostingConfig() { return _functionsHostingConfigOptions.Value ?.EnableProbingPathsForWorkers diff --git a/test/WebJobs.Script.Tests/Configuration/LanguageWorkerOptionsSetupTests.cs b/test/WebJobs.Script.Tests/Configuration/LanguageWorkerOptionsSetupTests.cs index cde90f93df..ed180831cc 100644 --- a/test/WebJobs.Script.Tests/Configuration/LanguageWorkerOptionsSetupTests.cs +++ b/test/WebJobs.Script.Tests/Configuration/LanguageWorkerOptionsSetupTests.cs @@ -73,5 +73,64 @@ public void LanguageWorkerOptions_Expected_ListOfConfigs(string workerRuntime) Assert.Equal(1, options.WorkerConfigs.Count); } } + + [Theory] + [InlineData("DotNet")] + [InlineData("dotnet")] + [InlineData(null)] + [InlineData("node")] + public void LanguageWorkerOptions_ProbingPaths_Expected_ListOfConfigs(string workerRuntime) + { + var testEnvironment = new TestEnvironment(); + var testMetricLogger = new TestMetricsLogger(); + var configurationBuilder = new ConfigurationBuilder() + .Add(new ScriptEnvironmentVariablesConfigurationSource()); + var configuration = configurationBuilder.Build(); + var testProfileManager = new Mock(); + var testScriptHostManager = new Mock(); + + if (!string.IsNullOrEmpty(workerRuntime)) + { + testEnvironment.SetEnvironmentVariable(RpcWorkerConstants.FunctionWorkerRuntimeSettingName, workerRuntime); + } + else + { + // The dotnet-isolated worker only runs in placeholder mode. Setting the placeholder environment to 1 for the test. + testEnvironment.SetEnvironmentVariable(EnvironmentSettingNames.AzureWebsitePlaceholderMode, "1"); + } + + testProfileManager.Setup(pm => pm.LoadWorkerDescriptionFromProfiles(It.IsAny(), out It.Ref.IsAny)) + .Callback((RpcWorkerDescription defaultDescription, out RpcWorkerDescription outDescription) => + { + // dotnet-isolated worker config does not have "DefaultExecutablePath" in the parent level.So, we should set it from a profile. + if (defaultDescription.Language == "dotnet-isolated") + { + outDescription = new RpcWorkerDescription() { DefaultExecutablePath = "testPath", Language = "dotnet-isolated" }; + } + else + { + // for other workers, we should return the default description as they have the "DefaultExecutablePath" in the parent level. + outDescription = defaultDescription; + } + }); + + LanguageWorkerOptionsSetup setup = new LanguageWorkerOptionsSetup(configuration, NullLoggerFactory.Instance, testEnvironment, testMetricLogger, testProfileManager.Object, testScriptHostManager.Object, new OptionsWrapper(new FunctionsHostingConfigOptions())); + LanguageWorkerOptions options = new LanguageWorkerOptions(); + + setup.Configure(options); + + if (string.IsNullOrEmpty(workerRuntime)) + { + Assert.Equal(5, options.WorkerConfigs.Count); + } + else if (workerRuntime.Equals(RpcWorkerConstants.DotNetLanguageWorkerName, StringComparison.OrdinalIgnoreCase)) + { + Assert.Empty(options.WorkerConfigs); + } + else + { + Assert.Equal(1, options.WorkerConfigs.Count); + } + } } } \ No newline at end of file diff --git a/test/WebJobs.Script.Tests/Workers/Rpc/RpcWorkerConfigFactoryTests.cs b/test/WebJobs.Script.Tests/Workers/Rpc/RpcWorkerConfigFactoryTests.cs index a15352f7ab..5f33a38b0a 100644 --- a/test/WebJobs.Script.Tests/Workers/Rpc/RpcWorkerConfigFactoryTests.cs +++ b/test/WebJobs.Script.Tests/Workers/Rpc/RpcWorkerConfigFactoryTests.cs @@ -152,25 +152,15 @@ public void JavaPath_FromEnvVars() public void GetConfigs_ProbingPaths() { string probingPath1 = Path.GetFullPath("..\\..\\..\\..\\test\\TestWorkers\\ProbingPaths\\workers\\"); - string fallbackPath = Path.GetFullPath("..\\..\\..\\..\\test\\TestWorkers\\FallbackPath\\workers\\"); var mockEnvironment = new Mock(); mockEnvironment.Setup(p => p.GetEnvironmentVariable(EnvironmentSettingNames.FunctionWorkerRuntime)).Returns("java"); - var inMemorySettings = new Dictionary - { - ["languageWorkers:probingPaths:0"] = probingPath1, - }; - - IConfiguration config = new ConfigurationBuilder() - .AddInMemoryCollection(inMemorySettings) - .Build(); - - var scriptSettingsManager = new ScriptSettingsManager(config); + IConfiguration config = new ConfigurationBuilder().Build(); var testLogger = new TestLogger("test"); var workerConfigurationResolver = new WorkerConfigurationResolver(config, testLogger, mockEnvironment.Object, _testWorkerProfileManager, new HashSet() { "java" }); - var configFactory = new RpcWorkerConfigFactory(config, testLogger, _testSysRuntimeInfo, mockEnvironment.Object, new TestMetricsLogger(), _testWorkerProfileManager, workerConfigurationResolver, true); + var configFactory = new RpcWorkerConfigFactory(config, testLogger, _testSysRuntimeInfo, mockEnvironment.Object, new TestMetricsLogger(), _testWorkerProfileManager, workerConfigurationResolver, true, new List () { probingPath1 }); var workerConfigs = configFactory.GetConfigs(); // check log messages @@ -222,21 +212,15 @@ public void DefaultWorkerConfigs_Overrides_VersionAppSetting(string runtimeSetti var testEnvironment = new TestEnvironment(); testEnvironment.SetEnvironmentVariable("FUNCTIONS_WORKER_RUNTIME_VERSION", runtimeSettingVersion); testEnvironment.SetEnvironmentVariable("FUNCTIONS_WORKER_RUNTIME", "powershell"); - testEnvironment.SetEnvironmentVariable(EnvironmentSettingNames.WorkerProbingPaths, $"{probingPath}"); - // testEnvironment.SetEnvironmentVariable(EnvironmentSettingNames.AzureWebJobsFeatureFlags, enableProbingPath); - var configBuilder = ScriptSettingsManager.CreateDefaultConfigurationBuilder(); - configBuilder.AddInMemoryCollection(new Dictionary - { - { $"{RpcWorkerConstants.LanguageWorkersSectionName}:probingPaths:0", probingPath } - }); - var config = configBuilder.Build(); - var scriptSettingsManager = new ScriptSettingsManager(config); + var config = ScriptSettingsManager.CreateDefaultConfigurationBuilder().Build(); + var testLogger = new TestLogger("test"); var workerConfigurationResolver = new WorkerConfigurationResolver(config, testLogger, testEnvironment, _testWorkerProfileManager, new HashSet() { "powershell" }); - var configFactory = new RpcWorkerConfigFactory(config, testLogger, _testSysRuntimeInfo, testEnvironment, new TestMetricsLogger(), _testWorkerProfileManager, workerConfigurationResolver, enableProbingPath is null ? false : true); + var configFactory = new RpcWorkerConfigFactory(config, testLogger, _testSysRuntimeInfo, testEnvironment, new TestMetricsLogger(), _testWorkerProfileManager, workerConfigurationResolver, enableProbingPath is null ? false : true, new List() { probingPath }); var workerConfigs = configFactory.GetConfigs(); var powershellWorkerConfig = workerConfigs.FirstOrDefault(w => w.Description.Language.Equals("powershell", StringComparison.OrdinalIgnoreCase)); + Assert.Equal(1, workerConfigs.Count); Assert.NotNull(powershellWorkerConfig); Assert.Equal(outputVersion, powershellWorkerConfig.Description.DefaultRuntimeVersion); From 1588711ce070f7682a35209d6767f75b779071bc Mon Sep 17 00:00:00 2001 From: Surbhi Gupta Date: Mon, 26 May 2025 18:40:03 -0500 Subject: [PATCH 07/55] Added test for LanguageWorkerOptionsSetup --- .../WorkerConfigurationResolver.cs | 5 +- .../LanguageWorkerOptionsSetupTests.cs | 50 ++++++++----------- .../Rpc/RpcWorkerConfigFactoryTests.cs | 2 +- 3 files changed, 23 insertions(+), 34 deletions(-) diff --git a/src/WebJobs.Script/Workers/Rpc/Configuration/WorkerConfigurationResolver.cs b/src/WebJobs.Script/Workers/Rpc/Configuration/WorkerConfigurationResolver.cs index 063d2066bf..1a41457eed 100644 --- a/src/WebJobs.Script/Workers/Rpc/Configuration/WorkerConfigurationResolver.cs +++ b/src/WebJobs.Script/Workers/Rpc/Configuration/WorkerConfigurationResolver.cs @@ -21,10 +21,7 @@ internal sealed class WorkerConfigurationResolver : IWorkerConfigurationResolver private readonly IWorkerProfileManager _profileManager; private readonly IEnvironment _environment; private readonly HashSet _probingPathsEnabledWorkersViaHostingConfig; - private readonly JsonSerializerOptions _jsonSerializerOptions = new() - { - PropertyNameCaseInsensitive = true - }; + private readonly JsonSerializerOptions _jsonSerializerOptions = new() { PropertyNameCaseInsensitive = true }; public WorkerConfigurationResolver(IConfiguration config, ILogger logger, diff --git a/test/WebJobs.Script.Tests/Configuration/LanguageWorkerOptionsSetupTests.cs b/test/WebJobs.Script.Tests/Configuration/LanguageWorkerOptionsSetupTests.cs index ed180831cc..c819b609ee 100644 --- a/test/WebJobs.Script.Tests/Configuration/LanguageWorkerOptionsSetupTests.cs +++ b/test/WebJobs.Script.Tests/Configuration/LanguageWorkerOptionsSetupTests.cs @@ -2,12 +2,16 @@ // Licensed under the MIT License. See License.txt in the project root for license information. using System; +using System.IO; +using System.Linq; using Microsoft.Azure.WebJobs.Script.Config; using Microsoft.Azure.WebJobs.Script.Workers.Profiles; using Microsoft.Azure.WebJobs.Script.Workers.Rpc; using Microsoft.Extensions.Configuration; +using Microsoft.Extensions.Logging; using Microsoft.Extensions.Logging.Abstractions; using Microsoft.Extensions.Options; +using Microsoft.WebJobs.Script.Tests; using Moq; using Xunit; @@ -75,12 +79,13 @@ public void LanguageWorkerOptions_Expected_ListOfConfigs(string workerRuntime) } [Theory] - [InlineData("DotNet")] - [InlineData("dotnet")] - [InlineData(null)] - [InlineData("node")] - public void LanguageWorkerOptions_ProbingPaths_Expected_ListOfConfigs(string workerRuntime) + [InlineData("java", "EnableWorkerProbingPaths", "java", "..\\..\\..\\..\\test\\TestWorkers\\ProbingPaths\\workers\\", "LATEST", "2.19.0")] + public void LanguageWorkerOptions_ProbingPaths_Expected_ListOfConfigs(string workerRuntime, string enableProbingPaths, string hostingOptionsSetting, string probingPath, string releaseChannel, string expectedVersion) { + var loggerProvider = new TestLoggerProvider(); + var loggerFactory = new LoggerFactory(); + loggerFactory.AddProvider(loggerProvider); + var testEnvironment = new TestEnvironment(); var testMetricLogger = new TestMetricsLogger(); var configurationBuilder = new ConfigurationBuilder() @@ -88,33 +93,17 @@ public void LanguageWorkerOptions_ProbingPaths_Expected_ListOfConfigs(string wor var configuration = configurationBuilder.Build(); var testProfileManager = new Mock(); var testScriptHostManager = new Mock(); + string probingPath1 = Path.GetFullPath(probingPath); - if (!string.IsNullOrEmpty(workerRuntime)) - { - testEnvironment.SetEnvironmentVariable(RpcWorkerConstants.FunctionWorkerRuntimeSettingName, workerRuntime); - } - else - { - // The dotnet-isolated worker only runs in placeholder mode. Setting the placeholder environment to 1 for the test. - testEnvironment.SetEnvironmentVariable(EnvironmentSettingNames.AzureWebsitePlaceholderMode, "1"); - } + testEnvironment.SetEnvironmentVariable(RpcWorkerConstants.FunctionWorkerRuntimeSettingName, workerRuntime); + testEnvironment.SetEnvironmentVariable(EnvironmentSettingNames.AzureWebJobsFeatureFlags, enableProbingPaths); + testEnvironment.SetEnvironmentVariable(EnvironmentSettingNames.AntaresPlatformReleaseChannel, releaseChannel); + testEnvironment.SetEnvironmentVariable(EnvironmentSettingNames.WorkerProbingPaths, probingPath1); - testProfileManager.Setup(pm => pm.LoadWorkerDescriptionFromProfiles(It.IsAny(), out It.Ref.IsAny)) - .Callback((RpcWorkerDescription defaultDescription, out RpcWorkerDescription outDescription) => - { - // dotnet-isolated worker config does not have "DefaultExecutablePath" in the parent level.So, we should set it from a profile. - if (defaultDescription.Language == "dotnet-isolated") - { - outDescription = new RpcWorkerDescription() { DefaultExecutablePath = "testPath", Language = "dotnet-isolated" }; - } - else - { - // for other workers, we should return the default description as they have the "DefaultExecutablePath" in the parent level. - outDescription = defaultDescription; - } - }); + var hostingOptions = new FunctionsHostingConfigOptions(); + hostingOptions.Features.Add(RpcWorkerConstants.EnableProbingPathsForWorkers, hostingOptionsSetting); - LanguageWorkerOptionsSetup setup = new LanguageWorkerOptionsSetup(configuration, NullLoggerFactory.Instance, testEnvironment, testMetricLogger, testProfileManager.Object, testScriptHostManager.Object, new OptionsWrapper(new FunctionsHostingConfigOptions())); + LanguageWorkerOptionsSetup setup = new LanguageWorkerOptionsSetup(configuration, loggerFactory, testEnvironment, testMetricLogger, testProfileManager.Object, testScriptHostManager.Object, new OptionsWrapper(hostingOptions)); LanguageWorkerOptions options = new LanguageWorkerOptions(); setup.Configure(options); @@ -130,6 +119,9 @@ public void LanguageWorkerOptions_ProbingPaths_Expected_ListOfConfigs(string wor else { Assert.Equal(1, options.WorkerConfigs.Count); + Assert.True(options.WorkerConfigs.First().Arguments.WorkerPath.Contains(expectedVersion)); + + var logs = loggerProvider.GetAllLogMessages(); } } } diff --git a/test/WebJobs.Script.Tests/Workers/Rpc/RpcWorkerConfigFactoryTests.cs b/test/WebJobs.Script.Tests/Workers/Rpc/RpcWorkerConfigFactoryTests.cs index 5f33a38b0a..0fe8504d34 100644 --- a/test/WebJobs.Script.Tests/Workers/Rpc/RpcWorkerConfigFactoryTests.cs +++ b/test/WebJobs.Script.Tests/Workers/Rpc/RpcWorkerConfigFactoryTests.cs @@ -160,7 +160,7 @@ public void GetConfigs_ProbingPaths() var testLogger = new TestLogger("test"); var workerConfigurationResolver = new WorkerConfigurationResolver(config, testLogger, mockEnvironment.Object, _testWorkerProfileManager, new HashSet() { "java" }); - var configFactory = new RpcWorkerConfigFactory(config, testLogger, _testSysRuntimeInfo, mockEnvironment.Object, new TestMetricsLogger(), _testWorkerProfileManager, workerConfigurationResolver, true, new List () { probingPath1 }); + var configFactory = new RpcWorkerConfigFactory(config, testLogger, _testSysRuntimeInfo, mockEnvironment.Object, new TestMetricsLogger(), _testWorkerProfileManager, workerConfigurationResolver, true, new List() { probingPath1 }); var workerConfigs = configFactory.GetConfigs(); // check log messages From 2606f377fcbf6b1be5abdd27a245ad8e3c59a1d6 Mon Sep 17 00:00:00 2001 From: Surbhi Gupta Date: Mon, 26 May 2025 22:18:23 -0500 Subject: [PATCH 08/55] Adding Tests for Utility class --- src/WebJobs.Script/ScriptConstants.cs | 4 +- .../WorkerConfigurationResolver.cs | 106 ++++++++---------- .../Rpc/IWorkerConfigurationResolver.cs | 2 +- .../Workers/Rpc/RpcWorkerConstants.cs | 1 + test/WebJobs.Script.Tests/UtilityTests.cs | 46 ++++++++ 5 files changed, 100 insertions(+), 59 deletions(-) diff --git a/src/WebJobs.Script/ScriptConstants.cs b/src/WebJobs.Script/ScriptConstants.cs index a55d1c7a50..e21a2c8c75 100644 --- a/src/WebJobs.Script/ScriptConstants.cs +++ b/src/WebJobs.Script/ScriptConstants.cs @@ -1,9 +1,10 @@ // Copyright (c) .NET Foundation. All rights reserved. // Licensed under the MIT License. See License.txt in the project root for license information. +using NuGet.Versioning; using System; +using System.Collections.Generic; using System.Collections.Immutable; -using NuGet.Versioning; namespace Microsoft.Azure.WebJobs.Script { @@ -251,6 +252,7 @@ public static class ScriptConstants public static readonly long DefaultMaxRequestBodySize = 104857600; public static readonly ImmutableArray SystemLogCategoryPrefixes = ImmutableArray.Create("Microsoft.Azure.WebJobs.", "Function.", "Worker.", "Host."); + public static readonly HashSet HostCapabilities = []; public static readonly string FunctionMetadataDirectTypeKey = "DirectType"; public static readonly string LiveLogsSessionAIKey = "#AzFuncLiveLogsSessionId"; diff --git a/src/WebJobs.Script/Workers/Rpc/Configuration/WorkerConfigurationResolver.cs b/src/WebJobs.Script/Workers/Rpc/Configuration/WorkerConfigurationResolver.cs index 1a41457eed..5c75143be6 100644 --- a/src/WebJobs.Script/Workers/Rpc/Configuration/WorkerConfigurationResolver.cs +++ b/src/WebJobs.Script/Workers/Rpc/Configuration/WorkerConfigurationResolver.cs @@ -6,6 +6,7 @@ using System.Collections.Generic; using System.IO; using System.Linq; +using System.Runtime.CompilerServices; using System.Text.Json; using Microsoft.Azure.AppService.Proxy.Common.Extensions; using Microsoft.Azure.WebJobs.Script.Workers.Profiles; @@ -38,49 +39,46 @@ public WorkerConfigurationResolver(IConfiguration config, public List GetWorkerConfigs(List probingPaths, string fallbackPath) { - var workerRuntime = _environment.GetEnvironmentVariable(EnvironmentSettingNames.FunctionWorkerRuntime); - - // Dictionary of language-name : workerConfig + // Dictionary of { language-name : path of workerConfig } ConcurrentDictionary outputDict = new ConcurrentDictionary(); - - // check worker release channel + var workerRuntime = _environment.GetEnvironmentVariable(EnvironmentSettingNames.FunctionWorkerRuntime); string releaseChannel = Utility.GetPlatformReleaseChannel(_environment); if (!probingPaths.IsNullOrEmpty()) { + // probing path directory structure is: /// foreach (var probingPath in probingPaths) { if (!string.IsNullOrEmpty(probingPath) && Directory.Exists(probingPath)) { foreach (var languageWorkerPath in Directory.EnumerateDirectories(probingPath)) { - string languageWorkerFolder = Path.GetFileName(languageWorkerPath); + string languageWorkerDir = Path.GetFileName(languageWorkerPath).ToLower(); - _logger.LogInformation("Probing for language worker in path: {LanguageWorkerPath}", languageWorkerPath); - - // Only skip worker directories that don't match the current runtime. - // Do not skip non-worker directories like the function app payload directory - // && languageWorkerPath.StartsWith(fallbackPath)) - if (!_probingPathsEnabledWorkersViaHostingConfig.Contains(languageWorkerFolder) || - (!_environment.IsMultiLanguageRuntimeEnvironment() && - workerRuntime is not null && - !workerRuntime.Equals(languageWorkerFolder, StringComparison.OrdinalIgnoreCase))) + // If probing paths are malformed and have duplicate directories of the same language worker (eg. due to different casing) + if (outputDict.ContainsKey(languageWorkerDir)) { continue; } - IEnumerable workerVersions = Directory.EnumerateDirectories(languageWorkerPath); - var versionsList = ParseWorkerVersions(workerVersions); - var versions = versionsList.OrderDescending(); - - if (outputDict.ContainsKey(languageWorkerFolder)) + // Only skip worker directories that don't match the current runtime or are not enabled via hosting config + // Do not skip non-worker directories like the function app payload directory + if (languageWorkerPath.StartsWith(fallbackPath) || languageWorkerPath.StartsWith(probingPath)) { - continue; + if ((_probingPathsEnabledWorkersViaHostingConfig is not null && + !_probingPathsEnabledWorkersViaHostingConfig.Contains(languageWorkerDir)) || + (!_environment.IsMultiLanguageRuntimeEnvironment() && + workerRuntime is not null && + !workerRuntime.Equals(languageWorkerDir, StringComparison.OrdinalIgnoreCase))) + { + continue; + } } - GetWorkerConfigsFromProbingPaths(versions, languageWorkerPath, languageWorkerFolder, releaseChannel, outputDict); + IEnumerable workerVersions = Directory.EnumerateDirectories(languageWorkerPath); + var versions = ParseWorkerVersionsInDescending(workerVersions); - _logger.LogInformation("Found worker config for {LanguageWorkerFolder} at {LanguageWorkerPath}", languageWorkerFolder, outputDict.GetValueOrDefault(languageWorkerFolder)); + GetWorkerConfigsFromProbingPaths(versions, languageWorkerPath, languageWorkerDir, releaseChannel, outputDict); } } } @@ -93,7 +91,7 @@ workerRuntime is not null && return outputDict.Values.ToList(); } - // fallback path + // Search in fallback path if worker cannot be found in probing paths GetWorkerConfigsFromWithinHost(fallbackPath, workerRuntime, outputDict); return outputDict.Values.ToList(); @@ -107,7 +105,8 @@ private void GetWorkerConfigsFromProbingPaths(IEnumerable versions, str foreach (Version versionFolder in versions) { string languageWorkerVersionPath = Path.Combine(languageWorkerPath, versionFolder.ToString()); - if (IsCompatibleWithHost(languageWorkerVersionPath)) + + if (IsWorkerCompatibleWithHost(languageWorkerVersionPath)) { found++; outputDict[languageWorkerFolder] = languageWorkerVersionPath; @@ -129,28 +128,27 @@ private void GetWorkerConfigsFromProbingPaths(IEnumerable versions, str private void GetWorkerConfigsFromWithinHost(string fallbackPath, string workerRuntime, ConcurrentDictionary outputDict) { - _logger.LogInformation("Searching for worker configs in fallback path: {FallbackPath}", fallbackPath); - - // fallback path if (Directory.Exists(fallbackPath)) { - foreach (var workerDir in Directory.EnumerateDirectories(fallbackPath)) + foreach (var workerPath in Directory.EnumerateDirectories(fallbackPath)) { - string workerFolder = Path.GetFileName(workerDir); + string workerDir = Path.GetFileName(workerPath).ToLower(); - if (outputDict.ContainsKey(workerFolder) || + if (workerPath.StartsWith(fallbackPath)) + { + if (outputDict.ContainsKey(workerDir) || (!_environment.IsMultiLanguageRuntimeEnvironment() && workerRuntime is not null && - !workerRuntime.Equals(workerFolder, StringComparison.OrdinalIgnoreCase))) - { - continue; + !workerRuntime.Equals(workerDir, StringComparison.OrdinalIgnoreCase))) + { + continue; + } } - string workerConfigPath = Path.Combine(workerDir, RpcWorkerConstants.WorkerConfigFileName); + string workerConfigPath = Path.Combine(workerPath, RpcWorkerConstants.WorkerConfigFileName); if (File.Exists(workerConfigPath)) { - outputDict[workerFolder] = workerDir; - _logger.LogInformation("Found worker in fallback path workerfolder = {workerFolder} and dir = {workerDir}", workerFolder, workerDir); + outputDict[workerDir] = workerPath; } if (!_environment.IsMultiLanguageRuntimeEnvironment() && @@ -163,40 +161,35 @@ workerRuntime is not null && } } - private List ParseWorkerVersions(IEnumerable workerVersions) + private static IEnumerable ParseWorkerVersionsInDescending(IEnumerable workerVersions) { var versions = new List(); foreach (var workerVersion in workerVersions) { - string versionFolder = Path.GetFileName(workerVersion); - - if (versionFolder.Length == 1) - { - versionFolder = versionFolder + ".0"; // Handle single digit versions like '1' as '1.0' - } + string versionDir = Path.GetFileName(workerVersion); + string formattedVersion = FormatVersion(versionDir); - if (Version.TryParse(versionFolder, out Version version)) + if (Version.TryParse(formattedVersion, out Version version)) { versions.Add(version); } - else - { - Console.WriteLine($"Failed to parse version: '{versionFolder}'"); - } } - return versions; + return versions.OrderDescending(); } - private HashSet GetHostCapabilities() + private static string FormatVersion(string version) { - HashSet hostCapabilites = ["test-capability-1", "test-capability-2"]; + if (!version.Contains('.')) + { + version = version + ".0"; // Handle versions like '1' as '1.0' + } - return hostCapabilites; + return version; } - private bool IsCompatibleWithHost(string workerDir) + private bool IsWorkerCompatibleWithHost(string workerDir) { string workerConfigPath = Path.Combine(workerDir, RpcWorkerConstants.WorkerConfigFileName); if (!File.Exists(workerConfigPath)) @@ -236,7 +229,7 @@ private HashSet GetHostRequirements(JsonElement workerConfig) { HashSet hostRequirements = new HashSet(); - if (workerConfig.TryGetProperty("hostRequirements", out JsonElement configSection)) + if (workerConfig.TryGetProperty(RpcWorkerConstants.HostRequirementsSectionName, out JsonElement configSection)) { var requirements = configSection.EnumerateArray(); @@ -251,10 +244,9 @@ private HashSet GetHostRequirements(JsonElement workerConfig) private bool DoesHostRequirementMeet(JsonElement workerConfig) { + HashSet hostCapabilities = ScriptConstants.HostCapabilities; HashSet hostRequirements = GetHostRequirements(workerConfig); - HashSet hostCapabilities = GetHostCapabilities(); - foreach (var hostRequirement in hostRequirements) { if (!hostCapabilities.Contains(hostRequirement)) diff --git a/src/WebJobs.Script/Workers/Rpc/IWorkerConfigurationResolver.cs b/src/WebJobs.Script/Workers/Rpc/IWorkerConfigurationResolver.cs index baaf4e2ad8..f55562fbea 100644 --- a/src/WebJobs.Script/Workers/Rpc/IWorkerConfigurationResolver.cs +++ b/src/WebJobs.Script/Workers/Rpc/IWorkerConfigurationResolver.cs @@ -13,6 +13,6 @@ namespace Microsoft.Azure.WebJobs.Script.Workers.Rpc.Configuration /// A list of paths to worker configuration files. public interface IWorkerConfigurationResolver { - internal List GetWorkerConfigs(List probingPaths, string fallbackPath); + List GetWorkerConfigs(List probingPaths, string fallbackPath); } } \ No newline at end of file diff --git a/src/WebJobs.Script/Workers/Rpc/RpcWorkerConstants.cs b/src/WebJobs.Script/Workers/Rpc/RpcWorkerConstants.cs index 18d56c348e..032156fdd9 100644 --- a/src/WebJobs.Script/Workers/Rpc/RpcWorkerConstants.cs +++ b/src/WebJobs.Script/Workers/Rpc/RpcWorkerConstants.cs @@ -27,6 +27,7 @@ public static class RpcWorkerConstants // Section names in host.json or AppSettings public const string LanguageWorkersSectionName = "languageWorkers"; public const string WorkerProbingPathsSectionName = "probingPaths"; + public const string HostRequirementsSectionName = "hostRequirements"; // Worker description constants public const string WorkerDescriptionLanguage = "language"; diff --git a/test/WebJobs.Script.Tests/UtilityTests.cs b/test/WebJobs.Script.Tests/UtilityTests.cs index 4a44afc296..132149c9d5 100644 --- a/test/WebJobs.Script.Tests/UtilityTests.cs +++ b/test/WebJobs.Script.Tests/UtilityTests.cs @@ -1016,6 +1016,52 @@ public void WorkerIndexingDecisionLogic_NullWorkerIndexingProperty(bool workerIn Assert.Equal(expected, workerShouldIndex); } + [Theory] + [InlineData(null, "node", true)] + [InlineData(null, "java|node", true)] + [InlineData(null, "", false)] + [InlineData(null, "| ", false)] + [InlineData(null, null, false)] + [InlineData(ScriptConstants.FeatureFlagEnableWorkerProbingPaths, "node", true)] + [InlineData(ScriptConstants.FeatureFlagEnableWorkerProbingPaths, "java|node", true)] + [InlineData(ScriptConstants.FeatureFlagEnableWorkerProbingPaths, "", true)] + [InlineData(ScriptConstants.FeatureFlagEnableWorkerProbingPaths, "| ", true)] + [InlineData(ScriptConstants.FeatureFlagEnableWorkerProbingPaths, null, true)] + [InlineData(ScriptConstants.FeatureFlagDisableWorkerProbingPaths, "node", false)] + [InlineData(ScriptConstants.FeatureFlagDisableWorkerProbingPaths, "java|node", false)] + [InlineData(ScriptConstants.FeatureFlagDisableWorkerProbingPaths, "| ", false)] + + public void AreProbingPathsEnabled_HostingConfigAndFeatureFlags_WorksAsExpected(string featureFlagValue, string hostingConfigSetting, bool expected) + { + HashSet hostingConfigEnabledWorkers = hostingConfigSetting?.Split('|', StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries).ToHashSet(); + + var testEnvironment = new TestEnvironment(); + testEnvironment.SetEnvironmentVariable(EnvironmentSettingNames.AzureWebJobsFeatureFlags, featureFlagValue); + + bool result = Utility.AreWorkerProbingPathsEnabled(testEnvironment, hostingConfigEnabledWorkers, null); + + Assert.Equal(expected, result); + } + + [Theory] + [InlineData("node", "node", null, true)] + [InlineData("node", "java", null, false)] + [InlineData("java|node", null, null, true)] + [InlineData("node", "node", "workflowapp", true)] + [InlineData("java|node", null, "workflowapp", true)] + [InlineData("| ", null, "workflowapp", false)] + public void AreProbingPathsEnabled_WorkerRuntimeAndMultiLanguage_WorksAsExpected(string hostingConfigSetting, string workerRuntime, string multilanguageApp , bool expected) + { + var testEnvironment = new TestEnvironment(); + testEnvironment.SetEnvironmentVariable(EnvironmentSettingNames.AppKind, multilanguageApp); + + HashSet hostingConfigEnabledWorkers = hostingConfigSetting?.Split('|', StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries).ToHashSet(); + + bool result = Utility.AreWorkerProbingPathsEnabled(testEnvironment, hostingConfigEnabledWorkers, workerRuntime); + + Assert.Equal(expected, result); + } + [Theory] [InlineData("True", true, true)] [InlineData("False", false, true)] From 08850c9d2bd6a5777e28a1cbf5a07931ff7918d2 Mon Sep 17 00:00:00 2001 From: Surbhi Gupta Date: Mon, 26 May 2025 23:41:08 -0500 Subject: [PATCH 09/55] Adding more tests --- .../testfile.txt} | 0 .../workers/java/2.18.0/worker.config.json | 2 +- .../2.19.0/testfile.txt} | 0 .../workers/java/2.19.0/worker.config.json | 2 +- .../3.10.1/testfile.txt} | 0 .../workers/node/3.10.1/worker.config.json | 4 +-- .../testfile.txt} | 0 .../workers/powershell/7.2/worker.config.json | 2 +- .../workers/powershell/7.4/testfile.txt | 1 + .../workers/powershell/7.4/worker.config.json | 2 +- .../workers/powershell/7/testfile.txt | 1 + .../workers/powershell/7/worker.config.json | 2 +- .../LanguageWorkerOptionsSetupTests.cs | 25 +++++++------------ .../Rpc/RpcWorkerConfigFactoryTests.cs | 21 ---------------- .../Rpc/WorkerConfigurationResolverTests.cs | 5 ++-- 15 files changed, 20 insertions(+), 47 deletions(-) rename test/TestWorkers/Probingpaths/workers/java/{2.19.0/azure-functions-java-worker.jar => 2.18.0/testfile.txt} (100%) rename test/TestWorkers/Probingpaths/workers/{powershell/7.2/Microsoft.Azure.Functions.PowerShellWorker.dll => java/2.19.0/testfile.txt} (100%) rename test/TestWorkers/Probingpaths/workers/{powershell/7.4/Microsoft.Azure.Functions.PowerShellWorker.dll => node/3.10.1/testfile.txt} (100%) rename test/TestWorkers/Probingpaths/workers/powershell/{7/Microsoft.Azure.Functions.PowerShellWorker.dll => 7.2/testfile.txt} (100%) create mode 100644 test/TestWorkers/Probingpaths/workers/powershell/7.4/testfile.txt create mode 100644 test/TestWorkers/Probingpaths/workers/powershell/7/testfile.txt diff --git a/test/TestWorkers/Probingpaths/workers/java/2.19.0/azure-functions-java-worker.jar b/test/TestWorkers/Probingpaths/workers/java/2.18.0/testfile.txt similarity index 100% rename from test/TestWorkers/Probingpaths/workers/java/2.19.0/azure-functions-java-worker.jar rename to test/TestWorkers/Probingpaths/workers/java/2.18.0/testfile.txt diff --git a/test/TestWorkers/Probingpaths/workers/java/2.18.0/worker.config.json b/test/TestWorkers/Probingpaths/workers/java/2.18.0/worker.config.json index 5e65e09614..52dc378929 100644 --- a/test/TestWorkers/Probingpaths/workers/java/2.18.0/worker.config.json +++ b/test/TestWorkers/Probingpaths/workers/java/2.18.0/worker.config.json @@ -3,7 +3,7 @@ "language": "java", "extensions": [".jar"], "defaultExecutablePath": "%JAVA_HOME%/bin/java", - "defaultWorkerPath": "azure-functions-java-worker.jar", + "defaultWorkerPath": "testfile.txt", "arguments": [] }, "profiles": [] diff --git a/test/TestWorkers/Probingpaths/workers/powershell/7.2/Microsoft.Azure.Functions.PowerShellWorker.dll b/test/TestWorkers/Probingpaths/workers/java/2.19.0/testfile.txt similarity index 100% rename from test/TestWorkers/Probingpaths/workers/powershell/7.2/Microsoft.Azure.Functions.PowerShellWorker.dll rename to test/TestWorkers/Probingpaths/workers/java/2.19.0/testfile.txt diff --git a/test/TestWorkers/Probingpaths/workers/java/2.19.0/worker.config.json b/test/TestWorkers/Probingpaths/workers/java/2.19.0/worker.config.json index d801782347..02b2835324 100644 --- a/test/TestWorkers/Probingpaths/workers/java/2.19.0/worker.config.json +++ b/test/TestWorkers/Probingpaths/workers/java/2.19.0/worker.config.json @@ -3,7 +3,7 @@ "language": "java", "extensions": [".jar"], "defaultExecutablePath": "%JAVA_HOME%/bin/java", - "defaultWorkerPath": "azure-functions-java-worker.jar", + "defaultWorkerPath": "testfile.txt", "arguments": [] }, "hostRequirements": [], diff --git a/test/TestWorkers/Probingpaths/workers/powershell/7.4/Microsoft.Azure.Functions.PowerShellWorker.dll b/test/TestWorkers/Probingpaths/workers/node/3.10.1/testfile.txt similarity index 100% rename from test/TestWorkers/Probingpaths/workers/powershell/7.4/Microsoft.Azure.Functions.PowerShellWorker.dll rename to test/TestWorkers/Probingpaths/workers/node/3.10.1/testfile.txt diff --git a/test/TestWorkers/Probingpaths/workers/node/3.10.1/worker.config.json b/test/TestWorkers/Probingpaths/workers/node/3.10.1/worker.config.json index c20e0b4e45..dacc607a55 100644 --- a/test/TestWorkers/Probingpaths/workers/node/3.10.1/worker.config.json +++ b/test/TestWorkers/Probingpaths/workers/node/3.10.1/worker.config.json @@ -3,10 +3,10 @@ "language": "node", "extensions": [".js", ".mjs", ".cjs"], "defaultExecutablePath": "node", - "defaultWorkerPath": "dist/src/nodejsWorker.js", + "defaultWorkerPath": "testfile.txt", "workerIndexing": "true" }, - "hostRequirements": ["test-capability-1", "test-capability-2"], + "hostRequirements": [], "processOptions": { "initializationTimeout": "00:02:00", "environmentReloadTimeout": "00:02:00" diff --git a/test/TestWorkers/Probingpaths/workers/powershell/7/Microsoft.Azure.Functions.PowerShellWorker.dll b/test/TestWorkers/Probingpaths/workers/powershell/7.2/testfile.txt similarity index 100% rename from test/TestWorkers/Probingpaths/workers/powershell/7/Microsoft.Azure.Functions.PowerShellWorker.dll rename to test/TestWorkers/Probingpaths/workers/powershell/7.2/testfile.txt diff --git a/test/TestWorkers/Probingpaths/workers/powershell/7.2/worker.config.json b/test/TestWorkers/Probingpaths/workers/powershell/7.2/worker.config.json index 52cef391ed..555a222a88 100644 --- a/test/TestWorkers/Probingpaths/workers/powershell/7.2/worker.config.json +++ b/test/TestWorkers/Probingpaths/workers/powershell/7.2/worker.config.json @@ -3,7 +3,7 @@ "language":"powershell", "extensions":[".ps1", ".psm1"], "defaultExecutablePath":"dotnet", - "defaultWorkerPath":"%FUNCTIONS_WORKER_RUNTIME_VERSION%/Microsoft.Azure.Functions.PowerShellWorker.dll", + "defaultWorkerPath":"%FUNCTIONS_WORKER_RUNTIME_VERSION%/testfile.txt", "supportedRuntimeVersions":["7", "7.2"], "defaultRuntimeVersion": "7.2", "sanitizeRuntimeVersionRegex":"\\d+\\.?\\d*" diff --git a/test/TestWorkers/Probingpaths/workers/powershell/7.4/testfile.txt b/test/TestWorkers/Probingpaths/workers/powershell/7.4/testfile.txt new file mode 100644 index 0000000000..07b06b15b8 --- /dev/null +++ b/test/TestWorkers/Probingpaths/workers/powershell/7.4/testfile.txt @@ -0,0 +1 @@ +This is empty file needed to run the unit tests for RPCWorkerConfigFactory. It checks if the path mentions in DefaultWorkerPath exists or not before it loads the workerconfig. \ No newline at end of file diff --git a/test/TestWorkers/Probingpaths/workers/powershell/7.4/worker.config.json b/test/TestWorkers/Probingpaths/workers/powershell/7.4/worker.config.json index 02fab11d36..73448885d9 100644 --- a/test/TestWorkers/Probingpaths/workers/powershell/7.4/worker.config.json +++ b/test/TestWorkers/Probingpaths/workers/powershell/7.4/worker.config.json @@ -3,7 +3,7 @@ "language":"powershell", "extensions":[".ps1", ".psm1"], "defaultExecutablePath":"dotnet", - "defaultWorkerPath":"%FUNCTIONS_WORKER_RUNTIME_VERSION%/Microsoft.Azure.Functions.PowerShellWorker.dll", + "defaultWorkerPath":"%FUNCTIONS_WORKER_RUNTIME_VERSION%/testfile.txt", "supportedRuntimeVersions":["7", "7.2", "7.4"], "defaultRuntimeVersion": "7.4", "sanitizeRuntimeVersionRegex":"\\d+\\.?\\d*" diff --git a/test/TestWorkers/Probingpaths/workers/powershell/7/testfile.txt b/test/TestWorkers/Probingpaths/workers/powershell/7/testfile.txt new file mode 100644 index 0000000000..07b06b15b8 --- /dev/null +++ b/test/TestWorkers/Probingpaths/workers/powershell/7/testfile.txt @@ -0,0 +1 @@ +This is empty file needed to run the unit tests for RPCWorkerConfigFactory. It checks if the path mentions in DefaultWorkerPath exists or not before it loads the workerconfig. \ No newline at end of file diff --git a/test/TestWorkers/Probingpaths/workers/powershell/7/worker.config.json b/test/TestWorkers/Probingpaths/workers/powershell/7/worker.config.json index 4a871e741b..b664b0ac5a 100644 --- a/test/TestWorkers/Probingpaths/workers/powershell/7/worker.config.json +++ b/test/TestWorkers/Probingpaths/workers/powershell/7/worker.config.json @@ -3,7 +3,7 @@ "language":"powershell", "extensions":[".ps1", ".psm1"], "defaultExecutablePath":"dotnet", - "defaultWorkerPath":"%FUNCTIONS_WORKER_RUNTIME_VERSION%/Microsoft.Azure.Functions.PowerShellWorker.dll", + "defaultWorkerPath":"%FUNCTIONS_WORKER_RUNTIME_VERSION%/testfile.txt", "supportedRuntimeVersions":["7", "7.2"], "defaultRuntimeVersion":"7", "sanitizeRuntimeVersionRegex":"\\d+\\.?\\d*" diff --git a/test/WebJobs.Script.Tests/Configuration/LanguageWorkerOptionsSetupTests.cs b/test/WebJobs.Script.Tests/Configuration/LanguageWorkerOptionsSetupTests.cs index c819b609ee..f83fbd2172 100644 --- a/test/WebJobs.Script.Tests/Configuration/LanguageWorkerOptionsSetupTests.cs +++ b/test/WebJobs.Script.Tests/Configuration/LanguageWorkerOptionsSetupTests.cs @@ -93,12 +93,12 @@ public void LanguageWorkerOptions_ProbingPaths_Expected_ListOfConfigs(string wor var configuration = configurationBuilder.Build(); var testProfileManager = new Mock(); var testScriptHostManager = new Mock(); - string probingPath1 = Path.GetFullPath(probingPath); + string fullProbingPath = Path.GetFullPath(probingPath); testEnvironment.SetEnvironmentVariable(RpcWorkerConstants.FunctionWorkerRuntimeSettingName, workerRuntime); testEnvironment.SetEnvironmentVariable(EnvironmentSettingNames.AzureWebJobsFeatureFlags, enableProbingPaths); testEnvironment.SetEnvironmentVariable(EnvironmentSettingNames.AntaresPlatformReleaseChannel, releaseChannel); - testEnvironment.SetEnvironmentVariable(EnvironmentSettingNames.WorkerProbingPaths, probingPath1); + testEnvironment.SetEnvironmentVariable(EnvironmentSettingNames.WorkerProbingPaths, fullProbingPath); var hostingOptions = new FunctionsHostingConfigOptions(); hostingOptions.Features.Add(RpcWorkerConstants.EnableProbingPathsForWorkers, hostingOptionsSetting); @@ -108,21 +108,14 @@ public void LanguageWorkerOptions_ProbingPaths_Expected_ListOfConfigs(string wor setup.Configure(options); - if (string.IsNullOrEmpty(workerRuntime)) - { - Assert.Equal(5, options.WorkerConfigs.Count); - } - else if (workerRuntime.Equals(RpcWorkerConstants.DotNetLanguageWorkerName, StringComparison.OrdinalIgnoreCase)) - { - Assert.Empty(options.WorkerConfigs); - } - else - { - Assert.Equal(1, options.WorkerConfigs.Count); - Assert.True(options.WorkerConfigs.First().Arguments.WorkerPath.Contains(expectedVersion)); + Assert.Equal(1, options.WorkerConfigs.Count); + Assert.True(options.WorkerConfigs.First().Arguments.WorkerPath.Contains(expectedVersion)); - var logs = loggerProvider.GetAllLogMessages(); - } + var logs = loggerProvider.GetAllLogMessages(); + + string path = Path.Combine(fullProbingPath, workerRuntime, expectedVersion); + string expectedLog = $"Added WorkerConfig for language: {workerRuntime} with DefaultWorkerPath: {path}"; + Assert.True(logs.Any(l => l.FormattedMessage.Contains(expectedLog))); } } } \ No newline at end of file diff --git a/test/WebJobs.Script.Tests/Workers/Rpc/RpcWorkerConfigFactoryTests.cs b/test/WebJobs.Script.Tests/Workers/Rpc/RpcWorkerConfigFactoryTests.cs index 0fe8504d34..09585281cd 100644 --- a/test/WebJobs.Script.Tests/Workers/Rpc/RpcWorkerConfigFactoryTests.cs +++ b/test/WebJobs.Script.Tests/Workers/Rpc/RpcWorkerConfigFactoryTests.cs @@ -148,27 +148,6 @@ public void JavaPath_FromEnvVars() Assert.Contains(@"/bin/java", javaPath); } - [Fact] - public void GetConfigs_ProbingPaths() - { - string probingPath1 = Path.GetFullPath("..\\..\\..\\..\\test\\TestWorkers\\ProbingPaths\\workers\\"); - - var mockEnvironment = new Mock(); - mockEnvironment.Setup(p => p.GetEnvironmentVariable(EnvironmentSettingNames.FunctionWorkerRuntime)).Returns("java"); - - IConfiguration config = new ConfigurationBuilder().Build(); - var testLogger = new TestLogger("test"); - - var workerConfigurationResolver = new WorkerConfigurationResolver(config, testLogger, mockEnvironment.Object, _testWorkerProfileManager, new HashSet() { "java" }); - var configFactory = new RpcWorkerConfigFactory(config, testLogger, _testSysRuntimeInfo, mockEnvironment.Object, new TestMetricsLogger(), _testWorkerProfileManager, workerConfigurationResolver, true, new List() { probingPath1 }); - var workerConfigs = configFactory.GetConfigs(); - - // check log messages - var logs = testLogger.GetLogMessages(); - Assert.Equal(1, workerConfigs.Count); - Assert.True(logs.Any(p => p.FormattedMessage.Contains("Probing paths set to: " + probingPath1))); - } - [Fact] public void DefaultWorkerConfigs_Overrides_DefaultWorkerRuntimeVersion_AppSetting() { diff --git a/test/WebJobs.Script.Tests/Workers/Rpc/WorkerConfigurationResolverTests.cs b/test/WebJobs.Script.Tests/Workers/Rpc/WorkerConfigurationResolverTests.cs index 4070ab201c..381fe4f4c5 100644 --- a/test/WebJobs.Script.Tests/Workers/Rpc/WorkerConfigurationResolverTests.cs +++ b/test/WebJobs.Script.Tests/Workers/Rpc/WorkerConfigurationResolverTests.cs @@ -10,7 +10,6 @@ using Microsoft.Azure.WebJobs.Script.Workers.Rpc.Configuration; using Microsoft.Extensions.Configuration; using Microsoft.Extensions.Logging; -using Microsoft.Extensions.Options; using Moq; using Xunit; @@ -35,8 +34,8 @@ public WorkerConfigurationResolverTests() } [Theory] - [InlineData("java", "LATEST", "2.19.0")] - [InlineData("java", "STANDARD", "2.18.0")] + // [InlineData("java", "LATEST", "2.19.0")] + // [InlineData("java", "STANDARD", "2.18.0")] [InlineData("node", "STANDARD", "3.10.1")] public void GetWorkerConfigs_ReturnsExpectedConfigs(string languageWorker, string releaseChannel, string version) { From 10d2e52b4fa099a26f940a5d7ec1e098f6aab194 Mon Sep 17 00:00:00 2001 From: Surbhi Gupta Date: Tue, 27 May 2025 00:17:45 -0500 Subject: [PATCH 10/55] Changes in Tests --- .../LanguageWorkerOptionsSetupTests.cs | 13 +++-- .../Rpc/WorkerConfigurationResolverTests.cs | 48 +++++-------------- 2 files changed, 21 insertions(+), 40 deletions(-) diff --git a/test/WebJobs.Script.Tests/Configuration/LanguageWorkerOptionsSetupTests.cs b/test/WebJobs.Script.Tests/Configuration/LanguageWorkerOptionsSetupTests.cs index f83fbd2172..b0cee51bf2 100644 --- a/test/WebJobs.Script.Tests/Configuration/LanguageWorkerOptionsSetupTests.cs +++ b/test/WebJobs.Script.Tests/Configuration/LanguageWorkerOptionsSetupTests.cs @@ -19,6 +19,9 @@ namespace Microsoft.Azure.WebJobs.Script.Tests.Configuration { public class LanguageWorkerOptionsSetupTests { + private readonly string _probingPath1 = Path.GetFullPath("..\\..\\..\\..\\test\\TestWorkers\\ProbingPaths\\workers\\"); + private readonly string _fallbackPath = Path.GetFullPath("workers"); + [Theory] [InlineData("DotNet")] [InlineData("dotnet")] @@ -80,6 +83,9 @@ public void LanguageWorkerOptions_Expected_ListOfConfigs(string workerRuntime) [Theory] [InlineData("java", "EnableWorkerProbingPaths", "java", "..\\..\\..\\..\\test\\TestWorkers\\ProbingPaths\\workers\\", "LATEST", "2.19.0")] + [InlineData("java", "EnableWorkerProbingPaths", "java", "..\\..\\..\\..\\test\\TestWorkers\\ProbingPaths\\workers\\", "STANDARD", "2.18.0")] + [InlineData("node", "EnableWorkerProbingPaths", "node", "..\\..\\..\\..\\test\\TestWorkers\\ProbingPaths\\workers\\", "LATEST", "3.10.1")] + [InlineData("node", "EnableWorkerProbingPaths", "java|node", "..\\..\\..\\..\\test\\TestWorkers\\ProbingPaths\\workers\\", "STANDARD", "3.10.1")] public void LanguageWorkerOptions_ProbingPaths_Expected_ListOfConfigs(string workerRuntime, string enableProbingPaths, string hostingOptionsSetting, string probingPath, string releaseChannel, string expectedVersion) { var loggerProvider = new TestLoggerProvider(); @@ -93,12 +99,12 @@ public void LanguageWorkerOptions_ProbingPaths_Expected_ListOfConfigs(string wor var configuration = configurationBuilder.Build(); var testProfileManager = new Mock(); var testScriptHostManager = new Mock(); - string fullProbingPath = Path.GetFullPath(probingPath); + string probingPathValue = string.Join(';', _probingPath1, string.Empty, "path-not-exists" ); testEnvironment.SetEnvironmentVariable(RpcWorkerConstants.FunctionWorkerRuntimeSettingName, workerRuntime); testEnvironment.SetEnvironmentVariable(EnvironmentSettingNames.AzureWebJobsFeatureFlags, enableProbingPaths); testEnvironment.SetEnvironmentVariable(EnvironmentSettingNames.AntaresPlatformReleaseChannel, releaseChannel); - testEnvironment.SetEnvironmentVariable(EnvironmentSettingNames.WorkerProbingPaths, fullProbingPath); + testEnvironment.SetEnvironmentVariable(EnvironmentSettingNames.WorkerProbingPaths, probingPathValue); var hostingOptions = new FunctionsHostingConfigOptions(); hostingOptions.Features.Add(RpcWorkerConstants.EnableProbingPathsForWorkers, hostingOptionsSetting); @@ -113,9 +119,10 @@ public void LanguageWorkerOptions_ProbingPaths_Expected_ListOfConfigs(string wor var logs = loggerProvider.GetAllLogMessages(); - string path = Path.Combine(fullProbingPath, workerRuntime, expectedVersion); + string path = Path.Combine(_probingPath1, workerRuntime, expectedVersion); string expectedLog = $"Added WorkerConfig for language: {workerRuntime} with DefaultWorkerPath: {path}"; Assert.True(logs.Any(l => l.FormattedMessage.Contains(expectedLog))); + Assert.True(logs.Any(l => l.FormattedMessage.Contains("Probing paths set to:"))); } } } \ No newline at end of file diff --git a/test/WebJobs.Script.Tests/Workers/Rpc/WorkerConfigurationResolverTests.cs b/test/WebJobs.Script.Tests/Workers/Rpc/WorkerConfigurationResolverTests.cs index 381fe4f4c5..0eac1578ab 100644 --- a/test/WebJobs.Script.Tests/Workers/Rpc/WorkerConfigurationResolverTests.cs +++ b/test/WebJobs.Script.Tests/Workers/Rpc/WorkerConfigurationResolverTests.cs @@ -4,9 +4,7 @@ using System.Collections.Generic; using System.IO; using System.Linq; -using Microsoft.Azure.WebJobs.Script.Config; using Microsoft.Azure.WebJobs.Script.Workers.Profiles; -using Microsoft.Azure.WebJobs.Script.Workers.Rpc; using Microsoft.Azure.WebJobs.Script.Workers.Rpc.Configuration; using Microsoft.Extensions.Configuration; using Microsoft.Extensions.Logging; @@ -33,30 +31,6 @@ public WorkerConfigurationResolverTests() _probingPaths = new List { _probingPath1, string.Empty, null, "path-not-exists" }; } - [Theory] - // [InlineData("java", "LATEST", "2.19.0")] - // [InlineData("java", "STANDARD", "2.18.0")] - [InlineData("node", "STANDARD", "3.10.1")] - public void GetWorkerConfigs_ReturnsExpectedConfigs(string languageWorker, string releaseChannel, string version) - { - // Arrange - var mockEnvironment = new Mock(); - mockEnvironment.Setup(p => p.GetEnvironmentVariable(EnvironmentSettingNames.FunctionWorkerRuntime)).Returns(languageWorker); - mockEnvironment.Setup(p => p.GetEnvironmentVariable(EnvironmentSettingNames.AntaresPlatformReleaseChannel)).Returns(releaseChannel); - mockEnvironment.Setup(p => p.GetEnvironmentVariable(EnvironmentSettingNames.AppKind)).Returns((string)null); - - // Act - var hostingOptionsValue = new FunctionsHostingConfigOptions(); - hostingOptionsValue.Features.Add(RpcWorkerConstants.EnableProbingPathsForWorkers, languageWorker); - var workerConfigurationResolver = new WorkerConfigurationResolver(_mockConfig.Object, _mockLogger.Object, mockEnvironment.Object, _mockProfileManager.Object, new HashSet() { languageWorker }); - - var result = workerConfigurationResolver.GetWorkerConfigs(_probingPaths, _fallbackPath); - - // Assert - Assert.Single(result); - Assert.Contains(_probingPath1 + languageWorker + "\\" + version, result); - } - [Theory] [InlineData("LATEST", "java\\2.19.0", "node\\3.10.1", "powershell\\7.4", "dotnet-isolated", "python")] [InlineData("STANDARD", "java\\2.18.0", "node\\3.10.1", "powershell\\7.2", "dotnet-isolated", "python")] @@ -74,11 +48,11 @@ public void GetWorkerConfigs_MultiLanguageWorker_ReturnsExpectedConfigs(string r // Assert Assert.Equal(result.Count, 5); - Assert.True(result.Any(r => r.Contains(_probingPath1 + java))); - Assert.True(result.Any(r => r.Contains(_probingPath1 + node))); - Assert.True(result.Any(r => r.Contains(_probingPath1 + powershell))); - Assert.True(result.Any(r => r.Contains(_fallbackPath + "\\" + dotnetIsolated))); - Assert.True(result.Any(r => r.Contains(_fallbackPath + "\\" + python))); + Assert.True(result.Any(r => r.Contains(Path.Combine(_probingPath1, java)))); + Assert.True(result.Any(r => r.Contains(Path.Combine(_probingPath1, node)))); + Assert.True(result.Any(r => r.Contains(Path.Combine(_probingPath1, powershell)))); + Assert.True(result.Any(r => r.Contains(Path.Combine(_fallbackPath, dotnetIsolated)))); + Assert.True(result.Any(r => r.Contains(Path.Combine(_fallbackPath, python)))); } [Theory] @@ -108,11 +82,11 @@ public void GetWorkerConfigs_MultiLanguageWorker_NullOREmptyProbingPath_ReturnsE // Assert Assert.Equal(result.Count, 5); - Assert.True(result.Any(r => r.Contains(_fallbackPath + "\\java"))); - Assert.True(result.Any(r => r.Contains(_fallbackPath + "\\node"))); - Assert.True(result.Any(r => r.Contains(_fallbackPath + "\\powershell"))); - Assert.True(result.Any(r => r.Contains(_fallbackPath + "\\dotnet-isolated"))); - Assert.True(result.Any(r => r.Contains(_fallbackPath + "\\python"))); + Assert.True(result.Any(r => r.Contains(Path.Combine(_fallbackPath, "java")))); + Assert.True(result.Any(r => r.Contains(Path.Combine(_fallbackPath, "node")))); + Assert.True(result.Any(r => r.Contains(Path.Combine(_fallbackPath, "powershell")))); + Assert.True(result.Any(r => r.Contains(Path.Combine(_fallbackPath, "dotnet-isolated")))); + Assert.True(result.Any(r => r.Contains(Path.Combine(_fallbackPath, "python")))); } [Theory] @@ -150,7 +124,7 @@ public void GetWorkerConfigs_NullOREmptyProbingPath_ReturnsExpectedConfigs(strin // Assert Assert.Equal(result.Count, 1); - Assert.True(result.Any(r => r.Contains(_fallbackPath + "\\" + languageWorker))); + Assert.True(result.Any(r => r.Contains(Path.Combine(_fallbackPath, languageWorker)))); } } } \ No newline at end of file From 95af17e294251d664012ffdc60818d4a79c69fe1 Mon Sep 17 00:00:00 2001 From: Surbhi Gupta Date: Tue, 27 May 2025 00:42:12 -0500 Subject: [PATCH 11/55] Updating LanguageWorkerOptionsSetupTests --- .../LanguageWorkerOptionsSetupTests.cs | 96 +++++++++++++++++-- 1 file changed, 90 insertions(+), 6 deletions(-) diff --git a/test/WebJobs.Script.Tests/Configuration/LanguageWorkerOptionsSetupTests.cs b/test/WebJobs.Script.Tests/Configuration/LanguageWorkerOptionsSetupTests.cs index b0cee51bf2..00973e128d 100644 --- a/test/WebJobs.Script.Tests/Configuration/LanguageWorkerOptionsSetupTests.cs +++ b/test/WebJobs.Script.Tests/Configuration/LanguageWorkerOptionsSetupTests.cs @@ -82,11 +82,11 @@ public void LanguageWorkerOptions_Expected_ListOfConfigs(string workerRuntime) } [Theory] - [InlineData("java", "EnableWorkerProbingPaths", "java", "..\\..\\..\\..\\test\\TestWorkers\\ProbingPaths\\workers\\", "LATEST", "2.19.0")] - [InlineData("java", "EnableWorkerProbingPaths", "java", "..\\..\\..\\..\\test\\TestWorkers\\ProbingPaths\\workers\\", "STANDARD", "2.18.0")] - [InlineData("node", "EnableWorkerProbingPaths", "node", "..\\..\\..\\..\\test\\TestWorkers\\ProbingPaths\\workers\\", "LATEST", "3.10.1")] - [InlineData("node", "EnableWorkerProbingPaths", "java|node", "..\\..\\..\\..\\test\\TestWorkers\\ProbingPaths\\workers\\", "STANDARD", "3.10.1")] - public void LanguageWorkerOptions_ProbingPaths_Expected_ListOfConfigs(string workerRuntime, string enableProbingPaths, string hostingOptionsSetting, string probingPath, string releaseChannel, string expectedVersion) + [InlineData("java", "EnableWorkerProbingPaths", "java", "LATEST", "2.19.0")] + [InlineData("java", "EnableWorkerProbingPaths", "java", "STANDARD", "2.18.0")] + [InlineData("node", "EnableWorkerProbingPaths", "node", "LATEST", "3.10.1")] + [InlineData("node", "EnableWorkerProbingPaths", "java|node", "STANDARD", "3.10.1")] + public void LanguageWorkerOptions_ProbingPaths_Expected_ListOfConfigs(string workerRuntime, string enableProbingPaths, string hostingOptionsSetting, string releaseChannel, string expectedVersion) { var loggerProvider = new TestLoggerProvider(); var loggerFactory = new LoggerFactory(); @@ -99,7 +99,7 @@ public void LanguageWorkerOptions_ProbingPaths_Expected_ListOfConfigs(string wor var configuration = configurationBuilder.Build(); var testProfileManager = new Mock(); var testScriptHostManager = new Mock(); - string probingPathValue = string.Join(';', _probingPath1, string.Empty, "path-not-exists" ); + string probingPathValue = string.Join(';', _probingPath1, string.Empty, "path-not-exists"); testEnvironment.SetEnvironmentVariable(RpcWorkerConstants.FunctionWorkerRuntimeSettingName, workerRuntime); testEnvironment.SetEnvironmentVariable(EnvironmentSettingNames.AzureWebJobsFeatureFlags, enableProbingPaths); @@ -124,5 +124,89 @@ public void LanguageWorkerOptions_ProbingPaths_Expected_ListOfConfigs(string wor Assert.True(logs.Any(l => l.FormattedMessage.Contains(expectedLog))); Assert.True(logs.Any(l => l.FormattedMessage.Contains("Probing paths set to:"))); } + + [Theory] + [InlineData("java", "EnableWorkerProbingPaths", "java", "LATEST")] + [InlineData("java", "EnableWorkerProbingPaths", "java", "STANDARD")] + [InlineData("node", "EnableWorkerProbingPaths", "node", "LATEST")] + [InlineData("node", "EnableWorkerProbingPaths", "java|node", "STANDARD")] + public void LanguageWorkerOptions_FallbackPath_Expected_ListOfConfigs(string workerRuntime, string enableProbingPaths, string hostingOptionsSetting, string releaseChannel) + { + var loggerProvider = new TestLoggerProvider(); + var loggerFactory = new LoggerFactory(); + loggerFactory.AddProvider(loggerProvider); + + var testEnvironment = new TestEnvironment(); + var testMetricLogger = new TestMetricsLogger(); + var configurationBuilder = new ConfigurationBuilder() + .Add(new ScriptEnvironmentVariablesConfigurationSource()); + var configuration = configurationBuilder.Build(); + var testProfileManager = new Mock(); + var testScriptHostManager = new Mock(); + string probingPathValue = null; + + testEnvironment.SetEnvironmentVariable(RpcWorkerConstants.FunctionWorkerRuntimeSettingName, workerRuntime); + testEnvironment.SetEnvironmentVariable(EnvironmentSettingNames.AzureWebJobsFeatureFlags, enableProbingPaths); + testEnvironment.SetEnvironmentVariable(EnvironmentSettingNames.AntaresPlatformReleaseChannel, releaseChannel); + testEnvironment.SetEnvironmentVariable(EnvironmentSettingNames.WorkerProbingPaths, probingPathValue); + + var hostingOptions = new FunctionsHostingConfigOptions(); + hostingOptions.Features.Add(RpcWorkerConstants.EnableProbingPathsForWorkers, hostingOptionsSetting); + + LanguageWorkerOptionsSetup setup = new LanguageWorkerOptionsSetup(configuration, loggerFactory, testEnvironment, testMetricLogger, testProfileManager.Object, testScriptHostManager.Object, new OptionsWrapper(hostingOptions)); + LanguageWorkerOptions options = new LanguageWorkerOptions(); + + setup.Configure(options); + + Assert.Equal(1, options.WorkerConfigs.Count); + + var logs = loggerProvider.GetAllLogMessages(); + + string path = Path.Combine(_fallbackPath, workerRuntime); + string expectedLog = $"Added WorkerConfig for language: {workerRuntime} with DefaultWorkerPath: {path}"; + Assert.True(logs.Any(l => l.FormattedMessage.Contains(expectedLog))); + Assert.True(logs.Any(l => l.FormattedMessage.Contains("Probing paths set to:"))); + } + + [Theory] + [InlineData("java", "LATEST")] + [InlineData("java", "STANDARD")] + [InlineData("node", "LATEST")] + [InlineData("node", "STANDARD")] + public void LanguageWorkerOptions_DisabledProbingPaths_Expected_ListOfConfigs(string workerRuntime, string releaseChannel) + { + var loggerProvider = new TestLoggerProvider(); + var loggerFactory = new LoggerFactory(); + loggerFactory.AddProvider(loggerProvider); + + var testEnvironment = new TestEnvironment(); + var testMetricLogger = new TestMetricsLogger(); + var configurationBuilder = new ConfigurationBuilder() + .Add(new ScriptEnvironmentVariablesConfigurationSource()); + var configuration = configurationBuilder.Build(); + var testProfileManager = new Mock(); + var testScriptHostManager = new Mock(); + string probingPathValue = null; + + testEnvironment.SetEnvironmentVariable(RpcWorkerConstants.FunctionWorkerRuntimeSettingName, workerRuntime); + testEnvironment.SetEnvironmentVariable(EnvironmentSettingNames.AntaresPlatformReleaseChannel, releaseChannel); + testEnvironment.SetEnvironmentVariable(EnvironmentSettingNames.WorkerProbingPaths, probingPathValue); + + var hostingOptions = new FunctionsHostingConfigOptions(); + + LanguageWorkerOptionsSetup setup = new LanguageWorkerOptionsSetup(configuration, loggerFactory, testEnvironment, testMetricLogger, testProfileManager.Object, testScriptHostManager.Object, new OptionsWrapper(hostingOptions)); + LanguageWorkerOptions options = new LanguageWorkerOptions(); + + setup.Configure(options); + + Assert.Equal(1, options.WorkerConfigs.Count); + + var logs = loggerProvider.GetAllLogMessages(); + + string path = Path.Combine(_fallbackPath, workerRuntime); + string expectedLog = $"Added WorkerConfig for language: {workerRuntime} with DefaultWorkerPath: {path}"; + Assert.True(logs.Any(l => l.FormattedMessage.Contains(expectedLog))); + Assert.True(logs.Any(l => l.FormattedMessage.Contains("Workers Directory set to:"))); + } } } \ No newline at end of file From 150aaf3991348d5db4cf3347b48aa3e429a3d8f1 Mon Sep 17 00:00:00 2001 From: Surbhi Gupta Date: Tue, 27 May 2025 01:20:10 -0500 Subject: [PATCH 12/55] Minor refactoring and code cleanup --- src/WebJobs.Script/ScriptConstants.cs | 2 +- src/WebJobs.Script/Utility.cs | 8 +++++--- .../Rpc/Configuration/LanguageWorkerOptionsSetup.cs | 11 +++++------ .../Workers/Rpc/RpcWorkerDescription.cs | 2 +- .../Probingpaths/workers/java/2.18.0/testfile.txt | 2 +- .../Probingpaths/workers/java/2.19.0/testfile.txt | 2 +- .../Probingpaths/workers/node/3.10.1/testfile.txt | 2 +- .../Probingpaths/workers/powershell/7.2/testfile.txt | 2 +- .../Probingpaths/workers/powershell/7.4/testfile.txt | 2 +- .../Probingpaths/workers/powershell/7/testfile.txt | 2 +- test/WebJobs.Script.Tests/UtilityTests.cs | 5 +++-- 11 files changed, 21 insertions(+), 19 deletions(-) diff --git a/src/WebJobs.Script/ScriptConstants.cs b/src/WebJobs.Script/ScriptConstants.cs index e21a2c8c75..80a291b99d 100644 --- a/src/WebJobs.Script/ScriptConstants.cs +++ b/src/WebJobs.Script/ScriptConstants.cs @@ -1,10 +1,10 @@ // Copyright (c) .NET Foundation. All rights reserved. // Licensed under the MIT License. See License.txt in the project root for license information. -using NuGet.Versioning; using System; using System.Collections.Generic; using System.Collections.Immutable; +using NuGet.Versioning; namespace Microsoft.Azure.WebJobs.Script { diff --git a/src/WebJobs.Script/Utility.cs b/src/WebJobs.Script/Utility.cs index 18c9d935fe..b759dd41d7 100644 --- a/src/WebJobs.Script/Utility.cs +++ b/src/WebJobs.Script/Utility.cs @@ -1069,7 +1069,7 @@ public static bool CanWorkerIndex(IEnumerable workerConfigs, IE // Worker probing paths can be enabled or disabled via feature flags or hosting config options. Feature flags take precedence over hosting config options. // Users can enable or disable probing paths via setting the appropriate feature flags. // Probing paths can also be enabled for specific workers at stamp level via the hosting config options. - public static bool AreWorkerProbingPathsEnabled(IEnvironment environment, HashSet probingPathsEnabledWorkersViaHostingConfig, string workerRuntime) + public static bool AreWorkerProbingPathsEnabled(IEnvironment environment, HashSet probingPathsEnabledWorkersViaHostingConfig) { bool areProbingPathsDisabled = FeatureFlags.IsEnabled(ScriptConstants.FeatureFlagDisableWorkerProbingPaths, environment); @@ -1085,12 +1085,14 @@ public static bool AreWorkerProbingPathsEnabled(IEnvironment environment, HashSe return true; } + string workerRuntime = environment.GetEnvironmentVariable(RpcWorkerConstants.FunctionWorkerRuntimeSettingName); + if (!environment.IsMultiLanguageRuntimeEnvironment() && !string.IsNullOrWhiteSpace(workerRuntime)) { - return probingPathsEnabledWorkersViaHostingConfig.Contains(workerRuntime); + return probingPathsEnabledWorkersViaHostingConfig?.Contains(workerRuntime) ?? false; } - return probingPathsEnabledWorkersViaHostingConfig is not null ? probingPathsEnabledWorkersViaHostingConfig.Any() : false; + return probingPathsEnabledWorkersViaHostingConfig?.Any() ?? false; } public static void LogAutorestGeneratedJsonIfExists(string rootScriptPath, ILogger logger) diff --git a/src/WebJobs.Script/Workers/Rpc/Configuration/LanguageWorkerOptionsSetup.cs b/src/WebJobs.Script/Workers/Rpc/Configuration/LanguageWorkerOptionsSetup.cs index 75554e16f4..ede89518cc 100644 --- a/src/WebJobs.Script/Workers/Rpc/Configuration/LanguageWorkerOptionsSetup.cs +++ b/src/WebJobs.Script/Workers/Rpc/Configuration/LanguageWorkerOptionsSetup.cs @@ -21,7 +21,7 @@ namespace Microsoft.Azure.WebJobs.Script.Workers.Rpc { internal class LanguageWorkerOptionsSetup : IConfigureOptions { - private const string _windowsWorkerProbingPath = "c:\\home\\SiteExtensions\\workers"; // Harcoded site extensions path for Windows until Antares starts setting this as Environment variable. + private const string _windowsWorkerProbingPath = "c:\\home\\SiteExtensions\\workers"; // Harcoded site extensions path for Windows until sets it as an Environment variable. private readonly IConfiguration _configuration; private readonly ILogger _logger; private readonly IEnvironment _environment; @@ -83,7 +83,7 @@ public void Configure(LanguageWorkerOptions options) } HashSet probingPathsEnabledWorkersViaHostingConfig = GetProbingPathsEnabledWorkersFromHostingConfig(); - bool probingPathsEnabled = Utility.AreWorkerProbingPathsEnabled(_environment, probingPathsEnabledWorkersViaHostingConfig, workerRuntime); + bool probingPathsEnabled = Utility.AreWorkerProbingPathsEnabled(_environment, probingPathsEnabledWorkersViaHostingConfig); List probingPaths = null; if (probingPathsEnabled) @@ -101,16 +101,15 @@ internal HashSet GetProbingPathsEnabledWorkersFromHostingConfig() return _functionsHostingConfigOptions.Value ?.EnableProbingPathsForWorkers ?.ToLowerInvariant() - ?.Split("|", StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries) - ?.ToHashSet(); + .Split("|", StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries) + .ToHashSet(); } internal List GetWorkerProbingPaths() { var probingPaths = new List(); - string probingPathsString = string.Empty; - // If Environment variable is set, read probing paths (works for linux) + // If Environment variable is set, read probing paths from Env (works for linux) string probingPathsEnvValue = _environment.GetEnvironmentVariableOrDefault(EnvironmentSettingNames.WorkerProbingPaths, null); if (!string.IsNullOrEmpty(probingPathsEnvValue)) diff --git a/src/WebJobs.Script/Workers/Rpc/RpcWorkerDescription.cs b/src/WebJobs.Script/Workers/Rpc/RpcWorkerDescription.cs index 53adf78c1f..fbd3034151 100644 --- a/src/WebJobs.Script/Workers/Rpc/RpcWorkerDescription.cs +++ b/src/WebJobs.Script/Workers/Rpc/RpcWorkerDescription.cs @@ -99,7 +99,7 @@ public override void ApplyDefaultsAndValidate(string workerDirectory, ILogger lo if (!string.IsNullOrEmpty(DefaultWorkerPath) && !Path.IsPathRooted(DefaultWorkerPath)) { // If probing paths are enabled and DefaultWorkerPath contains FunctionWorkerRuntimeVersionSettingName - // then version becomes redundant in the path. Replacing version with an empty string to avoid deuplication. + // then version becomes redundant in the path. Replacing version with an empty string to avoid duplication. if (probingPathsEnabled && DefaultWorkerPath.Contains(RpcWorkerConstants.RuntimeVersionPlaceholder)) { var versionDir = Path.GetFileName(WorkerDirectory); diff --git a/test/TestWorkers/Probingpaths/workers/java/2.18.0/testfile.txt b/test/TestWorkers/Probingpaths/workers/java/2.18.0/testfile.txt index 07b06b15b8..ac4738bec6 100644 --- a/test/TestWorkers/Probingpaths/workers/java/2.18.0/testfile.txt +++ b/test/TestWorkers/Probingpaths/workers/java/2.18.0/testfile.txt @@ -1 +1 @@ -This is empty file needed to run the unit tests for RPCWorkerConfigFactory. It checks if the path mentions in DefaultWorkerPath exists or not before it loads the workerconfig. \ No newline at end of file +This is an empty file used for running tests, as the code checks if the file path specified in DefaultWorkerPath exists before loading the workerconfig. \ No newline at end of file diff --git a/test/TestWorkers/Probingpaths/workers/java/2.19.0/testfile.txt b/test/TestWorkers/Probingpaths/workers/java/2.19.0/testfile.txt index 07b06b15b8..ac4738bec6 100644 --- a/test/TestWorkers/Probingpaths/workers/java/2.19.0/testfile.txt +++ b/test/TestWorkers/Probingpaths/workers/java/2.19.0/testfile.txt @@ -1 +1 @@ -This is empty file needed to run the unit tests for RPCWorkerConfigFactory. It checks if the path mentions in DefaultWorkerPath exists or not before it loads the workerconfig. \ No newline at end of file +This is an empty file used for running tests, as the code checks if the file path specified in DefaultWorkerPath exists before loading the workerconfig. \ No newline at end of file diff --git a/test/TestWorkers/Probingpaths/workers/node/3.10.1/testfile.txt b/test/TestWorkers/Probingpaths/workers/node/3.10.1/testfile.txt index 07b06b15b8..ac4738bec6 100644 --- a/test/TestWorkers/Probingpaths/workers/node/3.10.1/testfile.txt +++ b/test/TestWorkers/Probingpaths/workers/node/3.10.1/testfile.txt @@ -1 +1 @@ -This is empty file needed to run the unit tests for RPCWorkerConfigFactory. It checks if the path mentions in DefaultWorkerPath exists or not before it loads the workerconfig. \ No newline at end of file +This is an empty file used for running tests, as the code checks if the file path specified in DefaultWorkerPath exists before loading the workerconfig. \ No newline at end of file diff --git a/test/TestWorkers/Probingpaths/workers/powershell/7.2/testfile.txt b/test/TestWorkers/Probingpaths/workers/powershell/7.2/testfile.txt index 07b06b15b8..ac4738bec6 100644 --- a/test/TestWorkers/Probingpaths/workers/powershell/7.2/testfile.txt +++ b/test/TestWorkers/Probingpaths/workers/powershell/7.2/testfile.txt @@ -1 +1 @@ -This is empty file needed to run the unit tests for RPCWorkerConfigFactory. It checks if the path mentions in DefaultWorkerPath exists or not before it loads the workerconfig. \ No newline at end of file +This is an empty file used for running tests, as the code checks if the file path specified in DefaultWorkerPath exists before loading the workerconfig. \ No newline at end of file diff --git a/test/TestWorkers/Probingpaths/workers/powershell/7.4/testfile.txt b/test/TestWorkers/Probingpaths/workers/powershell/7.4/testfile.txt index 07b06b15b8..ac4738bec6 100644 --- a/test/TestWorkers/Probingpaths/workers/powershell/7.4/testfile.txt +++ b/test/TestWorkers/Probingpaths/workers/powershell/7.4/testfile.txt @@ -1 +1 @@ -This is empty file needed to run the unit tests for RPCWorkerConfigFactory. It checks if the path mentions in DefaultWorkerPath exists or not before it loads the workerconfig. \ No newline at end of file +This is an empty file used for running tests, as the code checks if the file path specified in DefaultWorkerPath exists before loading the workerconfig. \ No newline at end of file diff --git a/test/TestWorkers/Probingpaths/workers/powershell/7/testfile.txt b/test/TestWorkers/Probingpaths/workers/powershell/7/testfile.txt index 07b06b15b8..ac4738bec6 100644 --- a/test/TestWorkers/Probingpaths/workers/powershell/7/testfile.txt +++ b/test/TestWorkers/Probingpaths/workers/powershell/7/testfile.txt @@ -1 +1 @@ -This is empty file needed to run the unit tests for RPCWorkerConfigFactory. It checks if the path mentions in DefaultWorkerPath exists or not before it loads the workerconfig. \ No newline at end of file +This is an empty file used for running tests, as the code checks if the file path specified in DefaultWorkerPath exists before loading the workerconfig. \ No newline at end of file diff --git a/test/WebJobs.Script.Tests/UtilityTests.cs b/test/WebJobs.Script.Tests/UtilityTests.cs index 132149c9d5..527a448c11 100644 --- a/test/WebJobs.Script.Tests/UtilityTests.cs +++ b/test/WebJobs.Script.Tests/UtilityTests.cs @@ -1038,7 +1038,7 @@ public void AreProbingPathsEnabled_HostingConfigAndFeatureFlags_WorksAsExpected( var testEnvironment = new TestEnvironment(); testEnvironment.SetEnvironmentVariable(EnvironmentSettingNames.AzureWebJobsFeatureFlags, featureFlagValue); - bool result = Utility.AreWorkerProbingPathsEnabled(testEnvironment, hostingConfigEnabledWorkers, null); + bool result = Utility.AreWorkerProbingPathsEnabled(testEnvironment, hostingConfigEnabledWorkers); Assert.Equal(expected, result); } @@ -1054,10 +1054,11 @@ public void AreProbingPathsEnabled_WorkerRuntimeAndMultiLanguage_WorksAsExpected { var testEnvironment = new TestEnvironment(); testEnvironment.SetEnvironmentVariable(EnvironmentSettingNames.AppKind, multilanguageApp); + testEnvironment.SetEnvironmentVariable(EnvironmentSettingNames.FunctionWorkerRuntime, workerRuntime); HashSet hostingConfigEnabledWorkers = hostingConfigSetting?.Split('|', StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries).ToHashSet(); - bool result = Utility.AreWorkerProbingPathsEnabled(testEnvironment, hostingConfigEnabledWorkers, workerRuntime); + bool result = Utility.AreWorkerProbingPathsEnabled(testEnvironment, hostingConfigEnabledWorkers); Assert.Equal(expected, result); } From 0def60022f4ae82c28d00fcef45ff768da175d0d Mon Sep 17 00:00:00 2001 From: Surbhi Gupta Date: Tue, 27 May 2025 12:45:48 -0500 Subject: [PATCH 13/55] Fixing build warning --- src/WebJobs.Script/Workers/Rpc/RpcWorkerConstants.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/WebJobs.Script/Workers/Rpc/RpcWorkerConstants.cs b/src/WebJobs.Script/Workers/Rpc/RpcWorkerConstants.cs index 032156fdd9..16c7686eb8 100644 --- a/src/WebJobs.Script/Workers/Rpc/RpcWorkerConstants.cs +++ b/src/WebJobs.Script/Workers/Rpc/RpcWorkerConstants.cs @@ -92,7 +92,7 @@ public static class RpcWorkerConstants public const string WorkerIndexingEnabled = "WORKER_INDEXING_ENABLED"; public const string WorkerIndexingDisabledApps = "WORKER_INDEXING_DISABLED_APPS"; - public const string EnableProbingPathsForWorkers = "ENABLE_PROBING_PATHS_FOR_WORKERS"; + public const string EnableProbingPathsForWorkers = "ENABLE_PROBING_PATHS_FOR_WORKERS"; public const string RevertWorkerShutdownBehavior = "REVERT_WORKER_SHUTDOWN_BEHAVIOR"; public const string ShutdownWebhostWorkerChannelsOnHostShutdown = "ShutdownWebhostWorkerChannelsOnHostShutdown"; public const string ThrowOnMissingFunctionsWorkerRuntime = "THROW_ON_MISSING_FUNCTIONS_WORKER_RUNTIME"; From cc3e0444b119a3b6f87b88ad22cbf5d8f5b3e9d9 Mon Sep 17 00:00:00 2001 From: Surbhi Gupta Date: Tue, 27 May 2025 13:04:06 -0500 Subject: [PATCH 14/55] Fixing build issue in tests --- test/WebJobs.Script.Tests/UtilityTests.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/WebJobs.Script.Tests/UtilityTests.cs b/test/WebJobs.Script.Tests/UtilityTests.cs index 527a448c11..8f89eefd92 100644 --- a/test/WebJobs.Script.Tests/UtilityTests.cs +++ b/test/WebJobs.Script.Tests/UtilityTests.cs @@ -1050,7 +1050,7 @@ public void AreProbingPathsEnabled_HostingConfigAndFeatureFlags_WorksAsExpected( [InlineData("node", "node", "workflowapp", true)] [InlineData("java|node", null, "workflowapp", true)] [InlineData("| ", null, "workflowapp", false)] - public void AreProbingPathsEnabled_WorkerRuntimeAndMultiLanguage_WorksAsExpected(string hostingConfigSetting, string workerRuntime, string multilanguageApp , bool expected) + public void AreProbingPathsEnabled_WorkerRuntimeAndMultiLanguage_WorksAsExpected(string hostingConfigSetting, string workerRuntime, string multilanguageApp, bool expected) { var testEnvironment = new TestEnvironment(); testEnvironment.SetEnvironmentVariable(EnvironmentSettingNames.AppKind, multilanguageApp); From 26e858069cd8351a1b85457fbf399327a3c2bdac Mon Sep 17 00:00:00 2001 From: Surbhi Gupta Date: Tue, 27 May 2025 14:12:25 -0500 Subject: [PATCH 15/55] Unit test fixing --- .../Configuration/FunctionsHostingConfigOptionsTest.cs | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/test/WebJobs.Script.Tests/Configuration/FunctionsHostingConfigOptionsTest.cs b/test/WebJobs.Script.Tests/Configuration/FunctionsHostingConfigOptionsTest.cs index b54a868089..f8bf38a7a0 100644 --- a/test/WebJobs.Script.Tests/Configuration/FunctionsHostingConfigOptionsTest.cs +++ b/test/WebJobs.Script.Tests/Configuration/FunctionsHostingConfigOptionsTest.cs @@ -74,13 +74,10 @@ public static IEnumerable PropertyValues yield return [nameof(FunctionsHostingConfigOptions.IsDotNetInProcDisabled), "DotNetInProcDisabled=True", true]; yield return [nameof(FunctionsHostingConfigOptions.IsDotNetInProcDisabled), "DotNetInProcDisabled=1", true]; yield return [nameof(FunctionsHostingConfigOptions.IsDotNetInProcDisabled), "DotNetInProcDisabled=0", false]; - - yield return [nameof(FunctionsHostingConfigOptions.EnableProbingPathsForWorkers), "EnableProbingPathsForWorkers=java|node", "java|node"]; - yield return [nameof(FunctionsHostingConfigOptions.EnableProbingPathsForWorkers), "EnableProbingPathsForWorkers=java", "java"]; - + yield return [nameof(FunctionsHostingConfigOptions.EnableProbingPathsForWorkers), "ENABLE_PROBING_PATHS_FOR_WORKERS=java|node", "java|node"]; + yield return [nameof(FunctionsHostingConfigOptions.EnableProbingPathsForWorkers), "ENABLE_PROBING_PATHS_FOR_WORKERS=java", "java"]; yield return [nameof(FunctionsHostingConfigOptions.IsTestDataSuppressionEnabled), "EnableTestDataSuppression=1", true]; yield return [nameof(FunctionsHostingConfigOptions.IsCGroupMemoryMetricsEnabled), "EnableCGroupMemoryMetrics=1", true]; - #pragma warning restore SA1011 // Closing square brackets should be spaced correctly #pragma warning restore SA1010 // Opening square brackets should be spaced correctly } From 9b6a945fae6aec374d390b15199aae32eadf08ad Mon Sep 17 00:00:00 2001 From: Surbhi Gupta Date: Mon, 2 Jun 2025 11:29:54 -0700 Subject: [PATCH 16/55] Stylecop fixes --- .../Rpc/Configuration/WorkerConfigurationHelper.cs | 9 ++++----- .../Rpc/Configuration/WorkerConfigurationResolver.cs | 4 ++-- 2 files changed, 6 insertions(+), 7 deletions(-) diff --git a/src/WebJobs.Script/Workers/Rpc/Configuration/WorkerConfigurationHelper.cs b/src/WebJobs.Script/Workers/Rpc/Configuration/WorkerConfigurationHelper.cs index 60f7c1b87a..1a04f20f0b 100644 --- a/src/WebJobs.Script/Workers/Rpc/Configuration/WorkerConfigurationHelper.cs +++ b/src/WebJobs.Script/Workers/Rpc/Configuration/WorkerConfigurationHelper.cs @@ -65,11 +65,10 @@ internal static JsonElement GetWorkerConfigJsonElement(string workerConfigPath) return doc.RootElement.Clone(); } - private static List ReadWorkerDescriptionProfiles( - JsonElement profilesElement, - JsonSerializerOptions jsonSerializerOptions, - IWorkerProfileManager profileManager, - ILogger logger) + private static List ReadWorkerDescriptionProfiles(JsonElement profilesElement, + JsonSerializerOptions jsonSerializerOptions, + IWorkerProfileManager profileManager, + ILogger logger) { var profiles = profilesElement.Deserialize>(jsonSerializerOptions); diff --git a/src/WebJobs.Script/Workers/Rpc/Configuration/WorkerConfigurationResolver.cs b/src/WebJobs.Script/Workers/Rpc/Configuration/WorkerConfigurationResolver.cs index 5c75143be6..1bfb5987f3 100644 --- a/src/WebJobs.Script/Workers/Rpc/Configuration/WorkerConfigurationResolver.cs +++ b/src/WebJobs.Script/Workers/Rpc/Configuration/WorkerConfigurationResolver.cs @@ -6,7 +6,6 @@ using System.Collections.Generic; using System.IO; using System.Linq; -using System.Runtime.CompilerServices; using System.Text.Json; using Microsoft.Azure.AppService.Proxy.Common.Extensions; using Microsoft.Azure.WebJobs.Script.Workers.Profiles; @@ -39,8 +38,9 @@ public WorkerConfigurationResolver(IConfiguration config, public List GetWorkerConfigs(List probingPaths, string fallbackPath) { - // Dictionary of { language-name : path of workerConfig } + // Dictionary of { FUNCTIONS_WORKER_RUNTIME environment variable value : path of workerConfig } ConcurrentDictionary outputDict = new ConcurrentDictionary(); + var workerRuntime = _environment.GetEnvironmentVariable(EnvironmentSettingNames.FunctionWorkerRuntime); string releaseChannel = Utility.GetPlatformReleaseChannel(_environment); From ee60ec4dfc68aba92ff269ad87360572d9ad42f3 Mon Sep 17 00:00:00 2001 From: Surbhi Gupta Date: Thu, 5 Jun 2025 22:15:07 -0700 Subject: [PATCH 17/55] Test data cleanup --- .../dotnet-isolated/1.0.0/worker.config.json | 8 +++++++ .../workers/java/2.18.0/testfile.txt | 1 - .../workers/java/2.18.0/worker.config.json | 2 +- .../workers/java/2.19.0/testfile.txt | 1 - .../workers/java/2.19.0/worker.config.json | 2 +- .../workers/node/3.10.1/testfile.txt | 1 - .../workers/node/3.10.1/worker.config.json | 2 +- .../workers/powershell/7.4/testfile.txt | 1 - .../workers/powershell/7.4/worker.config.json | 2 +- .../Rpc/WorkerConfigurationResolverTests.cs | 24 +++++++++++++++++++ 10 files changed, 36 insertions(+), 8 deletions(-) create mode 100644 test/TestWorkers/ProbingPaths/workers/dotnet-isolated/1.0.0/worker.config.json delete mode 100644 test/TestWorkers/Probingpaths/workers/java/2.18.0/testfile.txt delete mode 100644 test/TestWorkers/Probingpaths/workers/java/2.19.0/testfile.txt delete mode 100644 test/TestWorkers/Probingpaths/workers/node/3.10.1/testfile.txt delete mode 100644 test/TestWorkers/Probingpaths/workers/powershell/7.4/testfile.txt diff --git a/test/TestWorkers/ProbingPaths/workers/dotnet-isolated/1.0.0/worker.config.json b/test/TestWorkers/ProbingPaths/workers/dotnet-isolated/1.0.0/worker.config.json new file mode 100644 index 0000000000..aef262654c --- /dev/null +++ b/test/TestWorkers/ProbingPaths/workers/dotnet-isolated/1.0.0/worker.config.json @@ -0,0 +1,8 @@ +{ + "description": { + "language": "dotnet-isolated", + "extensions": [ ".dll" ], + "workerIndexing": "true", + "defaultExecutablePath": "worker.config.json" + } +} \ No newline at end of file diff --git a/test/TestWorkers/Probingpaths/workers/java/2.18.0/testfile.txt b/test/TestWorkers/Probingpaths/workers/java/2.18.0/testfile.txt deleted file mode 100644 index ac4738bec6..0000000000 --- a/test/TestWorkers/Probingpaths/workers/java/2.18.0/testfile.txt +++ /dev/null @@ -1 +0,0 @@ -This is an empty file used for running tests, as the code checks if the file path specified in DefaultWorkerPath exists before loading the workerconfig. \ No newline at end of file diff --git a/test/TestWorkers/Probingpaths/workers/java/2.18.0/worker.config.json b/test/TestWorkers/Probingpaths/workers/java/2.18.0/worker.config.json index 52dc378929..f38fd2dd93 100644 --- a/test/TestWorkers/Probingpaths/workers/java/2.18.0/worker.config.json +++ b/test/TestWorkers/Probingpaths/workers/java/2.18.0/worker.config.json @@ -3,7 +3,7 @@ "language": "java", "extensions": [".jar"], "defaultExecutablePath": "%JAVA_HOME%/bin/java", - "defaultWorkerPath": "testfile.txt", + "defaultWorkerPath": "worker.config.json", "arguments": [] }, "profiles": [] diff --git a/test/TestWorkers/Probingpaths/workers/java/2.19.0/testfile.txt b/test/TestWorkers/Probingpaths/workers/java/2.19.0/testfile.txt deleted file mode 100644 index ac4738bec6..0000000000 --- a/test/TestWorkers/Probingpaths/workers/java/2.19.0/testfile.txt +++ /dev/null @@ -1 +0,0 @@ -This is an empty file used for running tests, as the code checks if the file path specified in DefaultWorkerPath exists before loading the workerconfig. \ No newline at end of file diff --git a/test/TestWorkers/Probingpaths/workers/java/2.19.0/worker.config.json b/test/TestWorkers/Probingpaths/workers/java/2.19.0/worker.config.json index 02b2835324..c50bb3919b 100644 --- a/test/TestWorkers/Probingpaths/workers/java/2.19.0/worker.config.json +++ b/test/TestWorkers/Probingpaths/workers/java/2.19.0/worker.config.json @@ -3,7 +3,7 @@ "language": "java", "extensions": [".jar"], "defaultExecutablePath": "%JAVA_HOME%/bin/java", - "defaultWorkerPath": "testfile.txt", + "defaultWorkerPath": "worker.config.json", "arguments": [] }, "hostRequirements": [], diff --git a/test/TestWorkers/Probingpaths/workers/node/3.10.1/testfile.txt b/test/TestWorkers/Probingpaths/workers/node/3.10.1/testfile.txt deleted file mode 100644 index ac4738bec6..0000000000 --- a/test/TestWorkers/Probingpaths/workers/node/3.10.1/testfile.txt +++ /dev/null @@ -1 +0,0 @@ -This is an empty file used for running tests, as the code checks if the file path specified in DefaultWorkerPath exists before loading the workerconfig. \ No newline at end of file diff --git a/test/TestWorkers/Probingpaths/workers/node/3.10.1/worker.config.json b/test/TestWorkers/Probingpaths/workers/node/3.10.1/worker.config.json index dacc607a55..564977d92c 100644 --- a/test/TestWorkers/Probingpaths/workers/node/3.10.1/worker.config.json +++ b/test/TestWorkers/Probingpaths/workers/node/3.10.1/worker.config.json @@ -3,7 +3,7 @@ "language": "node", "extensions": [".js", ".mjs", ".cjs"], "defaultExecutablePath": "node", - "defaultWorkerPath": "testfile.txt", + "defaultWorkerPath": "worker.config.json", "workerIndexing": "true" }, "hostRequirements": [], diff --git a/test/TestWorkers/Probingpaths/workers/powershell/7.4/testfile.txt b/test/TestWorkers/Probingpaths/workers/powershell/7.4/testfile.txt deleted file mode 100644 index ac4738bec6..0000000000 --- a/test/TestWorkers/Probingpaths/workers/powershell/7.4/testfile.txt +++ /dev/null @@ -1 +0,0 @@ -This is an empty file used for running tests, as the code checks if the file path specified in DefaultWorkerPath exists before loading the workerconfig. \ No newline at end of file diff --git a/test/TestWorkers/Probingpaths/workers/powershell/7.4/worker.config.json b/test/TestWorkers/Probingpaths/workers/powershell/7.4/worker.config.json index 73448885d9..3734952966 100644 --- a/test/TestWorkers/Probingpaths/workers/powershell/7.4/worker.config.json +++ b/test/TestWorkers/Probingpaths/workers/powershell/7.4/worker.config.json @@ -3,7 +3,7 @@ "language":"powershell", "extensions":[".ps1", ".psm1"], "defaultExecutablePath":"dotnet", - "defaultWorkerPath":"%FUNCTIONS_WORKER_RUNTIME_VERSION%/testfile.txt", + "defaultWorkerPath":"%FUNCTIONS_WORKER_RUNTIME_VERSION%/worker.config.json", "supportedRuntimeVersions":["7", "7.2", "7.4"], "defaultRuntimeVersion": "7.4", "sanitizeRuntimeVersionRegex":"\\d+\\.?\\d*" diff --git a/test/WebJobs.Script.Tests/Workers/Rpc/WorkerConfigurationResolverTests.cs b/test/WebJobs.Script.Tests/Workers/Rpc/WorkerConfigurationResolverTests.cs index 0eac1578ab..fe4ee8ca59 100644 --- a/test/WebJobs.Script.Tests/Workers/Rpc/WorkerConfigurationResolverTests.cs +++ b/test/WebJobs.Script.Tests/Workers/Rpc/WorkerConfigurationResolverTests.cs @@ -55,6 +55,30 @@ public void GetWorkerConfigs_MultiLanguageWorker_ReturnsExpectedConfigs(string r Assert.True(result.Any(r => r.Contains(Path.Combine(_fallbackPath, python)))); } + [Theory] + [InlineData("LATEST", "java\\2.19.0", "node\\3.10.1", "powershell\\7.4", "dotnet-isolated\\1.0.0", "python")] + [InlineData("STANDARD", "java\\2.18.0", "node\\3.10.1", "powershell\\7.2", "dotnet-isolated\\1.0.0", "python")] + public void GetWorkerConfigs_MultiLanguageWorker_Dotnet_ReturnsExpectedConfigs(string releaseChannel, string java, string node, string powershell, string dotnetIsolated, string python) + { + // Arrange + var mockEnvironment = new Mock(); + mockEnvironment.Setup(p => p.GetEnvironmentVariable(EnvironmentSettingNames.AntaresPlatformReleaseChannel)).Returns(releaseChannel); + mockEnvironment.Setup(p => p.GetEnvironmentVariable(EnvironmentSettingNames.AppKind)).Returns(ScriptConstants.WorkFlowAppKind); + mockEnvironment.Setup(p => p.GetEnvironmentVariable(EnvironmentSettingNames.FunctionWorkerRuntime)).Returns((string)null); + + var workerConfigurationResolver = new WorkerConfigurationResolver(_mockConfig.Object, _mockLogger.Object, mockEnvironment.Object, _mockProfileManager.Object, new HashSet() { "java", "node", "powershell", "dotnet-isolated" }); + + var result = workerConfigurationResolver.GetWorkerConfigs(_probingPaths, _fallbackPath); + + // Assert + Assert.Equal(result.Count, 5); + Assert.True(result.Any(r => r.Contains(Path.Combine(_probingPath1, java)))); + Assert.True(result.Any(r => r.Contains(Path.Combine(_probingPath1, node)))); + Assert.True(result.Any(r => r.Contains(Path.Combine(_probingPath1, powershell)))); + Assert.True(result.Any(r => r.Contains(Path.Combine(_probingPath1, dotnetIsolated)))); + Assert.True(result.Any(r => r.Contains(Path.Combine(_fallbackPath, python)))); + } + [Theory] [InlineData(null, "LATEST")] [InlineData(null, "STANDARD")] From 1495f31eb9140fda227363ef7407d0a7534c6778 Mon Sep 17 00:00:00 2001 From: Surbhi Gupta Date: Thu, 5 Jun 2025 22:22:21 -0700 Subject: [PATCH 18/55] Test data cleanup --- .../Probingpaths/workers/powershell/7.2/testfile.txt | 1 - .../Probingpaths/workers/powershell/7.2/worker.config.json | 2 +- test/TestWorkers/Probingpaths/workers/powershell/7/testfile.txt | 1 - .../Probingpaths/workers/powershell/7/worker.config.json | 2 +- 4 files changed, 2 insertions(+), 4 deletions(-) delete mode 100644 test/TestWorkers/Probingpaths/workers/powershell/7.2/testfile.txt delete mode 100644 test/TestWorkers/Probingpaths/workers/powershell/7/testfile.txt diff --git a/test/TestWorkers/Probingpaths/workers/powershell/7.2/testfile.txt b/test/TestWorkers/Probingpaths/workers/powershell/7.2/testfile.txt deleted file mode 100644 index ac4738bec6..0000000000 --- a/test/TestWorkers/Probingpaths/workers/powershell/7.2/testfile.txt +++ /dev/null @@ -1 +0,0 @@ -This is an empty file used for running tests, as the code checks if the file path specified in DefaultWorkerPath exists before loading the workerconfig. \ No newline at end of file diff --git a/test/TestWorkers/Probingpaths/workers/powershell/7.2/worker.config.json b/test/TestWorkers/Probingpaths/workers/powershell/7.2/worker.config.json index 555a222a88..cc51b2769d 100644 --- a/test/TestWorkers/Probingpaths/workers/powershell/7.2/worker.config.json +++ b/test/TestWorkers/Probingpaths/workers/powershell/7.2/worker.config.json @@ -3,7 +3,7 @@ "language":"powershell", "extensions":[".ps1", ".psm1"], "defaultExecutablePath":"dotnet", - "defaultWorkerPath":"%FUNCTIONS_WORKER_RUNTIME_VERSION%/testfile.txt", + "defaultWorkerPath":"%FUNCTIONS_WORKER_RUNTIME_VERSION%/worker.config.json", "supportedRuntimeVersions":["7", "7.2"], "defaultRuntimeVersion": "7.2", "sanitizeRuntimeVersionRegex":"\\d+\\.?\\d*" diff --git a/test/TestWorkers/Probingpaths/workers/powershell/7/testfile.txt b/test/TestWorkers/Probingpaths/workers/powershell/7/testfile.txt deleted file mode 100644 index ac4738bec6..0000000000 --- a/test/TestWorkers/Probingpaths/workers/powershell/7/testfile.txt +++ /dev/null @@ -1 +0,0 @@ -This is an empty file used for running tests, as the code checks if the file path specified in DefaultWorkerPath exists before loading the workerconfig. \ No newline at end of file diff --git a/test/TestWorkers/Probingpaths/workers/powershell/7/worker.config.json b/test/TestWorkers/Probingpaths/workers/powershell/7/worker.config.json index b664b0ac5a..1b318f929a 100644 --- a/test/TestWorkers/Probingpaths/workers/powershell/7/worker.config.json +++ b/test/TestWorkers/Probingpaths/workers/powershell/7/worker.config.json @@ -3,7 +3,7 @@ "language":"powershell", "extensions":[".ps1", ".psm1"], "defaultExecutablePath":"dotnet", - "defaultWorkerPath":"%FUNCTIONS_WORKER_RUNTIME_VERSION%/testfile.txt", + "defaultWorkerPath":"%FUNCTIONS_WORKER_RUNTIME_VERSION%/worker.config.json", "supportedRuntimeVersions":["7", "7.2"], "defaultRuntimeVersion":"7", "sanitizeRuntimeVersionRegex":"\\d+\\.?\\d*" From 4ee497b5c1042a19fd24e01ea497a92e2f7692bf Mon Sep 17 00:00:00 2001 From: Surbhi Gupta Date: Mon, 9 Jun 2025 16:57:00 -0500 Subject: [PATCH 19/55] Updated release notes --- release_notes.md | 1 + .../Workers/Rpc/Configuration/LanguageWorkerOptionsSetup.cs | 4 ++-- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/release_notes.md b/release_notes.md index 3ae6a5c80c..7163c5e845 100644 --- a/release_notes.md +++ b/release_notes.md @@ -17,3 +17,4 @@ - Improvements to coldstart pipeline (#11102). - Update Python Worker Version to [4.38.0](https://github.com/Azure/azure-functions-python-worker/releases/tag/4.38.0) - Only start the Diagnostic Events flush logs timer when events are present, preventing unnecessary flush attempts (#11100). +- Implement changes needed in the Host to decouple workers from the Host release (#11111) \ No newline at end of file diff --git a/src/WebJobs.Script/Workers/Rpc/Configuration/LanguageWorkerOptionsSetup.cs b/src/WebJobs.Script/Workers/Rpc/Configuration/LanguageWorkerOptionsSetup.cs index ede89518cc..2647c1ad63 100644 --- a/src/WebJobs.Script/Workers/Rpc/Configuration/LanguageWorkerOptionsSetup.cs +++ b/src/WebJobs.Script/Workers/Rpc/Configuration/LanguageWorkerOptionsSetup.cs @@ -21,7 +21,7 @@ namespace Microsoft.Azure.WebJobs.Script.Workers.Rpc { internal class LanguageWorkerOptionsSetup : IConfigureOptions { - private const string _windowsWorkerProbingPath = "c:\\home\\SiteExtensions\\workers"; // Harcoded site extensions path for Windows until sets it as an Environment variable. + private const string _windowsWorkerProbingPath = "c:\\home\\SiteExtensions\\workers"; // Harcoded site extensions path for Windows until Antares sets it as an Environment variable. private readonly IConfiguration _configuration; private readonly ILogger _logger; private readonly IEnvironment _environment; @@ -109,7 +109,7 @@ internal List GetWorkerProbingPaths() { var probingPaths = new List(); - // If Environment variable is set, read probing paths from Env (works for linux) + // If Environment variable is set, read probing paths from Environment string probingPathsEnvValue = _environment.GetEnvironmentVariableOrDefault(EnvironmentSettingNames.WorkerProbingPaths, null); if (!string.IsNullOrEmpty(probingPathsEnvValue)) From 5168090d69b6a7c7b3c4c765d96cf6e19ec6ada8 Mon Sep 17 00:00:00 2001 From: Surbhi Gupta Date: Thu, 12 Jun 2025 01:07:55 -0500 Subject: [PATCH 20/55] Renaming feature name --- .../Config/FunctionsHostingConfigOptions.cs | 4 +-- src/WebJobs.Script/ScriptConstants.cs | 4 +-- src/WebJobs.Script/Utility.cs | 20 +++++++------- .../LanguageWorkerOptionsSetup.cs | 14 +++++----- .../Configuration/RpcWorkerConfigFactory.cs | 16 ++++++------ .../WorkerConfigurationHelper.cs | 4 +-- .../WorkerConfigurationResolver.cs | 10 +++---- .../Workers/Rpc/RpcWorkerConstants.cs | 2 +- .../Workers/Rpc/RpcWorkerDescription.cs | 6 ++--- .../FunctionsHostingConfigOptionsTest.cs | 4 +-- .../LanguageWorkerOptionsSetupTests.cs | 20 +++++++------- test/WebJobs.Script.Tests/UtilityTests.cs | 26 +++++++++---------- .../Rpc/RpcWorkerConfigFactoryTests.cs | 8 +++--- 13 files changed, 69 insertions(+), 69 deletions(-) diff --git a/src/WebJobs.Script/Config/FunctionsHostingConfigOptions.cs b/src/WebJobs.Script/Config/FunctionsHostingConfigOptions.cs index 5d2bbb9b11..7debd0dd24 100644 --- a/src/WebJobs.Script/Config/FunctionsHostingConfigOptions.cs +++ b/src/WebJobs.Script/Config/FunctionsHostingConfigOptions.cs @@ -126,11 +126,11 @@ internal string WorkerIndexingDisabledApps /// /// Gets a string delimited by '|' that contains the names of the language workers available through Probing paths outside of the Host. /// - internal string EnableProbingPathsForWorkers + internal string WorkersAvailableForDynamicResolution { get { - return GetFeature(RpcWorkerConstants.EnableProbingPathsForWorkers) ?? string.Empty; + return GetFeature(RpcWorkerConstants.WorkersAvailableForDynamicResolution) ?? string.Empty; } } diff --git a/src/WebJobs.Script/ScriptConstants.cs b/src/WebJobs.Script/ScriptConstants.cs index 5b23623710..a86fc079b7 100644 --- a/src/WebJobs.Script/ScriptConstants.cs +++ b/src/WebJobs.Script/ScriptConstants.cs @@ -134,8 +134,8 @@ public static class ScriptConstants public const string FeatureFlagDisableWebHostLogForwarding = "DisableWebHostLogForwarding"; public const string FeatureFlagDisableMergedWebHostScriptHostConfiguration = "DisableMergedConfiguration"; public const string FeatureFlagEnableWorkerIndexing = "EnableWorkerIndexing"; - public const string FeatureFlagDisableWorkerProbingPaths = "DisableWorkerProbingPaths"; - public const string FeatureFlagEnableWorkerProbingPaths = "EnableWorkerProbingPaths"; + public const string FeatureFlagDisableDynamicWorkerResolution = "DisableDynamicWorkerResolution"; + public const string FeatureFlagEnableDynamicWorkerResolution = "EnableDynamicWorkerResolution"; public const string FeatureFlagEnableDebugTracing = "EnableDebugTracing"; public const string FeatureFlagEnableProxies = "EnableProxies"; public const string FeatureFlagStrictHISModeEnabled = "StrictHISModeEnabled"; diff --git a/src/WebJobs.Script/Utility.cs b/src/WebJobs.Script/Utility.cs index b6127f1367..ef11811086 100644 --- a/src/WebJobs.Script/Utility.cs +++ b/src/WebJobs.Script/Utility.cs @@ -1066,21 +1066,21 @@ public static bool CanWorkerIndex(IEnumerable workerConfigs, IE return workerIndexingEnabled && workerIndexingAvailable; } - // Worker probing paths can be enabled or disabled via feature flags or hosting config options. Feature flags take precedence over hosting config options. - // Users can enable or disable probing paths via setting the appropriate feature flags. - // Probing paths can also be enabled for specific workers at stamp level via the hosting config options. - public static bool AreWorkerProbingPathsEnabled(IEnvironment environment, HashSet probingPathsEnabledWorkersViaHostingConfig) + // Dynamic Worker Resolution can be enabled or disabled via feature flags or hosting config options. Feature flags take precedence over hosting config options. + // Users can enable or disable worker resolution via setting the appropriate feature flags. + // Worker resolution can also be enabled for specific workers at stamp level via the hosting config options. + public static bool IsDynamicWorkerResolutionEnabled(IEnvironment environment, HashSet workersAvailableForResolutionViaHostingConfig) { - bool areProbingPathsDisabled = FeatureFlags.IsEnabled(ScriptConstants.FeatureFlagDisableWorkerProbingPaths, environment); + bool isDynamicWorkerResolutionDisabled = FeatureFlags.IsEnabled(ScriptConstants.FeatureFlagDisableDynamicWorkerResolution, environment); - if (areProbingPathsDisabled) + if (isDynamicWorkerResolutionDisabled) { return false; } - bool areProbingPathsEnabled = FeatureFlags.IsEnabled(ScriptConstants.FeatureFlagEnableWorkerProbingPaths, environment); + bool isDynamicWorkerResolutionEnabled = FeatureFlags.IsEnabled(ScriptConstants.FeatureFlagEnableDynamicWorkerResolution, environment); - if (areProbingPathsEnabled) + if (isDynamicWorkerResolutionEnabled) { return true; } @@ -1089,10 +1089,10 @@ public static bool AreWorkerProbingPathsEnabled(IEnvironment environment, HashSe if (!environment.IsMultiLanguageRuntimeEnvironment() && !string.IsNullOrWhiteSpace(workerRuntime)) { - return probingPathsEnabledWorkersViaHostingConfig?.Contains(workerRuntime) ?? false; + return workersAvailableForResolutionViaHostingConfig?.Contains(workerRuntime) ?? false; } - return probingPathsEnabledWorkersViaHostingConfig?.Any() ?? false; + return workersAvailableForResolutionViaHostingConfig?.Any() ?? false; } public static void LogAutorestGeneratedJsonIfExists(string rootScriptPath, ILogger logger) diff --git a/src/WebJobs.Script/Workers/Rpc/Configuration/LanguageWorkerOptionsSetup.cs b/src/WebJobs.Script/Workers/Rpc/Configuration/LanguageWorkerOptionsSetup.cs index 2647c1ad63..bf3738ed1e 100644 --- a/src/WebJobs.Script/Workers/Rpc/Configuration/LanguageWorkerOptionsSetup.cs +++ b/src/WebJobs.Script/Workers/Rpc/Configuration/LanguageWorkerOptionsSetup.cs @@ -82,24 +82,24 @@ public void Configure(LanguageWorkerOptions options) } } - HashSet probingPathsEnabledWorkersViaHostingConfig = GetProbingPathsEnabledWorkersFromHostingConfig(); - bool probingPathsEnabled = Utility.AreWorkerProbingPathsEnabled(_environment, probingPathsEnabledWorkersViaHostingConfig); + HashSet workers = GetWorkersAvailableForResolutionViaHostingConfig(); + bool dynamicWorkerResolutionEnabled = Utility.IsDynamicWorkerResolutionEnabled(_environment, workers); List probingPaths = null; - if (probingPathsEnabled) + if (dynamicWorkerResolutionEnabled) { probingPaths = GetWorkerProbingPaths(); } - var workerConfigurationResolver = new WorkerConfigurationResolver(configuration, _logger, _environment, _workerProfileManager, probingPathsEnabledWorkersViaHostingConfig); - var configFactory = new RpcWorkerConfigFactory(configuration, _logger, SystemRuntimeInformation.Instance, _environment, _metricsLogger, _workerProfileManager, workerConfigurationResolver, probingPathsEnabled, probingPaths); + var workerConfigurationResolver = new WorkerConfigurationResolver(configuration, _logger, _environment, _workerProfileManager, workers); + var configFactory = new RpcWorkerConfigFactory(configuration, _logger, SystemRuntimeInformation.Instance, _environment, _metricsLogger, _workerProfileManager, workerConfigurationResolver, dynamicWorkerResolutionEnabled, probingPaths); options.WorkerConfigs = configFactory.GetConfigs(); } - internal HashSet GetProbingPathsEnabledWorkersFromHostingConfig() + internal HashSet GetWorkersAvailableForResolutionViaHostingConfig() { return _functionsHostingConfigOptions.Value - ?.EnableProbingPathsForWorkers + ?.WorkersAvailableForDynamicResolution ?.ToLowerInvariant() .Split("|", StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries) .ToHashSet(); diff --git a/src/WebJobs.Script/Workers/Rpc/Configuration/RpcWorkerConfigFactory.cs b/src/WebJobs.Script/Workers/Rpc/Configuration/RpcWorkerConfigFactory.cs index 4baa89966b..e3029a5a46 100644 --- a/src/WebJobs.Script/Workers/Rpc/Configuration/RpcWorkerConfigFactory.cs +++ b/src/WebJobs.Script/Workers/Rpc/Configuration/RpcWorkerConfigFactory.cs @@ -44,8 +44,8 @@ public RpcWorkerConfigFactory(IConfiguration config, IMetricsLogger metricsLogger, IWorkerProfileManager workerProfileManager, IWorkerConfigurationResolver workerConfigurationResolver, - bool probingPathsEnabled = false, - List probingPaths = null) + bool dynamicWorkerResolutionEnabled = false, + List workerProbingPaths = null) { _config = config ?? throw new ArgumentNullException(nameof(config)); _logger = logger ?? throw new ArgumentNullException(nameof(logger)); @@ -59,8 +59,8 @@ public RpcWorkerConfigFactory(IConfiguration config, WorkersDirPath = GetDefaultWorkersDirectory(Directory.Exists); var workersDirectorySection = _config.GetSection($"{RpcWorkerConstants.LanguageWorkersSectionName}:{WorkerConstants.WorkersDirectorySectionName}"); - ProbingPathsEnabled = probingPathsEnabled; - WorkerProbingPaths = probingPaths; + DynamicWorkerResolutionEnabled = dynamicWorkerResolutionEnabled; + WorkerProbingPaths = workerProbingPaths; if (!string.IsNullOrEmpty(workersDirectorySection.Value)) { @@ -70,7 +70,7 @@ public RpcWorkerConfigFactory(IConfiguration config, public string WorkersDirPath { get; } - internal bool ProbingPathsEnabled { get; } + internal bool DynamicWorkerResolutionEnabled { get; } internal List WorkerProbingPaths { get; } @@ -105,7 +105,7 @@ internal void BuildWorkerProviderDictionary() internal void AddProviders() { - if (ProbingPathsEnabled && WorkerProbingPaths is not null) + if (DynamicWorkerResolutionEnabled && WorkerProbingPaths is not null) { _logger.LogDebug("Probing paths set to: {probingPaths} and Workers Directory set as fallback path: {WorkersDirPath}", string.Join(", ", WorkerProbingPaths), WorkersDirPath); @@ -155,7 +155,7 @@ internal void AddProvider(string workerDir) if (!string.IsNullOrWhiteSpace(_workerRuntime) && !_environment.IsPlaceholderModeEnabled() && !_environment.IsMultiLanguageRuntimeEnvironment() && - !ProbingPathsEnabled) + !DynamicWorkerResolutionEnabled) { string workerRuntime = Path.GetFileName(workerDir); // Only skip worker directories that don't match the current runtime. @@ -182,7 +182,7 @@ internal void AddProvider(string workerDir) workerConfig, _jsonSerializerOptions, workerDir, - ProbingPathsEnabled, + DynamicWorkerResolutionEnabled, _profileManager, _config, _logger); diff --git a/src/WebJobs.Script/Workers/Rpc/Configuration/WorkerConfigurationHelper.cs b/src/WebJobs.Script/Workers/Rpc/Configuration/WorkerConfigurationHelper.cs index 1a04f20f0b..ae8d88b16a 100644 --- a/src/WebJobs.Script/Workers/Rpc/Configuration/WorkerConfigurationHelper.cs +++ b/src/WebJobs.Script/Workers/Rpc/Configuration/WorkerConfigurationHelper.cs @@ -18,7 +18,7 @@ internal static RpcWorkerDescription GetWorkerDescription( JsonElement workerConfig, JsonSerializerOptions jsonSerializerOptions, string workerDir, - bool probingPathsEnabled, + bool dynamicWorkerResolutionEnabled, IWorkerProfileManager profileManager, IConfiguration config, ILogger logger) @@ -45,7 +45,7 @@ internal static RpcWorkerDescription GetWorkerDescription( AddArgumentsFromAppSettings(workerDescription, languageSection); // Validate workerDescription - workerDescription.ApplyDefaultsAndValidate(Directory.GetCurrentDirectory(), logger, probingPathsEnabled); + workerDescription.ApplyDefaultsAndValidate(Directory.GetCurrentDirectory(), logger, dynamicWorkerResolutionEnabled); return workerDescription; } diff --git a/src/WebJobs.Script/Workers/Rpc/Configuration/WorkerConfigurationResolver.cs b/src/WebJobs.Script/Workers/Rpc/Configuration/WorkerConfigurationResolver.cs index 1bfb5987f3..a680ab8bfb 100644 --- a/src/WebJobs.Script/Workers/Rpc/Configuration/WorkerConfigurationResolver.cs +++ b/src/WebJobs.Script/Workers/Rpc/Configuration/WorkerConfigurationResolver.cs @@ -20,20 +20,20 @@ internal sealed class WorkerConfigurationResolver : IWorkerConfigurationResolver private readonly ILogger _logger; private readonly IWorkerProfileManager _profileManager; private readonly IEnvironment _environment; - private readonly HashSet _probingPathsEnabledWorkersViaHostingConfig; + private readonly HashSet _workersAvailableForResolutionViaHostingConfig; private readonly JsonSerializerOptions _jsonSerializerOptions = new() { PropertyNameCaseInsensitive = true }; public WorkerConfigurationResolver(IConfiguration config, ILogger logger, IEnvironment environment, IWorkerProfileManager workerProfileManager, - HashSet probingPathsEnabledWorkersViaHostingConfig) + HashSet workersAvailableForResolutionViaHostingConfig) { _config = config ?? throw new ArgumentNullException(nameof(config)); _logger = logger ?? throw new ArgumentNullException(nameof(logger)); _environment = environment ?? throw new ArgumentNullException(nameof(environment)); _profileManager = workerProfileManager ?? throw new ArgumentNullException(nameof(workerProfileManager)); - _probingPathsEnabledWorkersViaHostingConfig = probingPathsEnabledWorkersViaHostingConfig; + _workersAvailableForResolutionViaHostingConfig = workersAvailableForResolutionViaHostingConfig; } public List GetWorkerConfigs(List probingPaths, string fallbackPath) @@ -65,8 +65,8 @@ public List GetWorkerConfigs(List probingPaths, string fallbackP // Do not skip non-worker directories like the function app payload directory if (languageWorkerPath.StartsWith(fallbackPath) || languageWorkerPath.StartsWith(probingPath)) { - if ((_probingPathsEnabledWorkersViaHostingConfig is not null && - !_probingPathsEnabledWorkersViaHostingConfig.Contains(languageWorkerDir)) || + if ((_workersAvailableForResolutionViaHostingConfig is not null && + !_workersAvailableForResolutionViaHostingConfig.Contains(languageWorkerDir)) || (!_environment.IsMultiLanguageRuntimeEnvironment() && workerRuntime is not null && !workerRuntime.Equals(languageWorkerDir, StringComparison.OrdinalIgnoreCase))) diff --git a/src/WebJobs.Script/Workers/Rpc/RpcWorkerConstants.cs b/src/WebJobs.Script/Workers/Rpc/RpcWorkerConstants.cs index 16c7686eb8..3976c7f54c 100644 --- a/src/WebJobs.Script/Workers/Rpc/RpcWorkerConstants.cs +++ b/src/WebJobs.Script/Workers/Rpc/RpcWorkerConstants.cs @@ -92,7 +92,7 @@ public static class RpcWorkerConstants public const string WorkerIndexingEnabled = "WORKER_INDEXING_ENABLED"; public const string WorkerIndexingDisabledApps = "WORKER_INDEXING_DISABLED_APPS"; - public const string EnableProbingPathsForWorkers = "ENABLE_PROBING_PATHS_FOR_WORKERS"; + public const string WorkersAvailableForDynamicResolution = "WORKERS_AVAILABLE_FOR_DYNAMIC_RESOLUTION"; public const string RevertWorkerShutdownBehavior = "REVERT_WORKER_SHUTDOWN_BEHAVIOR"; public const string ShutdownWebhostWorkerChannelsOnHostShutdown = "ShutdownWebhostWorkerChannelsOnHostShutdown"; public const string ThrowOnMissingFunctionsWorkerRuntime = "THROW_ON_MISSING_FUNCTIONS_WORKER_RUNTIME"; diff --git a/src/WebJobs.Script/Workers/Rpc/RpcWorkerDescription.cs b/src/WebJobs.Script/Workers/Rpc/RpcWorkerDescription.cs index fbd3034151..e4088e8787 100644 --- a/src/WebJobs.Script/Workers/Rpc/RpcWorkerDescription.cs +++ b/src/WebJobs.Script/Workers/Rpc/RpcWorkerDescription.cs @@ -87,7 +87,7 @@ public List Extensions public override bool UseStdErrorStreamForErrorsOnly { get; set; } = false; - public override void ApplyDefaultsAndValidate(string workerDirectory, ILogger logger, bool probingPathsEnabled = false) + public override void ApplyDefaultsAndValidate(string workerDirectory, ILogger logger, bool dynamicWorkerResolutionEnabled = false) { if (workerDirectory == null) { @@ -98,9 +98,9 @@ public override void ApplyDefaultsAndValidate(string workerDirectory, ILogger lo if (!string.IsNullOrEmpty(DefaultWorkerPath) && !Path.IsPathRooted(DefaultWorkerPath)) { - // If probing paths are enabled and DefaultWorkerPath contains FunctionWorkerRuntimeVersionSettingName + // If dynamic worker resolution enabled and DefaultWorkerPath contains FunctionWorkerRuntimeVersionSettingName // then version becomes redundant in the path. Replacing version with an empty string to avoid duplication. - if (probingPathsEnabled && DefaultWorkerPath.Contains(RpcWorkerConstants.RuntimeVersionPlaceholder)) + if (dynamicWorkerResolutionEnabled && DefaultWorkerPath.Contains(RpcWorkerConstants.RuntimeVersionPlaceholder)) { var versionDir = Path.GetFileName(WorkerDirectory); WorkerDirectory = WorkerDirectory.Replace(versionDir, string.Empty); diff --git a/test/WebJobs.Script.Tests/Configuration/FunctionsHostingConfigOptionsTest.cs b/test/WebJobs.Script.Tests/Configuration/FunctionsHostingConfigOptionsTest.cs index f8bf38a7a0..434857a704 100644 --- a/test/WebJobs.Script.Tests/Configuration/FunctionsHostingConfigOptionsTest.cs +++ b/test/WebJobs.Script.Tests/Configuration/FunctionsHostingConfigOptionsTest.cs @@ -74,8 +74,8 @@ public static IEnumerable PropertyValues yield return [nameof(FunctionsHostingConfigOptions.IsDotNetInProcDisabled), "DotNetInProcDisabled=True", true]; yield return [nameof(FunctionsHostingConfigOptions.IsDotNetInProcDisabled), "DotNetInProcDisabled=1", true]; yield return [nameof(FunctionsHostingConfigOptions.IsDotNetInProcDisabled), "DotNetInProcDisabled=0", false]; - yield return [nameof(FunctionsHostingConfigOptions.EnableProbingPathsForWorkers), "ENABLE_PROBING_PATHS_FOR_WORKERS=java|node", "java|node"]; - yield return [nameof(FunctionsHostingConfigOptions.EnableProbingPathsForWorkers), "ENABLE_PROBING_PATHS_FOR_WORKERS=java", "java"]; + yield return [nameof(FunctionsHostingConfigOptions.WorkersAvailableForDynamicResolution), "WORKERS_AVAILABLE_FOR_DYNAMIC_RESOLUTION=java|node", "java|node"]; + yield return [nameof(FunctionsHostingConfigOptions.WorkersAvailableForDynamicResolution), "WORKERS_AVAILABLE_FOR_DYNAMIC_RESOLUTION=java", "java"]; yield return [nameof(FunctionsHostingConfigOptions.IsTestDataSuppressionEnabled), "EnableTestDataSuppression=1", true]; yield return [nameof(FunctionsHostingConfigOptions.IsCGroupMemoryMetricsEnabled), "EnableCGroupMemoryMetrics=1", true]; #pragma warning restore SA1011 // Closing square brackets should be spaced correctly diff --git a/test/WebJobs.Script.Tests/Configuration/LanguageWorkerOptionsSetupTests.cs b/test/WebJobs.Script.Tests/Configuration/LanguageWorkerOptionsSetupTests.cs index 00973e128d..fb1a940487 100644 --- a/test/WebJobs.Script.Tests/Configuration/LanguageWorkerOptionsSetupTests.cs +++ b/test/WebJobs.Script.Tests/Configuration/LanguageWorkerOptionsSetupTests.cs @@ -82,10 +82,10 @@ public void LanguageWorkerOptions_Expected_ListOfConfigs(string workerRuntime) } [Theory] - [InlineData("java", "EnableWorkerProbingPaths", "java", "LATEST", "2.19.0")] - [InlineData("java", "EnableWorkerProbingPaths", "java", "STANDARD", "2.18.0")] - [InlineData("node", "EnableWorkerProbingPaths", "node", "LATEST", "3.10.1")] - [InlineData("node", "EnableWorkerProbingPaths", "java|node", "STANDARD", "3.10.1")] + [InlineData("java", ScriptConstants.FeatureFlagEnableDynamicWorkerResolution, "java", "LATEST", "2.19.0")] + [InlineData("java", ScriptConstants.FeatureFlagEnableDynamicWorkerResolution, "java", "STANDARD", "2.18.0")] + [InlineData("node", ScriptConstants.FeatureFlagEnableDynamicWorkerResolution, "node", "LATEST", "3.10.1")] + [InlineData("node", ScriptConstants.FeatureFlagEnableDynamicWorkerResolution, "java|node", "STANDARD", "3.10.1")] public void LanguageWorkerOptions_ProbingPaths_Expected_ListOfConfigs(string workerRuntime, string enableProbingPaths, string hostingOptionsSetting, string releaseChannel, string expectedVersion) { var loggerProvider = new TestLoggerProvider(); @@ -107,7 +107,7 @@ public void LanguageWorkerOptions_ProbingPaths_Expected_ListOfConfigs(string wor testEnvironment.SetEnvironmentVariable(EnvironmentSettingNames.WorkerProbingPaths, probingPathValue); var hostingOptions = new FunctionsHostingConfigOptions(); - hostingOptions.Features.Add(RpcWorkerConstants.EnableProbingPathsForWorkers, hostingOptionsSetting); + hostingOptions.Features.Add(RpcWorkerConstants.WorkersAvailableForDynamicResolution, hostingOptionsSetting); LanguageWorkerOptionsSetup setup = new LanguageWorkerOptionsSetup(configuration, loggerFactory, testEnvironment, testMetricLogger, testProfileManager.Object, testScriptHostManager.Object, new OptionsWrapper(hostingOptions)); LanguageWorkerOptions options = new LanguageWorkerOptions(); @@ -126,10 +126,10 @@ public void LanguageWorkerOptions_ProbingPaths_Expected_ListOfConfigs(string wor } [Theory] - [InlineData("java", "EnableWorkerProbingPaths", "java", "LATEST")] - [InlineData("java", "EnableWorkerProbingPaths", "java", "STANDARD")] - [InlineData("node", "EnableWorkerProbingPaths", "node", "LATEST")] - [InlineData("node", "EnableWorkerProbingPaths", "java|node", "STANDARD")] + [InlineData("java", ScriptConstants.FeatureFlagEnableDynamicWorkerResolution, "java", "LATEST")] + [InlineData("java", ScriptConstants.FeatureFlagEnableDynamicWorkerResolution, "java", "STANDARD")] + [InlineData("node", ScriptConstants.FeatureFlagEnableDynamicWorkerResolution, "node", "LATEST")] + [InlineData("node", ScriptConstants.FeatureFlagEnableDynamicWorkerResolution, "java|node", "STANDARD")] public void LanguageWorkerOptions_FallbackPath_Expected_ListOfConfigs(string workerRuntime, string enableProbingPaths, string hostingOptionsSetting, string releaseChannel) { var loggerProvider = new TestLoggerProvider(); @@ -151,7 +151,7 @@ public void LanguageWorkerOptions_FallbackPath_Expected_ListOfConfigs(string wor testEnvironment.SetEnvironmentVariable(EnvironmentSettingNames.WorkerProbingPaths, probingPathValue); var hostingOptions = new FunctionsHostingConfigOptions(); - hostingOptions.Features.Add(RpcWorkerConstants.EnableProbingPathsForWorkers, hostingOptionsSetting); + hostingOptions.Features.Add(RpcWorkerConstants.WorkersAvailableForDynamicResolution, hostingOptionsSetting); LanguageWorkerOptionsSetup setup = new LanguageWorkerOptionsSetup(configuration, loggerFactory, testEnvironment, testMetricLogger, testProfileManager.Object, testScriptHostManager.Object, new OptionsWrapper(hostingOptions)); LanguageWorkerOptions options = new LanguageWorkerOptions(); diff --git a/test/WebJobs.Script.Tests/UtilityTests.cs b/test/WebJobs.Script.Tests/UtilityTests.cs index 8f89eefd92..7c04c00ab7 100644 --- a/test/WebJobs.Script.Tests/UtilityTests.cs +++ b/test/WebJobs.Script.Tests/UtilityTests.cs @@ -1022,23 +1022,23 @@ public void WorkerIndexingDecisionLogic_NullWorkerIndexingProperty(bool workerIn [InlineData(null, "", false)] [InlineData(null, "| ", false)] [InlineData(null, null, false)] - [InlineData(ScriptConstants.FeatureFlagEnableWorkerProbingPaths, "node", true)] - [InlineData(ScriptConstants.FeatureFlagEnableWorkerProbingPaths, "java|node", true)] - [InlineData(ScriptConstants.FeatureFlagEnableWorkerProbingPaths, "", true)] - [InlineData(ScriptConstants.FeatureFlagEnableWorkerProbingPaths, "| ", true)] - [InlineData(ScriptConstants.FeatureFlagEnableWorkerProbingPaths, null, true)] - [InlineData(ScriptConstants.FeatureFlagDisableWorkerProbingPaths, "node", false)] - [InlineData(ScriptConstants.FeatureFlagDisableWorkerProbingPaths, "java|node", false)] - [InlineData(ScriptConstants.FeatureFlagDisableWorkerProbingPaths, "| ", false)] - - public void AreProbingPathsEnabled_HostingConfigAndFeatureFlags_WorksAsExpected(string featureFlagValue, string hostingConfigSetting, bool expected) + [InlineData(ScriptConstants.FeatureFlagEnableDynamicWorkerResolution, "node", true)] + [InlineData(ScriptConstants.FeatureFlagEnableDynamicWorkerResolution, "java|node", true)] + [InlineData(ScriptConstants.FeatureFlagEnableDynamicWorkerResolution, "", true)] + [InlineData(ScriptConstants.FeatureFlagEnableDynamicWorkerResolution, "| ", true)] + [InlineData(ScriptConstants.FeatureFlagEnableDynamicWorkerResolution, null, true)] + [InlineData(ScriptConstants.FeatureFlagDisableDynamicWorkerResolution, "node", false)] + [InlineData(ScriptConstants.FeatureFlagDisableDynamicWorkerResolution, "java|node", false)] + [InlineData(ScriptConstants.FeatureFlagDisableDynamicWorkerResolution, "| ", false)] + + public void IsDynamicWorkerResolutionEnabled_HostingConfigAndFeatureFlags_WorksAsExpected(string featureFlagValue, string hostingConfigSetting, bool expected) { HashSet hostingConfigEnabledWorkers = hostingConfigSetting?.Split('|', StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries).ToHashSet(); var testEnvironment = new TestEnvironment(); testEnvironment.SetEnvironmentVariable(EnvironmentSettingNames.AzureWebJobsFeatureFlags, featureFlagValue); - bool result = Utility.AreWorkerProbingPathsEnabled(testEnvironment, hostingConfigEnabledWorkers); + bool result = Utility.IsDynamicWorkerResolutionEnabled(testEnvironment, hostingConfigEnabledWorkers); Assert.Equal(expected, result); } @@ -1050,7 +1050,7 @@ public void AreProbingPathsEnabled_HostingConfigAndFeatureFlags_WorksAsExpected( [InlineData("node", "node", "workflowapp", true)] [InlineData("java|node", null, "workflowapp", true)] [InlineData("| ", null, "workflowapp", false)] - public void AreProbingPathsEnabled_WorkerRuntimeAndMultiLanguage_WorksAsExpected(string hostingConfigSetting, string workerRuntime, string multilanguageApp, bool expected) + public void IsDynamicWorkerResolutionEnabled_WorkerRuntimeAndMultiLanguage_WorksAsExpected(string hostingConfigSetting, string workerRuntime, string multilanguageApp, bool expected) { var testEnvironment = new TestEnvironment(); testEnvironment.SetEnvironmentVariable(EnvironmentSettingNames.AppKind, multilanguageApp); @@ -1058,7 +1058,7 @@ public void AreProbingPathsEnabled_WorkerRuntimeAndMultiLanguage_WorksAsExpected HashSet hostingConfigEnabledWorkers = hostingConfigSetting?.Split('|', StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries).ToHashSet(); - bool result = Utility.AreWorkerProbingPathsEnabled(testEnvironment, hostingConfigEnabledWorkers); + bool result = Utility.IsDynamicWorkerResolutionEnabled(testEnvironment, hostingConfigEnabledWorkers); Assert.Equal(expected, result); } diff --git a/test/WebJobs.Script.Tests/Workers/Rpc/RpcWorkerConfigFactoryTests.cs b/test/WebJobs.Script.Tests/Workers/Rpc/RpcWorkerConfigFactoryTests.cs index 09585281cd..dc2e158762 100644 --- a/test/WebJobs.Script.Tests/Workers/Rpc/RpcWorkerConfigFactoryTests.cs +++ b/test/WebJobs.Script.Tests/Workers/Rpc/RpcWorkerConfigFactoryTests.cs @@ -178,12 +178,12 @@ public void DefaultWorkerConfigs_Overrides_DefaultWorkerRuntimeVersion_AppSettin [Theory] [InlineData("7.4", "7.4", null, null)] [InlineData(null, "7.4", null, null)] - [InlineData("7.4", "7.4", "..\\..\\..\\..\\test\\TestWorkers\\ProbingPaths\\workers\\", "EnableWorkerProbingPaths")] - [InlineData(null, "7.4", "..\\..\\..\\..\\test\\TestWorkers\\ProbingPaths\\workers\\", "EnableWorkerProbingPaths")] + [InlineData("7.4", "7.4", "..\\..\\..\\..\\test\\TestWorkers\\ProbingPaths\\workers\\", ScriptConstants.FeatureFlagEnableDynamicWorkerResolution)] + [InlineData(null, "7.4", "..\\..\\..\\..\\test\\TestWorkers\\ProbingPaths\\workers\\", ScriptConstants.FeatureFlagEnableDynamicWorkerResolution)] [InlineData("7.2", "7.2", null, null)] - [InlineData("7.2", "7.2", "..\\..\\..\\..\\test\\TestWorkers\\ProbingPaths\\workers\\", "EnableWorkerProbingPaths")] + [InlineData("7.2", "7.2", "..\\..\\..\\..\\test\\TestWorkers\\ProbingPaths\\workers\\", ScriptConstants.FeatureFlagEnableDynamicWorkerResolution)] [InlineData("7", "7", null, null)] - [InlineData("7", "7", "..\\..\\..\\..\\test\\TestWorkers\\ProbingPaths\\workers\\", "EnableWorkerProbingPaths")] + [InlineData("7", "7", "..\\..\\..\\..\\test\\TestWorkers\\ProbingPaths\\workers\\", ScriptConstants.FeatureFlagEnableDynamicWorkerResolution)] public void DefaultWorkerConfigs_Overrides_VersionAppSetting(string runtimeSettingVersion, string outputVersion, string probingPathValue, string enableProbingPath) { var probingPath = string.IsNullOrEmpty(probingPathValue) ? probingPathValue : Path.GetFullPath(probingPathValue); From 1ad3d14908e48115e8bfa8d3493c82afd2a21d64 Mon Sep 17 00:00:00 2001 From: Surbhi Gupta Date: Thu, 12 Jun 2025 01:43:14 -0500 Subject: [PATCH 21/55] Updated tests --- .../workers/powershell/7.2/worker.config.json | 11 ------ .../workers/powershell/7/worker.config.json | 11 ------ .../Rpc/RpcWorkerConfigFactoryTests.cs | 4 -- .../Rpc/WorkerConfigurationResolverTests.cs | 39 +++++-------------- 4 files changed, 9 insertions(+), 56 deletions(-) delete mode 100644 test/TestWorkers/Probingpaths/workers/powershell/7.2/worker.config.json delete mode 100644 test/TestWorkers/Probingpaths/workers/powershell/7/worker.config.json diff --git a/test/TestWorkers/Probingpaths/workers/powershell/7.2/worker.config.json b/test/TestWorkers/Probingpaths/workers/powershell/7.2/worker.config.json deleted file mode 100644 index cc51b2769d..0000000000 --- a/test/TestWorkers/Probingpaths/workers/powershell/7.2/worker.config.json +++ /dev/null @@ -1,11 +0,0 @@ -{ - "description":{ - "language":"powershell", - "extensions":[".ps1", ".psm1"], - "defaultExecutablePath":"dotnet", - "defaultWorkerPath":"%FUNCTIONS_WORKER_RUNTIME_VERSION%/worker.config.json", - "supportedRuntimeVersions":["7", "7.2"], - "defaultRuntimeVersion": "7.2", - "sanitizeRuntimeVersionRegex":"\\d+\\.?\\d*" - } -} diff --git a/test/TestWorkers/Probingpaths/workers/powershell/7/worker.config.json b/test/TestWorkers/Probingpaths/workers/powershell/7/worker.config.json deleted file mode 100644 index 1b318f929a..0000000000 --- a/test/TestWorkers/Probingpaths/workers/powershell/7/worker.config.json +++ /dev/null @@ -1,11 +0,0 @@ -{ - "description":{ - "language":"powershell", - "extensions":[".ps1", ".psm1"], - "defaultExecutablePath":"dotnet", - "defaultWorkerPath":"%FUNCTIONS_WORKER_RUNTIME_VERSION%/worker.config.json", - "supportedRuntimeVersions":["7", "7.2"], - "defaultRuntimeVersion":"7", - "sanitizeRuntimeVersionRegex":"\\d+\\.?\\d*" - } -} diff --git a/test/WebJobs.Script.Tests/Workers/Rpc/RpcWorkerConfigFactoryTests.cs b/test/WebJobs.Script.Tests/Workers/Rpc/RpcWorkerConfigFactoryTests.cs index dc2e158762..77b58b7e6e 100644 --- a/test/WebJobs.Script.Tests/Workers/Rpc/RpcWorkerConfigFactoryTests.cs +++ b/test/WebJobs.Script.Tests/Workers/Rpc/RpcWorkerConfigFactoryTests.cs @@ -180,10 +180,6 @@ public void DefaultWorkerConfigs_Overrides_DefaultWorkerRuntimeVersion_AppSettin [InlineData(null, "7.4", null, null)] [InlineData("7.4", "7.4", "..\\..\\..\\..\\test\\TestWorkers\\ProbingPaths\\workers\\", ScriptConstants.FeatureFlagEnableDynamicWorkerResolution)] [InlineData(null, "7.4", "..\\..\\..\\..\\test\\TestWorkers\\ProbingPaths\\workers\\", ScriptConstants.FeatureFlagEnableDynamicWorkerResolution)] - [InlineData("7.2", "7.2", null, null)] - [InlineData("7.2", "7.2", "..\\..\\..\\..\\test\\TestWorkers\\ProbingPaths\\workers\\", ScriptConstants.FeatureFlagEnableDynamicWorkerResolution)] - [InlineData("7", "7", null, null)] - [InlineData("7", "7", "..\\..\\..\\..\\test\\TestWorkers\\ProbingPaths\\workers\\", ScriptConstants.FeatureFlagEnableDynamicWorkerResolution)] public void DefaultWorkerConfigs_Overrides_VersionAppSetting(string runtimeSettingVersion, string outputVersion, string probingPathValue, string enableProbingPath) { var probingPath = string.IsNullOrEmpty(probingPathValue) ? probingPathValue : Path.GetFullPath(probingPathValue); diff --git a/test/WebJobs.Script.Tests/Workers/Rpc/WorkerConfigurationResolverTests.cs b/test/WebJobs.Script.Tests/Workers/Rpc/WorkerConfigurationResolverTests.cs index fe4ee8ca59..d564b485a6 100644 --- a/test/WebJobs.Script.Tests/Workers/Rpc/WorkerConfigurationResolverTests.cs +++ b/test/WebJobs.Script.Tests/Workers/Rpc/WorkerConfigurationResolverTests.cs @@ -33,7 +33,7 @@ public WorkerConfigurationResolverTests() [Theory] [InlineData("LATEST", "java\\2.19.0", "node\\3.10.1", "powershell\\7.4", "dotnet-isolated", "python")] - [InlineData("STANDARD", "java\\2.18.0", "node\\3.10.1", "powershell\\7.2", "dotnet-isolated", "python")] + [InlineData("STANDARD", "java\\2.18.0", "node\\3.10.1", "powershell\\7.4", "dotnet-isolated", "python")] public void GetWorkerConfigs_MultiLanguageWorker_ReturnsExpectedConfigs(string releaseChannel, string java, string node, string powershell, string dotnetIsolated, string python) { // Arrange @@ -55,30 +55,6 @@ public void GetWorkerConfigs_MultiLanguageWorker_ReturnsExpectedConfigs(string r Assert.True(result.Any(r => r.Contains(Path.Combine(_fallbackPath, python)))); } - [Theory] - [InlineData("LATEST", "java\\2.19.0", "node\\3.10.1", "powershell\\7.4", "dotnet-isolated\\1.0.0", "python")] - [InlineData("STANDARD", "java\\2.18.0", "node\\3.10.1", "powershell\\7.2", "dotnet-isolated\\1.0.0", "python")] - public void GetWorkerConfigs_MultiLanguageWorker_Dotnet_ReturnsExpectedConfigs(string releaseChannel, string java, string node, string powershell, string dotnetIsolated, string python) - { - // Arrange - var mockEnvironment = new Mock(); - mockEnvironment.Setup(p => p.GetEnvironmentVariable(EnvironmentSettingNames.AntaresPlatformReleaseChannel)).Returns(releaseChannel); - mockEnvironment.Setup(p => p.GetEnvironmentVariable(EnvironmentSettingNames.AppKind)).Returns(ScriptConstants.WorkFlowAppKind); - mockEnvironment.Setup(p => p.GetEnvironmentVariable(EnvironmentSettingNames.FunctionWorkerRuntime)).Returns((string)null); - - var workerConfigurationResolver = new WorkerConfigurationResolver(_mockConfig.Object, _mockLogger.Object, mockEnvironment.Object, _mockProfileManager.Object, new HashSet() { "java", "node", "powershell", "dotnet-isolated" }); - - var result = workerConfigurationResolver.GetWorkerConfigs(_probingPaths, _fallbackPath); - - // Assert - Assert.Equal(result.Count, 5); - Assert.True(result.Any(r => r.Contains(Path.Combine(_probingPath1, java)))); - Assert.True(result.Any(r => r.Contains(Path.Combine(_probingPath1, node)))); - Assert.True(result.Any(r => r.Contains(Path.Combine(_probingPath1, powershell)))); - Assert.True(result.Any(r => r.Contains(Path.Combine(_probingPath1, dotnetIsolated)))); - Assert.True(result.Any(r => r.Contains(Path.Combine(_fallbackPath, python)))); - } - [Theory] [InlineData(null, "LATEST")] [InlineData(null, "STANDARD")] @@ -129,10 +105,9 @@ public void GetWorkerConfigs_MultiLanguageWorker_NullOREmptyProbingPath_ReturnsE public void GetWorkerConfigs_NullOREmptyProbingPath_ReturnsExpectedConfigs(string probingPathValue, string releaseChannel, string languageWorker) { // Arrange - var mockEnvironment = new Mock(); - mockEnvironment.Setup(p => p.GetEnvironmentVariable(EnvironmentSettingNames.FunctionWorkerRuntime)).Returns(languageWorker); - mockEnvironment.Setup(p => p.GetEnvironmentVariable(EnvironmentSettingNames.AntaresPlatformReleaseChannel)).Returns(releaseChannel); - mockEnvironment.Setup(p => p.GetEnvironmentVariable(EnvironmentSettingNames.AppKind)).Returns((string)null); + var mockEnv = new Mock(); + mockEnv.Setup(p => p.GetEnvironmentVariable(EnvironmentSettingNames.FunctionWorkerRuntime)).Returns(languageWorker); + mockEnv.Setup(p => p.GetEnvironmentVariable(EnvironmentSettingNames.AntaresPlatformReleaseChannel)).Returns(releaseChannel); List probingPaths = null; @@ -141,8 +116,12 @@ public void GetWorkerConfigs_NullOREmptyProbingPath_ReturnsExpectedConfigs(strin probingPaths = new List(); } + var mockProfileManager = new Mock(); + var mockConfig = new Mock(); + var mockLogger = new Mock(); + // Act - var workerConfigurationResolver = new WorkerConfigurationResolver(_mockConfig.Object, _mockLogger.Object, mockEnvironment.Object, _mockProfileManager.Object, new HashSet() { "java", "node", "powershell" }); + var workerConfigurationResolver = new WorkerConfigurationResolver(mockConfig.Object, mockLogger.Object, mockEnv.Object, mockProfileManager.Object, new HashSet() { "java", "node", "powershell" }); var result = workerConfigurationResolver.GetWorkerConfigs(probingPaths, _fallbackPath); From d67a25489a0d2568781809fa58195e908d14cdf0 Mon Sep 17 00:00:00 2001 From: Surbhi Gupta Date: Fri, 13 Jun 2025 00:01:34 -0500 Subject: [PATCH 22/55] logging update --- .../Workers/Rpc/Configuration/LanguageWorkerOptionsSetup.cs | 2 +- .../Workers/Rpc/Configuration/RpcWorkerConfigFactory.cs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/WebJobs.Script/Workers/Rpc/Configuration/LanguageWorkerOptionsSetup.cs b/src/WebJobs.Script/Workers/Rpc/Configuration/LanguageWorkerOptionsSetup.cs index bf3738ed1e..48ab161899 100644 --- a/src/WebJobs.Script/Workers/Rpc/Configuration/LanguageWorkerOptionsSetup.cs +++ b/src/WebJobs.Script/Workers/Rpc/Configuration/LanguageWorkerOptionsSetup.cs @@ -21,7 +21,7 @@ namespace Microsoft.Azure.WebJobs.Script.Workers.Rpc { internal class LanguageWorkerOptionsSetup : IConfigureOptions { - private const string _windowsWorkerProbingPath = "c:\\home\\SiteExtensions\\workers"; // Harcoded site extensions path for Windows until Antares sets it as an Environment variable. + private const string _windowsWorkerProbingPath = "C:\\home\\SiteExtensions\\workers"; // Harcoded site extensions path for Windows until Antares sets it as an Environment variable. private readonly IConfiguration _configuration; private readonly ILogger _logger; private readonly IEnvironment _environment; diff --git a/src/WebJobs.Script/Workers/Rpc/Configuration/RpcWorkerConfigFactory.cs b/src/WebJobs.Script/Workers/Rpc/Configuration/RpcWorkerConfigFactory.cs index e3029a5a46..a23ae599f1 100644 --- a/src/WebJobs.Script/Workers/Rpc/Configuration/RpcWorkerConfigFactory.cs +++ b/src/WebJobs.Script/Workers/Rpc/Configuration/RpcWorkerConfigFactory.cs @@ -107,7 +107,7 @@ internal void AddProviders() { if (DynamicWorkerResolutionEnabled && WorkerProbingPaths is not null) { - _logger.LogDebug("Probing paths set to: {probingPaths} and Workers Directory set as fallback path: {WorkersDirPath}", string.Join(", ", WorkerProbingPaths), WorkersDirPath); + _logger.LogDebug("Worker probing paths set to: {probingPaths} and Workers Directory set as fallback path: {WorkersDirPath}", string.Join(", ", WorkerProbingPaths), WorkersDirPath); List workerConfigs = _workerConfigurationResolver.GetWorkerConfigs(WorkerProbingPaths, WorkersDirPath); From 584405340947b42ea333331d379b86a4a8031e19 Mon Sep 17 00:00:00 2001 From: Surbhi Gupta Date: Fri, 13 Jun 2025 00:03:23 -0500 Subject: [PATCH 23/55] Logging update --- .../Workers/Rpc/Configuration/RpcWorkerConfigFactory.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/WebJobs.Script/Workers/Rpc/Configuration/RpcWorkerConfigFactory.cs b/src/WebJobs.Script/Workers/Rpc/Configuration/RpcWorkerConfigFactory.cs index a23ae599f1..162f08de1e 100644 --- a/src/WebJobs.Script/Workers/Rpc/Configuration/RpcWorkerConfigFactory.cs +++ b/src/WebJobs.Script/Workers/Rpc/Configuration/RpcWorkerConfigFactory.cs @@ -107,7 +107,7 @@ internal void AddProviders() { if (DynamicWorkerResolutionEnabled && WorkerProbingPaths is not null) { - _logger.LogDebug("Worker probing paths set to: {probingPaths} and Workers Directory set as fallback path: {WorkersDirPath}", string.Join(", ", WorkerProbingPaths), WorkersDirPath); + _logger.LogDebug("Workers probing paths set to: {probingPaths} and Workers Directory set as fallback path: {WorkersDirPath}", string.Join(", ", WorkerProbingPaths), WorkersDirPath); List workerConfigs = _workerConfigurationResolver.GetWorkerConfigs(WorkerProbingPaths, WorkersDirPath); From 3bf3f794e43c313b3712df1a745bb53a6ea3b60d Mon Sep 17 00:00:00 2001 From: Surbhi Gupta Date: Fri, 13 Jun 2025 23:24:09 -0500 Subject: [PATCH 24/55] Placeholder mode check --- .../Workers/Rpc/Configuration/WorkerConfigurationResolver.cs | 1 + 1 file changed, 1 insertion(+) diff --git a/src/WebJobs.Script/Workers/Rpc/Configuration/WorkerConfigurationResolver.cs b/src/WebJobs.Script/Workers/Rpc/Configuration/WorkerConfigurationResolver.cs index a680ab8bfb..f8538f3ebe 100644 --- a/src/WebJobs.Script/Workers/Rpc/Configuration/WorkerConfigurationResolver.cs +++ b/src/WebJobs.Script/Workers/Rpc/Configuration/WorkerConfigurationResolver.cs @@ -68,6 +68,7 @@ public List GetWorkerConfigs(List probingPaths, string fallbackP if ((_workersAvailableForResolutionViaHostingConfig is not null && !_workersAvailableForResolutionViaHostingConfig.Contains(languageWorkerDir)) || (!_environment.IsMultiLanguageRuntimeEnvironment() && + !_environment.IsPlaceholderModeEnabled() && workerRuntime is not null && !workerRuntime.Equals(languageWorkerDir, StringComparison.OrdinalIgnoreCase))) { From 1171fe805b1b8e94c14f2950cdc2100d9b0dab24 Mon Sep 17 00:00:00 2001 From: Surbhi Gupta Date: Sun, 15 Jun 2025 01:56:06 -0500 Subject: [PATCH 25/55] Handling DefaultWorkerPath update scenario --- .../Workers/Http/HttpWorkerDescription.cs | 2 +- .../ProcessManagement/WorkerDescription.cs | 2 +- .../LanguageWorkerOptionsSetup.cs | 2 +- .../Configuration/RpcWorkerConfigFactory.cs | 28 ++++++++++--------- .../WorkerConfigurationHelper.cs | 7 +++-- .../WorkerConfigurationResolver.cs | 5 ++-- .../Workers/Rpc/RpcWorkerDescription.cs | 6 ++-- 7 files changed, 29 insertions(+), 23 deletions(-) diff --git a/src/WebJobs.Script/Workers/Http/HttpWorkerDescription.cs b/src/WebJobs.Script/Workers/Http/HttpWorkerDescription.cs index db59eea2db..15ef28beb3 100644 --- a/src/WebJobs.Script/Workers/Http/HttpWorkerDescription.cs +++ b/src/WebJobs.Script/Workers/Http/HttpWorkerDescription.cs @@ -19,7 +19,7 @@ public class HttpWorkerDescription : WorkerDescription public override bool UseStdErrorStreamForErrorsOnly { get; set; } = true; - public override void ApplyDefaultsAndValidate(string inputWorkerDirectory, ILogger logger, bool probingPathEnabled = false) + public override void ApplyDefaultsAndValidate(string inputWorkerDirectory, ILogger logger, bool probingPathEnabled = false, HashSet workersAvailableForResolutionViaHostingConfig = null) { if (inputWorkerDirectory == null) { diff --git a/src/WebJobs.Script/Workers/ProcessManagement/WorkerDescription.cs b/src/WebJobs.Script/Workers/ProcessManagement/WorkerDescription.cs index abae2c9278..6f8f438955 100644 --- a/src/WebJobs.Script/Workers/ProcessManagement/WorkerDescription.cs +++ b/src/WebJobs.Script/Workers/ProcessManagement/WorkerDescription.cs @@ -57,7 +57,7 @@ public abstract class WorkerDescription /// public bool? IsDisabled { get; set; } - public abstract void ApplyDefaultsAndValidate(string workerDirectory, ILogger logger, bool probingPathEnabled = false); + public abstract void ApplyDefaultsAndValidate(string workerDirectory, ILogger logger, bool probingPathEnabled = false, HashSet workersAvailableForResolutionViaHostingConfig = null); internal void ThrowIfFileNotExists(string inputFile, string paramName) { diff --git a/src/WebJobs.Script/Workers/Rpc/Configuration/LanguageWorkerOptionsSetup.cs b/src/WebJobs.Script/Workers/Rpc/Configuration/LanguageWorkerOptionsSetup.cs index 48ab161899..046884c169 100644 --- a/src/WebJobs.Script/Workers/Rpc/Configuration/LanguageWorkerOptionsSetup.cs +++ b/src/WebJobs.Script/Workers/Rpc/Configuration/LanguageWorkerOptionsSetup.cs @@ -92,7 +92,7 @@ public void Configure(LanguageWorkerOptions options) } var workerConfigurationResolver = new WorkerConfigurationResolver(configuration, _logger, _environment, _workerProfileManager, workers); - var configFactory = new RpcWorkerConfigFactory(configuration, _logger, SystemRuntimeInformation.Instance, _environment, _metricsLogger, _workerProfileManager, workerConfigurationResolver, dynamicWorkerResolutionEnabled, probingPaths); + var configFactory = new RpcWorkerConfigFactory(configuration, _logger, SystemRuntimeInformation.Instance, _environment, _metricsLogger, _workerProfileManager, workerConfigurationResolver, dynamicWorkerResolutionEnabled, probingPaths, workers); options.WorkerConfigs = configFactory.GetConfigs(); } diff --git a/src/WebJobs.Script/Workers/Rpc/Configuration/RpcWorkerConfigFactory.cs b/src/WebJobs.Script/Workers/Rpc/Configuration/RpcWorkerConfigFactory.cs index 162f08de1e..5c8c10b83e 100644 --- a/src/WebJobs.Script/Workers/Rpc/Configuration/RpcWorkerConfigFactory.cs +++ b/src/WebJobs.Script/Workers/Rpc/Configuration/RpcWorkerConfigFactory.cs @@ -30,12 +30,15 @@ internal class RpcWorkerConfigFactory private readonly string _workerRuntime; private readonly IEnvironment _environment; private readonly IWorkerConfigurationResolver _workerConfigurationResolver; + private readonly HashSet _workersAvailableForResolutionViaHostingConfig; + private readonly bool _dynamicWorkerResolutionEnabled; private readonly JsonSerializerOptions _jsonSerializerOptions = new() { PropertyNameCaseInsensitive = true }; private Dictionary _workerDescriptionDictionary = new Dictionary(); + private List _workerProbingPaths; public RpcWorkerConfigFactory(IConfiguration config, ILogger logger, @@ -45,7 +48,8 @@ public RpcWorkerConfigFactory(IConfiguration config, IWorkerProfileManager workerProfileManager, IWorkerConfigurationResolver workerConfigurationResolver, bool dynamicWorkerResolutionEnabled = false, - List workerProbingPaths = null) + List workerProbingPaths = null, + HashSet workersAvailableForResolutionViaHostingConfig = null) { _config = config ?? throw new ArgumentNullException(nameof(config)); _logger = logger ?? throw new ArgumentNullException(nameof(logger)); @@ -59,8 +63,9 @@ public RpcWorkerConfigFactory(IConfiguration config, WorkersDirPath = GetDefaultWorkersDirectory(Directory.Exists); var workersDirectorySection = _config.GetSection($"{RpcWorkerConstants.LanguageWorkersSectionName}:{WorkerConstants.WorkersDirectorySectionName}"); - DynamicWorkerResolutionEnabled = dynamicWorkerResolutionEnabled; - WorkerProbingPaths = workerProbingPaths; + _workerProbingPaths = workerProbingPaths; + _dynamicWorkerResolutionEnabled = dynamicWorkerResolutionEnabled; + _workersAvailableForResolutionViaHostingConfig = workersAvailableForResolutionViaHostingConfig; if (!string.IsNullOrEmpty(workersDirectorySection.Value)) { @@ -70,10 +75,6 @@ public RpcWorkerConfigFactory(IConfiguration config, public string WorkersDirPath { get; } - internal bool DynamicWorkerResolutionEnabled { get; } - - internal List WorkerProbingPaths { get; } - public IList GetConfigs() { using (_metricsLogger.LatencyEvent(MetricEventNames.GetConfigs)) @@ -105,11 +106,11 @@ internal void BuildWorkerProviderDictionary() internal void AddProviders() { - if (DynamicWorkerResolutionEnabled && WorkerProbingPaths is not null) + if (_dynamicWorkerResolutionEnabled) { - _logger.LogDebug("Workers probing paths set to: {probingPaths} and Workers Directory set as fallback path: {WorkersDirPath}", string.Join(", ", WorkerProbingPaths), WorkersDirPath); + _logger.LogDebug("Workers probing paths set to: {probingPaths} and Workers Directory set as fallback path: {WorkersDirPath}", string.Join(", ", _workerProbingPaths), WorkersDirPath); - List workerConfigs = _workerConfigurationResolver.GetWorkerConfigs(WorkerProbingPaths, WorkersDirPath); + List workerConfigs = _workerConfigurationResolver.GetWorkerConfigs(_workerProbingPaths, WorkersDirPath); foreach (var workerConfig in workerConfigs) { @@ -155,7 +156,7 @@ internal void AddProvider(string workerDir) if (!string.IsNullOrWhiteSpace(_workerRuntime) && !_environment.IsPlaceholderModeEnabled() && !_environment.IsMultiLanguageRuntimeEnvironment() && - !DynamicWorkerResolutionEnabled) + !_dynamicWorkerResolutionEnabled) { string workerRuntime = Path.GetFileName(workerDir); // Only skip worker directories that don't match the current runtime. @@ -182,10 +183,11 @@ internal void AddProvider(string workerDir) workerConfig, _jsonSerializerOptions, workerDir, - DynamicWorkerResolutionEnabled, _profileManager, _config, - _logger); + _logger, + _dynamicWorkerResolutionEnabled, + _workersAvailableForResolutionViaHostingConfig); if (workerDescription.IsDisabled == true) { diff --git a/src/WebJobs.Script/Workers/Rpc/Configuration/WorkerConfigurationHelper.cs b/src/WebJobs.Script/Workers/Rpc/Configuration/WorkerConfigurationHelper.cs index ae8d88b16a..63fb217ad4 100644 --- a/src/WebJobs.Script/Workers/Rpc/Configuration/WorkerConfigurationHelper.cs +++ b/src/WebJobs.Script/Workers/Rpc/Configuration/WorkerConfigurationHelper.cs @@ -18,10 +18,11 @@ internal static RpcWorkerDescription GetWorkerDescription( JsonElement workerConfig, JsonSerializerOptions jsonSerializerOptions, string workerDir, - bool dynamicWorkerResolutionEnabled, IWorkerProfileManager profileManager, IConfiguration config, - ILogger logger) + ILogger logger, + bool dynamicWorkerResolutionEnabled, + HashSet workersAvailableForResolutionViaHostingConfig) { var workerDescriptionElement = workerConfig.GetProperty(WorkerConstants.WorkerDescription); var workerDescription = workerDescriptionElement.Deserialize(jsonSerializerOptions); @@ -45,7 +46,7 @@ internal static RpcWorkerDescription GetWorkerDescription( AddArgumentsFromAppSettings(workerDescription, languageSection); // Validate workerDescription - workerDescription.ApplyDefaultsAndValidate(Directory.GetCurrentDirectory(), logger, dynamicWorkerResolutionEnabled); + workerDescription.ApplyDefaultsAndValidate(Directory.GetCurrentDirectory(), logger, dynamicWorkerResolutionEnabled, workersAvailableForResolutionViaHostingConfig); return workerDescription; } diff --git a/src/WebJobs.Script/Workers/Rpc/Configuration/WorkerConfigurationResolver.cs b/src/WebJobs.Script/Workers/Rpc/Configuration/WorkerConfigurationResolver.cs index f8538f3ebe..2363eb0944 100644 --- a/src/WebJobs.Script/Workers/Rpc/Configuration/WorkerConfigurationResolver.cs +++ b/src/WebJobs.Script/Workers/Rpc/Configuration/WorkerConfigurationResolver.cs @@ -213,10 +213,11 @@ private bool IsWorkerCompatibleWithHost(string workerDir) workerConfig, _jsonSerializerOptions, workerDir, - true, _profileManager, _config, - _logger); + _logger, + true, + _workersAvailableForResolutionViaHostingConfig); if (workerDescription.IsDisabled == true) { diff --git a/src/WebJobs.Script/Workers/Rpc/RpcWorkerDescription.cs b/src/WebJobs.Script/Workers/Rpc/RpcWorkerDescription.cs index e4088e8787..6e189ba634 100644 --- a/src/WebJobs.Script/Workers/Rpc/RpcWorkerDescription.cs +++ b/src/WebJobs.Script/Workers/Rpc/RpcWorkerDescription.cs @@ -87,7 +87,7 @@ public List Extensions public override bool UseStdErrorStreamForErrorsOnly { get; set; } = false; - public override void ApplyDefaultsAndValidate(string workerDirectory, ILogger logger, bool dynamicWorkerResolutionEnabled = false) + public override void ApplyDefaultsAndValidate(string workerDirectory, ILogger logger, bool dynamicWorkerResolutionEnabled = false, HashSet workersAvailableForResolutionViaHostingConfig = null) { if (workerDirectory == null) { @@ -100,7 +100,9 @@ public override void ApplyDefaultsAndValidate(string workerDirectory, ILogger lo { // If dynamic worker resolution enabled and DefaultWorkerPath contains FunctionWorkerRuntimeVersionSettingName // then version becomes redundant in the path. Replacing version with an empty string to avoid duplication. - if (dynamicWorkerResolutionEnabled && DefaultWorkerPath.Contains(RpcWorkerConstants.RuntimeVersionPlaceholder)) + if (dynamicWorkerResolutionEnabled && + workersAvailableForResolutionViaHostingConfig.Contains(Language) && + DefaultWorkerPath.Contains(RpcWorkerConstants.RuntimeVersionPlaceholder)) { var versionDir = Path.GetFileName(WorkerDirectory); WorkerDirectory = WorkerDirectory.Replace(versionDir, string.Empty); From 3bdc8e1eafb3e5112277d1d37aaa5c8d23c1a75f Mon Sep 17 00:00:00 2001 From: Surbhi Gupta Date: Mon, 16 Jun 2025 22:23:44 -0500 Subject: [PATCH 26/55] Tests update after logs changes --- .../Configuration/LanguageWorkerOptionsSetupTests.cs | 4 ++-- .../Workers/Rpc/RpcWorkerConfigFactoryTests.cs | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/test/WebJobs.Script.Tests/Configuration/LanguageWorkerOptionsSetupTests.cs b/test/WebJobs.Script.Tests/Configuration/LanguageWorkerOptionsSetupTests.cs index fb1a940487..4f7f3daaf8 100644 --- a/test/WebJobs.Script.Tests/Configuration/LanguageWorkerOptionsSetupTests.cs +++ b/test/WebJobs.Script.Tests/Configuration/LanguageWorkerOptionsSetupTests.cs @@ -122,7 +122,7 @@ public void LanguageWorkerOptions_ProbingPaths_Expected_ListOfConfigs(string wor string path = Path.Combine(_probingPath1, workerRuntime, expectedVersion); string expectedLog = $"Added WorkerConfig for language: {workerRuntime} with DefaultWorkerPath: {path}"; Assert.True(logs.Any(l => l.FormattedMessage.Contains(expectedLog))); - Assert.True(logs.Any(l => l.FormattedMessage.Contains("Probing paths set to:"))); + Assert.True(logs.Any(l => l.FormattedMessage.Contains("Workers probing paths set to:"))); } [Theory] @@ -165,7 +165,7 @@ public void LanguageWorkerOptions_FallbackPath_Expected_ListOfConfigs(string wor string path = Path.Combine(_fallbackPath, workerRuntime); string expectedLog = $"Added WorkerConfig for language: {workerRuntime} with DefaultWorkerPath: {path}"; Assert.True(logs.Any(l => l.FormattedMessage.Contains(expectedLog))); - Assert.True(logs.Any(l => l.FormattedMessage.Contains("Probing paths set to:"))); + Assert.True(logs.Any(l => l.FormattedMessage.Contains("Workers probing paths set to:"))); } [Theory] diff --git a/test/WebJobs.Script.Tests/Workers/Rpc/RpcWorkerConfigFactoryTests.cs b/test/WebJobs.Script.Tests/Workers/Rpc/RpcWorkerConfigFactoryTests.cs index 77b58b7e6e..e453589bed 100644 --- a/test/WebJobs.Script.Tests/Workers/Rpc/RpcWorkerConfigFactoryTests.cs +++ b/test/WebJobs.Script.Tests/Workers/Rpc/RpcWorkerConfigFactoryTests.cs @@ -192,7 +192,7 @@ public void DefaultWorkerConfigs_Overrides_VersionAppSetting(string runtimeSetti var testLogger = new TestLogger("test"); var workerConfigurationResolver = new WorkerConfigurationResolver(config, testLogger, testEnvironment, _testWorkerProfileManager, new HashSet() { "powershell" }); - var configFactory = new RpcWorkerConfigFactory(config, testLogger, _testSysRuntimeInfo, testEnvironment, new TestMetricsLogger(), _testWorkerProfileManager, workerConfigurationResolver, enableProbingPath is null ? false : true, new List() { probingPath }); + var configFactory = new RpcWorkerConfigFactory(config, testLogger, _testSysRuntimeInfo, testEnvironment, new TestMetricsLogger(), _testWorkerProfileManager, workerConfigurationResolver, enableProbingPath is null ? false : true, new List() { probingPath }, new HashSet() { "powershell" }); var workerConfigs = configFactory.GetConfigs(); var powershellWorkerConfig = workerConfigs.FirstOrDefault(w => w.Description.Language.Equals("powershell", StringComparison.OrdinalIgnoreCase)); From ed80fd19fda96cc5f9e8090888c192c32ecde3b8 Mon Sep 17 00:00:00 2001 From: Surbhi Gupta Date: Sun, 22 Jun 2025 17:10:30 -0500 Subject: [PATCH 27/55] Logs and Tests updates --- .../Configuration/RpcWorkerConfigFactory.cs | 2 +- .../WorkerConfigurationResolver.cs | 10 ++- .../Workers/Rpc/RpcWorkerDescription.cs | 1 + .../LanguageWorkerOptionsSetupTests.cs | 76 +++++++++++++++---- .../Rpc/WorkerConfigurationResolverTests.cs | 1 + 5 files changed, 71 insertions(+), 19 deletions(-) diff --git a/src/WebJobs.Script/Workers/Rpc/Configuration/RpcWorkerConfigFactory.cs b/src/WebJobs.Script/Workers/Rpc/Configuration/RpcWorkerConfigFactory.cs index ede4ea9a9c..2c80c1e144 100644 --- a/src/WebJobs.Script/Workers/Rpc/Configuration/RpcWorkerConfigFactory.cs +++ b/src/WebJobs.Script/Workers/Rpc/Configuration/RpcWorkerConfigFactory.cs @@ -223,7 +223,7 @@ internal void AddProvider(string workerDir) _workerDescriptionDictionary[workerDescription.Language] = rpcWorkerConfig; ReadLanguageWorkerFile(arguments.WorkerPath); - _logger.LogDebug("Added WorkerConfig for language: {language} with DefaultWorkerPath: {path}", workerDescription.Language, workerDescription.DefaultWorkerPath); + _logger.LogDebug("Added WorkerConfig for language: {language} with Worker Path: {path}", workerDescription.Language, workerDescription.DefaultWorkerPath); } } catch (Exception ex) when (!ex.IsFatal()) diff --git a/src/WebJobs.Script/Workers/Rpc/Configuration/WorkerConfigurationResolver.cs b/src/WebJobs.Script/Workers/Rpc/Configuration/WorkerConfigurationResolver.cs index 2363eb0944..32d17744bf 100644 --- a/src/WebJobs.Script/Workers/Rpc/Configuration/WorkerConfigurationResolver.cs +++ b/src/WebJobs.Script/Workers/Rpc/Configuration/WorkerConfigurationResolver.cs @@ -92,6 +92,8 @@ workerRuntime is not null && return outputDict.Values.ToList(); } + _logger.LogDebug("Searching for worker configs in the fallback directory."); + // Search in fallback path if worker cannot be found in probing paths GetWorkerConfigsFromWithinHost(fallbackPath, workerRuntime, outputDict); @@ -112,13 +114,17 @@ private void GetWorkerConfigsFromProbingPaths(IEnumerable versions, str found++; outputDict[languageWorkerFolder] = languageWorkerVersionPath; - if (string.IsNullOrEmpty(releaseChannel) || !releaseChannel.Equals(ScriptConstants.StandardPlatformChannelNameUpper)) + if (string.IsNullOrEmpty(releaseChannel) || + !(releaseChannel.Equals(ScriptConstants.StandardPlatformChannelNameUpper) || + releaseChannel.Equals(ScriptConstants.ExtendedPlatformChannelNameUpper))) { // latest version is the default break; } - if (found > 1 && releaseChannel.Equals(ScriptConstants.StandardPlatformChannelNameUpper)) + if (found > 1 && + (releaseChannel.Equals(ScriptConstants.StandardPlatformChannelNameUpper) || + releaseChannel.Equals(ScriptConstants.ExtendedPlatformChannelNameUpper))) { outputDict[languageWorkerFolder] = languageWorkerVersionPath; break; diff --git a/src/WebJobs.Script/Workers/Rpc/RpcWorkerDescription.cs b/src/WebJobs.Script/Workers/Rpc/RpcWorkerDescription.cs index 6e189ba634..161d32b083 100644 --- a/src/WebJobs.Script/Workers/Rpc/RpcWorkerDescription.cs +++ b/src/WebJobs.Script/Workers/Rpc/RpcWorkerDescription.cs @@ -101,6 +101,7 @@ public override void ApplyDefaultsAndValidate(string workerDirectory, ILogger lo // If dynamic worker resolution enabled and DefaultWorkerPath contains FunctionWorkerRuntimeVersionSettingName // then version becomes redundant in the path. Replacing version with an empty string to avoid duplication. if (dynamicWorkerResolutionEnabled && + workersAvailableForResolutionViaHostingConfig is not null && workersAvailableForResolutionViaHostingConfig.Contains(Language) && DefaultWorkerPath.Contains(RpcWorkerConstants.RuntimeVersionPlaceholder)) { diff --git a/test/WebJobs.Script.Tests/Configuration/LanguageWorkerOptionsSetupTests.cs b/test/WebJobs.Script.Tests/Configuration/LanguageWorkerOptionsSetupTests.cs index 4f7f3daaf8..db0597da9a 100644 --- a/test/WebJobs.Script.Tests/Configuration/LanguageWorkerOptionsSetupTests.cs +++ b/test/WebJobs.Script.Tests/Configuration/LanguageWorkerOptionsSetupTests.cs @@ -82,11 +82,13 @@ public void LanguageWorkerOptions_Expected_ListOfConfigs(string workerRuntime) } [Theory] - [InlineData("java", ScriptConstants.FeatureFlagEnableDynamicWorkerResolution, "java", "LATEST", "2.19.0")] - [InlineData("java", ScriptConstants.FeatureFlagEnableDynamicWorkerResolution, "java", "STANDARD", "2.18.0")] - [InlineData("node", ScriptConstants.FeatureFlagEnableDynamicWorkerResolution, "node", "LATEST", "3.10.1")] - [InlineData("node", ScriptConstants.FeatureFlagEnableDynamicWorkerResolution, "java|node", "STANDARD", "3.10.1")] - public void LanguageWorkerOptions_ProbingPaths_Expected_ListOfConfigs(string workerRuntime, string enableProbingPaths, string hostingOptionsSetting, string releaseChannel, string expectedVersion) + [InlineData("java", "java", "LATEST", "2.19.0")] + [InlineData("java", "java", "STANDARD", "2.18.0")] + [InlineData("node", "node", "LATEST", "3.10.1")] + [InlineData("node", "java|node", "STANDARD", "3.10.1")] + [InlineData("java", "java", "EXTENDED", "2.18.0")] + [InlineData("node", "java|node", "EXTENDED", "3.10.1")] + public void LanguageWorkerOptions_EnabledWorkerResolution_Expected_ListOfConfigs(string workerRuntime, string hostingOptionsSetting, string releaseChannel, string expectedVersion) { var loggerProvider = new TestLoggerProvider(); var loggerFactory = new LoggerFactory(); @@ -102,7 +104,7 @@ public void LanguageWorkerOptions_ProbingPaths_Expected_ListOfConfigs(string wor string probingPathValue = string.Join(';', _probingPath1, string.Empty, "path-not-exists"); testEnvironment.SetEnvironmentVariable(RpcWorkerConstants.FunctionWorkerRuntimeSettingName, workerRuntime); - testEnvironment.SetEnvironmentVariable(EnvironmentSettingNames.AzureWebJobsFeatureFlags, enableProbingPaths); + testEnvironment.SetEnvironmentVariable(EnvironmentSettingNames.AzureWebJobsFeatureFlags, ScriptConstants.FeatureFlagEnableDynamicWorkerResolution); testEnvironment.SetEnvironmentVariable(EnvironmentSettingNames.AntaresPlatformReleaseChannel, releaseChannel); testEnvironment.SetEnvironmentVariable(EnvironmentSettingNames.WorkerProbingPaths, probingPathValue); @@ -120,17 +122,17 @@ public void LanguageWorkerOptions_ProbingPaths_Expected_ListOfConfigs(string wor var logs = loggerProvider.GetAllLogMessages(); string path = Path.Combine(_probingPath1, workerRuntime, expectedVersion); - string expectedLog = $"Added WorkerConfig for language: {workerRuntime} with DefaultWorkerPath: {path}"; + string expectedLog = $"Added WorkerConfig for language: {workerRuntime} with Worker Path: {path}"; Assert.True(logs.Any(l => l.FormattedMessage.Contains(expectedLog))); Assert.True(logs.Any(l => l.FormattedMessage.Contains("Workers probing paths set to:"))); } [Theory] - [InlineData("java", ScriptConstants.FeatureFlagEnableDynamicWorkerResolution, "java", "LATEST")] - [InlineData("java", ScriptConstants.FeatureFlagEnableDynamicWorkerResolution, "java", "STANDARD")] - [InlineData("node", ScriptConstants.FeatureFlagEnableDynamicWorkerResolution, "node", "LATEST")] - [InlineData("node", ScriptConstants.FeatureFlagEnableDynamicWorkerResolution, "java|node", "STANDARD")] - public void LanguageWorkerOptions_FallbackPath_Expected_ListOfConfigs(string workerRuntime, string enableProbingPaths, string hostingOptionsSetting, string releaseChannel) + [InlineData("java", "java", "LATEST")] + [InlineData("java", "java", "STANDARD")] + [InlineData("node", "node", "LATEST")] + [InlineData("node", "java|node", "STANDARD")] + public void LanguageWorkerOptions_FallbackPath_Expected_ListOfConfigs(string workerRuntime, string hostingOptionsSetting, string releaseChannel) { var loggerProvider = new TestLoggerProvider(); var loggerFactory = new LoggerFactory(); @@ -146,7 +148,7 @@ public void LanguageWorkerOptions_FallbackPath_Expected_ListOfConfigs(string wor string probingPathValue = null; testEnvironment.SetEnvironmentVariable(RpcWorkerConstants.FunctionWorkerRuntimeSettingName, workerRuntime); - testEnvironment.SetEnvironmentVariable(EnvironmentSettingNames.AzureWebJobsFeatureFlags, enableProbingPaths); + testEnvironment.SetEnvironmentVariable(EnvironmentSettingNames.AzureWebJobsFeatureFlags, ScriptConstants.FeatureFlagEnableDynamicWorkerResolution); testEnvironment.SetEnvironmentVariable(EnvironmentSettingNames.AntaresPlatformReleaseChannel, releaseChannel); testEnvironment.SetEnvironmentVariable(EnvironmentSettingNames.WorkerProbingPaths, probingPathValue); @@ -163,9 +165,10 @@ public void LanguageWorkerOptions_FallbackPath_Expected_ListOfConfigs(string wor var logs = loggerProvider.GetAllLogMessages(); string path = Path.Combine(_fallbackPath, workerRuntime); - string expectedLog = $"Added WorkerConfig for language: {workerRuntime} with DefaultWorkerPath: {path}"; + string expectedLog = $"Added WorkerConfig for language: {workerRuntime} with Worker Path: {path}"; Assert.True(logs.Any(l => l.FormattedMessage.Contains(expectedLog))); Assert.True(logs.Any(l => l.FormattedMessage.Contains("Workers probing paths set to:"))); + Assert.True(logs.Any(l => l.FormattedMessage.Contains("Searching for worker configs in the fallback directory."))); } [Theory] @@ -173,7 +176,7 @@ public void LanguageWorkerOptions_FallbackPath_Expected_ListOfConfigs(string wor [InlineData("java", "STANDARD")] [InlineData("node", "LATEST")] [InlineData("node", "STANDARD")] - public void LanguageWorkerOptions_DisabledProbingPaths_Expected_ListOfConfigs(string workerRuntime, string releaseChannel) + public void LanguageWorkerOptions_DisabledWorkerResolution_Expected_ListOfConfigs(string workerRuntime, string releaseChannel) { var loggerProvider = new TestLoggerProvider(); var loggerFactory = new LoggerFactory(); @@ -204,9 +207,50 @@ public void LanguageWorkerOptions_DisabledProbingPaths_Expected_ListOfConfigs(st var logs = loggerProvider.GetAllLogMessages(); string path = Path.Combine(_fallbackPath, workerRuntime); - string expectedLog = $"Added WorkerConfig for language: {workerRuntime} with DefaultWorkerPath: {path}"; + string expectedLog = $"Added WorkerConfig for language: {workerRuntime} with Worker Path: {path}"; Assert.True(logs.Any(l => l.FormattedMessage.Contains(expectedLog))); Assert.True(logs.Any(l => l.FormattedMessage.Contains("Workers Directory set to:"))); } + + [Theory] + [InlineData("java")] + [InlineData("node")] + public void LanguageWorkerOptions_FallbackPath_NullHostingConfig(string workerRuntime) + { + var loggerProvider = new TestLoggerProvider(); + var loggerFactory = new LoggerFactory(); + loggerFactory.AddProvider(loggerProvider); + + var testEnvironment = new TestEnvironment(); + var testMetricLogger = new TestMetricsLogger(); + var configurationBuilder = new ConfigurationBuilder() + .Add(new ScriptEnvironmentVariablesConfigurationSource()); + var configuration = configurationBuilder.Build(); + var testProfileManager = new Mock(); + var testScriptHostManager = new Mock(); + string probingPathValue = null; + + testEnvironment.SetEnvironmentVariable(RpcWorkerConstants.FunctionWorkerRuntimeSettingName, workerRuntime); + testEnvironment.SetEnvironmentVariable(EnvironmentSettingNames.AzureWebJobsFeatureFlags, ScriptConstants.FeatureFlagEnableDynamicWorkerResolution); + testEnvironment.SetEnvironmentVariable(EnvironmentSettingNames.WorkerProbingPaths, probingPathValue); + + var hostingOptions = new FunctionsHostingConfigOptions(); + hostingOptions.Features.Add(RpcWorkerConstants.WorkersAvailableForDynamicResolution, null); + + LanguageWorkerOptionsSetup setup = new LanguageWorkerOptionsSetup(configuration, loggerFactory, testEnvironment, testMetricLogger, testProfileManager.Object, testScriptHostManager.Object, new OptionsWrapper(hostingOptions)); + LanguageWorkerOptions options = new LanguageWorkerOptions(); + + setup.Configure(options); + + Assert.Equal(1, options.WorkerConfigs.Count); + + var logs = loggerProvider.GetAllLogMessages(); + + string path = Path.Combine(_fallbackPath, workerRuntime); + string expectedLog = $"Added WorkerConfig for language: {workerRuntime} with Worker Path: {path}"; + Assert.True(logs.Any(l => l.FormattedMessage.Contains(expectedLog))); + Assert.True(logs.Any(l => l.FormattedMessage.Contains("Workers probing paths set to:"))); + Assert.True(logs.Any(l => l.FormattedMessage.Contains("Searching for worker configs in the fallback directory."))); + } } } \ No newline at end of file diff --git a/test/WebJobs.Script.Tests/Workers/Rpc/WorkerConfigurationResolverTests.cs b/test/WebJobs.Script.Tests/Workers/Rpc/WorkerConfigurationResolverTests.cs index d564b485a6..ebfb4067e4 100644 --- a/test/WebJobs.Script.Tests/Workers/Rpc/WorkerConfigurationResolverTests.cs +++ b/test/WebJobs.Script.Tests/Workers/Rpc/WorkerConfigurationResolverTests.cs @@ -34,6 +34,7 @@ public WorkerConfigurationResolverTests() [Theory] [InlineData("LATEST", "java\\2.19.0", "node\\3.10.1", "powershell\\7.4", "dotnet-isolated", "python")] [InlineData("STANDARD", "java\\2.18.0", "node\\3.10.1", "powershell\\7.4", "dotnet-isolated", "python")] + [InlineData("EXTENDED", "java\\2.18.0", "node\\3.10.1", "powershell\\7.4", "dotnet-isolated", "python")] public void GetWorkerConfigs_MultiLanguageWorker_ReturnsExpectedConfigs(string releaseChannel, string java, string node, string powershell, string dotnetIsolated, string python) { // Arrange From dd70dfc5a2972d296f56a0e7f3ee82f7697c47ce Mon Sep 17 00:00:00 2001 From: Surbhi Gupta Date: Tue, 24 Jun 2025 23:57:55 -0500 Subject: [PATCH 28/55] Addressed PR feedback --- .../Configuration/RpcWorkerConfigFactory.cs | 16 ++--- .../WorkerConfigurationResolver.cs | 69 ++++++++++--------- .../Rpc/IWorkerConfigurationResolver.cs | 14 ++-- .../FunctionsHostingConfigOptionsTest.cs | 1 + .../Rpc/RpcWorkerConfigFactoryTests.cs | 2 +- 5 files changed, 55 insertions(+), 47 deletions(-) diff --git a/src/WebJobs.Script/Workers/Rpc/Configuration/RpcWorkerConfigFactory.cs b/src/WebJobs.Script/Workers/Rpc/Configuration/RpcWorkerConfigFactory.cs index 2c80c1e144..2b5832f4d6 100644 --- a/src/WebJobs.Script/Workers/Rpc/Configuration/RpcWorkerConfigFactory.cs +++ b/src/WebJobs.Script/Workers/Rpc/Configuration/RpcWorkerConfigFactory.cs @@ -180,14 +180,14 @@ internal void AddProvider(string workerDir) var workerConfig = WorkerConfigurationHelper.GetWorkerConfigJsonElement(workerConfigPath); RpcWorkerDescription workerDescription = WorkerConfigurationHelper.GetWorkerDescription( - workerConfig, - _jsonSerializerOptions, - workerDir, - _profileManager, - _config, - _logger, - _dynamicWorkerResolutionEnabled, - _workersAvailableForResolutionViaHostingConfig); + workerConfig: workerConfig, + jsonSerializerOptions: _jsonSerializerOptions, + workerDir: workerDir, + profileManager: _profileManager, + config: _config, + logger: _logger, + dynamicWorkerResolutionEnabled: _dynamicWorkerResolutionEnabled, + workersAvailableForResolutionViaHostingConfig: _workersAvailableForResolutionViaHostingConfig); if (workerDescription.IsDisabled == true) { diff --git a/src/WebJobs.Script/Workers/Rpc/Configuration/WorkerConfigurationResolver.cs b/src/WebJobs.Script/Workers/Rpc/Configuration/WorkerConfigurationResolver.cs index 32d17744bf..6db10661ed 100644 --- a/src/WebJobs.Script/Workers/Rpc/Configuration/WorkerConfigurationResolver.cs +++ b/src/WebJobs.Script/Workers/Rpc/Configuration/WorkerConfigurationResolver.cs @@ -39,7 +39,7 @@ public WorkerConfigurationResolver(IConfiguration config, public List GetWorkerConfigs(List probingPaths, string fallbackPath) { // Dictionary of { FUNCTIONS_WORKER_RUNTIME environment variable value : path of workerConfig } - ConcurrentDictionary outputDict = new ConcurrentDictionary(); + Dictionary outputDict = new Dictionary(StringComparer.OrdinalIgnoreCase); var workerRuntime = _environment.GetEnvironmentVariable(EnvironmentSettingNames.FunctionWorkerRuntime); string releaseChannel = Utility.GetPlatformReleaseChannel(_environment); @@ -53,7 +53,7 @@ public List GetWorkerConfigs(List probingPaths, string fallbackP { foreach (var languageWorkerPath in Directory.EnumerateDirectories(probingPath)) { - string languageWorkerDir = Path.GetFileName(languageWorkerPath).ToLower(); + string languageWorkerDir = Path.GetFileName(languageWorkerPath); // If probing paths are malformed and have duplicate directories of the same language worker (eg. due to different casing) if (outputDict.ContainsKey(languageWorkerDir)) @@ -62,22 +62,18 @@ public List GetWorkerConfigs(List probingPaths, string fallbackP } // Only skip worker directories that don't match the current runtime or are not enabled via hosting config - // Do not skip non-worker directories like the function app payload directory - if (languageWorkerPath.StartsWith(fallbackPath) || languageWorkerPath.StartsWith(probingPath)) + if ((_workersAvailableForResolutionViaHostingConfig is not null && + !_workersAvailableForResolutionViaHostingConfig.Contains(languageWorkerDir)) || + (!_environment.IsMultiLanguageRuntimeEnvironment() && + !_environment.IsPlaceholderModeEnabled() && + workerRuntime is not null && + !workerRuntime.Equals(languageWorkerDir, StringComparison.OrdinalIgnoreCase))) { - if ((_workersAvailableForResolutionViaHostingConfig is not null && - !_workersAvailableForResolutionViaHostingConfig.Contains(languageWorkerDir)) || - (!_environment.IsMultiLanguageRuntimeEnvironment() && - !_environment.IsPlaceholderModeEnabled() && - workerRuntime is not null && - !workerRuntime.Equals(languageWorkerDir, StringComparison.OrdinalIgnoreCase))) - { - continue; - } + continue; } IEnumerable workerVersions = Directory.EnumerateDirectories(languageWorkerPath); - var versions = ParseWorkerVersionsInDescending(workerVersions); + var versions = GetWorkerVersionsDescending(workerVersions); GetWorkerConfigsFromProbingPaths(versions, languageWorkerPath, languageWorkerDir, releaseChannel, outputDict); } @@ -100,7 +96,7 @@ workerRuntime is not null && return outputDict.Values.ToList(); } - private void GetWorkerConfigsFromProbingPaths(IEnumerable versions, string languageWorkerPath, string languageWorkerFolder, string releaseChannel, ConcurrentDictionary outputDict) + private void GetWorkerConfigsFromProbingPaths(IEnumerable versions, string languageWorkerPath, string languageWorkerFolder, string releaseChannel, Dictionary outputDict) { int found = 0; @@ -133,7 +129,7 @@ private void GetWorkerConfigsFromProbingPaths(IEnumerable versions, str } } - private void GetWorkerConfigsFromWithinHost(string fallbackPath, string workerRuntime, ConcurrentDictionary outputDict) + private void GetWorkerConfigsFromWithinHost(string fallbackPath, string workerRuntime, Dictionary outputDict) { if (Directory.Exists(fallbackPath)) { @@ -141,15 +137,12 @@ private void GetWorkerConfigsFromWithinHost(string fallbackPath, string workerRu { string workerDir = Path.GetFileName(workerPath).ToLower(); - if (workerPath.StartsWith(fallbackPath)) + if (outputDict.ContainsKey(workerDir) || + (!_environment.IsMultiLanguageRuntimeEnvironment() && + workerRuntime is not null && + !workerRuntime.Equals(workerDir, StringComparison.OrdinalIgnoreCase))) { - if (outputDict.ContainsKey(workerDir) || - (!_environment.IsMultiLanguageRuntimeEnvironment() && - workerRuntime is not null && - !workerRuntime.Equals(workerDir, StringComparison.OrdinalIgnoreCase))) - { - continue; - } + continue; } string workerConfigPath = Path.Combine(workerPath, RpcWorkerConstants.WorkerConfigFileName); @@ -168,8 +161,13 @@ workerRuntime is not null && } } - private static IEnumerable ParseWorkerVersionsInDescending(IEnumerable workerVersions) + private static IEnumerable GetWorkerVersionsDescending(IEnumerable workerVersions) { + if (!workerVersions.Any()) + { + return Enumerable.Empty(); + } + var versions = new List(); foreach (var workerVersion in workerVersions) @@ -216,14 +214,14 @@ private bool IsWorkerCompatibleWithHost(string workerDir) // profiles evaluation RpcWorkerDescription workerDescription = WorkerConfigurationHelper.GetWorkerDescription( - workerConfig, - _jsonSerializerOptions, - workerDir, - _profileManager, - _config, - _logger, - true, - _workersAvailableForResolutionViaHostingConfig); + workerConfig: workerConfig, + jsonSerializerOptions: _jsonSerializerOptions, + workerDir: workerDir, + profileManager: _profileManager, + config: _config, + logger: _logger, + dynamicWorkerResolutionEnabled: true, + workersAvailableForResolutionViaHostingConfig: _workersAvailableForResolutionViaHostingConfig); if (workerDescription.IsDisabled == true) { @@ -233,6 +231,11 @@ private bool IsWorkerCompatibleWithHost(string workerDir) return true; } + /// + /// Extracts host requirements from the worker configuration JSON element. + /// + /// Worker config: { "hostRequirements": [ "test-capability1", "test-capability2" ] }. + /// HashSet { "test-capability1", "test-capability2" }. private HashSet GetHostRequirements(JsonElement workerConfig) { HashSet hostRequirements = new HashSet(); diff --git a/src/WebJobs.Script/Workers/Rpc/IWorkerConfigurationResolver.cs b/src/WebJobs.Script/Workers/Rpc/IWorkerConfigurationResolver.cs index f55562fbea..4af0eb4805 100644 --- a/src/WebJobs.Script/Workers/Rpc/IWorkerConfigurationResolver.cs +++ b/src/WebJobs.Script/Workers/Rpc/IWorkerConfigurationResolver.cs @@ -6,13 +6,17 @@ namespace Microsoft.Azure.WebJobs.Script.Workers.Rpc.Configuration { /// - /// Interface to resolve WorkerConfigs using configured probing paths and fallback path. + /// Interface to resolve the Worker Configs. /// - /// List of probing paths where workers are located. - /// A fallback path to check when workers cannot be found in the probing paths. - /// A list of paths to worker configuration files. public interface IWorkerConfigurationResolver { - List GetWorkerConfigs(List probingPaths, string fallbackPath); + /// + /// Gets worker configs using configured probing paths and fallback path. + /// + /// List of probing paths where workers are located. + /// A fallback path to check when workers cannot be found in the probing paths. + /// A list of paths to worker configuration files. + + List GetWorkerConfigs(List probingPaths, string fallbackPath); } } \ No newline at end of file diff --git a/test/WebJobs.Script.Tests/Configuration/FunctionsHostingConfigOptionsTest.cs b/test/WebJobs.Script.Tests/Configuration/FunctionsHostingConfigOptionsTest.cs index 5ffc425e6f..3bdf79ba0c 100644 --- a/test/WebJobs.Script.Tests/Configuration/FunctionsHostingConfigOptionsTest.cs +++ b/test/WebJobs.Script.Tests/Configuration/FunctionsHostingConfigOptionsTest.cs @@ -76,6 +76,7 @@ public static IEnumerable PropertyValues yield return [nameof(FunctionsHostingConfigOptions.IsDotNetInProcDisabled), "DotNetInProcDisabled=0", false]; yield return [nameof(FunctionsHostingConfigOptions.WorkersAvailableForDynamicResolution), "WORKERS_AVAILABLE_FOR_DYNAMIC_RESOLUTION=java|node", "java|node"]; yield return [nameof(FunctionsHostingConfigOptions.WorkersAvailableForDynamicResolution), "WORKERS_AVAILABLE_FOR_DYNAMIC_RESOLUTION=java", "java"]; + yield return [nameof(FunctionsHostingConfigOptions.WorkersAvailableForDynamicResolution), "WORKERS_AVAILABLE_FOR_DYNAMIC_RESOLUTION=java|dotnet-isolated|node", "java|dotnet-isolated|node"]; yield return [nameof(FunctionsHostingConfigOptions.IsTestDataSuppressionEnabled), "EnableTestDataSuppression=1", true]; #pragma warning restore SA1011 // Closing square brackets should be spaced correctly diff --git a/test/WebJobs.Script.Tests/Workers/Rpc/RpcWorkerConfigFactoryTests.cs b/test/WebJobs.Script.Tests/Workers/Rpc/RpcWorkerConfigFactoryTests.cs index ca465b9bc3..53103e0efe 100644 --- a/test/WebJobs.Script.Tests/Workers/Rpc/RpcWorkerConfigFactoryTests.cs +++ b/test/WebJobs.Script.Tests/Workers/Rpc/RpcWorkerConfigFactoryTests.cs @@ -186,7 +186,7 @@ public void DefaultWorkerConfigs_Overrides_VersionAppSetting(string runtimeSetti var testEnvironment = new TestEnvironment(); testEnvironment.SetEnvironmentVariable("FUNCTIONS_WORKER_RUNTIME_VERSION", runtimeSettingVersion); - testEnvironment.SetEnvironmentVariable("FUNCTIONS_WORKER_RUNTIME", "powershell"); + testEnvironment.SetEnvironmentVariable("FUNCTIONS_WORKER_RUNTIME", "powerShell"); var config = ScriptSettingsManager.CreateDefaultConfigurationBuilder().Build(); From 2a5655034e40a66a96234f6fcc3c36f2b3935cb1 Mon Sep 17 00:00:00 2001 From: Surbhi Gupta Date: Wed, 25 Jun 2025 00:10:20 -0500 Subject: [PATCH 29/55] Corrected method return value --- .../Rpc/Configuration/WorkerConfigurationResolver.cs | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/src/WebJobs.Script/Workers/Rpc/Configuration/WorkerConfigurationResolver.cs b/src/WebJobs.Script/Workers/Rpc/Configuration/WorkerConfigurationResolver.cs index 6db10661ed..c39885e81a 100644 --- a/src/WebJobs.Script/Workers/Rpc/Configuration/WorkerConfigurationResolver.cs +++ b/src/WebJobs.Script/Workers/Rpc/Configuration/WorkerConfigurationResolver.cs @@ -75,7 +75,7 @@ workerRuntime is not null && IEnumerable workerVersions = Directory.EnumerateDirectories(languageWorkerPath); var versions = GetWorkerVersionsDescending(workerVersions); - GetWorkerConfigsFromProbingPaths(versions, languageWorkerPath, languageWorkerDir, releaseChannel, outputDict); + outputDict = GetWorkerConfigsFromProbingPaths(versions, languageWorkerPath, languageWorkerDir, releaseChannel, outputDict); } } } @@ -91,12 +91,12 @@ workerRuntime is not null && _logger.LogDebug("Searching for worker configs in the fallback directory."); // Search in fallback path if worker cannot be found in probing paths - GetWorkerConfigsFromWithinHost(fallbackPath, workerRuntime, outputDict); + outputDict = GetWorkerConfigsFromWithinHost(fallbackPath, workerRuntime, outputDict); return outputDict.Values.ToList(); } - private void GetWorkerConfigsFromProbingPaths(IEnumerable versions, string languageWorkerPath, string languageWorkerFolder, string releaseChannel, Dictionary outputDict) + private Dictionary GetWorkerConfigsFromProbingPaths(IEnumerable versions, string languageWorkerPath, string languageWorkerFolder, string releaseChannel, Dictionary outputDict) { int found = 0; @@ -127,9 +127,11 @@ private void GetWorkerConfigsFromProbingPaths(IEnumerable versions, str } } } + + return outputDict; } - private void GetWorkerConfigsFromWithinHost(string fallbackPath, string workerRuntime, Dictionary outputDict) + private Dictionary GetWorkerConfigsFromWithinHost(string fallbackPath, string workerRuntime, Dictionary outputDict) { if (Directory.Exists(fallbackPath)) { @@ -159,6 +161,8 @@ workerRuntime is not null && } } } + + return outputDict; } private static IEnumerable GetWorkerVersionsDescending(IEnumerable workerVersions) From 2c7b9ef5a4e64f982ce3ba226c18404ca5f395d9 Mon Sep 17 00:00:00 2001 From: Surbhi Gupta Date: Fri, 27 Jun 2025 16:34:40 -0500 Subject: [PATCH 30/55] Addressing PR feedback --- .../WorkerConfigurationResolver.cs | 32 +++++++++++-------- 1 file changed, 18 insertions(+), 14 deletions(-) diff --git a/src/WebJobs.Script/Workers/Rpc/Configuration/WorkerConfigurationResolver.cs b/src/WebJobs.Script/Workers/Rpc/Configuration/WorkerConfigurationResolver.cs index c39885e81a..9796861f54 100644 --- a/src/WebJobs.Script/Workers/Rpc/Configuration/WorkerConfigurationResolver.cs +++ b/src/WebJobs.Script/Workers/Rpc/Configuration/WorkerConfigurationResolver.cs @@ -75,7 +75,7 @@ workerRuntime is not null && IEnumerable workerVersions = Directory.EnumerateDirectories(languageWorkerPath); var versions = GetWorkerVersionsDescending(workerVersions); - outputDict = GetWorkerConfigsFromProbingPaths(versions, languageWorkerPath, languageWorkerDir, releaseChannel, outputDict); + PopulateWorkerConfigsFromProbingPaths(versions, languageWorkerPath, languageWorkerDir, releaseChannel, outputDict); } } } @@ -91,14 +91,19 @@ workerRuntime is not null && _logger.LogDebug("Searching for worker configs in the fallback directory."); // Search in fallback path if worker cannot be found in probing paths - outputDict = GetWorkerConfigsFromWithinHost(fallbackPath, workerRuntime, outputDict); + PopulateWorkerConfigsFromWithinHost(fallbackPath, workerRuntime, outputDict); return outputDict.Values.ToList(); } - private Dictionary GetWorkerConfigsFromProbingPaths(IEnumerable versions, string languageWorkerPath, string languageWorkerFolder, string releaseChannel, Dictionary outputDict) + private void PopulateWorkerConfigsFromProbingPaths(IEnumerable versions, string languageWorkerPath, string languageWorkerFolder, string releaseChannel, Dictionary outputDict) { - int found = 0; + int compatibleWorkerCount = 0; + + bool isStandardOrExtendedChannel = + releaseChannel != null && + (releaseChannel.Equals(ScriptConstants.StandardPlatformChannelNameUpper) || + releaseChannel.Equals(ScriptConstants.ExtendedPlatformChannelNameUpper)); // language worker version foreach (Version versionFolder in versions) @@ -107,20 +112,15 @@ private Dictionary GetWorkerConfigsFromProbingPaths(IEnumerable< if (IsWorkerCompatibleWithHost(languageWorkerVersionPath)) { - found++; + compatibleWorkerCount++; outputDict[languageWorkerFolder] = languageWorkerVersionPath; - if (string.IsNullOrEmpty(releaseChannel) || - !(releaseChannel.Equals(ScriptConstants.StandardPlatformChannelNameUpper) || - releaseChannel.Equals(ScriptConstants.ExtendedPlatformChannelNameUpper))) + if (string.IsNullOrEmpty(releaseChannel) || !isStandardOrExtendedChannel) { - // latest version is the default - break; + break; // latest version is the default } - if (found > 1 && - (releaseChannel.Equals(ScriptConstants.StandardPlatformChannelNameUpper) || - releaseChannel.Equals(ScriptConstants.ExtendedPlatformChannelNameUpper))) + if (compatibleWorkerCount > 1 && isStandardOrExtendedChannel) { outputDict[languageWorkerFolder] = languageWorkerVersionPath; break; @@ -131,7 +131,7 @@ private Dictionary GetWorkerConfigsFromProbingPaths(IEnumerable< return outputDict; } - private Dictionary GetWorkerConfigsFromWithinHost(string fallbackPath, string workerRuntime, Dictionary outputDict) + private void PopulateWorkerConfigsFromWithinHost(string fallbackPath, string workerRuntime, Dictionary outputDict) { if (Directory.Exists(fallbackPath)) { @@ -183,6 +183,10 @@ private static IEnumerable GetWorkerVersionsDescending(IEnumerable Date: Fri, 27 Jun 2025 23:44:53 -0500 Subject: [PATCH 31/55] Addressed PR feedback --- .../Configuration/RpcWorkerConfigFactory.cs | 2 +- .../WorkerConfigurationResolver.cs | 41 +++++++++---------- 2 files changed, 20 insertions(+), 23 deletions(-) diff --git a/src/WebJobs.Script/Workers/Rpc/Configuration/RpcWorkerConfigFactory.cs b/src/WebJobs.Script/Workers/Rpc/Configuration/RpcWorkerConfigFactory.cs index 2b5832f4d6..cffb156bc2 100644 --- a/src/WebJobs.Script/Workers/Rpc/Configuration/RpcWorkerConfigFactory.cs +++ b/src/WebJobs.Script/Workers/Rpc/Configuration/RpcWorkerConfigFactory.cs @@ -108,7 +108,7 @@ internal void AddProviders() { if (_dynamicWorkerResolutionEnabled) { - _logger.LogDebug("Workers probing paths set to: {probingPaths} and Workers Directory set as fallback path: {WorkersDirPath}", string.Join(", ", _workerProbingPaths), WorkersDirPath); + _logger.LogDebug("Workers probing paths set to: {probingPaths} and Workers Directory set as fallback path: {workersDirPath}", string.Join(", ", _workerProbingPaths), WorkersDirPath); List workerConfigs = _workerConfigurationResolver.GetWorkerConfigs(_workerProbingPaths, WorkersDirPath); diff --git a/src/WebJobs.Script/Workers/Rpc/Configuration/WorkerConfigurationResolver.cs b/src/WebJobs.Script/Workers/Rpc/Configuration/WorkerConfigurationResolver.cs index 9796861f54..80e14cfc0e 100644 --- a/src/WebJobs.Script/Workers/Rpc/Configuration/WorkerConfigurationResolver.cs +++ b/src/WebJobs.Script/Workers/Rpc/Configuration/WorkerConfigurationResolver.cs @@ -2,7 +2,6 @@ // Licensed under the MIT License. See License.txt in the project root for license information. using System; -using System.Collections.Concurrent; using System.Collections.Generic; using System.IO; using System.Linq; @@ -33,7 +32,7 @@ public WorkerConfigurationResolver(IConfiguration config, _logger = logger ?? throw new ArgumentNullException(nameof(logger)); _environment = environment ?? throw new ArgumentNullException(nameof(environment)); _profileManager = workerProfileManager ?? throw new ArgumentNullException(nameof(workerProfileManager)); - _workersAvailableForResolutionViaHostingConfig = workersAvailableForResolutionViaHostingConfig; + _workersAvailableForResolutionViaHostingConfig = workersAvailableForResolutionViaHostingConfig ?? throw new ArgumentNullException(nameof(workersAvailableForResolutionViaHostingConfig)); } public List GetWorkerConfigs(List probingPaths, string fallbackPath) @@ -46,36 +45,34 @@ public List GetWorkerConfigs(List probingPaths, string fallbackP if (!probingPaths.IsNullOrEmpty()) { - // probing path directory structure is: /// + // probing path directory structure is: /// foreach (var probingPath in probingPaths) { if (!string.IsNullOrEmpty(probingPath) && Directory.Exists(probingPath)) { - foreach (var languageWorkerPath in Directory.EnumerateDirectories(probingPath)) + foreach (var workerRuntimePath in Directory.EnumerateDirectories(probingPath)) { - string languageWorkerDir = Path.GetFileName(languageWorkerPath); + string workerRuntimeDir = Path.GetFileName(workerRuntimePath); // If probing paths are malformed and have duplicate directories of the same language worker (eg. due to different casing) - if (outputDict.ContainsKey(languageWorkerDir)) + if (outputDict.ContainsKey(workerRuntimeDir)) { continue; } // Only skip worker directories that don't match the current runtime or are not enabled via hosting config - if ((_workersAvailableForResolutionViaHostingConfig is not null && - !_workersAvailableForResolutionViaHostingConfig.Contains(languageWorkerDir)) || - (!_environment.IsMultiLanguageRuntimeEnvironment() && - !_environment.IsPlaceholderModeEnabled() && - workerRuntime is not null && - !workerRuntime.Equals(languageWorkerDir, StringComparison.OrdinalIgnoreCase))) + if (!_workersAvailableForResolutionViaHostingConfig.Contains(workerRuntimeDir) || + (!_environment.IsMultiLanguageRuntimeEnvironment() && + !_environment.IsPlaceholderModeEnabled() && + IsRequiredWorkerRuntime(workerRuntime, workerRuntimeDir))) { continue; } - IEnumerable workerVersions = Directory.EnumerateDirectories(languageWorkerPath); + IEnumerable workerVersions = Directory.EnumerateDirectories(workerRuntimeDir); var versions = GetWorkerVersionsDescending(workerVersions); - PopulateWorkerConfigsFromProbingPaths(versions, languageWorkerPath, languageWorkerDir, releaseChannel, outputDict); + PopulateWorkerConfigsFromProbingPaths(versions, workerRuntimeDir, workerRuntimeDir, releaseChannel, outputDict); } } } @@ -127,8 +124,6 @@ private void PopulateWorkerConfigsFromProbingPaths(IEnumerable versions } } } - - return outputDict; } private void PopulateWorkerConfigsFromWithinHost(string fallbackPath, string workerRuntime, Dictionary outputDict) @@ -140,9 +135,8 @@ private void PopulateWorkerConfigsFromWithinHost(string fallbackPath, string wor string workerDir = Path.GetFileName(workerPath).ToLower(); if (outputDict.ContainsKey(workerDir) || - (!_environment.IsMultiLanguageRuntimeEnvironment() && - workerRuntime is not null && - !workerRuntime.Equals(workerDir, StringComparison.OrdinalIgnoreCase))) + (!_environment.IsMultiLanguageRuntimeEnvironment() && + IsRequiredWorkerRuntime(workerRuntime, workerDir))) { continue; } @@ -161,11 +155,9 @@ workerRuntime is not null && } } } - - return outputDict; } - private static IEnumerable GetWorkerVersionsDescending(IEnumerable workerVersions) + private IEnumerable GetWorkerVersionsDescending(IEnumerable workerVersions) { if (!workerVersions.Any()) { @@ -276,5 +268,10 @@ private bool DoesHostRequirementMeet(JsonElement workerConfig) return true; } + + private bool IsRequiredWorkerRuntime(string workerRuntime, string workerDir) + { + return workerRuntime is not null && !workerRuntime.Equals(workerDir, StringComparison.OrdinalIgnoreCase); + } } } \ No newline at end of file From 6a943871518c79d032eab8e0cac753abafb16f90 Mon Sep 17 00:00:00 2001 From: Surbhi Gupta Date: Sun, 29 Jun 2025 16:41:03 -0500 Subject: [PATCH 32/55] Using IFileSystem instead of System.Directory --- .../LanguageWorkerOptionsSetup.cs | 2 +- .../WorkerConfigurationResolver.cs | 16 ++++++++------ .../Rpc/RpcWorkerConfigFactoryTests.cs | 21 ++++++++++--------- .../Workers/Rpc/RpcWorkerConfigTests.cs | 3 ++- .../Rpc/WorkerConfigurationResolverTests.cs | 7 ++++--- 5 files changed, 28 insertions(+), 21 deletions(-) diff --git a/src/WebJobs.Script/Workers/Rpc/Configuration/LanguageWorkerOptionsSetup.cs b/src/WebJobs.Script/Workers/Rpc/Configuration/LanguageWorkerOptionsSetup.cs index 046884c169..7774be5ecf 100644 --- a/src/WebJobs.Script/Workers/Rpc/Configuration/LanguageWorkerOptionsSetup.cs +++ b/src/WebJobs.Script/Workers/Rpc/Configuration/LanguageWorkerOptionsSetup.cs @@ -91,7 +91,7 @@ public void Configure(LanguageWorkerOptions options) probingPaths = GetWorkerProbingPaths(); } - var workerConfigurationResolver = new WorkerConfigurationResolver(configuration, _logger, _environment, _workerProfileManager, workers); + var workerConfigurationResolver = new WorkerConfigurationResolver(configuration, _logger, _environment, FileUtility.Instance, _workerProfileManager, workers); var configFactory = new RpcWorkerConfigFactory(configuration, _logger, SystemRuntimeInformation.Instance, _environment, _metricsLogger, _workerProfileManager, workerConfigurationResolver, dynamicWorkerResolutionEnabled, probingPaths, workers); options.WorkerConfigs = configFactory.GetConfigs(); } diff --git a/src/WebJobs.Script/Workers/Rpc/Configuration/WorkerConfigurationResolver.cs b/src/WebJobs.Script/Workers/Rpc/Configuration/WorkerConfigurationResolver.cs index 80e14cfc0e..6cedb12e12 100644 --- a/src/WebJobs.Script/Workers/Rpc/Configuration/WorkerConfigurationResolver.cs +++ b/src/WebJobs.Script/Workers/Rpc/Configuration/WorkerConfigurationResolver.cs @@ -4,6 +4,7 @@ using System; using System.Collections.Generic; using System.IO; +using System.IO.Abstractions; using System.Linq; using System.Text.Json; using Microsoft.Azure.AppService.Proxy.Common.Extensions; @@ -19,18 +20,21 @@ internal sealed class WorkerConfigurationResolver : IWorkerConfigurationResolver private readonly ILogger _logger; private readonly IWorkerProfileManager _profileManager; private readonly IEnvironment _environment; + private readonly IFileSystem _fileSystem; private readonly HashSet _workersAvailableForResolutionViaHostingConfig; private readonly JsonSerializerOptions _jsonSerializerOptions = new() { PropertyNameCaseInsensitive = true }; public WorkerConfigurationResolver(IConfiguration config, ILogger logger, IEnvironment environment, + IFileSystem fileSystem, IWorkerProfileManager workerProfileManager, HashSet workersAvailableForResolutionViaHostingConfig) { _config = config ?? throw new ArgumentNullException(nameof(config)); _logger = logger ?? throw new ArgumentNullException(nameof(logger)); _environment = environment ?? throw new ArgumentNullException(nameof(environment)); + _fileSystem = fileSystem ?? throw new ArgumentNullException(nameof(fileSystem)); _profileManager = workerProfileManager ?? throw new ArgumentNullException(nameof(workerProfileManager)); _workersAvailableForResolutionViaHostingConfig = workersAvailableForResolutionViaHostingConfig ?? throw new ArgumentNullException(nameof(workersAvailableForResolutionViaHostingConfig)); } @@ -48,9 +52,9 @@ public List GetWorkerConfigs(List probingPaths, string fallbackP // probing path directory structure is: /// foreach (var probingPath in probingPaths) { - if (!string.IsNullOrEmpty(probingPath) && Directory.Exists(probingPath)) + if (!string.IsNullOrEmpty(probingPath) && _fileSystem.Directory.Exists(probingPath)) { - foreach (var workerRuntimePath in Directory.EnumerateDirectories(probingPath)) + foreach (var workerRuntimePath in _fileSystem.Directory.EnumerateDirectories(probingPath)) { string workerRuntimeDir = Path.GetFileName(workerRuntimePath); @@ -69,10 +73,10 @@ public List GetWorkerConfigs(List probingPaths, string fallbackP continue; } - IEnumerable workerVersions = Directory.EnumerateDirectories(workerRuntimeDir); + IEnumerable workerVersions = _fileSystem.Directory.EnumerateDirectories(workerRuntimePath); var versions = GetWorkerVersionsDescending(workerVersions); - PopulateWorkerConfigsFromProbingPaths(versions, workerRuntimeDir, workerRuntimeDir, releaseChannel, outputDict); + PopulateWorkerConfigsFromProbingPaths(versions, workerRuntimePath, workerRuntimeDir, releaseChannel, outputDict); } } } @@ -128,9 +132,9 @@ private void PopulateWorkerConfigsFromProbingPaths(IEnumerable versions private void PopulateWorkerConfigsFromWithinHost(string fallbackPath, string workerRuntime, Dictionary outputDict) { - if (Directory.Exists(fallbackPath)) + if (_fileSystem.Directory.Exists(fallbackPath)) { - foreach (var workerPath in Directory.EnumerateDirectories(fallbackPath)) + foreach (var workerPath in _fileSystem.Directory.EnumerateDirectories(fallbackPath)) { string workerDir = Path.GetFileName(workerPath).ToLower(); diff --git a/test/WebJobs.Script.Tests/Workers/Rpc/RpcWorkerConfigFactoryTests.cs b/test/WebJobs.Script.Tests/Workers/Rpc/RpcWorkerConfigFactoryTests.cs index 53103e0efe..38dd58d826 100644 --- a/test/WebJobs.Script.Tests/Workers/Rpc/RpcWorkerConfigFactoryTests.cs +++ b/test/WebJobs.Script.Tests/Workers/Rpc/RpcWorkerConfigFactoryTests.cs @@ -6,6 +6,7 @@ using System.IO; using System.Linq; using System.Text.Json; +using Microsoft.Azure.WebJobs.Script; using Microsoft.Azure.WebJobs.Script.Config; using Microsoft.Azure.WebJobs.Script.Workers; using Microsoft.Azure.WebJobs.Script.Workers.Profiles; @@ -41,7 +42,7 @@ public void DefaultLanguageWorkersDir() var expectedWorkersDir = Path.Combine(Path.GetDirectoryName(new Uri(typeof(RpcWorkerConfigFactory).Assembly.Location).LocalPath), RpcWorkerConstants.DefaultWorkersDirectoryName); var config = new ConfigurationBuilder().Build(); var testLogger = new TestLogger("test"); - var workerConfigurationResolver = new WorkerConfigurationResolver(config, testLogger, _testEnvironment, _testWorkerProfileManager, new HashSet()); + var workerConfigurationResolver = new WorkerConfigurationResolver(config, testLogger, _testEnvironment, FileUtility.Instance, _testWorkerProfileManager, new HashSet()); var configFactory = new RpcWorkerConfigFactory(config, testLogger, _testSysRuntimeInfo, _testEnvironment, new TestMetricsLogger(), _testWorkerProfileManager, workerConfigurationResolver); Assert.Equal(expectedWorkersDir, configFactory.WorkersDirPath); } @@ -75,7 +76,7 @@ public void LanguageWorker_WorkersDir_Set() }) .Build(); var testLogger = new TestLogger("test"); - var workerConfigurationResolver = new WorkerConfigurationResolver(config, testLogger, _testEnvironment, _testWorkerProfileManager, new HashSet()); + var workerConfigurationResolver = new WorkerConfigurationResolver(config, testLogger, _testEnvironment, FileUtility.Instance, _testWorkerProfileManager, new HashSet()); var configFactory = new RpcWorkerConfigFactory(config, testLogger, _testSysRuntimeInfo, _testEnvironment, new TestMetricsLogger(), _testWorkerProfileManager, workerConfigurationResolver); Assert.Equal(expectedWorkersDir, configFactory.WorkersDirPath); } @@ -92,7 +93,7 @@ public void LanguageWorker_WorkersDir_NotSet() var config = configBuilder.Build(); var scriptSettingsManager = new ScriptSettingsManager(config); var testLogger = new TestLogger("test"); - var workerConfigurationResolver = new WorkerConfigurationResolver(config, testLogger, _testEnvironment, _testWorkerProfileManager, new HashSet()); + var workerConfigurationResolver = new WorkerConfigurationResolver(config, testLogger, _testEnvironment, FileUtility.Instance, _testWorkerProfileManager, new HashSet()); var configFactory = new RpcWorkerConfigFactory(config, testLogger, _testSysRuntimeInfo, _testEnvironment, new TestMetricsLogger(), _testWorkerProfileManager, workerConfigurationResolver); Assert.Equal(expectedWorkersDir, configFactory.WorkersDirPath); } @@ -117,7 +118,7 @@ public void WorkerDescription_Skipped_When_Profile_Disables_Worker() var scriptSettingsManager = new ScriptSettingsManager(config); var testLogger = new TestLogger("test"); _testEnvironment.SetEnvironmentVariable("ENV_VAR_BAR", "True"); - var workerConfigurationResolver = new WorkerConfigurationResolver(config, testLogger, _testEnvironment, _testWorkerProfileManager, new HashSet()); + var workerConfigurationResolver = new WorkerConfigurationResolver(config, testLogger, _testEnvironment, FileUtility.Instance, _testWorkerProfileManager, new HashSet()); var configFactory = new RpcWorkerConfigFactory(config, testLogger, _testSysRuntimeInfo, _testEnvironment, new TestMetricsLogger(), _testWorkerProfileManager, workerConfigurationResolver); var workerConfigs = configFactory.GetConfigs(); @@ -140,7 +141,7 @@ public void JavaPath_FromEnvVars() var config = configBuilder.Build(); var scriptSettingsManager = new ScriptSettingsManager(config); var testLogger = new TestLogger("test"); - var workerConfigurationResolver = new WorkerConfigurationResolver(config, testLogger, _testEnvironment, _testWorkerProfileManager, new HashSet()); + var workerConfigurationResolver = new WorkerConfigurationResolver(config, testLogger, _testEnvironment, FileUtility.Instance, _testWorkerProfileManager, new HashSet()); var configFactory = new RpcWorkerConfigFactory(config, testLogger, _testSysRuntimeInfo, _testEnvironment, new TestMetricsLogger(), _testWorkerProfileManager, workerConfigurationResolver); var workerConfigs = configFactory.GetConfigs(); var javaPath = workerConfigs.FirstOrDefault(c => c.Description.Language.Equals("java", StringComparison.OrdinalIgnoreCase)).Description.DefaultExecutablePath; @@ -163,7 +164,7 @@ public void DefaultWorkerConfigs_Overrides_DefaultWorkerRuntimeVersion_AppSettin _testEnvironment.SetEnvironmentVariable(EnvironmentSettingNames.AzureWebsitePlaceholderMode, "1"); using var variables = new TestScopedSettings(scriptSettingsManager, testEnvVariables); - var workerConfigurationResolver = new WorkerConfigurationResolver(config, testLogger, _testEnvironment, _testWorkerProfileManager, new HashSet()); + var workerConfigurationResolver = new WorkerConfigurationResolver(config, testLogger, _testEnvironment, FileUtility.Instance, _testWorkerProfileManager, new HashSet()); var configFactory = new RpcWorkerConfigFactory(config, testLogger, _testSysRuntimeInfo, _testEnvironment, new TestMetricsLogger(), _testWorkerProfileManager, workerConfigurationResolver); var workerConfigs = configFactory.GetConfigs(); var pythonWorkerConfig = workerConfigs.FirstOrDefault(w => w.Description.Language.Equals("python", StringComparison.OrdinalIgnoreCase)); @@ -191,7 +192,7 @@ public void DefaultWorkerConfigs_Overrides_VersionAppSetting(string runtimeSetti var config = ScriptSettingsManager.CreateDefaultConfigurationBuilder().Build(); var testLogger = new TestLogger("test"); - var workerConfigurationResolver = new WorkerConfigurationResolver(config, testLogger, testEnvironment, _testWorkerProfileManager, new HashSet() { "powershell" }); + var workerConfigurationResolver = new WorkerConfigurationResolver(config, testLogger, testEnvironment, FileUtility.Instance, _testWorkerProfileManager, new HashSet() { "powershell" }); var configFactory = new RpcWorkerConfigFactory(config, testLogger, _testSysRuntimeInfo, testEnvironment, new TestMetricsLogger(), _testWorkerProfileManager, workerConfigurationResolver, enableProbingPath is null ? false : true, new List() { probingPath }, new HashSet() { "powershell" }); var workerConfigs = configFactory.GetConfigs(); var powershellWorkerConfig = workerConfigs.FirstOrDefault(w => w.Description.Language.Equals("powershell", StringComparison.OrdinalIgnoreCase)); @@ -225,7 +226,7 @@ public void ShouldAddProvider_Returns_Expected(string workerLanguage, string wor } var config = new ConfigurationBuilder().Build(); var testLogger = new TestLogger("test"); - var workerConfigurationResolver = new WorkerConfigurationResolver(config, testLogger, _testEnvironment, _testWorkerProfileManager, new HashSet()); + var workerConfigurationResolver = new WorkerConfigurationResolver(config, testLogger, _testEnvironment, FileUtility.Instance, _testWorkerProfileManager, new HashSet()); var rpcWorkerConfigFactory = new RpcWorkerConfigFactory(config, testLogger, _testSysRuntimeInfo, _testEnvironment, new TestMetricsLogger(), _testWorkerProfileManager, workerConfigurationResolver); _testEnvironment.SetEnvironmentVariable(RpcWorkerConstants.FunctionWorkerRuntimeSettingName, workerRuntime); Assert.Equal(expectedResult, rpcWorkerConfigFactory.ShouldAddWorkerConfig(workerLanguage)); @@ -270,7 +271,7 @@ public void GetWorkerProcessCount_Tests(bool defaultWorkerConfig, bool setProces var config = new ConfigurationBuilder().Build(); var testLogger = new TestLogger("test"); - var workerConfigurationResolver = new WorkerConfigurationResolver(config, testLogger, _testEnvironment, _testWorkerProfileManager, new HashSet()); + var workerConfigurationResolver = new WorkerConfigurationResolver(config, testLogger, _testEnvironment, FileUtility.Instance, _testWorkerProfileManager, new HashSet()); var rpcWorkerConfigFactory = new RpcWorkerConfigFactory(config, testLogger, _testSysRuntimeInfo, _testEnvironment, new TestMetricsLogger(), _testWorkerProfileManager, workerConfigurationResolver); var result = rpcWorkerConfigFactory.GetWorkerProcessCount(workerConfig); @@ -311,7 +312,7 @@ public void GetWorkerProcessCount_ThrowsException_Tests() var config = new ConfigurationBuilder().Build(); var testLogger = new TestLogger("test"); - var workerConfigurationResolver = new WorkerConfigurationResolver(config, testLogger, _testEnvironment, _testWorkerProfileManager, new HashSet()); + var workerConfigurationResolver = new WorkerConfigurationResolver(config, testLogger, _testEnvironment, FileUtility.Instance, _testWorkerProfileManager, new HashSet()); var rpcWorkerConfigFactory = new RpcWorkerConfigFactory(config, testLogger, _testSysRuntimeInfo, _testEnvironment, new TestMetricsLogger(), _testWorkerProfileManager, workerConfigurationResolver); var resultEx1 = Assert.Throws(() => rpcWorkerConfigFactory.GetWorkerProcessCount(workerConfig)); Assert.Contains("ProcessCount must be greater than 0", resultEx1.Message); diff --git a/test/WebJobs.Script.Tests/Workers/Rpc/RpcWorkerConfigTests.cs b/test/WebJobs.Script.Tests/Workers/Rpc/RpcWorkerConfigTests.cs index 51b2e97572..c2fe5bd3f3 100644 --- a/test/WebJobs.Script.Tests/Workers/Rpc/RpcWorkerConfigTests.cs +++ b/test/WebJobs.Script.Tests/Workers/Rpc/RpcWorkerConfigTests.cs @@ -7,6 +7,7 @@ using System.IO; using System.Linq; using System.Runtime.InteropServices; +using Microsoft.Azure.WebJobs.Script; using Microsoft.Azure.WebJobs.Script.Config; using Microsoft.Azure.WebJobs.Script.Diagnostics; using Microsoft.Azure.WebJobs.Script.Workers; @@ -685,7 +686,7 @@ private IEnumerable TestReadWorkerProviderFromConfig(IEnumerabl var scriptHostOptions = new ScriptJobHostOptions(); var scriptSettingsManager = new ScriptSettingsManager(config); var workerProfileManager = new Mock(); - var workerConfigurationResolver = new WorkerConfigurationResolver(config, testLogger, _testEnvironment, workerProfileManager.Object, new HashSet()); + var workerConfigurationResolver = new WorkerConfigurationResolver(config, testLogger, _testEnvironment, FileUtility.Instance, workerProfileManager.Object, new HashSet()); var configFactory = new RpcWorkerConfigFactory(config, testLogger, _testSysRuntimeInfo, _testEnvironment, new TestMetricsLogger(), workerProfileManager.Object, workerConfigurationResolver); if (appSvcEnv) diff --git a/test/WebJobs.Script.Tests/Workers/Rpc/WorkerConfigurationResolverTests.cs b/test/WebJobs.Script.Tests/Workers/Rpc/WorkerConfigurationResolverTests.cs index ebfb4067e4..c25e6053c2 100644 --- a/test/WebJobs.Script.Tests/Workers/Rpc/WorkerConfigurationResolverTests.cs +++ b/test/WebJobs.Script.Tests/Workers/Rpc/WorkerConfigurationResolverTests.cs @@ -4,6 +4,7 @@ using System.Collections.Generic; using System.IO; using System.Linq; +using Microsoft.Azure.WebJobs.Script; using Microsoft.Azure.WebJobs.Script.Workers.Profiles; using Microsoft.Azure.WebJobs.Script.Workers.Rpc.Configuration; using Microsoft.Extensions.Configuration; @@ -43,7 +44,7 @@ public void GetWorkerConfigs_MultiLanguageWorker_ReturnsExpectedConfigs(string r mockEnvironment.Setup(p => p.GetEnvironmentVariable(EnvironmentSettingNames.AppKind)).Returns(ScriptConstants.WorkFlowAppKind); mockEnvironment.Setup(p => p.GetEnvironmentVariable(EnvironmentSettingNames.FunctionWorkerRuntime)).Returns((string)null); - var workerConfigurationResolver = new WorkerConfigurationResolver(_mockConfig.Object, _mockLogger.Object, mockEnvironment.Object, _mockProfileManager.Object, new HashSet() { "java", "node", "powershell" }); + var workerConfigurationResolver = new WorkerConfigurationResolver(_mockConfig.Object, _mockLogger.Object, mockEnvironment.Object, FileUtility.Instance, _mockProfileManager.Object, new HashSet() { "java", "node", "powershell" }); var result = workerConfigurationResolver.GetWorkerConfigs(_probingPaths, _fallbackPath); @@ -77,7 +78,7 @@ public void GetWorkerConfigs_MultiLanguageWorker_NullOREmptyProbingPath_ReturnsE } // Act - var workerConfigurationResolver = new WorkerConfigurationResolver(_mockConfig.Object, _mockLogger.Object, mockEnvironment.Object, _mockProfileManager.Object, new HashSet() { "java", "node", "powershell" }); + var workerConfigurationResolver = new WorkerConfigurationResolver(_mockConfig.Object, _mockLogger.Object, mockEnvironment.Object, FileUtility.Instance, _mockProfileManager.Object, new HashSet() { "java", "node", "powershell" }); var result = workerConfigurationResolver.GetWorkerConfigs(probingPaths, _fallbackPath); @@ -122,7 +123,7 @@ public void GetWorkerConfigs_NullOREmptyProbingPath_ReturnsExpectedConfigs(strin var mockLogger = new Mock(); // Act - var workerConfigurationResolver = new WorkerConfigurationResolver(mockConfig.Object, mockLogger.Object, mockEnv.Object, mockProfileManager.Object, new HashSet() { "java", "node", "powershell" }); + var workerConfigurationResolver = new WorkerConfigurationResolver(mockConfig.Object, mockLogger.Object, mockEnv.Object, FileUtility.Instance, mockProfileManager.Object, new HashSet() { "java", "node", "powershell" }); var result = workerConfigurationResolver.GetWorkerConfigs(probingPaths, _fallbackPath); From 275831ce10907326c375a87c86bf43c8ea6d258f Mon Sep 17 00:00:00 2001 From: Surbhi Gupta Date: Mon, 30 Jun 2025 00:57:21 -0500 Subject: [PATCH 33/55] Addressing PR feedback. Updating method to get versions in descending order --- .../LanguageWorkerOptionsSetup.cs | 10 +++- .../WorkerConfigurationResolver.cs | 60 +++++++++++-------- 2 files changed, 42 insertions(+), 28 deletions(-) diff --git a/src/WebJobs.Script/Workers/Rpc/Configuration/LanguageWorkerOptionsSetup.cs b/src/WebJobs.Script/Workers/Rpc/Configuration/LanguageWorkerOptionsSetup.cs index 7774be5ecf..b165792eec 100644 --- a/src/WebJobs.Script/Workers/Rpc/Configuration/LanguageWorkerOptionsSetup.cs +++ b/src/WebJobs.Script/Workers/Rpc/Configuration/LanguageWorkerOptionsSetup.cs @@ -21,7 +21,6 @@ namespace Microsoft.Azure.WebJobs.Script.Workers.Rpc { internal class LanguageWorkerOptionsSetup : IConfigureOptions { - private const string _windowsWorkerProbingPath = "C:\\home\\SiteExtensions\\workers"; // Harcoded site extensions path for Windows until Antares sets it as an Environment variable. private readonly IConfiguration _configuration; private readonly ILogger _logger; private readonly IEnvironment _environment; @@ -120,7 +119,14 @@ internal List GetWorkerProbingPaths() { if (_environment.IsAnyWindows()) { - probingPaths.Add(_windowsWorkerProbingPath); + // Harcoded site extensions path for Windows until Antares sets it as an Environment variable. + var assemblyPath = System.Reflection.Assembly.GetExecutingAssembly().Location; + var assemblyDir = Path.GetDirectoryName(assemblyPath); + var parentDir = Directory.GetParent(assemblyDir).FullName; + + var windowsWorkerFullProbingPath = Path.Combine(parentDir, RpcWorkerConstants.DefaultWorkersDirectoryName); + probingPaths.Add(windowsWorkerFullProbingPath); + _logger.LogTrace("Windows worker probing path: {windowsWorkerFullProbingPath}", windowsWorkerFullProbingPath); } } diff --git a/src/WebJobs.Script/Workers/Rpc/Configuration/WorkerConfigurationResolver.cs b/src/WebJobs.Script/Workers/Rpc/Configuration/WorkerConfigurationResolver.cs index 6cedb12e12..9a563676e6 100644 --- a/src/WebJobs.Script/Workers/Rpc/Configuration/WorkerConfigurationResolver.cs +++ b/src/WebJobs.Script/Workers/Rpc/Configuration/WorkerConfigurationResolver.cs @@ -73,10 +73,7 @@ public List GetWorkerConfigs(List probingPaths, string fallbackP continue; } - IEnumerable workerVersions = _fileSystem.Directory.EnumerateDirectories(workerRuntimePath); - var versions = GetWorkerVersionsDescending(workerVersions); - - PopulateWorkerConfigsFromProbingPaths(versions, workerRuntimePath, workerRuntimeDir, releaseChannel, outputDict); + PopulateWorkerConfigsFromProbingPaths(workerRuntimePath, workerRuntimeDir, releaseChannel, outputDict); } } } @@ -97,8 +94,11 @@ workerRuntime is not null && return outputDict.Values.ToList(); } - private void PopulateWorkerConfigsFromProbingPaths(IEnumerable versions, string languageWorkerPath, string languageWorkerFolder, string releaseChannel, Dictionary outputDict) + private void PopulateWorkerConfigsFromProbingPaths(string languageWorkerPath, string languageWorkerFolder, string releaseChannel, Dictionary outputDict) { + var versionsDir = _fileSystem.Directory.EnumerateDirectories(languageWorkerPath); + var versionPathMap = GetWorkerVersionsDescending(versionsDir); + int compatibleWorkerCount = 0; bool isStandardOrExtendedChannel = @@ -107,9 +107,9 @@ private void PopulateWorkerConfigsFromProbingPaths(IEnumerable versions releaseChannel.Equals(ScriptConstants.ExtendedPlatformChannelNameUpper)); // language worker version - foreach (Version versionFolder in versions) + foreach (var versionPair in versionPathMap) { - string languageWorkerVersionPath = Path.Combine(languageWorkerPath, versionFolder.ToString()); + string languageWorkerVersionPath = versionPair.Value; if (IsWorkerCompatibleWithHost(languageWorkerVersionPath)) { @@ -161,23 +161,23 @@ workerRuntime is not null && } } - private IEnumerable GetWorkerVersionsDescending(IEnumerable workerVersions) + private SortedList GetWorkerVersionsDescending(IEnumerable workerVersionPaths) { - if (!workerVersions.Any()) + var versionPathMap = new SortedList(new DescendingVersionComparer()); + + if (!workerVersionPaths.Any()) { - return Enumerable.Empty(); + return versionPathMap; } - var versions = new List(); - - foreach (var workerVersion in workerVersions) + foreach (var workerVersionPath in workerVersionPaths) { - string versionDir = Path.GetFileName(workerVersion); + string versionDir = Path.GetFileName(workerVersionPath); string formattedVersion = FormatVersion(versionDir); if (Version.TryParse(formattedVersion, out Version version)) { - versions.Add(version); + versionPathMap[version] = workerVersionPath; } else { @@ -185,17 +185,7 @@ private IEnumerable GetWorkerVersionsDescending(IEnumerable wor } } - return versions.OrderDescending(); - } - - private static string FormatVersion(string version) - { - if (!version.Contains('.')) - { - version = version + ".0"; // Handle versions like '1' as '1.0' - } - - return version; + return versionPathMap; } private bool IsWorkerCompatibleWithHost(string workerDir) @@ -277,5 +267,23 @@ private bool IsRequiredWorkerRuntime(string workerRuntime, string workerDir) { return workerRuntime is not null && !workerRuntime.Equals(workerDir, StringComparison.OrdinalIgnoreCase); } + + private string FormatVersion(string version) + { + if (!version.Contains('.')) + { + version = version + ".0"; // Handle versions like '1' as '1.0' + } + + return version; + } + + private class DescendingVersionComparer : IComparer + { + public int Compare(Version x, Version y) + { + return y.CompareTo(x); // Inverted comparison for descending order + } + } } } \ No newline at end of file From f0a30aebe675e5876d66ac6691692a391474658d Mon Sep 17 00:00:00 2001 From: Surbhi Gupta Date: Tue, 1 Jul 2025 00:54:13 -0500 Subject: [PATCH 34/55] Getting parent directory for windows probing path --- .../Workers/Rpc/Configuration/LanguageWorkerOptionsSetup.cs | 6 +++--- .../Rpc/Configuration/WorkerConfigurationResolver.cs | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/WebJobs.Script/Workers/Rpc/Configuration/LanguageWorkerOptionsSetup.cs b/src/WebJobs.Script/Workers/Rpc/Configuration/LanguageWorkerOptionsSetup.cs index b165792eec..666007bca6 100644 --- a/src/WebJobs.Script/Workers/Rpc/Configuration/LanguageWorkerOptionsSetup.cs +++ b/src/WebJobs.Script/Workers/Rpc/Configuration/LanguageWorkerOptionsSetup.cs @@ -6,6 +6,7 @@ using System.Diagnostics; using System.IO; using System.Linq; +using System.Reflection; using System.Text; using System.Text.Json; using Microsoft.Azure.WebJobs.Script.Config; @@ -120,13 +121,12 @@ internal List GetWorkerProbingPaths() if (_environment.IsAnyWindows()) { // Harcoded site extensions path for Windows until Antares sets it as an Environment variable. - var assemblyPath = System.Reflection.Assembly.GetExecutingAssembly().Location; + var assemblyPath = Assembly.GetExecutingAssembly().Location; var assemblyDir = Path.GetDirectoryName(assemblyPath); - var parentDir = Directory.GetParent(assemblyDir).FullName; + var parentDir = Directory.GetParent(assemblyDir)?.Parent?.FullName; //Move 2 directories up to get to the SiteExtensions directory var windowsWorkerFullProbingPath = Path.Combine(parentDir, RpcWorkerConstants.DefaultWorkersDirectoryName); probingPaths.Add(windowsWorkerFullProbingPath); - _logger.LogTrace("Windows worker probing path: {windowsWorkerFullProbingPath}", windowsWorkerFullProbingPath); } } diff --git a/src/WebJobs.Script/Workers/Rpc/Configuration/WorkerConfigurationResolver.cs b/src/WebJobs.Script/Workers/Rpc/Configuration/WorkerConfigurationResolver.cs index 9a563676e6..2f66127ef5 100644 --- a/src/WebJobs.Script/Workers/Rpc/Configuration/WorkerConfigurationResolver.cs +++ b/src/WebJobs.Script/Workers/Rpc/Configuration/WorkerConfigurationResolver.cs @@ -64,7 +64,7 @@ public List GetWorkerConfigs(List probingPaths, string fallbackP continue; } - // Only skip worker directories that don't match the current runtime or are not enabled via hosting config + // Skip worker directories that don't match the current runtime or are not enabled via hosting config if (!_workersAvailableForResolutionViaHostingConfig.Contains(workerRuntimeDir) || (!_environment.IsMultiLanguageRuntimeEnvironment() && !_environment.IsPlaceholderModeEnabled() && From f92c521dd07a5bdd4d119f45e1891a8ab41c298f Mon Sep 17 00:00:00 2001 From: Surbhi Gupta Date: Wed, 2 Jul 2025 00:43:19 -0500 Subject: [PATCH 35/55] Refactored code and introduced static and dynamic worker config resolution --- ... => DynamicWorkerConfigurationResolver.cs} | 30 +++++++---- .../LanguageWorkerOptionsSetup.cs | 12 +++-- .../Configuration/RpcWorkerConfigFactory.cs | 53 ++----------------- .../StaticWorkerConfigurationResolver.cs | 43 +++++++++++++++ .../WorkerConfigurationHelper.cs | 28 ++++++++++ .../Rpc/IWorkerConfigurationResolver.cs | 9 +--- .../LanguageWorkerOptionsSetupTests.cs | 4 +- .../Rpc/RpcWorkerConfigFactoryTests.cs | 27 +++++----- .../Workers/Rpc/RpcWorkerConfigTests.cs | 2 +- .../Rpc/WorkerConfigurationResolverTests.cs | 13 +++-- 10 files changed, 125 insertions(+), 96 deletions(-) rename src/WebJobs.Script/Workers/Rpc/Configuration/{WorkerConfigurationResolver.cs => DynamicWorkerConfigurationResolver.cs} (91%) create mode 100644 src/WebJobs.Script/Workers/Rpc/Configuration/StaticWorkerConfigurationResolver.cs diff --git a/src/WebJobs.Script/Workers/Rpc/Configuration/WorkerConfigurationResolver.cs b/src/WebJobs.Script/Workers/Rpc/Configuration/DynamicWorkerConfigurationResolver.cs similarity index 91% rename from src/WebJobs.Script/Workers/Rpc/Configuration/WorkerConfigurationResolver.cs rename to src/WebJobs.Script/Workers/Rpc/Configuration/DynamicWorkerConfigurationResolver.cs index 2f66127ef5..13ad1a894c 100644 --- a/src/WebJobs.Script/Workers/Rpc/Configuration/WorkerConfigurationResolver.cs +++ b/src/WebJobs.Script/Workers/Rpc/Configuration/DynamicWorkerConfigurationResolver.cs @@ -14,7 +14,7 @@ namespace Microsoft.Azure.WebJobs.Script.Workers.Rpc.Configuration { - internal sealed class WorkerConfigurationResolver : IWorkerConfigurationResolver + internal sealed class DynamicWorkerConfigurationResolver : IWorkerConfigurationResolver { private readonly IConfiguration _config; private readonly ILogger _logger; @@ -22,14 +22,16 @@ internal sealed class WorkerConfigurationResolver : IWorkerConfigurationResolver private readonly IEnvironment _environment; private readonly IFileSystem _fileSystem; private readonly HashSet _workersAvailableForResolutionViaHostingConfig; + private readonly List _workerProbingPaths; private readonly JsonSerializerOptions _jsonSerializerOptions = new() { PropertyNameCaseInsensitive = true }; - public WorkerConfigurationResolver(IConfiguration config, + public DynamicWorkerConfigurationResolver(IConfiguration config, ILogger logger, IEnvironment environment, IFileSystem fileSystem, IWorkerProfileManager workerProfileManager, - HashSet workersAvailableForResolutionViaHostingConfig) + HashSet workersAvailableForResolutionViaHostingConfig, + List workerProbingPaths) { _config = config ?? throw new ArgumentNullException(nameof(config)); _logger = logger ?? throw new ArgumentNullException(nameof(logger)); @@ -37,20 +39,24 @@ public WorkerConfigurationResolver(IConfiguration config, _fileSystem = fileSystem ?? throw new ArgumentNullException(nameof(fileSystem)); _profileManager = workerProfileManager ?? throw new ArgumentNullException(nameof(workerProfileManager)); _workersAvailableForResolutionViaHostingConfig = workersAvailableForResolutionViaHostingConfig ?? throw new ArgumentNullException(nameof(workersAvailableForResolutionViaHostingConfig)); + _workerProbingPaths = workerProbingPaths; } - public List GetWorkerConfigs(List probingPaths, string fallbackPath) + public List GetWorkerConfigs() { + string probingPaths = _workerProbingPaths is not null ? string.Join(", ", _workerProbingPaths) : null; + _logger.LogDebug("Workers probing paths set to: {probingPaths}.", probingPaths); + // Dictionary of { FUNCTIONS_WORKER_RUNTIME environment variable value : path of workerConfig } Dictionary outputDict = new Dictionary(StringComparer.OrdinalIgnoreCase); var workerRuntime = _environment.GetEnvironmentVariable(EnvironmentSettingNames.FunctionWorkerRuntime); string releaseChannel = Utility.GetPlatformReleaseChannel(_environment); - if (!probingPaths.IsNullOrEmpty()) + if (_workerProbingPaths is not null) { // probing path directory structure is: /// - foreach (var probingPath in probingPaths) + foreach (var probingPath in _workerProbingPaths) { if (!string.IsNullOrEmpty(probingPath) && _fileSystem.Directory.Exists(probingPath)) { @@ -86,10 +92,8 @@ workerRuntime is not null && return outputDict.Values.ToList(); } - _logger.LogDebug("Searching for worker configs in the fallback directory."); - // Search in fallback path if worker cannot be found in probing paths - PopulateWorkerConfigsFromWithinHost(fallbackPath, workerRuntime, outputDict); + PopulateWorkerConfigsFromWithinHost(workerRuntime, outputDict); return outputDict.Values.ToList(); } @@ -130,9 +134,13 @@ private void PopulateWorkerConfigsFromProbingPaths(string languageWorkerPath, st } } - private void PopulateWorkerConfigsFromWithinHost(string fallbackPath, string workerRuntime, Dictionary outputDict) + private void PopulateWorkerConfigsFromWithinHost(string workerRuntime, Dictionary outputDict) { - if (_fileSystem.Directory.Exists(fallbackPath)) + var fallbackPath = WorkerConfigurationHelper.GetWorkersDirPath(_config); + + _logger.LogDebug("Searching for worker configs in the fallback directory: {fallbackPath}", fallbackPath); + + if (fallbackPath != null && _fileSystem.Directory.Exists(fallbackPath)) { foreach (var workerPath in _fileSystem.Directory.EnumerateDirectories(fallbackPath)) { diff --git a/src/WebJobs.Script/Workers/Rpc/Configuration/LanguageWorkerOptionsSetup.cs b/src/WebJobs.Script/Workers/Rpc/Configuration/LanguageWorkerOptionsSetup.cs index 666007bca6..d7f821236f 100644 --- a/src/WebJobs.Script/Workers/Rpc/Configuration/LanguageWorkerOptionsSetup.cs +++ b/src/WebJobs.Script/Workers/Rpc/Configuration/LanguageWorkerOptionsSetup.cs @@ -84,15 +84,19 @@ public void Configure(LanguageWorkerOptions options) HashSet workers = GetWorkersAvailableForResolutionViaHostingConfig(); bool dynamicWorkerResolutionEnabled = Utility.IsDynamicWorkerResolutionEnabled(_environment, workers); - List probingPaths = null; + IWorkerConfigurationResolver workerConfigurationResolver = null; if (dynamicWorkerResolutionEnabled) { - probingPaths = GetWorkerProbingPaths(); + List probingPaths = GetWorkerProbingPaths(); + workerConfigurationResolver = new DynamicWorkerConfigurationResolver(configuration, _logger, _environment, FileUtility.Instance, _workerProfileManager, workers, probingPaths); + } + else + { + workerConfigurationResolver = new StaticWorkerConfigurationResolver(configuration, _logger); } - var workerConfigurationResolver = new WorkerConfigurationResolver(configuration, _logger, _environment, FileUtility.Instance, _workerProfileManager, workers); - var configFactory = new RpcWorkerConfigFactory(configuration, _logger, SystemRuntimeInformation.Instance, _environment, _metricsLogger, _workerProfileManager, workerConfigurationResolver, dynamicWorkerResolutionEnabled, probingPaths, workers); + var configFactory = new RpcWorkerConfigFactory(configuration, _logger, SystemRuntimeInformation.Instance, _environment, _metricsLogger, _workerProfileManager, workerConfigurationResolver, dynamicWorkerResolutionEnabled, workers); options.WorkerConfigs = configFactory.GetConfigs(); } diff --git a/src/WebJobs.Script/Workers/Rpc/Configuration/RpcWorkerConfigFactory.cs b/src/WebJobs.Script/Workers/Rpc/Configuration/RpcWorkerConfigFactory.cs index cffb156bc2..c962184134 100644 --- a/src/WebJobs.Script/Workers/Rpc/Configuration/RpcWorkerConfigFactory.cs +++ b/src/WebJobs.Script/Workers/Rpc/Configuration/RpcWorkerConfigFactory.cs @@ -7,15 +7,12 @@ using System.IO; using System.Linq; using System.Text.Json; -using System.Text.RegularExpressions; using System.Threading.Tasks; -using Microsoft.Azure.WebJobs.Script.Config; using Microsoft.Azure.WebJobs.Script.Diagnostics; using Microsoft.Azure.WebJobs.Script.Workers.Profiles; using Microsoft.Azure.WebJobs.Script.Workers.Rpc.Configuration; using Microsoft.Extensions.Configuration; using Microsoft.Extensions.Logging; -using Microsoft.Extensions.Options; namespace Microsoft.Azure.WebJobs.Script.Workers.Rpc { @@ -38,7 +35,6 @@ internal class RpcWorkerConfigFactory }; private Dictionary _workerDescriptionDictionary = new Dictionary(); - private List _workerProbingPaths; public RpcWorkerConfigFactory(IConfiguration config, ILogger logger, @@ -48,7 +44,6 @@ public RpcWorkerConfigFactory(IConfiguration config, IWorkerProfileManager workerProfileManager, IWorkerConfigurationResolver workerConfigurationResolver, bool dynamicWorkerResolutionEnabled = false, - List workerProbingPaths = null, HashSet workersAvailableForResolutionViaHostingConfig = null) { _config = config ?? throw new ArgumentNullException(nameof(config)); @@ -60,17 +55,9 @@ public RpcWorkerConfigFactory(IConfiguration config, _workerRuntime = _environment.GetEnvironmentVariable(RpcWorkerConstants.FunctionWorkerRuntimeSettingName); _workerConfigurationResolver = workerConfigurationResolver ?? throw new ArgumentNullException(nameof(workerConfigurationResolver)); - WorkersDirPath = GetDefaultWorkersDirectory(Directory.Exists); - var workersDirectorySection = _config.GetSection($"{RpcWorkerConstants.LanguageWorkersSectionName}:{WorkerConstants.WorkersDirectorySectionName}"); - - _workerProbingPaths = workerProbingPaths; _dynamicWorkerResolutionEnabled = dynamicWorkerResolutionEnabled; _workersAvailableForResolutionViaHostingConfig = workersAvailableForResolutionViaHostingConfig; - - if (!string.IsNullOrEmpty(workersDirectorySection.Value)) - { - WorkersDirPath = workersDirectorySection.Value; - } + WorkersDirPath = WorkerConfigurationHelper.GetWorkersDirPath(config); } public string WorkersDirPath { get; } @@ -84,20 +71,6 @@ public IList GetConfigs() } } - internal static string GetDefaultWorkersDirectory(Func directoryExists) - { -#pragma warning disable SYSLIB0012 // Type or member is obsolete - string assemblyLocalPath = Path.GetDirectoryName(new Uri(typeof(RpcWorkerConfigFactory).Assembly.CodeBase).LocalPath); -#pragma warning restore SYSLIB0012 // Type or member is obsolete - string workersDirPath = Path.Combine(assemblyLocalPath, RpcWorkerConstants.DefaultWorkersDirectoryName); - if (!directoryExists(workersDirPath)) - { - // Site Extension. Default to parent directory - workersDirPath = Path.Combine(Directory.GetParent(assemblyLocalPath).FullName, RpcWorkerConstants.DefaultWorkersDirectoryName); - } - return workersDirPath; - } - internal void BuildWorkerProviderDictionary() { AddProviders(); @@ -106,29 +79,11 @@ internal void BuildWorkerProviderDictionary() internal void AddProviders() { - if (_dynamicWorkerResolutionEnabled) - { - _logger.LogDebug("Workers probing paths set to: {probingPaths} and Workers Directory set as fallback path: {workersDirPath}", string.Join(", ", _workerProbingPaths), WorkersDirPath); + List workerConfigs = _workerConfigurationResolver.GetWorkerConfigs(); - List workerConfigs = _workerConfigurationResolver.GetWorkerConfigs(_workerProbingPaths, WorkersDirPath); - - foreach (var workerConfig in workerConfigs) - { - AddProvider(workerConfig); - } - } - else + foreach (var workerConfig in workerConfigs) { - _logger.LogDebug("Workers Directory set to: {WorkersDirPath}", WorkersDirPath); - - foreach (var workerDir in Directory.EnumerateDirectories(WorkersDirPath)) - { - string workerConfigPath = Path.Combine(workerDir, RpcWorkerConstants.WorkerConfigFileName); - if (File.Exists(workerConfigPath)) - { - AddProvider(workerDir); - } - } + AddProvider(workerConfig); } } diff --git a/src/WebJobs.Script/Workers/Rpc/Configuration/StaticWorkerConfigurationResolver.cs b/src/WebJobs.Script/Workers/Rpc/Configuration/StaticWorkerConfigurationResolver.cs new file mode 100644 index 0000000000..6c8c0e87ce --- /dev/null +++ b/src/WebJobs.Script/Workers/Rpc/Configuration/StaticWorkerConfigurationResolver.cs @@ -0,0 +1,43 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the MIT License. See License.txt in the project root for license information. + +using System; +using System.Collections.Generic; +using System.IO; +using Microsoft.Extensions.Configuration; +using Microsoft.Extensions.Logging; + +namespace Microsoft.Azure.WebJobs.Script.Workers.Rpc.Configuration +{ + internal class StaticWorkerConfigurationResolver : IWorkerConfigurationResolver + { + private readonly ILogger _logger; + private readonly IConfiguration _configuration; + + public StaticWorkerConfigurationResolver(IConfiguration configuration, ILogger logger) + { + _logger = logger ?? throw new ArgumentNullException(nameof(logger)); + _configuration = configuration; + } + + public List GetWorkerConfigs() + { + string workersDirPath = WorkerConfigurationHelper.GetWorkersDirPath(_configuration); + _logger.LogDebug("Workers Directory set to: {WorkersDirPath}", workersDirPath); + + List workerConfigs = new(); + + foreach (var workerDir in Directory.EnumerateDirectories(workersDirPath)) + { + string workerConfigPath = Path.Combine(workerDir, RpcWorkerConstants.WorkerConfigFileName); + + if (File.Exists(workerConfigPath)) + { + workerConfigs.Add(workerDir); + } + } + + return workerConfigs; + } + } +} diff --git a/src/WebJobs.Script/Workers/Rpc/Configuration/WorkerConfigurationHelper.cs b/src/WebJobs.Script/Workers/Rpc/Configuration/WorkerConfigurationHelper.cs index 63fb217ad4..bee86e7c54 100644 --- a/src/WebJobs.Script/Workers/Rpc/Configuration/WorkerConfigurationHelper.cs +++ b/src/WebJobs.Script/Workers/Rpc/Configuration/WorkerConfigurationHelper.cs @@ -126,5 +126,33 @@ internal static void AddArgumentsFromAppSettings(RpcWorkerDescription workerDesc ((List)workerDescription.Arguments).AddRange(Regex.Split(argumentsSection.Value, @"\s+")); } } + + internal static string GetDefaultWorkersDirectory(Func directoryExists) + { +#pragma warning disable SYSLIB0012 // Type or member is obsolete + string assemblyLocalPath = Path.GetDirectoryName(new Uri(typeof(RpcWorkerConfigFactory).Assembly.CodeBase).LocalPath); +#pragma warning restore SYSLIB0012 // Type or member is obsolete + string workersDirPath = Path.Combine(assemblyLocalPath, RpcWorkerConstants.DefaultWorkersDirectoryName); + if (!directoryExists(workersDirPath)) + { + // Site Extension. Default to parent directory + workersDirPath = Path.Combine(Directory.GetParent(assemblyLocalPath).FullName, RpcWorkerConstants.DefaultWorkersDirectoryName); + } + return workersDirPath; + } + + internal static string GetWorkersDirPath(IConfiguration configuration) + { + var workersDirectorySection = configuration?.GetSection($"{RpcWorkerConstants.LanguageWorkersSectionName}:{WorkerConstants.WorkersDirectorySectionName}"); + + string workersDirPath = GetDefaultWorkersDirectory(Directory.Exists); + + if (!string.IsNullOrEmpty(workersDirectorySection?.Value)) + { + workersDirPath = workersDirectorySection.Value; + } + + return workersDirPath; + } } } \ No newline at end of file diff --git a/src/WebJobs.Script/Workers/Rpc/IWorkerConfigurationResolver.cs b/src/WebJobs.Script/Workers/Rpc/IWorkerConfigurationResolver.cs index 4af0eb4805..5b93b42a1a 100644 --- a/src/WebJobs.Script/Workers/Rpc/IWorkerConfigurationResolver.cs +++ b/src/WebJobs.Script/Workers/Rpc/IWorkerConfigurationResolver.cs @@ -10,13 +10,6 @@ namespace Microsoft.Azure.WebJobs.Script.Workers.Rpc.Configuration /// public interface IWorkerConfigurationResolver { - /// - /// Gets worker configs using configured probing paths and fallback path. - /// - /// List of probing paths where workers are located. - /// A fallback path to check when workers cannot be found in the probing paths. - /// A list of paths to worker configuration files. - - List GetWorkerConfigs(List probingPaths, string fallbackPath); + List GetWorkerConfigs(); } } \ No newline at end of file diff --git a/test/WebJobs.Script.Tests/Configuration/LanguageWorkerOptionsSetupTests.cs b/test/WebJobs.Script.Tests/Configuration/LanguageWorkerOptionsSetupTests.cs index db0597da9a..f73f211688 100644 --- a/test/WebJobs.Script.Tests/Configuration/LanguageWorkerOptionsSetupTests.cs +++ b/test/WebJobs.Script.Tests/Configuration/LanguageWorkerOptionsSetupTests.cs @@ -168,7 +168,7 @@ public void LanguageWorkerOptions_FallbackPath_Expected_ListOfConfigs(string wor string expectedLog = $"Added WorkerConfig for language: {workerRuntime} with Worker Path: {path}"; Assert.True(logs.Any(l => l.FormattedMessage.Contains(expectedLog))); Assert.True(logs.Any(l => l.FormattedMessage.Contains("Workers probing paths set to:"))); - Assert.True(logs.Any(l => l.FormattedMessage.Contains("Searching for worker configs in the fallback directory."))); + Assert.True(logs.Any(l => l.FormattedMessage.Contains("Searching for worker configs in the fallback directory"))); } [Theory] @@ -250,7 +250,7 @@ public void LanguageWorkerOptions_FallbackPath_NullHostingConfig(string workerRu string expectedLog = $"Added WorkerConfig for language: {workerRuntime} with Worker Path: {path}"; Assert.True(logs.Any(l => l.FormattedMessage.Contains(expectedLog))); Assert.True(logs.Any(l => l.FormattedMessage.Contains("Workers probing paths set to:"))); - Assert.True(logs.Any(l => l.FormattedMessage.Contains("Searching for worker configs in the fallback directory."))); + Assert.True(logs.Any(l => l.FormattedMessage.Contains("Searching for worker configs in the fallback directory"))); } } } \ No newline at end of file diff --git a/test/WebJobs.Script.Tests/Workers/Rpc/RpcWorkerConfigFactoryTests.cs b/test/WebJobs.Script.Tests/Workers/Rpc/RpcWorkerConfigFactoryTests.cs index 38dd58d826..0b6a29b9b7 100644 --- a/test/WebJobs.Script.Tests/Workers/Rpc/RpcWorkerConfigFactoryTests.cs +++ b/test/WebJobs.Script.Tests/Workers/Rpc/RpcWorkerConfigFactoryTests.cs @@ -6,7 +6,6 @@ using System.IO; using System.Linq; using System.Text.Json; -using Microsoft.Azure.WebJobs.Script; using Microsoft.Azure.WebJobs.Script.Config; using Microsoft.Azure.WebJobs.Script.Workers; using Microsoft.Azure.WebJobs.Script.Workers.Profiles; @@ -42,7 +41,7 @@ public void DefaultLanguageWorkersDir() var expectedWorkersDir = Path.Combine(Path.GetDirectoryName(new Uri(typeof(RpcWorkerConfigFactory).Assembly.Location).LocalPath), RpcWorkerConstants.DefaultWorkersDirectoryName); var config = new ConfigurationBuilder().Build(); var testLogger = new TestLogger("test"); - var workerConfigurationResolver = new WorkerConfigurationResolver(config, testLogger, _testEnvironment, FileUtility.Instance, _testWorkerProfileManager, new HashSet()); + var workerConfigurationResolver = new StaticWorkerConfigurationResolver(config, testLogger); var configFactory = new RpcWorkerConfigFactory(config, testLogger, _testSysRuntimeInfo, _testEnvironment, new TestMetricsLogger(), _testWorkerProfileManager, workerConfigurationResolver); Assert.Equal(expectedWorkersDir, configFactory.WorkersDirPath); } @@ -61,8 +60,8 @@ public void GetDefaultWorkersDirectory_Returns_Expected() var config = new ConfigurationBuilder().Build(); var testLogger = new TestLogger("test"); - Assert.Equal(expectedWorkersDirIsCurrentDir, RpcWorkerConfigFactory.GetDefaultWorkersDirectory(Directory.Exists)); - Assert.Equal(expectedWorkersDirIsParentDir, RpcWorkerConfigFactory.GetDefaultWorkersDirectory(testDirectoryExists)); + Assert.Equal(expectedWorkersDirIsCurrentDir, WorkerConfigurationHelper.GetDefaultWorkersDirectory(Directory.Exists)); + Assert.Equal(expectedWorkersDirIsParentDir, WorkerConfigurationHelper.GetDefaultWorkersDirectory(testDirectoryExists)); } [Fact] @@ -76,7 +75,7 @@ public void LanguageWorker_WorkersDir_Set() }) .Build(); var testLogger = new TestLogger("test"); - var workerConfigurationResolver = new WorkerConfigurationResolver(config, testLogger, _testEnvironment, FileUtility.Instance, _testWorkerProfileManager, new HashSet()); + var workerConfigurationResolver = new StaticWorkerConfigurationResolver(config, testLogger); var configFactory = new RpcWorkerConfigFactory(config, testLogger, _testSysRuntimeInfo, _testEnvironment, new TestMetricsLogger(), _testWorkerProfileManager, workerConfigurationResolver); Assert.Equal(expectedWorkersDir, configFactory.WorkersDirPath); } @@ -93,7 +92,7 @@ public void LanguageWorker_WorkersDir_NotSet() var config = configBuilder.Build(); var scriptSettingsManager = new ScriptSettingsManager(config); var testLogger = new TestLogger("test"); - var workerConfigurationResolver = new WorkerConfigurationResolver(config, testLogger, _testEnvironment, FileUtility.Instance, _testWorkerProfileManager, new HashSet()); + var workerConfigurationResolver = new StaticWorkerConfigurationResolver(config, testLogger); var configFactory = new RpcWorkerConfigFactory(config, testLogger, _testSysRuntimeInfo, _testEnvironment, new TestMetricsLogger(), _testWorkerProfileManager, workerConfigurationResolver); Assert.Equal(expectedWorkersDir, configFactory.WorkersDirPath); } @@ -118,7 +117,7 @@ public void WorkerDescription_Skipped_When_Profile_Disables_Worker() var scriptSettingsManager = new ScriptSettingsManager(config); var testLogger = new TestLogger("test"); _testEnvironment.SetEnvironmentVariable("ENV_VAR_BAR", "True"); - var workerConfigurationResolver = new WorkerConfigurationResolver(config, testLogger, _testEnvironment, FileUtility.Instance, _testWorkerProfileManager, new HashSet()); + var workerConfigurationResolver = new StaticWorkerConfigurationResolver(config, testLogger); var configFactory = new RpcWorkerConfigFactory(config, testLogger, _testSysRuntimeInfo, _testEnvironment, new TestMetricsLogger(), _testWorkerProfileManager, workerConfigurationResolver); var workerConfigs = configFactory.GetConfigs(); @@ -141,7 +140,7 @@ public void JavaPath_FromEnvVars() var config = configBuilder.Build(); var scriptSettingsManager = new ScriptSettingsManager(config); var testLogger = new TestLogger("test"); - var workerConfigurationResolver = new WorkerConfigurationResolver(config, testLogger, _testEnvironment, FileUtility.Instance, _testWorkerProfileManager, new HashSet()); + var workerConfigurationResolver = new StaticWorkerConfigurationResolver(config, testLogger); var configFactory = new RpcWorkerConfigFactory(config, testLogger, _testSysRuntimeInfo, _testEnvironment, new TestMetricsLogger(), _testWorkerProfileManager, workerConfigurationResolver); var workerConfigs = configFactory.GetConfigs(); var javaPath = workerConfigs.FirstOrDefault(c => c.Description.Language.Equals("java", StringComparison.OrdinalIgnoreCase)).Description.DefaultExecutablePath; @@ -164,7 +163,7 @@ public void DefaultWorkerConfigs_Overrides_DefaultWorkerRuntimeVersion_AppSettin _testEnvironment.SetEnvironmentVariable(EnvironmentSettingNames.AzureWebsitePlaceholderMode, "1"); using var variables = new TestScopedSettings(scriptSettingsManager, testEnvVariables); - var workerConfigurationResolver = new WorkerConfigurationResolver(config, testLogger, _testEnvironment, FileUtility.Instance, _testWorkerProfileManager, new HashSet()); + var workerConfigurationResolver = new StaticWorkerConfigurationResolver(config, testLogger); var configFactory = new RpcWorkerConfigFactory(config, testLogger, _testSysRuntimeInfo, _testEnvironment, new TestMetricsLogger(), _testWorkerProfileManager, workerConfigurationResolver); var workerConfigs = configFactory.GetConfigs(); var pythonWorkerConfig = workerConfigs.FirstOrDefault(w => w.Description.Language.Equals("python", StringComparison.OrdinalIgnoreCase)); @@ -192,8 +191,8 @@ public void DefaultWorkerConfigs_Overrides_VersionAppSetting(string runtimeSetti var config = ScriptSettingsManager.CreateDefaultConfigurationBuilder().Build(); var testLogger = new TestLogger("test"); - var workerConfigurationResolver = new WorkerConfigurationResolver(config, testLogger, testEnvironment, FileUtility.Instance, _testWorkerProfileManager, new HashSet() { "powershell" }); - var configFactory = new RpcWorkerConfigFactory(config, testLogger, _testSysRuntimeInfo, testEnvironment, new TestMetricsLogger(), _testWorkerProfileManager, workerConfigurationResolver, enableProbingPath is null ? false : true, new List() { probingPath }, new HashSet() { "powershell" }); + var workerConfigurationResolver = new DynamicWorkerConfigurationResolver(config, testLogger, testEnvironment, FileUtility.Instance, _testWorkerProfileManager, new HashSet() { "powershell" }, new List() { probingPath }); + var configFactory = new RpcWorkerConfigFactory(config, testLogger, _testSysRuntimeInfo, testEnvironment, new TestMetricsLogger(), _testWorkerProfileManager, workerConfigurationResolver, enableProbingPath is null ? false : true, new HashSet() { "powershell" }); var workerConfigs = configFactory.GetConfigs(); var powershellWorkerConfig = workerConfigs.FirstOrDefault(w => w.Description.Language.Equals("powershell", StringComparison.OrdinalIgnoreCase)); @@ -226,7 +225,7 @@ public void ShouldAddProvider_Returns_Expected(string workerLanguage, string wor } var config = new ConfigurationBuilder().Build(); var testLogger = new TestLogger("test"); - var workerConfigurationResolver = new WorkerConfigurationResolver(config, testLogger, _testEnvironment, FileUtility.Instance, _testWorkerProfileManager, new HashSet()); + var workerConfigurationResolver = new StaticWorkerConfigurationResolver(config, testLogger); var rpcWorkerConfigFactory = new RpcWorkerConfigFactory(config, testLogger, _testSysRuntimeInfo, _testEnvironment, new TestMetricsLogger(), _testWorkerProfileManager, workerConfigurationResolver); _testEnvironment.SetEnvironmentVariable(RpcWorkerConstants.FunctionWorkerRuntimeSettingName, workerRuntime); Assert.Equal(expectedResult, rpcWorkerConfigFactory.ShouldAddWorkerConfig(workerLanguage)); @@ -271,7 +270,7 @@ public void GetWorkerProcessCount_Tests(bool defaultWorkerConfig, bool setProces var config = new ConfigurationBuilder().Build(); var testLogger = new TestLogger("test"); - var workerConfigurationResolver = new WorkerConfigurationResolver(config, testLogger, _testEnvironment, FileUtility.Instance, _testWorkerProfileManager, new HashSet()); + var workerConfigurationResolver = new StaticWorkerConfigurationResolver(config, testLogger); var rpcWorkerConfigFactory = new RpcWorkerConfigFactory(config, testLogger, _testSysRuntimeInfo, _testEnvironment, new TestMetricsLogger(), _testWorkerProfileManager, workerConfigurationResolver); var result = rpcWorkerConfigFactory.GetWorkerProcessCount(workerConfig); @@ -312,7 +311,7 @@ public void GetWorkerProcessCount_ThrowsException_Tests() var config = new ConfigurationBuilder().Build(); var testLogger = new TestLogger("test"); - var workerConfigurationResolver = new WorkerConfigurationResolver(config, testLogger, _testEnvironment, FileUtility.Instance, _testWorkerProfileManager, new HashSet()); + var workerConfigurationResolver = new StaticWorkerConfigurationResolver(config, testLogger); var rpcWorkerConfigFactory = new RpcWorkerConfigFactory(config, testLogger, _testSysRuntimeInfo, _testEnvironment, new TestMetricsLogger(), _testWorkerProfileManager, workerConfigurationResolver); var resultEx1 = Assert.Throws(() => rpcWorkerConfigFactory.GetWorkerProcessCount(workerConfig)); Assert.Contains("ProcessCount must be greater than 0", resultEx1.Message); diff --git a/test/WebJobs.Script.Tests/Workers/Rpc/RpcWorkerConfigTests.cs b/test/WebJobs.Script.Tests/Workers/Rpc/RpcWorkerConfigTests.cs index c2fe5bd3f3..355aca0823 100644 --- a/test/WebJobs.Script.Tests/Workers/Rpc/RpcWorkerConfigTests.cs +++ b/test/WebJobs.Script.Tests/Workers/Rpc/RpcWorkerConfigTests.cs @@ -686,7 +686,7 @@ private IEnumerable TestReadWorkerProviderFromConfig(IEnumerabl var scriptHostOptions = new ScriptJobHostOptions(); var scriptSettingsManager = new ScriptSettingsManager(config); var workerProfileManager = new Mock(); - var workerConfigurationResolver = new WorkerConfigurationResolver(config, testLogger, _testEnvironment, FileUtility.Instance, workerProfileManager.Object, new HashSet()); + IWorkerConfigurationResolver workerConfigurationResolver = new StaticWorkerConfigurationResolver(config, testLogger); var configFactory = new RpcWorkerConfigFactory(config, testLogger, _testSysRuntimeInfo, _testEnvironment, new TestMetricsLogger(), workerProfileManager.Object, workerConfigurationResolver); if (appSvcEnv) diff --git a/test/WebJobs.Script.Tests/Workers/Rpc/WorkerConfigurationResolverTests.cs b/test/WebJobs.Script.Tests/Workers/Rpc/WorkerConfigurationResolverTests.cs index c25e6053c2..de1fc5e191 100644 --- a/test/WebJobs.Script.Tests/Workers/Rpc/WorkerConfigurationResolverTests.cs +++ b/test/WebJobs.Script.Tests/Workers/Rpc/WorkerConfigurationResolverTests.cs @@ -4,7 +4,6 @@ using System.Collections.Generic; using System.IO; using System.Linq; -using Microsoft.Azure.WebJobs.Script; using Microsoft.Azure.WebJobs.Script.Workers.Profiles; using Microsoft.Azure.WebJobs.Script.Workers.Rpc.Configuration; using Microsoft.Extensions.Configuration; @@ -44,9 +43,9 @@ public void GetWorkerConfigs_MultiLanguageWorker_ReturnsExpectedConfigs(string r mockEnvironment.Setup(p => p.GetEnvironmentVariable(EnvironmentSettingNames.AppKind)).Returns(ScriptConstants.WorkFlowAppKind); mockEnvironment.Setup(p => p.GetEnvironmentVariable(EnvironmentSettingNames.FunctionWorkerRuntime)).Returns((string)null); - var workerConfigurationResolver = new WorkerConfigurationResolver(_mockConfig.Object, _mockLogger.Object, mockEnvironment.Object, FileUtility.Instance, _mockProfileManager.Object, new HashSet() { "java", "node", "powershell" }); + var workerConfigurationResolver = new DynamicWorkerConfigurationResolver(_mockConfig.Object, _mockLogger.Object, mockEnvironment.Object, FileUtility.Instance, _mockProfileManager.Object, new HashSet() { "java", "node", "powershell" }, _probingPaths); - var result = workerConfigurationResolver.GetWorkerConfigs(_probingPaths, _fallbackPath); + var result = workerConfigurationResolver.GetWorkerConfigs(); // Assert Assert.Equal(result.Count, 5); @@ -78,9 +77,9 @@ public void GetWorkerConfigs_MultiLanguageWorker_NullOREmptyProbingPath_ReturnsE } // Act - var workerConfigurationResolver = new WorkerConfigurationResolver(_mockConfig.Object, _mockLogger.Object, mockEnvironment.Object, FileUtility.Instance, _mockProfileManager.Object, new HashSet() { "java", "node", "powershell" }); + var workerConfigurationResolver = new DynamicWorkerConfigurationResolver(_mockConfig.Object, _mockLogger.Object, mockEnvironment.Object, FileUtility.Instance, _mockProfileManager.Object, new HashSet() { "java", "node", "powershell" }, probingPaths); - var result = workerConfigurationResolver.GetWorkerConfigs(probingPaths, _fallbackPath); + var result = workerConfigurationResolver.GetWorkerConfigs(); // Assert Assert.Equal(result.Count, 5); @@ -123,9 +122,9 @@ public void GetWorkerConfigs_NullOREmptyProbingPath_ReturnsExpectedConfigs(strin var mockLogger = new Mock(); // Act - var workerConfigurationResolver = new WorkerConfigurationResolver(mockConfig.Object, mockLogger.Object, mockEnv.Object, FileUtility.Instance, mockProfileManager.Object, new HashSet() { "java", "node", "powershell" }); + var workerConfigurationResolver = new DynamicWorkerConfigurationResolver(mockConfig.Object, mockLogger.Object, mockEnv.Object, FileUtility.Instance, mockProfileManager.Object, new HashSet() { "java", "node", "powershell" }, probingPaths); - var result = workerConfigurationResolver.GetWorkerConfigs(probingPaths, _fallbackPath); + var result = workerConfigurationResolver.GetWorkerConfigs(); // Assert Assert.Equal(result.Count, 1); From bf3d12611a0a513c98725e58e9d0077962c2075d Mon Sep 17 00:00:00 2001 From: Surbhi Gupta Date: Wed, 2 Jul 2025 23:21:32 -0500 Subject: [PATCH 36/55] code cleanup --- src/WebJobs.Script/ScriptConstants.cs | 1 - src/WebJobs.Script/Utility.cs | 7 ----- .../Workers/Http/HttpWorkerDescription.cs | 2 +- .../ProcessManagement/WorkerDescription.cs | 2 +- .../DynamicWorkerConfigurationResolver.cs | 4 +-- .../LanguageWorkerOptionsSetup.cs | 6 ++-- .../Configuration/RpcWorkerConfigFactory.cs | 15 ++-------- .../WorkerConfigurationHelper.cs | 6 ++-- .../Workers/Rpc/RpcWorkerDescription.cs | 15 +--------- .../LanguageWorkerOptionsSetupTests.cs | 5 +--- test/WebJobs.Script.Tests/UtilityTests.cs | 5 ---- .../Rpc/RpcWorkerConfigFactoryTests.cs | 30 ++++++------------- 12 files changed, 21 insertions(+), 77 deletions(-) diff --git a/src/WebJobs.Script/ScriptConstants.cs b/src/WebJobs.Script/ScriptConstants.cs index 1b64881a51..b80b35fa8f 100644 --- a/src/WebJobs.Script/ScriptConstants.cs +++ b/src/WebJobs.Script/ScriptConstants.cs @@ -134,7 +134,6 @@ public static class ScriptConstants public const string FeatureFlagDisableMergedWebHostScriptHostConfiguration = "DisableMergedConfiguration"; public const string FeatureFlagEnableWorkerIndexing = "EnableWorkerIndexing"; public const string FeatureFlagDisableDynamicWorkerResolution = "DisableDynamicWorkerResolution"; - public const string FeatureFlagEnableDynamicWorkerResolution = "EnableDynamicWorkerResolution"; public const string FeatureFlagEnableDebugTracing = "EnableDebugTracing"; public const string FeatureFlagEnableProxies = "EnableProxies"; public const string FeatureFlagStrictHISModeEnabled = "StrictHISModeEnabled"; diff --git a/src/WebJobs.Script/Utility.cs b/src/WebJobs.Script/Utility.cs index ef11811086..7006e6e30a 100644 --- a/src/WebJobs.Script/Utility.cs +++ b/src/WebJobs.Script/Utility.cs @@ -1078,13 +1078,6 @@ public static bool IsDynamicWorkerResolutionEnabled(IEnvironment environment, Ha return false; } - bool isDynamicWorkerResolutionEnabled = FeatureFlags.IsEnabled(ScriptConstants.FeatureFlagEnableDynamicWorkerResolution, environment); - - if (isDynamicWorkerResolutionEnabled) - { - return true; - } - string workerRuntime = environment.GetEnvironmentVariable(RpcWorkerConstants.FunctionWorkerRuntimeSettingName); if (!environment.IsMultiLanguageRuntimeEnvironment() && !string.IsNullOrWhiteSpace(workerRuntime)) diff --git a/src/WebJobs.Script/Workers/Http/HttpWorkerDescription.cs b/src/WebJobs.Script/Workers/Http/HttpWorkerDescription.cs index 15ef28beb3..190433eea6 100644 --- a/src/WebJobs.Script/Workers/Http/HttpWorkerDescription.cs +++ b/src/WebJobs.Script/Workers/Http/HttpWorkerDescription.cs @@ -19,7 +19,7 @@ public class HttpWorkerDescription : WorkerDescription public override bool UseStdErrorStreamForErrorsOnly { get; set; } = true; - public override void ApplyDefaultsAndValidate(string inputWorkerDirectory, ILogger logger, bool probingPathEnabled = false, HashSet workersAvailableForResolutionViaHostingConfig = null) + public override void ApplyDefaultsAndValidate(string inputWorkerDirectory, ILogger logger) { if (inputWorkerDirectory == null) { diff --git a/src/WebJobs.Script/Workers/ProcessManagement/WorkerDescription.cs b/src/WebJobs.Script/Workers/ProcessManagement/WorkerDescription.cs index 6f8f438955..b950e33c3c 100644 --- a/src/WebJobs.Script/Workers/ProcessManagement/WorkerDescription.cs +++ b/src/WebJobs.Script/Workers/ProcessManagement/WorkerDescription.cs @@ -57,7 +57,7 @@ public abstract class WorkerDescription /// public bool? IsDisabled { get; set; } - public abstract void ApplyDefaultsAndValidate(string workerDirectory, ILogger logger, bool probingPathEnabled = false, HashSet workersAvailableForResolutionViaHostingConfig = null); + public abstract void ApplyDefaultsAndValidate(string workerDirectory, ILogger logger); internal void ThrowIfFileNotExists(string inputFile, string paramName) { diff --git a/src/WebJobs.Script/Workers/Rpc/Configuration/DynamicWorkerConfigurationResolver.cs b/src/WebJobs.Script/Workers/Rpc/Configuration/DynamicWorkerConfigurationResolver.cs index 13ad1a894c..ad87a06077 100644 --- a/src/WebJobs.Script/Workers/Rpc/Configuration/DynamicWorkerConfigurationResolver.cs +++ b/src/WebJobs.Script/Workers/Rpc/Configuration/DynamicWorkerConfigurationResolver.cs @@ -221,9 +221,7 @@ private bool IsWorkerCompatibleWithHost(string workerDir) workerDir: workerDir, profileManager: _profileManager, config: _config, - logger: _logger, - dynamicWorkerResolutionEnabled: true, - workersAvailableForResolutionViaHostingConfig: _workersAvailableForResolutionViaHostingConfig); + logger: _logger); if (workerDescription.IsDisabled == true) { diff --git a/src/WebJobs.Script/Workers/Rpc/Configuration/LanguageWorkerOptionsSetup.cs b/src/WebJobs.Script/Workers/Rpc/Configuration/LanguageWorkerOptionsSetup.cs index d7f821236f..424208ea47 100644 --- a/src/WebJobs.Script/Workers/Rpc/Configuration/LanguageWorkerOptionsSetup.cs +++ b/src/WebJobs.Script/Workers/Rpc/Configuration/LanguageWorkerOptionsSetup.cs @@ -7,8 +7,6 @@ using System.IO; using System.Linq; using System.Reflection; -using System.Text; -using System.Text.Json; using Microsoft.Azure.WebJobs.Script.Config; using Microsoft.Azure.WebJobs.Script.Diagnostics; using Microsoft.Azure.WebJobs.Script.Workers.Profiles; @@ -96,7 +94,7 @@ public void Configure(LanguageWorkerOptions options) workerConfigurationResolver = new StaticWorkerConfigurationResolver(configuration, _logger); } - var configFactory = new RpcWorkerConfigFactory(configuration, _logger, SystemRuntimeInformation.Instance, _environment, _metricsLogger, _workerProfileManager, workerConfigurationResolver, dynamicWorkerResolutionEnabled, workers); + var configFactory = new RpcWorkerConfigFactory(configuration, _logger, SystemRuntimeInformation.Instance, _environment, _metricsLogger, _workerProfileManager, workerConfigurationResolver, dynamicWorkerResolutionEnabled); options.WorkerConfigs = configFactory.GetConfigs(); } @@ -129,7 +127,9 @@ internal List GetWorkerProbingPaths() var assemblyDir = Path.GetDirectoryName(assemblyPath); var parentDir = Directory.GetParent(assemblyDir)?.Parent?.FullName; //Move 2 directories up to get to the SiteExtensions directory + // Example probing path for Windows: "c:\\home\\SiteExtensions\\workers" var windowsWorkerFullProbingPath = Path.Combine(parentDir, RpcWorkerConstants.DefaultWorkersDirectoryName); + probingPaths.Add(windowsWorkerFullProbingPath); } } diff --git a/src/WebJobs.Script/Workers/Rpc/Configuration/RpcWorkerConfigFactory.cs b/src/WebJobs.Script/Workers/Rpc/Configuration/RpcWorkerConfigFactory.cs index c962184134..b5b149f750 100644 --- a/src/WebJobs.Script/Workers/Rpc/Configuration/RpcWorkerConfigFactory.cs +++ b/src/WebJobs.Script/Workers/Rpc/Configuration/RpcWorkerConfigFactory.cs @@ -27,7 +27,6 @@ internal class RpcWorkerConfigFactory private readonly string _workerRuntime; private readonly IEnvironment _environment; private readonly IWorkerConfigurationResolver _workerConfigurationResolver; - private readonly HashSet _workersAvailableForResolutionViaHostingConfig; private readonly bool _dynamicWorkerResolutionEnabled; private readonly JsonSerializerOptions _jsonSerializerOptions = new() { @@ -43,8 +42,7 @@ public RpcWorkerConfigFactory(IConfiguration config, IMetricsLogger metricsLogger, IWorkerProfileManager workerProfileManager, IWorkerConfigurationResolver workerConfigurationResolver, - bool dynamicWorkerResolutionEnabled = false, - HashSet workersAvailableForResolutionViaHostingConfig = null) + bool dynamicWorkerResolutionEnabled = false) { _config = config ?? throw new ArgumentNullException(nameof(config)); _logger = logger ?? throw new ArgumentNullException(nameof(logger)); @@ -56,7 +54,6 @@ public RpcWorkerConfigFactory(IConfiguration config, _workerConfigurationResolver = workerConfigurationResolver ?? throw new ArgumentNullException(nameof(workerConfigurationResolver)); _dynamicWorkerResolutionEnabled = dynamicWorkerResolutionEnabled; - _workersAvailableForResolutionViaHostingConfig = workersAvailableForResolutionViaHostingConfig; WorkersDirPath = WorkerConfigurationHelper.GetWorkersDirPath(config); } @@ -134,15 +131,7 @@ internal void AddProvider(string workerDir) var workerConfig = WorkerConfigurationHelper.GetWorkerConfigJsonElement(workerConfigPath); - RpcWorkerDescription workerDescription = WorkerConfigurationHelper.GetWorkerDescription( - workerConfig: workerConfig, - jsonSerializerOptions: _jsonSerializerOptions, - workerDir: workerDir, - profileManager: _profileManager, - config: _config, - logger: _logger, - dynamicWorkerResolutionEnabled: _dynamicWorkerResolutionEnabled, - workersAvailableForResolutionViaHostingConfig: _workersAvailableForResolutionViaHostingConfig); + RpcWorkerDescription workerDescription = WorkerConfigurationHelper.GetWorkerDescription(workerConfig, _jsonSerializerOptions, workerDir, _profileManager, _config, _logger); if (workerDescription.IsDisabled == true) { diff --git a/src/WebJobs.Script/Workers/Rpc/Configuration/WorkerConfigurationHelper.cs b/src/WebJobs.Script/Workers/Rpc/Configuration/WorkerConfigurationHelper.cs index bee86e7c54..701f1eac49 100644 --- a/src/WebJobs.Script/Workers/Rpc/Configuration/WorkerConfigurationHelper.cs +++ b/src/WebJobs.Script/Workers/Rpc/Configuration/WorkerConfigurationHelper.cs @@ -20,9 +20,7 @@ internal static RpcWorkerDescription GetWorkerDescription( string workerDir, IWorkerProfileManager profileManager, IConfiguration config, - ILogger logger, - bool dynamicWorkerResolutionEnabled, - HashSet workersAvailableForResolutionViaHostingConfig) + ILogger logger) { var workerDescriptionElement = workerConfig.GetProperty(WorkerConstants.WorkerDescription); var workerDescription = workerDescriptionElement.Deserialize(jsonSerializerOptions); @@ -46,7 +44,7 @@ internal static RpcWorkerDescription GetWorkerDescription( AddArgumentsFromAppSettings(workerDescription, languageSection); // Validate workerDescription - workerDescription.ApplyDefaultsAndValidate(Directory.GetCurrentDirectory(), logger, dynamicWorkerResolutionEnabled, workersAvailableForResolutionViaHostingConfig); + workerDescription.ApplyDefaultsAndValidate(Directory.GetCurrentDirectory(), logger); return workerDescription; } diff --git a/src/WebJobs.Script/Workers/Rpc/RpcWorkerDescription.cs b/src/WebJobs.Script/Workers/Rpc/RpcWorkerDescription.cs index 161d32b083..570497343b 100644 --- a/src/WebJobs.Script/Workers/Rpc/RpcWorkerDescription.cs +++ b/src/WebJobs.Script/Workers/Rpc/RpcWorkerDescription.cs @@ -87,7 +87,7 @@ public List Extensions public override bool UseStdErrorStreamForErrorsOnly { get; set; } = false; - public override void ApplyDefaultsAndValidate(string workerDirectory, ILogger logger, bool dynamicWorkerResolutionEnabled = false, HashSet workersAvailableForResolutionViaHostingConfig = null) + public override void ApplyDefaultsAndValidate(string workerDirectory, ILogger logger) { if (workerDirectory == null) { @@ -95,23 +95,10 @@ public override void ApplyDefaultsAndValidate(string workerDirectory, ILogger lo } Arguments = Arguments ?? new List(); WorkerDirectory = WorkerDirectory ?? workerDirectory; - if (!string.IsNullOrEmpty(DefaultWorkerPath) && !Path.IsPathRooted(DefaultWorkerPath)) { - // If dynamic worker resolution enabled and DefaultWorkerPath contains FunctionWorkerRuntimeVersionSettingName - // then version becomes redundant in the path. Replacing version with an empty string to avoid duplication. - if (dynamicWorkerResolutionEnabled && - workersAvailableForResolutionViaHostingConfig is not null && - workersAvailableForResolutionViaHostingConfig.Contains(Language) && - DefaultWorkerPath.Contains(RpcWorkerConstants.RuntimeVersionPlaceholder)) - { - var versionDir = Path.GetFileName(WorkerDirectory); - WorkerDirectory = WorkerDirectory.Replace(versionDir, string.Empty); - } - DefaultWorkerPath = Path.Combine(WorkerDirectory, DefaultWorkerPath); } - if (string.IsNullOrEmpty(Language)) { throw new ValidationException($"WorkerDescription {nameof(Language)} cannot be empty"); diff --git a/test/WebJobs.Script.Tests/Configuration/LanguageWorkerOptionsSetupTests.cs b/test/WebJobs.Script.Tests/Configuration/LanguageWorkerOptionsSetupTests.cs index f73f211688..ac08635179 100644 --- a/test/WebJobs.Script.Tests/Configuration/LanguageWorkerOptionsSetupTests.cs +++ b/test/WebJobs.Script.Tests/Configuration/LanguageWorkerOptionsSetupTests.cs @@ -104,7 +104,6 @@ public void LanguageWorkerOptions_EnabledWorkerResolution_Expected_ListOfConfigs string probingPathValue = string.Join(';', _probingPath1, string.Empty, "path-not-exists"); testEnvironment.SetEnvironmentVariable(RpcWorkerConstants.FunctionWorkerRuntimeSettingName, workerRuntime); - testEnvironment.SetEnvironmentVariable(EnvironmentSettingNames.AzureWebJobsFeatureFlags, ScriptConstants.FeatureFlagEnableDynamicWorkerResolution); testEnvironment.SetEnvironmentVariable(EnvironmentSettingNames.AntaresPlatformReleaseChannel, releaseChannel); testEnvironment.SetEnvironmentVariable(EnvironmentSettingNames.WorkerProbingPaths, probingPathValue); @@ -148,7 +147,6 @@ public void LanguageWorkerOptions_FallbackPath_Expected_ListOfConfigs(string wor string probingPathValue = null; testEnvironment.SetEnvironmentVariable(RpcWorkerConstants.FunctionWorkerRuntimeSettingName, workerRuntime); - testEnvironment.SetEnvironmentVariable(EnvironmentSettingNames.AzureWebJobsFeatureFlags, ScriptConstants.FeatureFlagEnableDynamicWorkerResolution); testEnvironment.SetEnvironmentVariable(EnvironmentSettingNames.AntaresPlatformReleaseChannel, releaseChannel); testEnvironment.SetEnvironmentVariable(EnvironmentSettingNames.WorkerProbingPaths, probingPathValue); @@ -231,11 +229,10 @@ public void LanguageWorkerOptions_FallbackPath_NullHostingConfig(string workerRu string probingPathValue = null; testEnvironment.SetEnvironmentVariable(RpcWorkerConstants.FunctionWorkerRuntimeSettingName, workerRuntime); - testEnvironment.SetEnvironmentVariable(EnvironmentSettingNames.AzureWebJobsFeatureFlags, ScriptConstants.FeatureFlagEnableDynamicWorkerResolution); testEnvironment.SetEnvironmentVariable(EnvironmentSettingNames.WorkerProbingPaths, probingPathValue); var hostingOptions = new FunctionsHostingConfigOptions(); - hostingOptions.Features.Add(RpcWorkerConstants.WorkersAvailableForDynamicResolution, null); + hostingOptions.Features.Add(RpcWorkerConstants.WorkersAvailableForDynamicResolution, workerRuntime); LanguageWorkerOptionsSetup setup = new LanguageWorkerOptionsSetup(configuration, loggerFactory, testEnvironment, testMetricLogger, testProfileManager.Object, testScriptHostManager.Object, new OptionsWrapper(hostingOptions)); LanguageWorkerOptions options = new LanguageWorkerOptions(); diff --git a/test/WebJobs.Script.Tests/UtilityTests.cs b/test/WebJobs.Script.Tests/UtilityTests.cs index 7c04c00ab7..c184ab9e93 100644 --- a/test/WebJobs.Script.Tests/UtilityTests.cs +++ b/test/WebJobs.Script.Tests/UtilityTests.cs @@ -1022,11 +1022,6 @@ public void WorkerIndexingDecisionLogic_NullWorkerIndexingProperty(bool workerIn [InlineData(null, "", false)] [InlineData(null, "| ", false)] [InlineData(null, null, false)] - [InlineData(ScriptConstants.FeatureFlagEnableDynamicWorkerResolution, "node", true)] - [InlineData(ScriptConstants.FeatureFlagEnableDynamicWorkerResolution, "java|node", true)] - [InlineData(ScriptConstants.FeatureFlagEnableDynamicWorkerResolution, "", true)] - [InlineData(ScriptConstants.FeatureFlagEnableDynamicWorkerResolution, "| ", true)] - [InlineData(ScriptConstants.FeatureFlagEnableDynamicWorkerResolution, null, true)] [InlineData(ScriptConstants.FeatureFlagDisableDynamicWorkerResolution, "node", false)] [InlineData(ScriptConstants.FeatureFlagDisableDynamicWorkerResolution, "java|node", false)] [InlineData(ScriptConstants.FeatureFlagDisableDynamicWorkerResolution, "| ", false)] diff --git a/test/WebJobs.Script.Tests/Workers/Rpc/RpcWorkerConfigFactoryTests.cs b/test/WebJobs.Script.Tests/Workers/Rpc/RpcWorkerConfigFactoryTests.cs index 0b6a29b9b7..5704be1d79 100644 --- a/test/WebJobs.Script.Tests/Workers/Rpc/RpcWorkerConfigFactoryTests.cs +++ b/test/WebJobs.Script.Tests/Workers/Rpc/RpcWorkerConfigFactoryTests.cs @@ -175,35 +175,23 @@ public void DefaultWorkerConfigs_Overrides_DefaultWorkerRuntimeVersion_AppSettin Assert.Equal("7.4", powershellWorkerConfig.Description.DefaultRuntimeVersion); } - [Theory] - [InlineData("7.4", "7.4", null, null)] - [InlineData(null, "7.4", null, null)] - [InlineData("7.4", "7.4", "..\\..\\..\\..\\test\\TestWorkers\\ProbingPaths\\workers\\", ScriptConstants.FeatureFlagEnableDynamicWorkerResolution)] - [InlineData(null, "7.4", "..\\..\\..\\..\\test\\TestWorkers\\ProbingPaths\\workers\\", ScriptConstants.FeatureFlagEnableDynamicWorkerResolution)] - public void DefaultWorkerConfigs_Overrides_VersionAppSetting(string runtimeSettingVersion, string outputVersion, string probingPathValue, string enableProbingPath) + [Fact] + public void DefaultWorkerConfigs_Overrides_VersionAppSetting() { - var probingPath = string.IsNullOrEmpty(probingPathValue) ? probingPathValue : Path.GetFullPath(probingPathValue); - var testEnvironment = new TestEnvironment(); - testEnvironment.SetEnvironmentVariable("FUNCTIONS_WORKER_RUNTIME_VERSION", runtimeSettingVersion); + testEnvironment.SetEnvironmentVariable("FUNCTIONS_WORKER_RUNTIME_VERSION", "7.4"); testEnvironment.SetEnvironmentVariable("FUNCTIONS_WORKER_RUNTIME", "powerShell"); - - var config = ScriptSettingsManager.CreateDefaultConfigurationBuilder().Build(); - + var configBuilder = ScriptSettingsManager.CreateDefaultConfigurationBuilder(); + var config = configBuilder.Build(); + var scriptSettingsManager = new ScriptSettingsManager(config); var testLogger = new TestLogger("test"); - var workerConfigurationResolver = new DynamicWorkerConfigurationResolver(config, testLogger, testEnvironment, FileUtility.Instance, _testWorkerProfileManager, new HashSet() { "powershell" }, new List() { probingPath }); - var configFactory = new RpcWorkerConfigFactory(config, testLogger, _testSysRuntimeInfo, testEnvironment, new TestMetricsLogger(), _testWorkerProfileManager, workerConfigurationResolver, enableProbingPath is null ? false : true, new HashSet() { "powershell" }); + var resolver = new StaticWorkerConfigurationResolver(config, testLogger); + var configFactory = new RpcWorkerConfigFactory(config, testLogger, _testSysRuntimeInfo, testEnvironment, new TestMetricsLogger(), _testWorkerProfileManager, resolver); var workerConfigs = configFactory.GetConfigs(); var powershellWorkerConfig = workerConfigs.FirstOrDefault(w => w.Description.Language.Equals("powershell", StringComparison.OrdinalIgnoreCase)); - Assert.Equal(1, workerConfigs.Count); Assert.NotNull(powershellWorkerConfig); - Assert.Equal(outputVersion, powershellWorkerConfig.Description.DefaultRuntimeVersion); - - if (probingPath is not null) - { - Assert.True(powershellWorkerConfig.Arguments.WorkerPath.Contains(probingPath)); - } + Assert.Equal("7.4", powershellWorkerConfig.Description.DefaultRuntimeVersion); } [Theory] From 9e4b960e40ca5939045b785e18c3b74bd4704ed8 Mon Sep 17 00:00:00 2001 From: Surbhi Gupta Date: Thu, 3 Jul 2025 00:29:05 -0500 Subject: [PATCH 37/55] Code cleanup and added more comments --- src/WebJobs.Script/Utility.cs | 4 +-- .../DynamicWorkerConfigurationResolver.cs | 30 +++++++++++-------- 2 files changed, 19 insertions(+), 15 deletions(-) diff --git a/src/WebJobs.Script/Utility.cs b/src/WebJobs.Script/Utility.cs index 7006e6e30a..3b828a181e 100644 --- a/src/WebJobs.Script/Utility.cs +++ b/src/WebJobs.Script/Utility.cs @@ -1067,8 +1067,8 @@ public static bool CanWorkerIndex(IEnumerable workerConfigs, IE } // Dynamic Worker Resolution can be enabled or disabled via feature flags or hosting config options. Feature flags take precedence over hosting config options. - // Users can enable or disable worker resolution via setting the appropriate feature flags. - // Worker resolution can also be enabled for specific workers at stamp level via the hosting config options. + // Users can disable worker resolution via setting the appropriate feature flag. + // Worker resolution can be enabled for specific workers at stamp level via the hosting config options. public static bool IsDynamicWorkerResolutionEnabled(IEnvironment environment, HashSet workersAvailableForResolutionViaHostingConfig) { bool isDynamicWorkerResolutionDisabled = FeatureFlags.IsEnabled(ScriptConstants.FeatureFlagDisableDynamicWorkerResolution, environment); diff --git a/src/WebJobs.Script/Workers/Rpc/Configuration/DynamicWorkerConfigurationResolver.cs b/src/WebJobs.Script/Workers/Rpc/Configuration/DynamicWorkerConfigurationResolver.cs index ad87a06077..6f027c68a7 100644 --- a/src/WebJobs.Script/Workers/Rpc/Configuration/DynamicWorkerConfigurationResolver.cs +++ b/src/WebJobs.Script/Workers/Rpc/Configuration/DynamicWorkerConfigurationResolver.cs @@ -7,7 +7,6 @@ using System.IO.Abstractions; using System.Linq; using System.Text.Json; -using Microsoft.Azure.AppService.Proxy.Common.Extensions; using Microsoft.Azure.WebJobs.Script.Workers.Profiles; using Microsoft.Extensions.Configuration; using Microsoft.Extensions.Logging; @@ -38,21 +37,22 @@ public DynamicWorkerConfigurationResolver(IConfiguration config, _environment = environment ?? throw new ArgumentNullException(nameof(environment)); _fileSystem = fileSystem ?? throw new ArgumentNullException(nameof(fileSystem)); _profileManager = workerProfileManager ?? throw new ArgumentNullException(nameof(workerProfileManager)); - _workersAvailableForResolutionViaHostingConfig = workersAvailableForResolutionViaHostingConfig ?? throw new ArgumentNullException(nameof(workersAvailableForResolutionViaHostingConfig)); + _workersAvailableForResolutionViaHostingConfig = workersAvailableForResolutionViaHostingConfig; _workerProbingPaths = workerProbingPaths; } public List GetWorkerConfigs() { string probingPaths = _workerProbingPaths is not null ? string.Join(", ", _workerProbingPaths) : null; - _logger.LogDebug("Workers probing paths set to: {probingPaths}.", probingPaths); - - // Dictionary of { FUNCTIONS_WORKER_RUNTIME environment variable value : path of workerConfig } - Dictionary outputDict = new Dictionary(StringComparer.OrdinalIgnoreCase); + _logger.LogDebug("Workers probing paths set to: {probingPaths}", probingPaths); var workerRuntime = _environment.GetEnvironmentVariable(EnvironmentSettingNames.FunctionWorkerRuntime); string releaseChannel = Utility.GetPlatformReleaseChannel(_environment); + // Dictionary of { FUNCTIONS_WORKER_RUNTIME environment variable value : path of workerConfig } + // Example: outputDict = {"java": "path1", "node": "path2", "dotnet-isolated": "path3"} for multilanguage worker scenario + Dictionary outputDict = new Dictionary(StringComparer.OrdinalIgnoreCase); + if (_workerProbingPaths is not null) { // probing path directory structure is: /// @@ -70,11 +70,13 @@ public List GetWorkerConfigs() continue; } + bool workerUnavailableViaHostingConfig = _workersAvailableForResolutionViaHostingConfig is not null && !_workersAvailableForResolutionViaHostingConfig.Contains(workerRuntimeDir); + // Skip worker directories that don't match the current runtime or are not enabled via hosting config - if (!_workersAvailableForResolutionViaHostingConfig.Contains(workerRuntimeDir) || + if (workerUnavailableViaHostingConfig || (!_environment.IsMultiLanguageRuntimeEnvironment() && !_environment.IsPlaceholderModeEnabled() && - IsRequiredWorkerRuntime(workerRuntime, workerRuntimeDir))) + IsNotRequiredWorkerRuntime(workerRuntime, workerRuntimeDir))) { continue; } @@ -100,8 +102,10 @@ workerRuntime is not null && private void PopulateWorkerConfigsFromProbingPaths(string languageWorkerPath, string languageWorkerFolder, string releaseChannel, Dictionary outputDict) { - var versionsDir = _fileSystem.Directory.EnumerateDirectories(languageWorkerPath); - var versionPathMap = GetWorkerVersionsDescending(versionsDir); + var workerVersionPaths = _fileSystem.Directory.EnumerateDirectories(languageWorkerPath); + + // Map of: (parsed worker version, worker path) + var versionPathMap = GetWorkerVersionsDescending(workerVersionPaths); int compatibleWorkerCount = 0; @@ -110,7 +114,6 @@ private void PopulateWorkerConfigsFromProbingPaths(string languageWorkerPath, st (releaseChannel.Equals(ScriptConstants.StandardPlatformChannelNameUpper) || releaseChannel.Equals(ScriptConstants.ExtendedPlatformChannelNameUpper)); - // language worker version foreach (var versionPair in versionPathMap) { string languageWorkerVersionPath = versionPair.Value; @@ -148,7 +151,7 @@ private void PopulateWorkerConfigsFromWithinHost(string workerRuntime, Dictionar if (outputDict.ContainsKey(workerDir) || (!_environment.IsMultiLanguageRuntimeEnvironment() && - IsRequiredWorkerRuntime(workerRuntime, workerDir))) + IsNotRequiredWorkerRuntime(workerRuntime, workerDir))) { continue; } @@ -171,6 +174,7 @@ workerRuntime is not null && private SortedList GetWorkerVersionsDescending(IEnumerable workerVersionPaths) { + // Map of: (parsed worker version, worker path) var versionPathMap = new SortedList(new DescendingVersionComparer()); if (!workerVersionPaths.Any()) @@ -269,7 +273,7 @@ private bool DoesHostRequirementMeet(JsonElement workerConfig) return true; } - private bool IsRequiredWorkerRuntime(string workerRuntime, string workerDir) + private bool IsNotRequiredWorkerRuntime(string workerRuntime, string workerDir) { return workerRuntime is not null && !workerRuntime.Equals(workerDir, StringComparison.OrdinalIgnoreCase); } From 2585946f9c9bad60326c8f56034bbdd34419eab6 Mon Sep 17 00:00:00 2001 From: Surbhi Gupta Date: Thu, 3 Jul 2025 14:08:06 -0500 Subject: [PATCH 38/55] Adding condition if worker.config is empty --- .../Rpc/Configuration/DynamicWorkerConfigurationResolver.cs | 5 +++++ .../Workers/Rpc/Configuration/WorkerConfigurationHelper.cs | 5 +++++ 2 files changed, 10 insertions(+) diff --git a/src/WebJobs.Script/Workers/Rpc/Configuration/DynamicWorkerConfigurationResolver.cs b/src/WebJobs.Script/Workers/Rpc/Configuration/DynamicWorkerConfigurationResolver.cs index 6f027c68a7..e227bd58d5 100644 --- a/src/WebJobs.Script/Workers/Rpc/Configuration/DynamicWorkerConfigurationResolver.cs +++ b/src/WebJobs.Script/Workers/Rpc/Configuration/DynamicWorkerConfigurationResolver.cs @@ -210,6 +210,11 @@ private bool IsWorkerCompatibleWithHost(string workerDir) JsonElement workerConfig = WorkerConfigurationHelper.GetWorkerConfigJsonElement(workerConfigPath); + if (workerConfig.ValueKind == JsonValueKind.Undefined) + { + return false; + } + // static capability resolution bool doesHostRequirementMeet = DoesHostRequirementMeet(workerConfig); diff --git a/src/WebJobs.Script/Workers/Rpc/Configuration/WorkerConfigurationHelper.cs b/src/WebJobs.Script/Workers/Rpc/Configuration/WorkerConfigurationHelper.cs index 701f1eac49..baf4e1a3a3 100644 --- a/src/WebJobs.Script/Workers/Rpc/Configuration/WorkerConfigurationHelper.cs +++ b/src/WebJobs.Script/Workers/Rpc/Configuration/WorkerConfigurationHelper.cs @@ -58,6 +58,11 @@ internal static JsonElement GetWorkerConfigJsonElement(string workerConfigPath) jsonSpan = jsonSpan[3..]; // Skip UTF-8 Byte Order Mark (BOM) if present at the beginning of the file. } + if (jsonSpan.IsEmpty) + { + return default; // Return default JsonElement if the file is empty. + } + var reader = new Utf8JsonReader(jsonSpan, isFinalBlock: true, state: default); using var doc = JsonDocument.ParseValue(ref reader); From 3c55d1d9fa608a2932fce18193bf04a259efe703 Mon Sep 17 00:00:00 2001 From: Surbhi Gupta Date: Sat, 5 Jul 2025 18:38:45 -0500 Subject: [PATCH 39/55] Addressed PR feedback and code refactoring --- .../Config/FunctionsHostingConfigOptions.cs | 2 +- .../Environment/EnvironmentExtensions.cs | 31 +++++++++++- src/WebJobs.Script/ScriptConstants.cs | 2 +- src/WebJobs.Script/Utility.cs | 22 --------- .../DynamicWorkerConfigurationResolver.cs | 25 ++++------ .../LanguageWorkerOptionsSetup.cs | 37 ++++++++------ .../Configuration/RpcWorkerConfigFactory.cs | 2 +- .../WorkerConfigurationHelper.cs | 5 ++ .../Rpc/IWorkerConfigurationResolver.cs | 2 +- .../LanguageWorkerOptionsSetupTests.cs | 49 +++++++++++++++++-- .../Extensions/EnvironmentExtensionsTests.cs | 43 ++++++++++++++++ test/WebJobs.Script.Tests/UtilityTests.cs | 42 ---------------- .../Rpc/WorkerConfigurationResolverTests.cs | 5 +- 13 files changed, 162 insertions(+), 105 deletions(-) diff --git a/src/WebJobs.Script/Config/FunctionsHostingConfigOptions.cs b/src/WebJobs.Script/Config/FunctionsHostingConfigOptions.cs index 588bd0b553..818586517e 100644 --- a/src/WebJobs.Script/Config/FunctionsHostingConfigOptions.cs +++ b/src/WebJobs.Script/Config/FunctionsHostingConfigOptions.cs @@ -92,7 +92,7 @@ internal string WorkerIndexingDisabledApps } /// - /// Gets a string delimited by '|' that contains the names of the language workers available through Probing paths outside of the Host. + /// Gets a string delimited by '|' that contains the names of the language workers available through probing paths outside of the Host. /// internal string WorkersAvailableForDynamicResolution { diff --git a/src/WebJobs.Script/Environment/EnvironmentExtensions.cs b/src/WebJobs.Script/Environment/EnvironmentExtensions.cs index cfdb60c27d..4aca1c6d12 100644 --- a/src/WebJobs.Script/Environment/EnvironmentExtensions.cs +++ b/src/WebJobs.Script/Environment/EnvironmentExtensions.cs @@ -283,7 +283,7 @@ public static bool IsWindowsElasticPremium(this IEnvironment environment) /// /// The environment to verify. /// if running in a Windows App Service app; otherwise, false. - public static bool IsAnyWindows(this IEnvironment environment) + public static bool IsWindowsEnvironment(this IEnvironment environment) { return environment.IsWindowsAzureManagedHosting() || environment.IsWindowsConsumption() || environment.IsWindowsElasticPremium(); } @@ -697,6 +697,35 @@ public static bool IsInProc(this IEnvironment environment, string workerRuntime return string.IsNullOrEmpty(workerRuntime) || string.Equals(workerRuntime, RpcWorkerConstants.DotNetLanguageWorkerName, StringComparison.OrdinalIgnoreCase); } + /// + /// Checks if Dynamic Worker Resolution feature is disabled via feature flag. + /// Returns true if the feature is disabled, false otherwise. + /// + public static bool IsWorkerResolutionFeatureDisabled(this IEnvironment environment) + { + return FeatureFlags.IsEnabled(ScriptConstants.FeatureFlagDisableDynamicWorkerResolution, environment); + } + + // Dynamic Worker Resolution can be enabled or disabled via feature flags or hosting config options. Feature flags take precedence over hosting config options. + // Users can disable worker resolution via setting the appropriate feature flag. + // Worker resolution can be enabled for specific workers at stamp level via the hosting config options. + public static bool IsDynamicWorkerResolutionEnabled(this IEnvironment environment, HashSet workersAvailableForResolutionViaHostingConfig) + { + if (environment.IsWorkerResolutionFeatureDisabled()) + { + return false; + } + + string workerRuntime = environment.GetEnvironmentVariable(RpcWorkerConstants.FunctionWorkerRuntimeSettingName); + + if (!environment.IsMultiLanguageRuntimeEnvironment() && !string.IsNullOrWhiteSpace(workerRuntime)) + { + return workersAvailableForResolutionViaHostingConfig?.Contains(workerRuntime) ?? false; + } + + return workersAvailableForResolutionViaHostingConfig?.Any() ?? false; + } + public static bool IsApplicationInsightsAgentEnabled(this IEnvironment environment) { // cache the value of the environment variable diff --git a/src/WebJobs.Script/ScriptConstants.cs b/src/WebJobs.Script/ScriptConstants.cs index b80b35fa8f..0d101aab5b 100644 --- a/src/WebJobs.Script/ScriptConstants.cs +++ b/src/WebJobs.Script/ScriptConstants.cs @@ -249,7 +249,7 @@ public static class ScriptConstants public static readonly long DefaultMaxRequestBodySize = 104857600; public static readonly ImmutableArray SystemLogCategoryPrefixes = ImmutableArray.Create("Microsoft.Azure.WebJobs.", "Function.", "Worker.", "Host."); - public static readonly HashSet HostCapabilities = []; + public static readonly HashSet HostCapabilities = new HashSet(StringComparer.OrdinalIgnoreCase); public static readonly string FunctionMetadataDirectTypeKey = "DirectType"; public static readonly string LiveLogsSessionAIKey = "#AzFuncLiveLogsSessionId"; diff --git a/src/WebJobs.Script/Utility.cs b/src/WebJobs.Script/Utility.cs index 3b828a181e..79bb686058 100644 --- a/src/WebJobs.Script/Utility.cs +++ b/src/WebJobs.Script/Utility.cs @@ -1066,28 +1066,6 @@ public static bool CanWorkerIndex(IEnumerable workerConfigs, IE return workerIndexingEnabled && workerIndexingAvailable; } - // Dynamic Worker Resolution can be enabled or disabled via feature flags or hosting config options. Feature flags take precedence over hosting config options. - // Users can disable worker resolution via setting the appropriate feature flag. - // Worker resolution can be enabled for specific workers at stamp level via the hosting config options. - public static bool IsDynamicWorkerResolutionEnabled(IEnvironment environment, HashSet workersAvailableForResolutionViaHostingConfig) - { - bool isDynamicWorkerResolutionDisabled = FeatureFlags.IsEnabled(ScriptConstants.FeatureFlagDisableDynamicWorkerResolution, environment); - - if (isDynamicWorkerResolutionDisabled) - { - return false; - } - - string workerRuntime = environment.GetEnvironmentVariable(RpcWorkerConstants.FunctionWorkerRuntimeSettingName); - - if (!environment.IsMultiLanguageRuntimeEnvironment() && !string.IsNullOrWhiteSpace(workerRuntime)) - { - return workersAvailableForResolutionViaHostingConfig?.Contains(workerRuntime) ?? false; - } - - return workersAvailableForResolutionViaHostingConfig?.Any() ?? false; - } - public static void LogAutorestGeneratedJsonIfExists(string rootScriptPath, ILogger logger) { string autorestGeneratedJsonPath = Path.Combine(rootScriptPath, ScriptConstants.AutorestGeenratedMetadataFileName); diff --git a/src/WebJobs.Script/Workers/Rpc/Configuration/DynamicWorkerConfigurationResolver.cs b/src/WebJobs.Script/Workers/Rpc/Configuration/DynamicWorkerConfigurationResolver.cs index e227bd58d5..5c772d77d1 100644 --- a/src/WebJobs.Script/Workers/Rpc/Configuration/DynamicWorkerConfigurationResolver.cs +++ b/src/WebJobs.Script/Workers/Rpc/Configuration/DynamicWorkerConfigurationResolver.cs @@ -76,7 +76,7 @@ public List GetWorkerConfigs() if (workerUnavailableViaHostingConfig || (!_environment.IsMultiLanguageRuntimeEnvironment() && !_environment.IsPlaceholderModeEnabled() && - IsNotRequiredWorkerRuntime(workerRuntime, workerRuntimeDir))) + WorkerConfigurationHelper.ShouldSkipRuntime(workerRuntime, workerRuntimeDir))) { continue; } @@ -111,8 +111,8 @@ private void PopulateWorkerConfigsFromProbingPaths(string languageWorkerPath, st bool isStandardOrExtendedChannel = releaseChannel != null && - (releaseChannel.Equals(ScriptConstants.StandardPlatformChannelNameUpper) || - releaseChannel.Equals(ScriptConstants.ExtendedPlatformChannelNameUpper)); + (releaseChannel.Equals(ScriptConstants.StandardPlatformChannelNameUpper, StringComparison.OrdinalIgnoreCase) || + releaseChannel.Equals(ScriptConstants.ExtendedPlatformChannelNameUpper, StringComparison.OrdinalIgnoreCase)); foreach (var versionPair in versionPathMap) { @@ -151,7 +151,7 @@ private void PopulateWorkerConfigsFromWithinHost(string workerRuntime, Dictionar if (outputDict.ContainsKey(workerDir) || (!_environment.IsMultiLanguageRuntimeEnvironment() && - IsNotRequiredWorkerRuntime(workerRuntime, workerDir))) + WorkerConfigurationHelper.ShouldSkipRuntime(workerRuntime, workerDir))) { continue; } @@ -216,9 +216,9 @@ private bool IsWorkerCompatibleWithHost(string workerDir) } // static capability resolution - bool doesHostRequirementMeet = DoesHostRequirementMeet(workerConfig); + bool hostHasRequiredCapabilities = DoesHostHasRequiredCapabilities(workerConfig); - if (!doesHostRequirementMeet) + if (!hostHasRequiredCapabilities) { return false; } @@ -245,9 +245,9 @@ private bool IsWorkerCompatibleWithHost(string workerDir) /// /// Worker config: { "hostRequirements": [ "test-capability1", "test-capability2" ] }. /// HashSet { "test-capability1", "test-capability2" }. - private HashSet GetHostRequirements(JsonElement workerConfig) + private HashSet GetHostRequirementsFromWorker(JsonElement workerConfig) { - HashSet hostRequirements = new HashSet(); + HashSet hostRequirements = new HashSet(StringComparer.OrdinalIgnoreCase); if (workerConfig.TryGetProperty(RpcWorkerConstants.HostRequirementsSectionName, out JsonElement configSection)) { @@ -262,10 +262,10 @@ private HashSet GetHostRequirements(JsonElement workerConfig) return hostRequirements; } - private bool DoesHostRequirementMeet(JsonElement workerConfig) + private bool DoesHostHasRequiredCapabilities(JsonElement workerConfig) { HashSet hostCapabilities = ScriptConstants.HostCapabilities; - HashSet hostRequirements = GetHostRequirements(workerConfig); + HashSet hostRequirements = GetHostRequirementsFromWorker(workerConfig); foreach (var hostRequirement in hostRequirements) { @@ -278,11 +278,6 @@ private bool DoesHostRequirementMeet(JsonElement workerConfig) return true; } - private bool IsNotRequiredWorkerRuntime(string workerRuntime, string workerDir) - { - return workerRuntime is not null && !workerRuntime.Equals(workerDir, StringComparison.OrdinalIgnoreCase); - } - private string FormatVersion(string version) { if (!version.Contains('.')) diff --git a/src/WebJobs.Script/Workers/Rpc/Configuration/LanguageWorkerOptionsSetup.cs b/src/WebJobs.Script/Workers/Rpc/Configuration/LanguageWorkerOptionsSetup.cs index 424208ea47..222d9bd384 100644 --- a/src/WebJobs.Script/Workers/Rpc/Configuration/LanguageWorkerOptionsSetup.cs +++ b/src/WebJobs.Script/Workers/Rpc/Configuration/LanguageWorkerOptionsSetup.cs @@ -81,7 +81,7 @@ public void Configure(LanguageWorkerOptions options) } HashSet workers = GetWorkersAvailableForResolutionViaHostingConfig(); - bool dynamicWorkerResolutionEnabled = Utility.IsDynamicWorkerResolutionEnabled(_environment, workers); + bool dynamicWorkerResolutionEnabled = _environment.IsDynamicWorkerResolutionEnabled(workers); IWorkerConfigurationResolver workerConfigurationResolver = null; if (dynamicWorkerResolutionEnabled) @@ -98,14 +98,10 @@ public void Configure(LanguageWorkerOptions options) options.WorkerConfigs = configFactory.GetConfigs(); } - internal HashSet GetWorkersAvailableForResolutionViaHostingConfig() - { - return _functionsHostingConfigOptions.Value - ?.WorkersAvailableForDynamicResolution - ?.ToLowerInvariant() - .Split("|", StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries) - .ToHashSet(); - } + internal HashSet GetWorkersAvailableForResolutionViaHostingConfig() => + (_functionsHostingConfigOptions.Value?.WorkersAvailableForDynamicResolution ?? string.Empty) + .Split('|', StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries) + .ToHashSet(StringComparer.OrdinalIgnoreCase); internal List GetWorkerProbingPaths() { @@ -120,22 +116,31 @@ internal List GetWorkerProbingPaths() } else { - if (_environment.IsAnyWindows()) + if (_environment.IsWindowsEnvironment()) { // Harcoded site extensions path for Windows until Antares sets it as an Environment variable. - var assemblyPath = Assembly.GetExecutingAssembly().Location; - var assemblyDir = Path.GetDirectoryName(assemblyPath); - var parentDir = Directory.GetParent(assemblyDir)?.Parent?.FullName; //Move 2 directories up to get to the SiteExtensions directory - // Example probing path for Windows: "c:\\home\\SiteExtensions\\workers" - var windowsWorkerFullProbingPath = Path.Combine(parentDir, RpcWorkerConstants.DefaultWorkersDirectoryName); + string windowsSiteExtensionsPath = GetWindowsSiteExtensionsPath(); - probingPaths.Add(windowsWorkerFullProbingPath); + if (string.IsNullOrWhiteSpace(windowsSiteExtensionsPath)) + { + var windowsWorkerFullProbingPath = Path.Combine(windowsSiteExtensionsPath, RpcWorkerConstants.DefaultWorkersDirectoryName); + probingPaths.Add(windowsWorkerFullProbingPath); + } } } return probingPaths; } + + internal string GetWindowsSiteExtensionsPath() + { + var assemblyPath = Assembly.GetExecutingAssembly().Location; + var assemblyDir = Path.GetDirectoryName(assemblyPath); + + //Move 2 directories up to get to the SiteExtensions directory + return Directory.GetParent(assemblyDir)?.Parent?.FullName; + } } /// diff --git a/src/WebJobs.Script/Workers/Rpc/Configuration/RpcWorkerConfigFactory.cs b/src/WebJobs.Script/Workers/Rpc/Configuration/RpcWorkerConfigFactory.cs index b5b149f750..aafc165d76 100644 --- a/src/WebJobs.Script/Workers/Rpc/Configuration/RpcWorkerConfigFactory.cs +++ b/src/WebJobs.Script/Workers/Rpc/Configuration/RpcWorkerConfigFactory.cs @@ -167,7 +167,7 @@ internal void AddProvider(string workerDir) _workerDescriptionDictionary[workerDescription.Language] = rpcWorkerConfig; ReadLanguageWorkerFile(arguments.WorkerPath); - _logger.LogDebug("Added WorkerConfig for language: {language} with Worker Path: {path}", workerDescription.Language, workerDescription.DefaultWorkerPath); + _logger.LogDebug("Added WorkerConfig for language: {language} with worker path: {path}", workerDescription.Language, workerDescription.DefaultWorkerPath); } } catch (Exception ex) when (!ex.IsFatal()) diff --git a/src/WebJobs.Script/Workers/Rpc/Configuration/WorkerConfigurationHelper.cs b/src/WebJobs.Script/Workers/Rpc/Configuration/WorkerConfigurationHelper.cs index baf4e1a3a3..fea180373c 100644 --- a/src/WebJobs.Script/Workers/Rpc/Configuration/WorkerConfigurationHelper.cs +++ b/src/WebJobs.Script/Workers/Rpc/Configuration/WorkerConfigurationHelper.cs @@ -157,5 +157,10 @@ internal static string GetWorkersDirPath(IConfiguration configuration) return workersDirPath; } + + internal static bool ShouldSkipRuntime(string workerRuntime, string workerDir) + { + return workerRuntime is not null && !workerRuntime.Equals(workerDir, StringComparison.OrdinalIgnoreCase); + } } } \ No newline at end of file diff --git a/src/WebJobs.Script/Workers/Rpc/IWorkerConfigurationResolver.cs b/src/WebJobs.Script/Workers/Rpc/IWorkerConfigurationResolver.cs index 5b93b42a1a..66a95c32cf 100644 --- a/src/WebJobs.Script/Workers/Rpc/IWorkerConfigurationResolver.cs +++ b/src/WebJobs.Script/Workers/Rpc/IWorkerConfigurationResolver.cs @@ -8,7 +8,7 @@ namespace Microsoft.Azure.WebJobs.Script.Workers.Rpc.Configuration /// /// Interface to resolve the Worker Configs. /// - public interface IWorkerConfigurationResolver + internal interface IWorkerConfigurationResolver { List GetWorkerConfigs(); } diff --git a/test/WebJobs.Script.Tests/Configuration/LanguageWorkerOptionsSetupTests.cs b/test/WebJobs.Script.Tests/Configuration/LanguageWorkerOptionsSetupTests.cs index ac08635179..15703b5f72 100644 --- a/test/WebJobs.Script.Tests/Configuration/LanguageWorkerOptionsSetupTests.cs +++ b/test/WebJobs.Script.Tests/Configuration/LanguageWorkerOptionsSetupTests.cs @@ -121,7 +121,7 @@ public void LanguageWorkerOptions_EnabledWorkerResolution_Expected_ListOfConfigs var logs = loggerProvider.GetAllLogMessages(); string path = Path.Combine(_probingPath1, workerRuntime, expectedVersion); - string expectedLog = $"Added WorkerConfig for language: {workerRuntime} with Worker Path: {path}"; + string expectedLog = $"Added WorkerConfig for language: {workerRuntime} with worker path: {path}"; Assert.True(logs.Any(l => l.FormattedMessage.Contains(expectedLog))); Assert.True(logs.Any(l => l.FormattedMessage.Contains("Workers probing paths set to:"))); } @@ -163,12 +163,53 @@ public void LanguageWorkerOptions_FallbackPath_Expected_ListOfConfigs(string wor var logs = loggerProvider.GetAllLogMessages(); string path = Path.Combine(_fallbackPath, workerRuntime); - string expectedLog = $"Added WorkerConfig for language: {workerRuntime} with Worker Path: {path}"; + string expectedLog = $"Added WorkerConfig for language: {workerRuntime} with worker path: {path}"; Assert.True(logs.Any(l => l.FormattedMessage.Contains(expectedLog))); Assert.True(logs.Any(l => l.FormattedMessage.Contains("Workers probing paths set to:"))); Assert.True(logs.Any(l => l.FormattedMessage.Contains("Searching for worker configs in the fallback directory"))); } + [Theory] + [InlineData("java", null, "LATEST")] + [InlineData("java", "", "STANDARD")] + [InlineData("node", " ", "LATEST")] + public void LanguageWorkerOptions_NullHostingConfig_FeatureDisabled_ListOfConfigs(string workerRuntime, string hostingOptionsSetting, string releaseChannel) + { + var loggerProvider = new TestLoggerProvider(); + var loggerFactory = new LoggerFactory(); + loggerFactory.AddProvider(loggerProvider); + + var testEnvironment = new TestEnvironment(); + var testMetricLogger = new TestMetricsLogger(); + var configurationBuilder = new ConfigurationBuilder() + .Add(new ScriptEnvironmentVariablesConfigurationSource()); + var configuration = configurationBuilder.Build(); + var testProfileManager = new Mock(); + var testScriptHostManager = new Mock(); + string probingPathValue = string.Join(';', _probingPath1, string.Empty, "path-not-exists"); + + testEnvironment.SetEnvironmentVariable(RpcWorkerConstants.FunctionWorkerRuntimeSettingName, workerRuntime); + testEnvironment.SetEnvironmentVariable(EnvironmentSettingNames.AntaresPlatformReleaseChannel, releaseChannel); + testEnvironment.SetEnvironmentVariable(EnvironmentSettingNames.WorkerProbingPaths, probingPathValue); + + var hostingOptions = new FunctionsHostingConfigOptions(); + hostingOptions.Features.Add(RpcWorkerConstants.WorkersAvailableForDynamicResolution, hostingOptionsSetting); + + LanguageWorkerOptionsSetup setup = new LanguageWorkerOptionsSetup(configuration, loggerFactory, testEnvironment, testMetricLogger, testProfileManager.Object, testScriptHostManager.Object, new OptionsWrapper(hostingOptions)); + LanguageWorkerOptions options = new LanguageWorkerOptions(); + + setup.Configure(options); + + Assert.Equal(1, options.WorkerConfigs.Count); + + var logs = loggerProvider.GetAllLogMessages(); + + string path = Path.Combine(_fallbackPath, workerRuntime); + string expectedLog = $"Added WorkerConfig for language: {workerRuntime} with worker path: {path}"; + Assert.True(logs.Any(l => l.FormattedMessage.Contains(expectedLog))); + Assert.True(logs.Any(l => l.FormattedMessage.Contains("Workers Directory set to:"))); + } + [Theory] [InlineData("java", "LATEST")] [InlineData("java", "STANDARD")] @@ -205,7 +246,7 @@ public void LanguageWorkerOptions_DisabledWorkerResolution_Expected_ListOfConfig var logs = loggerProvider.GetAllLogMessages(); string path = Path.Combine(_fallbackPath, workerRuntime); - string expectedLog = $"Added WorkerConfig for language: {workerRuntime} with Worker Path: {path}"; + string expectedLog = $"Added WorkerConfig for language: {workerRuntime} with worker path: {path}"; Assert.True(logs.Any(l => l.FormattedMessage.Contains(expectedLog))); Assert.True(logs.Any(l => l.FormattedMessage.Contains("Workers Directory set to:"))); } @@ -244,7 +285,7 @@ public void LanguageWorkerOptions_FallbackPath_NullHostingConfig(string workerRu var logs = loggerProvider.GetAllLogMessages(); string path = Path.Combine(_fallbackPath, workerRuntime); - string expectedLog = $"Added WorkerConfig for language: {workerRuntime} with Worker Path: {path}"; + string expectedLog = $"Added WorkerConfig for language: {workerRuntime} with worker path: {path}"; Assert.True(logs.Any(l => l.FormattedMessage.Contains(expectedLog))); Assert.True(logs.Any(l => l.FormattedMessage.Contains("Workers probing paths set to:"))); Assert.True(logs.Any(l => l.FormattedMessage.Contains("Searching for worker configs in the fallback directory"))); diff --git a/test/WebJobs.Script.Tests/Extensions/EnvironmentExtensionsTests.cs b/test/WebJobs.Script.Tests/Extensions/EnvironmentExtensionsTests.cs index 9ea8bcf5d5..085658e4b9 100644 --- a/test/WebJobs.Script.Tests/Extensions/EnvironmentExtensionsTests.cs +++ b/test/WebJobs.Script.Tests/Extensions/EnvironmentExtensionsTests.cs @@ -3,6 +3,7 @@ using System; using System.Collections.Generic; +using System.Linq; using Microsoft.Azure.WebJobs.Script.Description; using Microsoft.Azure.WebJobs.Script.Workers.Rpc; using Microsoft.WebJobs.Script.Tests; @@ -548,5 +549,47 @@ public void IsInValidationMode_ReturnsExpectedResult(string functionGroup, bool Assert.Equal(expected, env.IsInValidationMode()); } + + [Theory] + [InlineData(null, "node", true)] + [InlineData(null, "java|node", true)] + [InlineData(null, "", false)] + [InlineData(null, "| ", false)] + [InlineData(null, null, false)] + [InlineData(ScriptConstants.FeatureFlagDisableDynamicWorkerResolution, "node", false)] + [InlineData(ScriptConstants.FeatureFlagDisableDynamicWorkerResolution, "java|node", false)] + [InlineData(ScriptConstants.FeatureFlagDisableDynamicWorkerResolution, "| ", false)] + + public void IsDynamicWorkerResolutionEnabled_HostingConfigAndFeatureFlags_WorksAsExpected(string featureFlagValue, string hostingConfigSetting, bool expected) + { + HashSet hostingConfigEnabledWorkers = hostingConfigSetting?.Split('|', StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries).ToHashSet(); + + var testEnvironment = new TestEnvironment(); + testEnvironment.SetEnvironmentVariable(AzureWebJobsFeatureFlags, featureFlagValue); + + bool result = testEnvironment.IsDynamicWorkerResolutionEnabled(hostingConfigEnabledWorkers); + + Assert.Equal(expected, result); + } + + [Theory] + [InlineData("node", "node", null, true)] + [InlineData("node", "java", null, false)] + [InlineData("java|node", null, null, true)] + [InlineData("node", "node", "workflowapp", true)] + [InlineData("java|node", null, "workflowapp", true)] + [InlineData("| ", null, "workflowapp", false)] + public void IsDynamicWorkerResolutionEnabled_WorkerRuntimeAndMultiLanguage_WorksAsExpected(string hostingConfigSetting, string workerRuntime, string multilanguageApp, bool expected) + { + var testEnvironment = new TestEnvironment(); + testEnvironment.SetEnvironmentVariable(AppKind, multilanguageApp); + testEnvironment.SetEnvironmentVariable(FunctionWorkerRuntime, workerRuntime); + + HashSet hostingConfigEnabledWorkers = hostingConfigSetting?.Split('|', StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries).ToHashSet(); + + bool result = testEnvironment.IsDynamicWorkerResolutionEnabled(hostingConfigEnabledWorkers); + + Assert.Equal(expected, result); + } } } diff --git a/test/WebJobs.Script.Tests/UtilityTests.cs b/test/WebJobs.Script.Tests/UtilityTests.cs index c184ab9e93..4a44afc296 100644 --- a/test/WebJobs.Script.Tests/UtilityTests.cs +++ b/test/WebJobs.Script.Tests/UtilityTests.cs @@ -1016,48 +1016,6 @@ public void WorkerIndexingDecisionLogic_NullWorkerIndexingProperty(bool workerIn Assert.Equal(expected, workerShouldIndex); } - [Theory] - [InlineData(null, "node", true)] - [InlineData(null, "java|node", true)] - [InlineData(null, "", false)] - [InlineData(null, "| ", false)] - [InlineData(null, null, false)] - [InlineData(ScriptConstants.FeatureFlagDisableDynamicWorkerResolution, "node", false)] - [InlineData(ScriptConstants.FeatureFlagDisableDynamicWorkerResolution, "java|node", false)] - [InlineData(ScriptConstants.FeatureFlagDisableDynamicWorkerResolution, "| ", false)] - - public void IsDynamicWorkerResolutionEnabled_HostingConfigAndFeatureFlags_WorksAsExpected(string featureFlagValue, string hostingConfigSetting, bool expected) - { - HashSet hostingConfigEnabledWorkers = hostingConfigSetting?.Split('|', StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries).ToHashSet(); - - var testEnvironment = new TestEnvironment(); - testEnvironment.SetEnvironmentVariable(EnvironmentSettingNames.AzureWebJobsFeatureFlags, featureFlagValue); - - bool result = Utility.IsDynamicWorkerResolutionEnabled(testEnvironment, hostingConfigEnabledWorkers); - - Assert.Equal(expected, result); - } - - [Theory] - [InlineData("node", "node", null, true)] - [InlineData("node", "java", null, false)] - [InlineData("java|node", null, null, true)] - [InlineData("node", "node", "workflowapp", true)] - [InlineData("java|node", null, "workflowapp", true)] - [InlineData("| ", null, "workflowapp", false)] - public void IsDynamicWorkerResolutionEnabled_WorkerRuntimeAndMultiLanguage_WorksAsExpected(string hostingConfigSetting, string workerRuntime, string multilanguageApp, bool expected) - { - var testEnvironment = new TestEnvironment(); - testEnvironment.SetEnvironmentVariable(EnvironmentSettingNames.AppKind, multilanguageApp); - testEnvironment.SetEnvironmentVariable(EnvironmentSettingNames.FunctionWorkerRuntime, workerRuntime); - - HashSet hostingConfigEnabledWorkers = hostingConfigSetting?.Split('|', StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries).ToHashSet(); - - bool result = Utility.IsDynamicWorkerResolutionEnabled(testEnvironment, hostingConfigEnabledWorkers); - - Assert.Equal(expected, result); - } - [Theory] [InlineData("True", true, true)] [InlineData("False", false, true)] diff --git a/test/WebJobs.Script.Tests/Workers/Rpc/WorkerConfigurationResolverTests.cs b/test/WebJobs.Script.Tests/Workers/Rpc/WorkerConfigurationResolverTests.cs index de1fc5e191..3c2b909de3 100644 --- a/test/WebJobs.Script.Tests/Workers/Rpc/WorkerConfigurationResolverTests.cs +++ b/test/WebJobs.Script.Tests/Workers/Rpc/WorkerConfigurationResolverTests.cs @@ -35,6 +35,9 @@ public WorkerConfigurationResolverTests() [InlineData("LATEST", "java\\2.19.0", "node\\3.10.1", "powershell\\7.4", "dotnet-isolated", "python")] [InlineData("STANDARD", "java\\2.18.0", "node\\3.10.1", "powershell\\7.4", "dotnet-isolated", "python")] [InlineData("EXTENDED", "java\\2.18.0", "node\\3.10.1", "powershell\\7.4", "dotnet-isolated", "python")] + [InlineData("laTest", "java\\2.19.0", "node\\3.10.1", "powershell\\7.4", "dotnet-isolated", "python")] + [InlineData("abc", "java\\2.19.0", "node\\3.10.1", "powershell\\7.4", "dotnet-isolated", "python")] + [InlineData("Standard", "java\\2.18.0", "node\\3.10.1", "powershell\\7.4", "dotnet-isolated", "python")] public void GetWorkerConfigs_MultiLanguageWorker_ReturnsExpectedConfigs(string releaseChannel, string java, string node, string powershell, string dotnetIsolated, string python) { // Arrange @@ -60,7 +63,7 @@ public void GetWorkerConfigs_MultiLanguageWorker_ReturnsExpectedConfigs(string r [InlineData(null, "LATEST")] [InlineData(null, "STANDARD")] [InlineData("Empty", "LATEST")] - [InlineData("Empty", "STANDARD")] + [InlineData("Empty", "abc")] public void GetWorkerConfigs_MultiLanguageWorker_NullOREmptyProbingPath_ReturnsExpectedConfigs(string probingPathValue, string releaseChannel) { // Arrange From c5a1b0ff3e1dd21129403769e9059277bdb7315e Mon Sep 17 00:00:00 2001 From: Surbhi Gupta Date: Sat, 5 Jul 2025 23:13:00 -0500 Subject: [PATCH 40/55] Creating WorkerConfigurationResolverFactory --- .../DynamicWorkerConfigurationResolver.cs | 47 ++++---- .../LanguageWorkerOptionsSetup.cs | 63 +---------- .../Configuration/RpcWorkerConfigFactory.cs | 8 +- .../WorkerConfigurationHelper.cs | 8 ++ .../WorkerConfigurationResolverFactory.cs | 101 ++++++++++++++++++ .../IWorkerConfigurationResolverFactory.cs | 19 ++++ 6 files changed, 160 insertions(+), 86 deletions(-) create mode 100644 src/WebJobs.Script/Workers/Rpc/Configuration/WorkerConfigurationResolverFactory.cs create mode 100644 src/WebJobs.Script/Workers/Rpc/IWorkerConfigurationResolverFactory.cs diff --git a/src/WebJobs.Script/Workers/Rpc/Configuration/DynamicWorkerConfigurationResolver.cs b/src/WebJobs.Script/Workers/Rpc/Configuration/DynamicWorkerConfigurationResolver.cs index 5c772d77d1..22d978d450 100644 --- a/src/WebJobs.Script/Workers/Rpc/Configuration/DynamicWorkerConfigurationResolver.cs +++ b/src/WebJobs.Script/Workers/Rpc/Configuration/DynamicWorkerConfigurationResolver.cs @@ -42,17 +42,36 @@ public DynamicWorkerConfigurationResolver(IConfiguration config, } public List GetWorkerConfigs() + { + // Dictionary of { FUNCTIONS_WORKER_RUNTIME environment variable value : path of workerConfig } + // Example: outputDict = {"java": "path1", "node": "path2", "dotnet-isolated": "path3"} for multilanguage worker scenario + Dictionary outputDict = new Dictionary(StringComparer.OrdinalIgnoreCase); + + var workerRuntime = _environment.GetEnvironmentVariable(EnvironmentSettingNames.FunctionWorkerRuntime); + + // Search for worker configs in probing paths + ResolveWorkerConfigsFromProbingPaths(workerRuntime, outputDict); + + if (!_environment.IsMultiLanguageRuntimeEnvironment() && + workerRuntime is not null && + outputDict.ContainsKey(workerRuntime)) + { + return outputDict.Values.ToList(); + } + + // Search in fallback path if worker cannot be found in probing paths + ResolveWorkerConfigsFromWithinHost(workerRuntime, outputDict); + + return outputDict.Values.ToList(); + } + + private void ResolveWorkerConfigsFromProbingPaths(string workerRuntime, Dictionary outputDict) { string probingPaths = _workerProbingPaths is not null ? string.Join(", ", _workerProbingPaths) : null; _logger.LogDebug("Workers probing paths set to: {probingPaths}", probingPaths); - var workerRuntime = _environment.GetEnvironmentVariable(EnvironmentSettingNames.FunctionWorkerRuntime); string releaseChannel = Utility.GetPlatformReleaseChannel(_environment); - // Dictionary of { FUNCTIONS_WORKER_RUNTIME environment variable value : path of workerConfig } - // Example: outputDict = {"java": "path1", "node": "path2", "dotnet-isolated": "path3"} for multilanguage worker scenario - Dictionary outputDict = new Dictionary(StringComparer.OrdinalIgnoreCase); - if (_workerProbingPaths is not null) { // probing path directory structure is: /// @@ -81,26 +100,14 @@ public List GetWorkerConfigs() continue; } - PopulateWorkerConfigsFromProbingPaths(workerRuntimePath, workerRuntimeDir, releaseChannel, outputDict); + ResolveWorkerConfigsFromVersionsDirs(workerRuntimePath, workerRuntimeDir, releaseChannel, outputDict); } } } } - - if (!_environment.IsMultiLanguageRuntimeEnvironment() && - workerRuntime is not null && - outputDict.ContainsKey(workerRuntime)) - { - return outputDict.Values.ToList(); - } - - // Search in fallback path if worker cannot be found in probing paths - PopulateWorkerConfigsFromWithinHost(workerRuntime, outputDict); - - return outputDict.Values.ToList(); } - private void PopulateWorkerConfigsFromProbingPaths(string languageWorkerPath, string languageWorkerFolder, string releaseChannel, Dictionary outputDict) + private void ResolveWorkerConfigsFromVersionsDirs(string languageWorkerPath, string languageWorkerFolder, string releaseChannel, Dictionary outputDict) { var workerVersionPaths = _fileSystem.Directory.EnumerateDirectories(languageWorkerPath); @@ -137,7 +144,7 @@ private void PopulateWorkerConfigsFromProbingPaths(string languageWorkerPath, st } } - private void PopulateWorkerConfigsFromWithinHost(string workerRuntime, Dictionary outputDict) + private void ResolveWorkerConfigsFromWithinHost(string workerRuntime, Dictionary outputDict) { var fallbackPath = WorkerConfigurationHelper.GetWorkersDirPath(_config); diff --git a/src/WebJobs.Script/Workers/Rpc/Configuration/LanguageWorkerOptionsSetup.cs b/src/WebJobs.Script/Workers/Rpc/Configuration/LanguageWorkerOptionsSetup.cs index 222d9bd384..537d3b40a6 100644 --- a/src/WebJobs.Script/Workers/Rpc/Configuration/LanguageWorkerOptionsSetup.cs +++ b/src/WebJobs.Script/Workers/Rpc/Configuration/LanguageWorkerOptionsSetup.cs @@ -4,9 +4,6 @@ using System; using System.Collections.Generic; using System.Diagnostics; -using System.IO; -using System.Linq; -using System.Reflection; using Microsoft.Azure.WebJobs.Script.Config; using Microsoft.Azure.WebJobs.Script.Diagnostics; using Microsoft.Azure.WebJobs.Script.Workers.Profiles; @@ -80,67 +77,13 @@ public void Configure(LanguageWorkerOptions options) } } - HashSet workers = GetWorkersAvailableForResolutionViaHostingConfig(); - bool dynamicWorkerResolutionEnabled = _environment.IsDynamicWorkerResolutionEnabled(workers); - IWorkerConfigurationResolver workerConfigurationResolver = null; + var resolverFactory = new WorkerConfigurationResolverFactory(configuration, _logger, _environment, _workerProfileManager, _functionsHostingConfigOptions); - if (dynamicWorkerResolutionEnabled) - { - List probingPaths = GetWorkerProbingPaths(); - workerConfigurationResolver = new DynamicWorkerConfigurationResolver(configuration, _logger, _environment, FileUtility.Instance, _workerProfileManager, workers, probingPaths); - } - else - { - workerConfigurationResolver = new StaticWorkerConfigurationResolver(configuration, _logger); - } + IWorkerConfigurationResolver workerConfigurationResolver = resolverFactory.CreateResolver(); - var configFactory = new RpcWorkerConfigFactory(configuration, _logger, SystemRuntimeInformation.Instance, _environment, _metricsLogger, _workerProfileManager, workerConfigurationResolver, dynamicWorkerResolutionEnabled); + var configFactory = new RpcWorkerConfigFactory(configuration, _logger, SystemRuntimeInformation.Instance, _environment, _metricsLogger, _workerProfileManager, workerConfigurationResolver); options.WorkerConfigs = configFactory.GetConfigs(); } - - internal HashSet GetWorkersAvailableForResolutionViaHostingConfig() => - (_functionsHostingConfigOptions.Value?.WorkersAvailableForDynamicResolution ?? string.Empty) - .Split('|', StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries) - .ToHashSet(StringComparer.OrdinalIgnoreCase); - - internal List GetWorkerProbingPaths() - { - var probingPaths = new List(); - - // If Environment variable is set, read probing paths from Environment - string probingPathsEnvValue = _environment.GetEnvironmentVariableOrDefault(EnvironmentSettingNames.WorkerProbingPaths, null); - - if (!string.IsNullOrEmpty(probingPathsEnvValue)) - { - probingPaths = probingPathsEnvValue.Split(';', StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries).ToList(); - } - else - { - if (_environment.IsWindowsEnvironment()) - { - // Harcoded site extensions path for Windows until Antares sets it as an Environment variable. - // Example probing path for Windows: "c:\\home\\SiteExtensions\\workers" - string windowsSiteExtensionsPath = GetWindowsSiteExtensionsPath(); - - if (string.IsNullOrWhiteSpace(windowsSiteExtensionsPath)) - { - var windowsWorkerFullProbingPath = Path.Combine(windowsSiteExtensionsPath, RpcWorkerConstants.DefaultWorkersDirectoryName); - probingPaths.Add(windowsWorkerFullProbingPath); - } - } - } - - return probingPaths; - } - - internal string GetWindowsSiteExtensionsPath() - { - var assemblyPath = Assembly.GetExecutingAssembly().Location; - var assemblyDir = Path.GetDirectoryName(assemblyPath); - - //Move 2 directories up to get to the SiteExtensions directory - return Directory.GetParent(assemblyDir)?.Parent?.FullName; - } } /// diff --git a/src/WebJobs.Script/Workers/Rpc/Configuration/RpcWorkerConfigFactory.cs b/src/WebJobs.Script/Workers/Rpc/Configuration/RpcWorkerConfigFactory.cs index aafc165d76..50610a5b6a 100644 --- a/src/WebJobs.Script/Workers/Rpc/Configuration/RpcWorkerConfigFactory.cs +++ b/src/WebJobs.Script/Workers/Rpc/Configuration/RpcWorkerConfigFactory.cs @@ -27,7 +27,6 @@ internal class RpcWorkerConfigFactory private readonly string _workerRuntime; private readonly IEnvironment _environment; private readonly IWorkerConfigurationResolver _workerConfigurationResolver; - private readonly bool _dynamicWorkerResolutionEnabled; private readonly JsonSerializerOptions _jsonSerializerOptions = new() { PropertyNameCaseInsensitive = true @@ -41,8 +40,7 @@ public RpcWorkerConfigFactory(IConfiguration config, IEnvironment environment, IMetricsLogger metricsLogger, IWorkerProfileManager workerProfileManager, - IWorkerConfigurationResolver workerConfigurationResolver, - bool dynamicWorkerResolutionEnabled = false) + IWorkerConfigurationResolver workerConfigurationResolver) { _config = config ?? throw new ArgumentNullException(nameof(config)); _logger = logger ?? throw new ArgumentNullException(nameof(logger)); @@ -53,7 +51,6 @@ public RpcWorkerConfigFactory(IConfiguration config, _workerRuntime = _environment.GetEnvironmentVariable(RpcWorkerConstants.FunctionWorkerRuntimeSettingName); _workerConfigurationResolver = workerConfigurationResolver ?? throw new ArgumentNullException(nameof(workerConfigurationResolver)); - _dynamicWorkerResolutionEnabled = dynamicWorkerResolutionEnabled; WorkersDirPath = WorkerConfigurationHelper.GetWorkersDirPath(config); } @@ -107,8 +104,7 @@ internal void AddProvider(string workerDir) // After specialization, load worker config only for the specified runtime unless it's a multi-language app. if (!string.IsNullOrWhiteSpace(_workerRuntime) && !_environment.IsPlaceholderModeEnabled() && - !_environment.IsMultiLanguageRuntimeEnvironment() && - !_dynamicWorkerResolutionEnabled) + !_environment.IsMultiLanguageRuntimeEnvironment()) { string workerRuntime = Path.GetFileName(workerDir); // Only skip worker directories that don't match the current runtime. diff --git a/src/WebJobs.Script/Workers/Rpc/Configuration/WorkerConfigurationHelper.cs b/src/WebJobs.Script/Workers/Rpc/Configuration/WorkerConfigurationHelper.cs index fea180373c..101660875c 100644 --- a/src/WebJobs.Script/Workers/Rpc/Configuration/WorkerConfigurationHelper.cs +++ b/src/WebJobs.Script/Workers/Rpc/Configuration/WorkerConfigurationHelper.cs @@ -4,11 +4,14 @@ using System; using System.Collections.Generic; using System.IO; +using System.Linq; using System.Text.Json; using System.Text.RegularExpressions; +using Microsoft.Azure.WebJobs.Script.Config; using Microsoft.Azure.WebJobs.Script.Workers.Profiles; using Microsoft.Extensions.Configuration; using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; namespace Microsoft.Azure.WebJobs.Script.Workers.Rpc.Configuration { @@ -162,5 +165,10 @@ internal static bool ShouldSkipRuntime(string workerRuntime, string workerDir) { return workerRuntime is not null && !workerRuntime.Equals(workerDir, StringComparison.OrdinalIgnoreCase); } + + internal static HashSet GetWorkersAvailableForResolutionViaHostingConfig(IOptions functionsHostingConfigOptions) => + (functionsHostingConfigOptions.Value?.WorkersAvailableForDynamicResolution ?? string.Empty) + .Split('|', StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries) + .ToHashSet(StringComparer.OrdinalIgnoreCase); } } \ No newline at end of file diff --git a/src/WebJobs.Script/Workers/Rpc/Configuration/WorkerConfigurationResolverFactory.cs b/src/WebJobs.Script/Workers/Rpc/Configuration/WorkerConfigurationResolverFactory.cs new file mode 100644 index 0000000000..b095381f60 --- /dev/null +++ b/src/WebJobs.Script/Workers/Rpc/Configuration/WorkerConfigurationResolverFactory.cs @@ -0,0 +1,101 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the MIT License. See License.txt in the project root for license information. + +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Reflection; +using Microsoft.Azure.WebJobs.Script.Config; +using Microsoft.Azure.WebJobs.Script.Workers.Profiles; +using Microsoft.Extensions.Configuration; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; + +namespace Microsoft.Azure.WebJobs.Script.Workers.Rpc.Configuration +{ + /// + /// Factory for creating worker configuration resolvers based on current configuration. + /// + internal class WorkerConfigurationResolverFactory : IWorkerConfigurationResolverFactory + { + private readonly IConfiguration _configuration; + private readonly ILogger _logger; + private readonly IEnvironment _environment; + private readonly IWorkerProfileManager _workerProfileManager; + private readonly IOptions _functionsHostingConfigOptions; + + public WorkerConfigurationResolverFactory( + IConfiguration configuration, + ILogger logger, IEnvironment environment, + IWorkerProfileManager workerProfileManager, + IOptions functionsHostingConfigOptions) + { + _configuration = configuration ?? throw new ArgumentNullException(nameof(configuration)); + _logger = logger ?? throw new ArgumentNullException(nameof(logger)); + _environment = environment ?? throw new ArgumentNullException(nameof(environment)); + _workerProfileManager = workerProfileManager ?? throw new ArgumentNullException(nameof(workerProfileManager)); + _functionsHostingConfigOptions = functionsHostingConfigOptions ?? throw new ArgumentNullException(nameof(functionsHostingConfigOptions)); + } + + public IWorkerConfigurationResolver CreateResolver() + { + HashSet workersAvailableForResolution = WorkerConfigurationHelper.GetWorkersAvailableForResolutionViaHostingConfig(_functionsHostingConfigOptions); + List probingPaths = GetWorkerProbingPaths(); + bool dynamicWorkerResolutionEnabled = _environment.IsDynamicWorkerResolutionEnabled(workersAvailableForResolution); + + if (dynamicWorkerResolutionEnabled) + { + return new DynamicWorkerConfigurationResolver( + _configuration, + _logger, + _environment, + FileUtility.Instance, + _workerProfileManager, + workersAvailableForResolution, + probingPaths); + } + + return new StaticWorkerConfigurationResolver(_configuration, _logger); + } + + internal List GetWorkerProbingPaths() + { + var probingPaths = new List(); + + // If Environment variable is set, read probing paths from Environment + string probingPathsEnvValue = _environment.GetEnvironmentVariableOrDefault(EnvironmentSettingNames.WorkerProbingPaths, null); + + if (!string.IsNullOrEmpty(probingPathsEnvValue)) + { + probingPaths = probingPathsEnvValue.Split(';', StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries).ToList(); + } + else + { + if (_environment.IsWindowsEnvironment()) + { + // Harcoded site extensions path for Windows until Antares sets it as an Environment variable. + // Example probing path for Windows: "c:\\home\\SiteExtensions\\workers" + string windowsSiteExtensionsPath = GetWindowsSiteExtensionsPath(); + + if (string.IsNullOrWhiteSpace(windowsSiteExtensionsPath)) + { + var windowsWorkerFullProbingPath = Path.Combine(windowsSiteExtensionsPath, RpcWorkerConstants.DefaultWorkersDirectoryName); + probingPaths.Add(windowsWorkerFullProbingPath); + } + } + } + + return probingPaths; + } + + internal string GetWindowsSiteExtensionsPath() + { + var assemblyPath = Assembly.GetExecutingAssembly().Location; + var assemblyDir = Path.GetDirectoryName(assemblyPath); + + //Move 2 directories up to get to the SiteExtensions directory + return Directory.GetParent(assemblyDir)?.Parent?.FullName; + } + } +} \ No newline at end of file diff --git a/src/WebJobs.Script/Workers/Rpc/IWorkerConfigurationResolverFactory.cs b/src/WebJobs.Script/Workers/Rpc/IWorkerConfigurationResolverFactory.cs new file mode 100644 index 0000000000..6a62fe0895 --- /dev/null +++ b/src/WebJobs.Script/Workers/Rpc/IWorkerConfigurationResolverFactory.cs @@ -0,0 +1,19 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the MIT License. See License.txt in the project root for license information. + +using Microsoft.Azure.WebJobs.Script.Workers.Rpc.Configuration; + +namespace Microsoft.Azure.WebJobs.Script.Workers.Rpc +{ + /// + /// Factory interface for creating worker configuration resolvers. + /// + internal interface IWorkerConfigurationResolverFactory + { + /// + /// Creates an appropriate worker configuration resolver based on the current environment and settings. + /// + /// An implementation of . + IWorkerConfigurationResolver CreateResolver(); + } +} \ No newline at end of file From 6449065c0b5bf91e7ff9a0e3b2f0e297c1e2793f Mon Sep 17 00:00:00 2001 From: Surbhi Gupta Date: Sun, 6 Jul 2025 01:05:39 -0500 Subject: [PATCH 41/55] code cleanup --- .../Environment/EnvironmentExtensions.cs | 23 ++++++++++++------- src/WebJobs.Script/Utility.cs | 6 ----- ... => DefaultWorkerConfigurationResolver.cs} | 9 ++++---- .../DynamicWorkerConfigurationResolver.cs | 13 ++++++++--- .../Configuration/RpcWorkerConfigFactory.cs | 4 +--- .../WorkerConfigurationHelper.cs | 5 ---- .../WorkerConfigurationResolverFactory.cs | 21 ++++++++--------- .../Rpc/RpcWorkerConfigFactoryTests.cs | 20 ++++++++-------- .../Workers/Rpc/RpcWorkerConfigTests.cs | 2 +- 9 files changed, 52 insertions(+), 51 deletions(-) rename src/WebJobs.Script/Workers/Rpc/Configuration/{StaticWorkerConfigurationResolver.cs => DefaultWorkerConfigurationResolver.cs} (72%) diff --git a/src/WebJobs.Script/Environment/EnvironmentExtensions.cs b/src/WebJobs.Script/Environment/EnvironmentExtensions.cs index 4aca1c6d12..fe9f8c9e2d 100644 --- a/src/WebJobs.Script/Environment/EnvironmentExtensions.cs +++ b/src/WebJobs.Script/Environment/EnvironmentExtensions.cs @@ -697,8 +697,13 @@ public static bool IsInProc(this IEnvironment environment, string workerRuntime return string.IsNullOrEmpty(workerRuntime) || string.Equals(workerRuntime, RpcWorkerConstants.DotNetLanguageWorkerName, StringComparison.OrdinalIgnoreCase); } + public static string GetPlatformReleaseChannel(this IEnvironment environment) + { + return environment.GetEnvironmentVariable(AntaresPlatformReleaseChannel) ?? ScriptConstants.LatestPlatformChannelNameUpper; + } + /// - /// Checks if Dynamic Worker Resolution feature is disabled via feature flag. + /// Checks if the Dynamic Worker Resolution feature is disabled via feature flag. /// Returns true if the feature is disabled, false otherwise. /// public static bool IsWorkerResolutionFeatureDisabled(this IEnvironment environment) @@ -706,24 +711,26 @@ public static bool IsWorkerResolutionFeatureDisabled(this IEnvironment environme return FeatureFlags.IsEnabled(ScriptConstants.FeatureFlagDisableDynamicWorkerResolution, environment); } - // Dynamic Worker Resolution can be enabled or disabled via feature flags or hosting config options. Feature flags take precedence over hosting config options. - // Users can disable worker resolution via setting the appropriate feature flag. - // Worker resolution can be enabled for specific workers at stamp level via the hosting config options. + // Users can disable dynamic worker resolution via setting the appropriate feature flag. + // Worker resolution can be enabled for specific workers at the stamp level via hosting config options. + // Feature flag takes precedence over hosting config options. public static bool IsDynamicWorkerResolutionEnabled(this IEnvironment environment, HashSet workersAvailableForResolutionViaHostingConfig) { - if (environment.IsWorkerResolutionFeatureDisabled()) + if (environment.IsWorkerResolutionFeatureDisabled() || workersAvailableForResolutionViaHostingConfig is null) { return false; } string workerRuntime = environment.GetEnvironmentVariable(RpcWorkerConstants.FunctionWorkerRuntimeSettingName); - if (!environment.IsMultiLanguageRuntimeEnvironment() && !string.IsNullOrWhiteSpace(workerRuntime)) + if (!environment.IsMultiLanguageRuntimeEnvironment() && + !string.IsNullOrWhiteSpace(workerRuntime) && + !environment.IsPlaceholderModeEnabled()) { - return workersAvailableForResolutionViaHostingConfig?.Contains(workerRuntime) ?? false; + return workersAvailableForResolutionViaHostingConfig.Contains(workerRuntime); } - return workersAvailableForResolutionViaHostingConfig?.Any() ?? false; + return workersAvailableForResolutionViaHostingConfig.Any(); } public static bool IsApplicationInsightsAgentEnabled(this IEnvironment environment) diff --git a/src/WebJobs.Script/Utility.cs b/src/WebJobs.Script/Utility.cs index 79bb686058..30c343535d 100644 --- a/src/WebJobs.Script/Utility.cs +++ b/src/WebJobs.Script/Utility.cs @@ -24,7 +24,6 @@ using Microsoft.Azure.WebJobs.Script.Models; using Microsoft.Azure.WebJobs.Script.Workers.Rpc; using Microsoft.Extensions.DependencyInjection; -using Microsoft.Extensions.Hosting; using Microsoft.Extensions.Logging; using Microsoft.Extensions.Options; using Newtonsoft.Json; @@ -1128,11 +1127,6 @@ public static FunctionAppContentEditingState GetFunctionAppContentEditingState(I } } - public static string GetPlatformReleaseChannel(IEnvironment environment) - { - return environment.GetEnvironmentVariable(EnvironmentSettingNames.AntaresPlatformReleaseChannel) ?? ScriptConstants.LatestPlatformChannelNameUpper; - } - public static bool TryReadAsBool(IDictionary properties, string propertyKey, out bool result) { if (properties.TryGetValue(propertyKey, out object valueObject)) diff --git a/src/WebJobs.Script/Workers/Rpc/Configuration/StaticWorkerConfigurationResolver.cs b/src/WebJobs.Script/Workers/Rpc/Configuration/DefaultWorkerConfigurationResolver.cs similarity index 72% rename from src/WebJobs.Script/Workers/Rpc/Configuration/StaticWorkerConfigurationResolver.cs rename to src/WebJobs.Script/Workers/Rpc/Configuration/DefaultWorkerConfigurationResolver.cs index 6c8c0e87ce..e116388854 100644 --- a/src/WebJobs.Script/Workers/Rpc/Configuration/StaticWorkerConfigurationResolver.cs +++ b/src/WebJobs.Script/Workers/Rpc/Configuration/DefaultWorkerConfigurationResolver.cs @@ -9,21 +9,22 @@ namespace Microsoft.Azure.WebJobs.Script.Workers.Rpc.Configuration { - internal class StaticWorkerConfigurationResolver : IWorkerConfigurationResolver + // This class resolves worker configurations by scanning the "workers" directory within the Host for worker config files. + internal class DefaultWorkerConfigurationResolver : IWorkerConfigurationResolver { private readonly ILogger _logger; private readonly IConfiguration _configuration; - public StaticWorkerConfigurationResolver(IConfiguration configuration, ILogger logger) + public DefaultWorkerConfigurationResolver(IConfiguration configuration, ILogger logger) { _logger = logger ?? throw new ArgumentNullException(nameof(logger)); - _configuration = configuration; + _configuration = configuration ?? throw new ArgumentNullException(nameof(configuration)); } public List GetWorkerConfigs() { string workersDirPath = WorkerConfigurationHelper.GetWorkersDirPath(_configuration); - _logger.LogDebug("Workers Directory set to: {WorkersDirPath}", workersDirPath); + _logger.LogDebug("Workers Directory set to: {workersDirPath}", workersDirPath); List workerConfigs = new(); diff --git a/src/WebJobs.Script/Workers/Rpc/Configuration/DynamicWorkerConfigurationResolver.cs b/src/WebJobs.Script/Workers/Rpc/Configuration/DynamicWorkerConfigurationResolver.cs index 22d978d450..0e5cc1fe83 100644 --- a/src/WebJobs.Script/Workers/Rpc/Configuration/DynamicWorkerConfigurationResolver.cs +++ b/src/WebJobs.Script/Workers/Rpc/Configuration/DynamicWorkerConfigurationResolver.cs @@ -13,6 +13,8 @@ namespace Microsoft.Azure.WebJobs.Script.Workers.Rpc.Configuration { + // This class resolves worker configurations dynamically based on the current environment and configuration settings. + // It searches for worker configs in specified probing paths and the fallback path, and returns a list of worker configuration paths. internal sealed class DynamicWorkerConfigurationResolver : IWorkerConfigurationResolver { private readonly IConfiguration _config; @@ -70,7 +72,7 @@ private void ResolveWorkerConfigsFromProbingPaths(string workerRuntime, Dictiona string probingPaths = _workerProbingPaths is not null ? string.Join(", ", _workerProbingPaths) : null; _logger.LogDebug("Workers probing paths set to: {probingPaths}", probingPaths); - string releaseChannel = Utility.GetPlatformReleaseChannel(_environment); + string releaseChannel = EnvironmentExtensions.GetPlatformReleaseChannel(_environment); if (_workerProbingPaths is not null) { @@ -95,7 +97,7 @@ private void ResolveWorkerConfigsFromProbingPaths(string workerRuntime, Dictiona if (workerUnavailableViaHostingConfig || (!_environment.IsMultiLanguageRuntimeEnvironment() && !_environment.IsPlaceholderModeEnabled() && - WorkerConfigurationHelper.ShouldSkipRuntime(workerRuntime, workerRuntimeDir))) + ShouldSkipWorkerDirectory(workerRuntime, workerRuntimeDir))) { continue; } @@ -158,7 +160,7 @@ private void ResolveWorkerConfigsFromWithinHost(string workerRuntime, Dictionary if (outputDict.ContainsKey(workerDir) || (!_environment.IsMultiLanguageRuntimeEnvironment() && - WorkerConfigurationHelper.ShouldSkipRuntime(workerRuntime, workerDir))) + ShouldSkipWorkerDirectory(workerRuntime, workerDir))) { continue; } @@ -285,6 +287,11 @@ private bool DoesHostHasRequiredCapabilities(JsonElement workerConfig) return true; } + internal static bool ShouldSkipWorkerDirectory(string workerRuntime, string workerDir) + { + return workerRuntime is not null && !workerRuntime.Equals(workerDir, StringComparison.OrdinalIgnoreCase); + } + private string FormatVersion(string version) { if (!version.Contains('.')) diff --git a/src/WebJobs.Script/Workers/Rpc/Configuration/RpcWorkerConfigFactory.cs b/src/WebJobs.Script/Workers/Rpc/Configuration/RpcWorkerConfigFactory.cs index 50610a5b6a..446542e8d9 100644 --- a/src/WebJobs.Script/Workers/Rpc/Configuration/RpcWorkerConfigFactory.cs +++ b/src/WebJobs.Script/Workers/Rpc/Configuration/RpcWorkerConfigFactory.cs @@ -102,9 +102,7 @@ internal void AddProvider(string workerDir) try { // After specialization, load worker config only for the specified runtime unless it's a multi-language app. - if (!string.IsNullOrWhiteSpace(_workerRuntime) && - !_environment.IsPlaceholderModeEnabled() && - !_environment.IsMultiLanguageRuntimeEnvironment()) + if (!string.IsNullOrWhiteSpace(_workerRuntime) && !_environment.IsPlaceholderModeEnabled() && !_environment.IsMultiLanguageRuntimeEnvironment()) { string workerRuntime = Path.GetFileName(workerDir); // Only skip worker directories that don't match the current runtime. diff --git a/src/WebJobs.Script/Workers/Rpc/Configuration/WorkerConfigurationHelper.cs b/src/WebJobs.Script/Workers/Rpc/Configuration/WorkerConfigurationHelper.cs index 101660875c..37d1199313 100644 --- a/src/WebJobs.Script/Workers/Rpc/Configuration/WorkerConfigurationHelper.cs +++ b/src/WebJobs.Script/Workers/Rpc/Configuration/WorkerConfigurationHelper.cs @@ -161,11 +161,6 @@ internal static string GetWorkersDirPath(IConfiguration configuration) return workersDirPath; } - internal static bool ShouldSkipRuntime(string workerRuntime, string workerDir) - { - return workerRuntime is not null && !workerRuntime.Equals(workerDir, StringComparison.OrdinalIgnoreCase); - } - internal static HashSet GetWorkersAvailableForResolutionViaHostingConfig(IOptions functionsHostingConfigOptions) => (functionsHostingConfigOptions.Value?.WorkersAvailableForDynamicResolution ?? string.Empty) .Split('|', StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries) diff --git a/src/WebJobs.Script/Workers/Rpc/Configuration/WorkerConfigurationResolverFactory.cs b/src/WebJobs.Script/Workers/Rpc/Configuration/WorkerConfigurationResolverFactory.cs index b095381f60..f0b1986fa2 100644 --- a/src/WebJobs.Script/Workers/Rpc/Configuration/WorkerConfigurationResolverFactory.cs +++ b/src/WebJobs.Script/Workers/Rpc/Configuration/WorkerConfigurationResolverFactory.cs @@ -15,7 +15,7 @@ namespace Microsoft.Azure.WebJobs.Script.Workers.Rpc.Configuration { /// - /// Factory for creating worker configuration resolvers based on current configuration. + /// Factory for creating worker configuration resolvers depending on if dynamic worker resolution is enabled or not. /// internal class WorkerConfigurationResolverFactory : IWorkerConfigurationResolverFactory { @@ -46,17 +46,16 @@ public IWorkerConfigurationResolver CreateResolver() if (dynamicWorkerResolutionEnabled) { - return new DynamicWorkerConfigurationResolver( - _configuration, - _logger, - _environment, - FileUtility.Instance, - _workerProfileManager, - workersAvailableForResolution, - probingPaths); + return new DynamicWorkerConfigurationResolver(_configuration, + _logger, + _environment, + FileUtility.Instance, + _workerProfileManager, + workersAvailableForResolution, + probingPaths); } - return new StaticWorkerConfigurationResolver(_configuration, _logger); + return new DefaultWorkerConfigurationResolver(_configuration, _logger); } internal List GetWorkerProbingPaths() @@ -89,7 +88,7 @@ internal List GetWorkerProbingPaths() return probingPaths; } - internal string GetWindowsSiteExtensionsPath() + internal static string GetWindowsSiteExtensionsPath() { var assemblyPath = Assembly.GetExecutingAssembly().Location; var assemblyDir = Path.GetDirectoryName(assemblyPath); diff --git a/test/WebJobs.Script.Tests/Workers/Rpc/RpcWorkerConfigFactoryTests.cs b/test/WebJobs.Script.Tests/Workers/Rpc/RpcWorkerConfigFactoryTests.cs index 5704be1d79..d78d930040 100644 --- a/test/WebJobs.Script.Tests/Workers/Rpc/RpcWorkerConfigFactoryTests.cs +++ b/test/WebJobs.Script.Tests/Workers/Rpc/RpcWorkerConfigFactoryTests.cs @@ -41,7 +41,7 @@ public void DefaultLanguageWorkersDir() var expectedWorkersDir = Path.Combine(Path.GetDirectoryName(new Uri(typeof(RpcWorkerConfigFactory).Assembly.Location).LocalPath), RpcWorkerConstants.DefaultWorkersDirectoryName); var config = new ConfigurationBuilder().Build(); var testLogger = new TestLogger("test"); - var workerConfigurationResolver = new StaticWorkerConfigurationResolver(config, testLogger); + var workerConfigurationResolver = new DefaultWorkerConfigurationResolver(config, testLogger); var configFactory = new RpcWorkerConfigFactory(config, testLogger, _testSysRuntimeInfo, _testEnvironment, new TestMetricsLogger(), _testWorkerProfileManager, workerConfigurationResolver); Assert.Equal(expectedWorkersDir, configFactory.WorkersDirPath); } @@ -75,7 +75,7 @@ public void LanguageWorker_WorkersDir_Set() }) .Build(); var testLogger = new TestLogger("test"); - var workerConfigurationResolver = new StaticWorkerConfigurationResolver(config, testLogger); + var workerConfigurationResolver = new DefaultWorkerConfigurationResolver(config, testLogger); var configFactory = new RpcWorkerConfigFactory(config, testLogger, _testSysRuntimeInfo, _testEnvironment, new TestMetricsLogger(), _testWorkerProfileManager, workerConfigurationResolver); Assert.Equal(expectedWorkersDir, configFactory.WorkersDirPath); } @@ -92,7 +92,7 @@ public void LanguageWorker_WorkersDir_NotSet() var config = configBuilder.Build(); var scriptSettingsManager = new ScriptSettingsManager(config); var testLogger = new TestLogger("test"); - var workerConfigurationResolver = new StaticWorkerConfigurationResolver(config, testLogger); + var workerConfigurationResolver = new DefaultWorkerConfigurationResolver(config, testLogger); var configFactory = new RpcWorkerConfigFactory(config, testLogger, _testSysRuntimeInfo, _testEnvironment, new TestMetricsLogger(), _testWorkerProfileManager, workerConfigurationResolver); Assert.Equal(expectedWorkersDir, configFactory.WorkersDirPath); } @@ -117,7 +117,7 @@ public void WorkerDescription_Skipped_When_Profile_Disables_Worker() var scriptSettingsManager = new ScriptSettingsManager(config); var testLogger = new TestLogger("test"); _testEnvironment.SetEnvironmentVariable("ENV_VAR_BAR", "True"); - var workerConfigurationResolver = new StaticWorkerConfigurationResolver(config, testLogger); + var workerConfigurationResolver = new DefaultWorkerConfigurationResolver(config, testLogger); var configFactory = new RpcWorkerConfigFactory(config, testLogger, _testSysRuntimeInfo, _testEnvironment, new TestMetricsLogger(), _testWorkerProfileManager, workerConfigurationResolver); var workerConfigs = configFactory.GetConfigs(); @@ -140,7 +140,7 @@ public void JavaPath_FromEnvVars() var config = configBuilder.Build(); var scriptSettingsManager = new ScriptSettingsManager(config); var testLogger = new TestLogger("test"); - var workerConfigurationResolver = new StaticWorkerConfigurationResolver(config, testLogger); + var workerConfigurationResolver = new DefaultWorkerConfigurationResolver(config, testLogger); var configFactory = new RpcWorkerConfigFactory(config, testLogger, _testSysRuntimeInfo, _testEnvironment, new TestMetricsLogger(), _testWorkerProfileManager, workerConfigurationResolver); var workerConfigs = configFactory.GetConfigs(); var javaPath = workerConfigs.FirstOrDefault(c => c.Description.Language.Equals("java", StringComparison.OrdinalIgnoreCase)).Description.DefaultExecutablePath; @@ -163,7 +163,7 @@ public void DefaultWorkerConfigs_Overrides_DefaultWorkerRuntimeVersion_AppSettin _testEnvironment.SetEnvironmentVariable(EnvironmentSettingNames.AzureWebsitePlaceholderMode, "1"); using var variables = new TestScopedSettings(scriptSettingsManager, testEnvVariables); - var workerConfigurationResolver = new StaticWorkerConfigurationResolver(config, testLogger); + var workerConfigurationResolver = new DefaultWorkerConfigurationResolver(config, testLogger); var configFactory = new RpcWorkerConfigFactory(config, testLogger, _testSysRuntimeInfo, _testEnvironment, new TestMetricsLogger(), _testWorkerProfileManager, workerConfigurationResolver); var workerConfigs = configFactory.GetConfigs(); var pythonWorkerConfig = workerConfigs.FirstOrDefault(w => w.Description.Language.Equals("python", StringComparison.OrdinalIgnoreCase)); @@ -185,7 +185,7 @@ public void DefaultWorkerConfigs_Overrides_VersionAppSetting() var config = configBuilder.Build(); var scriptSettingsManager = new ScriptSettingsManager(config); var testLogger = new TestLogger("test"); - var resolver = new StaticWorkerConfigurationResolver(config, testLogger); + var resolver = new DefaultWorkerConfigurationResolver(config, testLogger); var configFactory = new RpcWorkerConfigFactory(config, testLogger, _testSysRuntimeInfo, testEnvironment, new TestMetricsLogger(), _testWorkerProfileManager, resolver); var workerConfigs = configFactory.GetConfigs(); var powershellWorkerConfig = workerConfigs.FirstOrDefault(w => w.Description.Language.Equals("powershell", StringComparison.OrdinalIgnoreCase)); @@ -213,7 +213,7 @@ public void ShouldAddProvider_Returns_Expected(string workerLanguage, string wor } var config = new ConfigurationBuilder().Build(); var testLogger = new TestLogger("test"); - var workerConfigurationResolver = new StaticWorkerConfigurationResolver(config, testLogger); + var workerConfigurationResolver = new DefaultWorkerConfigurationResolver(config, testLogger); var rpcWorkerConfigFactory = new RpcWorkerConfigFactory(config, testLogger, _testSysRuntimeInfo, _testEnvironment, new TestMetricsLogger(), _testWorkerProfileManager, workerConfigurationResolver); _testEnvironment.SetEnvironmentVariable(RpcWorkerConstants.FunctionWorkerRuntimeSettingName, workerRuntime); Assert.Equal(expectedResult, rpcWorkerConfigFactory.ShouldAddWorkerConfig(workerLanguage)); @@ -258,7 +258,7 @@ public void GetWorkerProcessCount_Tests(bool defaultWorkerConfig, bool setProces var config = new ConfigurationBuilder().Build(); var testLogger = new TestLogger("test"); - var workerConfigurationResolver = new StaticWorkerConfigurationResolver(config, testLogger); + var workerConfigurationResolver = new DefaultWorkerConfigurationResolver(config, testLogger); var rpcWorkerConfigFactory = new RpcWorkerConfigFactory(config, testLogger, _testSysRuntimeInfo, _testEnvironment, new TestMetricsLogger(), _testWorkerProfileManager, workerConfigurationResolver); var result = rpcWorkerConfigFactory.GetWorkerProcessCount(workerConfig); @@ -299,7 +299,7 @@ public void GetWorkerProcessCount_ThrowsException_Tests() var config = new ConfigurationBuilder().Build(); var testLogger = new TestLogger("test"); - var workerConfigurationResolver = new StaticWorkerConfigurationResolver(config, testLogger); + var workerConfigurationResolver = new DefaultWorkerConfigurationResolver(config, testLogger); var rpcWorkerConfigFactory = new RpcWorkerConfigFactory(config, testLogger, _testSysRuntimeInfo, _testEnvironment, new TestMetricsLogger(), _testWorkerProfileManager, workerConfigurationResolver); var resultEx1 = Assert.Throws(() => rpcWorkerConfigFactory.GetWorkerProcessCount(workerConfig)); Assert.Contains("ProcessCount must be greater than 0", resultEx1.Message); diff --git a/test/WebJobs.Script.Tests/Workers/Rpc/RpcWorkerConfigTests.cs b/test/WebJobs.Script.Tests/Workers/Rpc/RpcWorkerConfigTests.cs index 355aca0823..80317bfb9c 100644 --- a/test/WebJobs.Script.Tests/Workers/Rpc/RpcWorkerConfigTests.cs +++ b/test/WebJobs.Script.Tests/Workers/Rpc/RpcWorkerConfigTests.cs @@ -686,7 +686,7 @@ private IEnumerable TestReadWorkerProviderFromConfig(IEnumerabl var scriptHostOptions = new ScriptJobHostOptions(); var scriptSettingsManager = new ScriptSettingsManager(config); var workerProfileManager = new Mock(); - IWorkerConfigurationResolver workerConfigurationResolver = new StaticWorkerConfigurationResolver(config, testLogger); + IWorkerConfigurationResolver workerConfigurationResolver = new DefaultWorkerConfigurationResolver(config, testLogger); var configFactory = new RpcWorkerConfigFactory(config, testLogger, _testSysRuntimeInfo, _testEnvironment, new TestMetricsLogger(), workerProfileManager.Object, workerConfigurationResolver); if (appSvcEnv) From 5ad1699d21f70f728d6dba22eca1e03901e47e78 Mon Sep 17 00:00:00 2001 From: Surbhi Gupta Date: Sun, 6 Jul 2025 18:28:19 -0500 Subject: [PATCH 42/55] Minor code refactoring --- .../Rpc/Configuration/WorkerConfigurationHelper.cs | 8 -------- .../Configuration/WorkerConfigurationResolverFactory.cs | 7 ++++++- 2 files changed, 6 insertions(+), 9 deletions(-) diff --git a/src/WebJobs.Script/Workers/Rpc/Configuration/WorkerConfigurationHelper.cs b/src/WebJobs.Script/Workers/Rpc/Configuration/WorkerConfigurationHelper.cs index 37d1199313..baf4e1a3a3 100644 --- a/src/WebJobs.Script/Workers/Rpc/Configuration/WorkerConfigurationHelper.cs +++ b/src/WebJobs.Script/Workers/Rpc/Configuration/WorkerConfigurationHelper.cs @@ -4,14 +4,11 @@ using System; using System.Collections.Generic; using System.IO; -using System.Linq; using System.Text.Json; using System.Text.RegularExpressions; -using Microsoft.Azure.WebJobs.Script.Config; using Microsoft.Azure.WebJobs.Script.Workers.Profiles; using Microsoft.Extensions.Configuration; using Microsoft.Extensions.Logging; -using Microsoft.Extensions.Options; namespace Microsoft.Azure.WebJobs.Script.Workers.Rpc.Configuration { @@ -160,10 +157,5 @@ internal static string GetWorkersDirPath(IConfiguration configuration) return workersDirPath; } - - internal static HashSet GetWorkersAvailableForResolutionViaHostingConfig(IOptions functionsHostingConfigOptions) => - (functionsHostingConfigOptions.Value?.WorkersAvailableForDynamicResolution ?? string.Empty) - .Split('|', StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries) - .ToHashSet(StringComparer.OrdinalIgnoreCase); } } \ No newline at end of file diff --git a/src/WebJobs.Script/Workers/Rpc/Configuration/WorkerConfigurationResolverFactory.cs b/src/WebJobs.Script/Workers/Rpc/Configuration/WorkerConfigurationResolverFactory.cs index f0b1986fa2..1407ed41a9 100644 --- a/src/WebJobs.Script/Workers/Rpc/Configuration/WorkerConfigurationResolverFactory.cs +++ b/src/WebJobs.Script/Workers/Rpc/Configuration/WorkerConfigurationResolverFactory.cs @@ -40,7 +40,7 @@ public WorkerConfigurationResolverFactory( public IWorkerConfigurationResolver CreateResolver() { - HashSet workersAvailableForResolution = WorkerConfigurationHelper.GetWorkersAvailableForResolutionViaHostingConfig(_functionsHostingConfigOptions); + HashSet workersAvailableForResolution = GetWorkersAvailableForResolutionViaHostingConfig(_functionsHostingConfigOptions); List probingPaths = GetWorkerProbingPaths(); bool dynamicWorkerResolutionEnabled = _environment.IsDynamicWorkerResolutionEnabled(workersAvailableForResolution); @@ -88,6 +88,11 @@ internal List GetWorkerProbingPaths() return probingPaths; } + internal static HashSet GetWorkersAvailableForResolutionViaHostingConfig(IOptions functionsHostingConfigOptions) => + (functionsHostingConfigOptions.Value?.WorkersAvailableForDynamicResolution ?? string.Empty) + .Split('|', StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries) + .ToHashSet(StringComparer.OrdinalIgnoreCase); + internal static string GetWindowsSiteExtensionsPath() { var assemblyPath = Assembly.GetExecutingAssembly().Location; From d15984c11b49ab2a907eb69af795c6a2204bf3ff Mon Sep 17 00:00:00 2001 From: Surbhi Gupta Date: Mon, 7 Jul 2025 15:18:24 -0500 Subject: [PATCH 43/55] Code cleanup and addressing PR feedback --- .../DynamicWorkerConfigurationResolver.cs | 63 ++++++++++--------- .../Workers/Rpc/RpcWorkerConstants.cs | 1 - 2 files changed, 32 insertions(+), 32 deletions(-) diff --git a/src/WebJobs.Script/Workers/Rpc/Configuration/DynamicWorkerConfigurationResolver.cs b/src/WebJobs.Script/Workers/Rpc/Configuration/DynamicWorkerConfigurationResolver.cs index 0e5cc1fe83..3cd930dcb8 100644 --- a/src/WebJobs.Script/Workers/Rpc/Configuration/DynamicWorkerConfigurationResolver.cs +++ b/src/WebJobs.Script/Workers/Rpc/Configuration/DynamicWorkerConfigurationResolver.cs @@ -69,41 +69,42 @@ workerRuntime is not null && private void ResolveWorkerConfigsFromProbingPaths(string workerRuntime, Dictionary outputDict) { - string probingPaths = _workerProbingPaths is not null ? string.Join(", ", _workerProbingPaths) : null; - _logger.LogDebug("Workers probing paths set to: {probingPaths}", probingPaths); + if (_workerProbingPaths is null) + { + return; + } + + _logger.LogDebug("Workers probing paths set to: {probingPaths}", string.Join(", ", _workerProbingPaths)); string releaseChannel = EnvironmentExtensions.GetPlatformReleaseChannel(_environment); - if (_workerProbingPaths is not null) + // probing path directory structure is: /// + foreach (var probingPath in _workerProbingPaths) { - // probing path directory structure is: /// - foreach (var probingPath in _workerProbingPaths) + if (!string.IsNullOrWhiteSpace(probingPath) && _fileSystem.Directory.Exists(probingPath)) { - if (!string.IsNullOrEmpty(probingPath) && _fileSystem.Directory.Exists(probingPath)) + foreach (var workerRuntimePath in _fileSystem.Directory.EnumerateDirectories(probingPath)) { - foreach (var workerRuntimePath in _fileSystem.Directory.EnumerateDirectories(probingPath)) + string workerRuntimeDir = Path.GetFileName(workerRuntimePath); + + // If probing paths are malformed and have duplicate directories of the same language worker (eg. due to different casing) + if (outputDict.ContainsKey(workerRuntimeDir)) { - string workerRuntimeDir = Path.GetFileName(workerRuntimePath); - - // If probing paths are malformed and have duplicate directories of the same language worker (eg. due to different casing) - if (outputDict.ContainsKey(workerRuntimeDir)) - { - continue; - } - - bool workerUnavailableViaHostingConfig = _workersAvailableForResolutionViaHostingConfig is not null && !_workersAvailableForResolutionViaHostingConfig.Contains(workerRuntimeDir); - - // Skip worker directories that don't match the current runtime or are not enabled via hosting config - if (workerUnavailableViaHostingConfig || - (!_environment.IsMultiLanguageRuntimeEnvironment() && - !_environment.IsPlaceholderModeEnabled() && - ShouldSkipWorkerDirectory(workerRuntime, workerRuntimeDir))) - { - continue; - } - - ResolveWorkerConfigsFromVersionsDirs(workerRuntimePath, workerRuntimeDir, releaseChannel, outputDict); + continue; } + + bool workerUnavailableViaHostingConfig = _workersAvailableForResolutionViaHostingConfig is not null && !_workersAvailableForResolutionViaHostingConfig.Contains(workerRuntimeDir); + + // Skip worker directories that don't match the current runtime or are not enabled via hosting config + if (workerUnavailableViaHostingConfig || + (!_environment.IsMultiLanguageRuntimeEnvironment() && + !_environment.IsPlaceholderModeEnabled() && + ShouldSkipWorkerDirectory(workerRuntime, workerRuntimeDir))) + { + continue; + } + + ResolveWorkerConfigsFromVersionsDirs(workerRuntimePath, workerRuntimeDir, releaseChannel, outputDict); } } } @@ -119,7 +120,7 @@ private void ResolveWorkerConfigsFromVersionsDirs(string languageWorkerPath, str int compatibleWorkerCount = 0; bool isStandardOrExtendedChannel = - releaseChannel != null && + !string.IsNullOrWhiteSpace(releaseChannel) && (releaseChannel.Equals(ScriptConstants.StandardPlatformChannelNameUpper, StringComparison.OrdinalIgnoreCase) || releaseChannel.Equals(ScriptConstants.ExtendedPlatformChannelNameUpper, StringComparison.OrdinalIgnoreCase)); @@ -152,15 +153,15 @@ private void ResolveWorkerConfigsFromWithinHost(string workerRuntime, Dictionary _logger.LogDebug("Searching for worker configs in the fallback directory: {fallbackPath}", fallbackPath); - if (fallbackPath != null && _fileSystem.Directory.Exists(fallbackPath)) + if (!string.IsNullOrEmpty(fallbackPath) && _fileSystem.Directory.Exists(fallbackPath)) { foreach (var workerPath in _fileSystem.Directory.EnumerateDirectories(fallbackPath)) { string workerDir = Path.GetFileName(workerPath).ToLower(); if (outputDict.ContainsKey(workerDir) || - (!_environment.IsMultiLanguageRuntimeEnvironment() && - ShouldSkipWorkerDirectory(workerRuntime, workerDir))) + (!_environment.IsMultiLanguageRuntimeEnvironment() && + ShouldSkipWorkerDirectory(workerRuntime, workerDir))) { continue; } diff --git a/src/WebJobs.Script/Workers/Rpc/RpcWorkerConstants.cs b/src/WebJobs.Script/Workers/Rpc/RpcWorkerConstants.cs index 3976c7f54c..e192944241 100644 --- a/src/WebJobs.Script/Workers/Rpc/RpcWorkerConstants.cs +++ b/src/WebJobs.Script/Workers/Rpc/RpcWorkerConstants.cs @@ -26,7 +26,6 @@ public static class RpcWorkerConstants // Section names in host.json or AppSettings public const string LanguageWorkersSectionName = "languageWorkers"; - public const string WorkerProbingPathsSectionName = "probingPaths"; public const string HostRequirementsSectionName = "hostRequirements"; // Worker description constants From c6b9e93164e858b609af7a2d6654dd09b6ddc544 Mon Sep 17 00:00:00 2001 From: Surbhi Gupta Date: Mon, 7 Jul 2025 20:14:25 -0500 Subject: [PATCH 44/55] Test folder update --- .../Rpc/Configuration/WorkerConfigurationResolverFactory.cs | 6 +++--- .../workers/dotnet-isolated/1.0.0/worker.config.json | 0 .../workers/java/2.18.0/worker.config.json | 0 .../workers/java/2.18.1/worker.config.json | 0 .../workers/java/2.18.2/worker.config.json | 0 .../workers/java/2.19.0/worker.config.json | 0 .../workers/node/3.10.1/worker.config.json | 0 .../workers/powershell/7.4/worker.config.json | 0 8 files changed, 3 insertions(+), 3 deletions(-) rename test/TestWorkers/{ProbingPaths => ProbingPath}/workers/dotnet-isolated/1.0.0/worker.config.json (100%) rename test/TestWorkers/{Probingpaths => ProbingPath}/workers/java/2.18.0/worker.config.json (100%) rename test/TestWorkers/{Probingpaths => ProbingPath}/workers/java/2.18.1/worker.config.json (100%) rename test/TestWorkers/{Probingpaths => ProbingPath}/workers/java/2.18.2/worker.config.json (100%) rename test/TestWorkers/{Probingpaths => ProbingPath}/workers/java/2.19.0/worker.config.json (100%) rename test/TestWorkers/{Probingpaths => ProbingPath}/workers/node/3.10.1/worker.config.json (100%) rename test/TestWorkers/{Probingpaths => ProbingPath}/workers/powershell/7.4/worker.config.json (100%) diff --git a/src/WebJobs.Script/Workers/Rpc/Configuration/WorkerConfigurationResolverFactory.cs b/src/WebJobs.Script/Workers/Rpc/Configuration/WorkerConfigurationResolverFactory.cs index 1407ed41a9..1d91674d84 100644 --- a/src/WebJobs.Script/Workers/Rpc/Configuration/WorkerConfigurationResolverFactory.cs +++ b/src/WebJobs.Script/Workers/Rpc/Configuration/WorkerConfigurationResolverFactory.cs @@ -89,9 +89,9 @@ internal List GetWorkerProbingPaths() } internal static HashSet GetWorkersAvailableForResolutionViaHostingConfig(IOptions functionsHostingConfigOptions) => - (functionsHostingConfigOptions.Value?.WorkersAvailableForDynamicResolution ?? string.Empty) - .Split('|', StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries) - .ToHashSet(StringComparer.OrdinalIgnoreCase); + (functionsHostingConfigOptions.Value?.WorkersAvailableForDynamicResolution ?? string.Empty) + .Split('|', StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries) + .ToHashSet(StringComparer.OrdinalIgnoreCase); internal static string GetWindowsSiteExtensionsPath() { diff --git a/test/TestWorkers/ProbingPaths/workers/dotnet-isolated/1.0.0/worker.config.json b/test/TestWorkers/ProbingPath/workers/dotnet-isolated/1.0.0/worker.config.json similarity index 100% rename from test/TestWorkers/ProbingPaths/workers/dotnet-isolated/1.0.0/worker.config.json rename to test/TestWorkers/ProbingPath/workers/dotnet-isolated/1.0.0/worker.config.json diff --git a/test/TestWorkers/Probingpaths/workers/java/2.18.0/worker.config.json b/test/TestWorkers/ProbingPath/workers/java/2.18.0/worker.config.json similarity index 100% rename from test/TestWorkers/Probingpaths/workers/java/2.18.0/worker.config.json rename to test/TestWorkers/ProbingPath/workers/java/2.18.0/worker.config.json diff --git a/test/TestWorkers/Probingpaths/workers/java/2.18.1/worker.config.json b/test/TestWorkers/ProbingPath/workers/java/2.18.1/worker.config.json similarity index 100% rename from test/TestWorkers/Probingpaths/workers/java/2.18.1/worker.config.json rename to test/TestWorkers/ProbingPath/workers/java/2.18.1/worker.config.json diff --git a/test/TestWorkers/Probingpaths/workers/java/2.18.2/worker.config.json b/test/TestWorkers/ProbingPath/workers/java/2.18.2/worker.config.json similarity index 100% rename from test/TestWorkers/Probingpaths/workers/java/2.18.2/worker.config.json rename to test/TestWorkers/ProbingPath/workers/java/2.18.2/worker.config.json diff --git a/test/TestWorkers/Probingpaths/workers/java/2.19.0/worker.config.json b/test/TestWorkers/ProbingPath/workers/java/2.19.0/worker.config.json similarity index 100% rename from test/TestWorkers/Probingpaths/workers/java/2.19.0/worker.config.json rename to test/TestWorkers/ProbingPath/workers/java/2.19.0/worker.config.json diff --git a/test/TestWorkers/Probingpaths/workers/node/3.10.1/worker.config.json b/test/TestWorkers/ProbingPath/workers/node/3.10.1/worker.config.json similarity index 100% rename from test/TestWorkers/Probingpaths/workers/node/3.10.1/worker.config.json rename to test/TestWorkers/ProbingPath/workers/node/3.10.1/worker.config.json diff --git a/test/TestWorkers/Probingpaths/workers/powershell/7.4/worker.config.json b/test/TestWorkers/ProbingPath/workers/powershell/7.4/worker.config.json similarity index 100% rename from test/TestWorkers/Probingpaths/workers/powershell/7.4/worker.config.json rename to test/TestWorkers/ProbingPath/workers/powershell/7.4/worker.config.json From be527c49687f23a1c55089ddd4fc84a2ba6e4995 Mon Sep 17 00:00:00 2001 From: Surbhi Gupta Date: Mon, 7 Jul 2025 20:15:10 -0500 Subject: [PATCH 45/55] Test folder name update --- .../workers/dotnet-isolated/1.0.0/worker.config.json | 0 .../workers/java/2.18.0/worker.config.json | 0 .../workers/java/2.18.1/worker.config.json | 0 .../workers/java/2.18.2/worker.config.json | 0 .../workers/java/2.19.0/worker.config.json | 0 .../workers/node/3.10.1/worker.config.json | 0 .../workers/powershell/7.4/worker.config.json | 0 7 files changed, 0 insertions(+), 0 deletions(-) rename test/TestWorkers/{ProbingPath => ProbingPaths}/workers/dotnet-isolated/1.0.0/worker.config.json (100%) rename test/TestWorkers/{ProbingPath => ProbingPaths}/workers/java/2.18.0/worker.config.json (100%) rename test/TestWorkers/{ProbingPath => ProbingPaths}/workers/java/2.18.1/worker.config.json (100%) rename test/TestWorkers/{ProbingPath => ProbingPaths}/workers/java/2.18.2/worker.config.json (100%) rename test/TestWorkers/{ProbingPath => ProbingPaths}/workers/java/2.19.0/worker.config.json (100%) rename test/TestWorkers/{ProbingPath => ProbingPaths}/workers/node/3.10.1/worker.config.json (100%) rename test/TestWorkers/{ProbingPath => ProbingPaths}/workers/powershell/7.4/worker.config.json (100%) diff --git a/test/TestWorkers/ProbingPath/workers/dotnet-isolated/1.0.0/worker.config.json b/test/TestWorkers/ProbingPaths/workers/dotnet-isolated/1.0.0/worker.config.json similarity index 100% rename from test/TestWorkers/ProbingPath/workers/dotnet-isolated/1.0.0/worker.config.json rename to test/TestWorkers/ProbingPaths/workers/dotnet-isolated/1.0.0/worker.config.json diff --git a/test/TestWorkers/ProbingPath/workers/java/2.18.0/worker.config.json b/test/TestWorkers/ProbingPaths/workers/java/2.18.0/worker.config.json similarity index 100% rename from test/TestWorkers/ProbingPath/workers/java/2.18.0/worker.config.json rename to test/TestWorkers/ProbingPaths/workers/java/2.18.0/worker.config.json diff --git a/test/TestWorkers/ProbingPath/workers/java/2.18.1/worker.config.json b/test/TestWorkers/ProbingPaths/workers/java/2.18.1/worker.config.json similarity index 100% rename from test/TestWorkers/ProbingPath/workers/java/2.18.1/worker.config.json rename to test/TestWorkers/ProbingPaths/workers/java/2.18.1/worker.config.json diff --git a/test/TestWorkers/ProbingPath/workers/java/2.18.2/worker.config.json b/test/TestWorkers/ProbingPaths/workers/java/2.18.2/worker.config.json similarity index 100% rename from test/TestWorkers/ProbingPath/workers/java/2.18.2/worker.config.json rename to test/TestWorkers/ProbingPaths/workers/java/2.18.2/worker.config.json diff --git a/test/TestWorkers/ProbingPath/workers/java/2.19.0/worker.config.json b/test/TestWorkers/ProbingPaths/workers/java/2.19.0/worker.config.json similarity index 100% rename from test/TestWorkers/ProbingPath/workers/java/2.19.0/worker.config.json rename to test/TestWorkers/ProbingPaths/workers/java/2.19.0/worker.config.json diff --git a/test/TestWorkers/ProbingPath/workers/node/3.10.1/worker.config.json b/test/TestWorkers/ProbingPaths/workers/node/3.10.1/worker.config.json similarity index 100% rename from test/TestWorkers/ProbingPath/workers/node/3.10.1/worker.config.json rename to test/TestWorkers/ProbingPaths/workers/node/3.10.1/worker.config.json diff --git a/test/TestWorkers/ProbingPath/workers/powershell/7.4/worker.config.json b/test/TestWorkers/ProbingPaths/workers/powershell/7.4/worker.config.json similarity index 100% rename from test/TestWorkers/ProbingPath/workers/powershell/7.4/worker.config.json rename to test/TestWorkers/ProbingPaths/workers/powershell/7.4/worker.config.json From e0cb000e8b8b79dfa18ecc59298e54a733c58f81 Mon Sep 17 00:00:00 2001 From: Surbhi Gupta Date: Tue, 8 Jul 2025 15:58:01 -0500 Subject: [PATCH 46/55] Added some code comments --- .../Configuration/DynamicWorkerConfigurationResolver.cs | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/src/WebJobs.Script/Workers/Rpc/Configuration/DynamicWorkerConfigurationResolver.cs b/src/WebJobs.Script/Workers/Rpc/Configuration/DynamicWorkerConfigurationResolver.cs index 3cd930dcb8..fd041ee5f4 100644 --- a/src/WebJobs.Script/Workers/Rpc/Configuration/DynamicWorkerConfigurationResolver.cs +++ b/src/WebJobs.Script/Workers/Rpc/Configuration/DynamicWorkerConfigurationResolver.cs @@ -47,6 +47,7 @@ public List GetWorkerConfigs() { // Dictionary of { FUNCTIONS_WORKER_RUNTIME environment variable value : path of workerConfig } // Example: outputDict = {"java": "path1", "node": "path2", "dotnet-isolated": "path3"} for multilanguage worker scenario + // Sample path: "///" Dictionary outputDict = new Dictionary(StringComparer.OrdinalIgnoreCase); var workerRuntime = _environment.GetEnvironmentVariable(EnvironmentSettingNames.FunctionWorkerRuntime); @@ -115,6 +116,7 @@ private void ResolveWorkerConfigsFromVersionsDirs(string languageWorkerPath, str var workerVersionPaths = _fileSystem.Directory.EnumerateDirectories(languageWorkerPath); // Map of: (parsed worker version, worker path) + // Example: [ (1.0.0, "/java/1.0.0"), (2.0.0, "/java/2.0.0") ] var versionPathMap = GetWorkerVersionsDescending(workerVersionPaths); int compatibleWorkerCount = 0; @@ -135,13 +137,13 @@ private void ResolveWorkerConfigsFromVersionsDirs(string languageWorkerPath, str if (string.IsNullOrEmpty(releaseChannel) || !isStandardOrExtendedChannel) { - break; // latest version is the default + return; // latest version is the default } if (compatibleWorkerCount > 1 && isStandardOrExtendedChannel) { outputDict[languageWorkerFolder] = languageWorkerVersionPath; - break; + return; } } } @@ -176,7 +178,7 @@ private void ResolveWorkerConfigsFromWithinHost(string workerRuntime, Dictionary workerRuntime is not null && outputDict.ContainsKey(workerRuntime)) { - break; + return; } } } @@ -185,6 +187,7 @@ workerRuntime is not null && private SortedList GetWorkerVersionsDescending(IEnumerable workerVersionPaths) { // Map of: (parsed worker version, worker path) + // Example: [ (1.0.0, "/java/1.0.0"), (2.0.0, "/java/2.0.0") ] var versionPathMap = new SortedList(new DescendingVersionComparer()); if (!workerVersionPaths.Any()) From 738cea94fd375710d59fce2c1092c80b3a4dd040 Mon Sep 17 00:00:00 2001 From: Surbhi Gupta Date: Tue, 8 Jul 2025 23:23:04 -0500 Subject: [PATCH 47/55] Addressing PR feedback --- .../DefaultWorkerConfigurationResolver.cs | 6 +- .../DynamicWorkerConfigurationResolver.cs | 56 ++++++++++--------- .../Configuration/RpcWorkerConfigFactory.cs | 2 +- .../WorkerConfigurationResolverFactory.cs | 2 +- .../Rpc/IWorkerConfigurationResolver.cs | 2 +- .../Rpc/WorkerConfigurationResolverTests.cs | 6 +- 6 files changed, 40 insertions(+), 34 deletions(-) diff --git a/src/WebJobs.Script/Workers/Rpc/Configuration/DefaultWorkerConfigurationResolver.cs b/src/WebJobs.Script/Workers/Rpc/Configuration/DefaultWorkerConfigurationResolver.cs index e116388854..9abf62eb05 100644 --- a/src/WebJobs.Script/Workers/Rpc/Configuration/DefaultWorkerConfigurationResolver.cs +++ b/src/WebJobs.Script/Workers/Rpc/Configuration/DefaultWorkerConfigurationResolver.cs @@ -10,7 +10,7 @@ namespace Microsoft.Azure.WebJobs.Script.Workers.Rpc.Configuration { // This class resolves worker configurations by scanning the "workers" directory within the Host for worker config files. - internal class DefaultWorkerConfigurationResolver : IWorkerConfigurationResolver + internal sealed class DefaultWorkerConfigurationResolver : IWorkerConfigurationResolver { private readonly ILogger _logger; private readonly IConfiguration _configuration; @@ -21,9 +21,9 @@ public DefaultWorkerConfigurationResolver(IConfiguration configuration, ILogger _configuration = configuration ?? throw new ArgumentNullException(nameof(configuration)); } - public List GetWorkerConfigs() + public List GetWorkerConfigPaths() { - string workersDirPath = WorkerConfigurationHelper.GetWorkersDirPath(_configuration); + var workersDirPath = WorkerConfigurationHelper.GetWorkersDirPath(_configuration); _logger.LogDebug("Workers Directory set to: {workersDirPath}", workersDirPath); List workerConfigs = new(); diff --git a/src/WebJobs.Script/Workers/Rpc/Configuration/DynamicWorkerConfigurationResolver.cs b/src/WebJobs.Script/Workers/Rpc/Configuration/DynamicWorkerConfigurationResolver.cs index fd041ee5f4..5a9f95fd5c 100644 --- a/src/WebJobs.Script/Workers/Rpc/Configuration/DynamicWorkerConfigurationResolver.cs +++ b/src/WebJobs.Script/Workers/Rpc/Configuration/DynamicWorkerConfigurationResolver.cs @@ -43,12 +43,12 @@ public DynamicWorkerConfigurationResolver(IConfiguration config, _workerProbingPaths = workerProbingPaths; } - public List GetWorkerConfigs() + public List GetWorkerConfigPaths() { // Dictionary of { FUNCTIONS_WORKER_RUNTIME environment variable value : path of workerConfig } // Example: outputDict = {"java": "path1", "node": "path2", "dotnet-isolated": "path3"} for multilanguage worker scenario // Sample path: "///" - Dictionary outputDict = new Dictionary(StringComparer.OrdinalIgnoreCase); + var outputDict = new Dictionary(StringComparer.OrdinalIgnoreCase); var workerRuntime = _environment.GetEnvironmentVariable(EnvironmentSettingNames.FunctionWorkerRuntime); @@ -82,31 +82,37 @@ private void ResolveWorkerConfigsFromProbingPaths(string workerRuntime, Dictiona // probing path directory structure is: /// foreach (var probingPath in _workerProbingPaths) { - if (!string.IsNullOrWhiteSpace(probingPath) && _fileSystem.Directory.Exists(probingPath)) + if (string.IsNullOrWhiteSpace(probingPath)) { - foreach (var workerRuntimePath in _fileSystem.Directory.EnumerateDirectories(probingPath)) + continue; + } + + if (!_fileSystem.Directory.Exists(probingPath)) + { + _logger.LogDebug("Worker probing path directory does not exist: {probingPath}", probingPath); + } + + foreach (var workerRuntimePath in _fileSystem.Directory.EnumerateDirectories(probingPath)) + { + string workerRuntimeDir = Path.GetFileName(workerRuntimePath); + + // If probing paths are malformed and have duplicate directories of the same language worker (eg. due to different casing) + if (outputDict.ContainsKey(workerRuntimeDir)) { - string workerRuntimeDir = Path.GetFileName(workerRuntimePath); - - // If probing paths are malformed and have duplicate directories of the same language worker (eg. due to different casing) - if (outputDict.ContainsKey(workerRuntimeDir)) - { - continue; - } - - bool workerUnavailableViaHostingConfig = _workersAvailableForResolutionViaHostingConfig is not null && !_workersAvailableForResolutionViaHostingConfig.Contains(workerRuntimeDir); - - // Skip worker directories that don't match the current runtime or are not enabled via hosting config - if (workerUnavailableViaHostingConfig || - (!_environment.IsMultiLanguageRuntimeEnvironment() && - !_environment.IsPlaceholderModeEnabled() && - ShouldSkipWorkerDirectory(workerRuntime, workerRuntimeDir))) - { - continue; - } - - ResolveWorkerConfigsFromVersionsDirs(workerRuntimePath, workerRuntimeDir, releaseChannel, outputDict); + continue; + } + + // Skip worker directories that don't match the current runtime or are not enabled via hosting config + // Do not load all worker directories after the specialization is done and if it is not a multi-language runtime environment + if (!_workersAvailableForResolutionViaHostingConfig.Contains(workerRuntimeDir) || + (!_environment.IsMultiLanguageRuntimeEnvironment() && + !_environment.IsPlaceholderModeEnabled() && + ShouldSkipWorkerDirectory(workerRuntime, workerRuntimeDir))) + { + continue; } + + ResolveWorkerConfigsFromVersionsDirs(workerRuntimePath, workerRuntimeDir, releaseChannel, outputDict); } } } @@ -140,7 +146,7 @@ private void ResolveWorkerConfigsFromVersionsDirs(string languageWorkerPath, str return; // latest version is the default } - if (compatibleWorkerCount > 1 && isStandardOrExtendedChannel) + if (compatibleWorkerCount > 1) { outputDict[languageWorkerFolder] = languageWorkerVersionPath; return; diff --git a/src/WebJobs.Script/Workers/Rpc/Configuration/RpcWorkerConfigFactory.cs b/src/WebJobs.Script/Workers/Rpc/Configuration/RpcWorkerConfigFactory.cs index 446542e8d9..3f4893971b 100644 --- a/src/WebJobs.Script/Workers/Rpc/Configuration/RpcWorkerConfigFactory.cs +++ b/src/WebJobs.Script/Workers/Rpc/Configuration/RpcWorkerConfigFactory.cs @@ -73,7 +73,7 @@ internal void BuildWorkerProviderDictionary() internal void AddProviders() { - List workerConfigs = _workerConfigurationResolver.GetWorkerConfigs(); + List workerConfigs = _workerConfigurationResolver.GetWorkerConfigPaths(); foreach (var workerConfig in workerConfigs) { diff --git a/src/WebJobs.Script/Workers/Rpc/Configuration/WorkerConfigurationResolverFactory.cs b/src/WebJobs.Script/Workers/Rpc/Configuration/WorkerConfigurationResolverFactory.cs index 1d91674d84..d1ec4f021b 100644 --- a/src/WebJobs.Script/Workers/Rpc/Configuration/WorkerConfigurationResolverFactory.cs +++ b/src/WebJobs.Script/Workers/Rpc/Configuration/WorkerConfigurationResolverFactory.cs @@ -17,7 +17,7 @@ namespace Microsoft.Azure.WebJobs.Script.Workers.Rpc.Configuration /// /// Factory for creating worker configuration resolvers depending on if dynamic worker resolution is enabled or not. /// - internal class WorkerConfigurationResolverFactory : IWorkerConfigurationResolverFactory + internal sealed class WorkerConfigurationResolverFactory : IWorkerConfigurationResolverFactory { private readonly IConfiguration _configuration; private readonly ILogger _logger; diff --git a/src/WebJobs.Script/Workers/Rpc/IWorkerConfigurationResolver.cs b/src/WebJobs.Script/Workers/Rpc/IWorkerConfigurationResolver.cs index 66a95c32cf..a11b384d94 100644 --- a/src/WebJobs.Script/Workers/Rpc/IWorkerConfigurationResolver.cs +++ b/src/WebJobs.Script/Workers/Rpc/IWorkerConfigurationResolver.cs @@ -10,6 +10,6 @@ namespace Microsoft.Azure.WebJobs.Script.Workers.Rpc.Configuration /// internal interface IWorkerConfigurationResolver { - List GetWorkerConfigs(); + List GetWorkerConfigPaths(); } } \ No newline at end of file diff --git a/test/WebJobs.Script.Tests/Workers/Rpc/WorkerConfigurationResolverTests.cs b/test/WebJobs.Script.Tests/Workers/Rpc/WorkerConfigurationResolverTests.cs index 3c2b909de3..61a84ed45b 100644 --- a/test/WebJobs.Script.Tests/Workers/Rpc/WorkerConfigurationResolverTests.cs +++ b/test/WebJobs.Script.Tests/Workers/Rpc/WorkerConfigurationResolverTests.cs @@ -48,7 +48,7 @@ public void GetWorkerConfigs_MultiLanguageWorker_ReturnsExpectedConfigs(string r var workerConfigurationResolver = new DynamicWorkerConfigurationResolver(_mockConfig.Object, _mockLogger.Object, mockEnvironment.Object, FileUtility.Instance, _mockProfileManager.Object, new HashSet() { "java", "node", "powershell" }, _probingPaths); - var result = workerConfigurationResolver.GetWorkerConfigs(); + var result = workerConfigurationResolver.GetWorkerConfigPaths(); // Assert Assert.Equal(result.Count, 5); @@ -82,7 +82,7 @@ public void GetWorkerConfigs_MultiLanguageWorker_NullOREmptyProbingPath_ReturnsE // Act var workerConfigurationResolver = new DynamicWorkerConfigurationResolver(_mockConfig.Object, _mockLogger.Object, mockEnvironment.Object, FileUtility.Instance, _mockProfileManager.Object, new HashSet() { "java", "node", "powershell" }, probingPaths); - var result = workerConfigurationResolver.GetWorkerConfigs(); + var result = workerConfigurationResolver.GetWorkerConfigPaths(); // Assert Assert.Equal(result.Count, 5); @@ -127,7 +127,7 @@ public void GetWorkerConfigs_NullOREmptyProbingPath_ReturnsExpectedConfigs(strin // Act var workerConfigurationResolver = new DynamicWorkerConfigurationResolver(mockConfig.Object, mockLogger.Object, mockEnv.Object, FileUtility.Instance, mockProfileManager.Object, new HashSet() { "java", "node", "powershell" }, probingPaths); - var result = workerConfigurationResolver.GetWorkerConfigs(); + var result = workerConfigurationResolver.GetWorkerConfigPaths(); // Assert Assert.Equal(result.Count, 1); From 710ae357ec0b923aaf6cc8b5e14d55d3d7adc8cd Mon Sep 17 00:00:00 2001 From: Surbhi Gupta Date: Wed, 9 Jul 2025 18:06:30 -0500 Subject: [PATCH 48/55] Log improvement --- .../Rpc/Configuration/DynamicWorkerConfigurationResolver.cs | 4 ++-- .../Workers/Rpc/Configuration/LanguageWorkerOptionsSetup.cs | 2 +- .../Rpc/Configuration/WorkerConfigurationResolverFactory.cs | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/WebJobs.Script/Workers/Rpc/Configuration/DynamicWorkerConfigurationResolver.cs b/src/WebJobs.Script/Workers/Rpc/Configuration/DynamicWorkerConfigurationResolver.cs index 5a9f95fd5c..51a3d34a5f 100644 --- a/src/WebJobs.Script/Workers/Rpc/Configuration/DynamicWorkerConfigurationResolver.cs +++ b/src/WebJobs.Script/Workers/Rpc/Configuration/DynamicWorkerConfigurationResolver.cs @@ -70,13 +70,13 @@ workerRuntime is not null && private void ResolveWorkerConfigsFromProbingPaths(string workerRuntime, Dictionary outputDict) { + _logger.LogDebug("Workers probing paths set to: {probingPaths}", _workerProbingPaths is null ? null : string.Join(", ", _workerProbingPaths)); + if (_workerProbingPaths is null) { return; } - _logger.LogDebug("Workers probing paths set to: {probingPaths}", string.Join(", ", _workerProbingPaths)); - string releaseChannel = EnvironmentExtensions.GetPlatformReleaseChannel(_environment); // probing path directory structure is: /// diff --git a/src/WebJobs.Script/Workers/Rpc/Configuration/LanguageWorkerOptionsSetup.cs b/src/WebJobs.Script/Workers/Rpc/Configuration/LanguageWorkerOptionsSetup.cs index 537d3b40a6..67814c8c66 100644 --- a/src/WebJobs.Script/Workers/Rpc/Configuration/LanguageWorkerOptionsSetup.cs +++ b/src/WebJobs.Script/Workers/Rpc/Configuration/LanguageWorkerOptionsSetup.cs @@ -109,4 +109,4 @@ public void PostConfigure(string name, LanguageWorkerOptions options) logger.LogInformation(message); } } -} \ No newline at end of file +} diff --git a/src/WebJobs.Script/Workers/Rpc/Configuration/WorkerConfigurationResolverFactory.cs b/src/WebJobs.Script/Workers/Rpc/Configuration/WorkerConfigurationResolverFactory.cs index d1ec4f021b..03b593f642 100644 --- a/src/WebJobs.Script/Workers/Rpc/Configuration/WorkerConfigurationResolverFactory.cs +++ b/src/WebJobs.Script/Workers/Rpc/Configuration/WorkerConfigurationResolverFactory.cs @@ -77,7 +77,7 @@ internal List GetWorkerProbingPaths() // Example probing path for Windows: "c:\\home\\SiteExtensions\\workers" string windowsSiteExtensionsPath = GetWindowsSiteExtensionsPath(); - if (string.IsNullOrWhiteSpace(windowsSiteExtensionsPath)) + if (!string.IsNullOrWhiteSpace(windowsSiteExtensionsPath)) { var windowsWorkerFullProbingPath = Path.Combine(windowsSiteExtensionsPath, RpcWorkerConstants.DefaultWorkersDirectoryName); probingPaths.Add(windowsWorkerFullProbingPath); From 90e0ad24de583aad30d8d096c4599c784ac39714 Mon Sep 17 00:00:00 2001 From: Surbhi Gupta Date: Fri, 11 Jul 2025 22:08:54 -0500 Subject: [PATCH 49/55] Condition update to include placeholder mode --- .../Rpc/Configuration/DynamicWorkerConfigurationResolver.cs | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/WebJobs.Script/Workers/Rpc/Configuration/DynamicWorkerConfigurationResolver.cs b/src/WebJobs.Script/Workers/Rpc/Configuration/DynamicWorkerConfigurationResolver.cs index 51a3d34a5f..1d3dc7556f 100644 --- a/src/WebJobs.Script/Workers/Rpc/Configuration/DynamicWorkerConfigurationResolver.cs +++ b/src/WebJobs.Script/Workers/Rpc/Configuration/DynamicWorkerConfigurationResolver.cs @@ -56,6 +56,7 @@ public List GetWorkerConfigPaths() ResolveWorkerConfigsFromProbingPaths(workerRuntime, outputDict); if (!_environment.IsMultiLanguageRuntimeEnvironment() && + !_environment.IsPlaceholderModeEnabled() && workerRuntime is not null && outputDict.ContainsKey(workerRuntime)) { @@ -169,6 +170,7 @@ private void ResolveWorkerConfigsFromWithinHost(string workerRuntime, Dictionary if (outputDict.ContainsKey(workerDir) || (!_environment.IsMultiLanguageRuntimeEnvironment() && + !_environment.IsPlaceholderModeEnabled() && ShouldSkipWorkerDirectory(workerRuntime, workerDir))) { continue; @@ -181,6 +183,7 @@ private void ResolveWorkerConfigsFromWithinHost(string workerRuntime, Dictionary } if (!_environment.IsMultiLanguageRuntimeEnvironment() && + !_environment.IsPlaceholderModeEnabled() && workerRuntime is not null && outputDict.ContainsKey(workerRuntime)) { From fbb3c10143aa9fa5b7c8e416233ce5525974b274 Mon Sep 17 00:00:00 2001 From: Surbhi Gupta Date: Tue, 15 Jul 2025 01:04:46 -0500 Subject: [PATCH 50/55] Test fix --- .../Rpc/Configuration/DynamicWorkerConfigurationResolver.cs | 1 + 1 file changed, 1 insertion(+) diff --git a/src/WebJobs.Script/Workers/Rpc/Configuration/DynamicWorkerConfigurationResolver.cs b/src/WebJobs.Script/Workers/Rpc/Configuration/DynamicWorkerConfigurationResolver.cs index 1d3dc7556f..24e3f4bdbb 100644 --- a/src/WebJobs.Script/Workers/Rpc/Configuration/DynamicWorkerConfigurationResolver.cs +++ b/src/WebJobs.Script/Workers/Rpc/Configuration/DynamicWorkerConfigurationResolver.cs @@ -91,6 +91,7 @@ private void ResolveWorkerConfigsFromProbingPaths(string workerRuntime, Dictiona if (!_fileSystem.Directory.Exists(probingPath)) { _logger.LogDebug("Worker probing path directory does not exist: {probingPath}", probingPath); + continue; } foreach (var workerRuntimePath in _fileSystem.Directory.EnumerateDirectories(probingPath)) From 9f1a9affb0f16570ef3a87d75a2efd972c50c175 Mon Sep 17 00:00:00 2001 From: Surbhi Gupta Date: Mon, 28 Jul 2025 15:23:55 -0500 Subject: [PATCH 51/55] Improve WorkerConfigurationResolver to leverage Options pattern (#11203) --- .../WebHostServiceCollectionExtensions.cs | 18 ++- .../WebJobsScriptHostService.cs | 11 +- .../Environment/EnvironmentExtensions.cs | 10 +- src/WebJobs.Script/ScriptConstants.cs | 1 + .../ScriptHostBuilderExtensions.cs | 8 +- .../DefaultWorkerConfigurationResolver.cs | 17 ++- .../DynamicWorkerConfigurationResolver.cs | 49 +++--- .../LanguageWorkerOptionsSetup.cs | 17 +-- .../Configuration/RpcWorkerConfigFactory.cs | 8 +- .../WorkerConfigurationHelper.cs | 34 +++-- .../WorkerConfigurationResolverFactory.cs | 105 ------------- .../WorkerConfigurationResolverOptions.cs | 26 ++++ ...WorkerConfigurationResolverOptionsSetup.cs | 129 ++++++++++++++++ .../Workers/Rpc/RpcWorkerConstants.cs | 1 + .../TestHostBuilderExtensions.cs | 3 + .../LanguageWorkerOptionsSetupTests.cs | 64 ++++++-- .../Extensions/EnvironmentExtensionsTests.cs | 26 +++- .../WebJobsScriptHostServiceTests.cs | 24 +-- .../WorkerConfigurationResolverTestsHelper.cs | 30 ++++ .../Rpc/RpcWorkerConfigFactoryTests.cs | 142 ++++++++++++++---- .../Workers/Rpc/RpcWorkerConfigTests.cs | 15 +- ...rConfigurationResolverOptionsSetupTests.cs | 63 ++++++++ .../Rpc/WorkerConfigurationResolverTests.cs | 94 +++++++++--- 23 files changed, 652 insertions(+), 243 deletions(-) delete mode 100644 src/WebJobs.Script/Workers/Rpc/Configuration/WorkerConfigurationResolverFactory.cs create mode 100644 src/WebJobs.Script/Workers/Rpc/Configuration/WorkerConfigurationResolverOptions.cs create mode 100644 src/WebJobs.Script/Workers/Rpc/Configuration/WorkerConfigurationResolverOptionsSetup.cs create mode 100644 test/WebJobs.Script.Tests/WorkerConfigurationResolverTestsHelper.cs create mode 100644 test/WebJobs.Script.Tests/Workers/Rpc/WorkerConfigurationResolverOptionsSetupTests.cs diff --git a/src/WebJobs.Script.WebHost/WebHostServiceCollectionExtensions.cs b/src/WebJobs.Script.WebHost/WebHostServiceCollectionExtensions.cs index 0d5eb6d79e..3e00032b3d 100644 --- a/src/WebJobs.Script.WebHost/WebHostServiceCollectionExtensions.cs +++ b/src/WebJobs.Script.WebHost/WebHostServiceCollectionExtensions.cs @@ -29,7 +29,9 @@ using Microsoft.Azure.WebJobs.Script.WebHost.Security.Authorization.Policies; using Microsoft.Azure.WebJobs.Script.WebHost.Standby; using Microsoft.Azure.WebJobs.Script.Workers.FunctionDataCache; +using Microsoft.Azure.WebJobs.Script.Workers.Profiles; using Microsoft.Azure.WebJobs.Script.Workers.Rpc; +using Microsoft.Azure.WebJobs.Script.Workers.Rpc.Configuration; using Microsoft.Azure.WebJobs.Script.Workers.SharedMemoryDataTransfer; using Microsoft.Extensions.Azure; using Microsoft.Extensions.Configuration; @@ -217,6 +219,7 @@ public static void AddWebJobsScriptHost(this IServiceCollection services, IConfi services.ConfigureOptions(); services.AddSingleton, ScriptApplicationHostOptionsChangeTokenSource>(); services.ConfigureOptions(); + services.ConfigureOptions(); services.ConfigureOptions(); services.ConfigureOptionsWithChangeTokenSource>(); services.ConfigureOptionsWithChangeTokenSource>(); @@ -228,9 +231,22 @@ public static void AddWebJobsScriptHost(this IServiceCollection services, IConfi services.AddHostingConfigOptions(configuration); services.ConfigureOptions(); - // Refresh LanguageWorkerOptions when HostBuiltChangeTokenSource is triggered. + // Refresh WorkerConfigurationResolverOptions and LanguageWorkerOptions when HostBuiltChangeTokenSource is triggered. + services.ConfigureOptionsWithChangeTokenSource>(); services.ConfigureOptionsWithChangeTokenSource>(); + services.AddSingleton(p => + { + var environment = p.GetService(); + var workerConfigurationResolverOptions = p.GetService>(); + var workerProfileManager = p.GetService(); + var loggerFactory = p.GetService(); + + return environment.IsDynamicWorkerResolutionEnabled(workerConfigurationResolverOptions) ? + new DynamicWorkerConfigurationResolver(loggerFactory, FileUtility.Instance, workerProfileManager, workerConfigurationResolverOptions) : + new DefaultWorkerConfigurationResolver(loggerFactory, workerConfigurationResolverOptions); + }); + services.TryAddSingleton(); services.TryAddSingleton(s => DefaultMiddlewarePipeline.Empty); diff --git a/src/WebJobs.Script.WebHost/WebJobsScriptHostService.cs b/src/WebJobs.Script.WebHost/WebJobsScriptHostService.cs index 8fb10c55db..71ef8ee2a7 100644 --- a/src/WebJobs.Script.WebHost/WebJobsScriptHostService.cs +++ b/src/WebJobs.Script.WebHost/WebJobsScriptHostService.cs @@ -30,6 +30,7 @@ using Microsoft.Azure.WebJobs.Script.WebHost.Diagnostics.Extensions; using Microsoft.Azure.WebJobs.Script.Workers; using Microsoft.Azure.WebJobs.Script.Workers.Rpc; +using Microsoft.Azure.WebJobs.Script.Workers.Rpc.Configuration; using Microsoft.Extensions.Configuration; using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Hosting; @@ -66,6 +67,7 @@ public class WebJobsScriptHostService : IHostedService, IScriptHostManager, ISer private readonly bool _originalStandbyModeValue; private readonly string _originalFunctionsWorkerRuntime; private readonly string _originalFunctionsWorkerRuntimeVersion; + private readonly IOptionsChangeTokenSource _workerConfigResolverOptionsChangeTokenSource; private readonly IOptionsChangeTokenSource _languageWorkerOptionsChangeTokenSource; // we're only using this dictionary's keys so it acts as a "ConcurrentHashSet" @@ -89,7 +91,8 @@ public WebJobsScriptHostService(IOptionsMonitor ap HostPerformanceManager hostPerformanceManager, IOptions healthMonitorOptions, IMetricsLogger metricsLogger, IApplicationLifetime applicationLifetime, IConfiguration config, IScriptEventManager eventManager, IHostMetrics hostMetrics, IOptions hostingConfigOptions, - IOptionsChangeTokenSource languageWorkerOptionsChangeTokenSource) + IOptionsChangeTokenSource languageWorkerOptionsChangeTokenSource, + IOptionsChangeTokenSource workerConfigResolverOptionsChangeTokenSource) { ArgumentNullException.ThrowIfNull(loggerFactory); @@ -100,6 +103,7 @@ public WebJobsScriptHostService(IOptionsMonitor ap RegisterApplicationLifetimeEvents(); _metricsLogger = metricsLogger; + _workerConfigResolverOptionsChangeTokenSource = workerConfigResolverOptionsChangeTokenSource ?? throw new ArgumentNullException(nameof(workerConfigResolverOptionsChangeTokenSource)); _languageWorkerOptionsChangeTokenSource = languageWorkerOptionsChangeTokenSource ?? throw new ArgumentNullException(nameof(languageWorkerOptionsChangeTokenSource)); _applicationHostOptions = applicationHostOptions ?? throw new ArgumentNullException(nameof(applicationHostOptions)); _scriptWebHostEnvironment = scriptWebHostEnvironment ?? throw new ArgumentNullException(nameof(scriptWebHostEnvironment)); @@ -378,6 +382,11 @@ private async Task UnsynchronizedStartHostAsync(ScriptHostStartupOperation activ } } + if (_workerConfigResolverOptionsChangeTokenSource is HostBuiltChangeTokenSource { } hostBuiltChangeToken) + { + hostBuiltChangeToken.TriggerChange(); + } + if (_languageWorkerOptionsChangeTokenSource is HostBuiltChangeTokenSource { } hostBuiltChangeTokenSource) { hostBuiltChangeTokenSource.TriggerChange(); diff --git a/src/WebJobs.Script/Environment/EnvironmentExtensions.cs b/src/WebJobs.Script/Environment/EnvironmentExtensions.cs index fe9f8c9e2d..da51cdd059 100644 --- a/src/WebJobs.Script/Environment/EnvironmentExtensions.cs +++ b/src/WebJobs.Script/Environment/EnvironmentExtensions.cs @@ -10,6 +10,8 @@ using Microsoft.Azure.WebJobs.Script.Config; using Microsoft.Azure.WebJobs.Script.Description; using Microsoft.Azure.WebJobs.Script.Workers.Rpc; +using Microsoft.Azure.WebJobs.Script.Workers.Rpc.Configuration; +using Microsoft.Extensions.Options; using static Microsoft.Azure.WebJobs.Script.EnvironmentSettingNames; using static Microsoft.Azure.WebJobs.Script.Utility; @@ -714,9 +716,9 @@ public static bool IsWorkerResolutionFeatureDisabled(this IEnvironment environme // Users can disable dynamic worker resolution via setting the appropriate feature flag. // Worker resolution can be enabled for specific workers at the stamp level via hosting config options. // Feature flag takes precedence over hosting config options. - public static bool IsDynamicWorkerResolutionEnabled(this IEnvironment environment, HashSet workersAvailableForResolutionViaHostingConfig) + public static bool IsDynamicWorkerResolutionEnabled(this IEnvironment environment, IOptionsMonitor options) { - if (environment.IsWorkerResolutionFeatureDisabled() || workersAvailableForResolutionViaHostingConfig is null) + if (environment.IsWorkerResolutionFeatureDisabled() || options.CurrentValue.WorkersAvailableForResolution is null) { return false; } @@ -727,10 +729,10 @@ public static bool IsDynamicWorkerResolutionEnabled(this IEnvironment environmen !string.IsNullOrWhiteSpace(workerRuntime) && !environment.IsPlaceholderModeEnabled()) { - return workersAvailableForResolutionViaHostingConfig.Contains(workerRuntime); + return options.CurrentValue.WorkersAvailableForResolution.Contains(workerRuntime); } - return workersAvailableForResolutionViaHostingConfig.Any(); + return options.CurrentValue.WorkersAvailableForResolution.Any(); } public static bool IsApplicationInsightsAgentEnabled(this IEnvironment environment) diff --git a/src/WebJobs.Script/ScriptConstants.cs b/src/WebJobs.Script/ScriptConstants.cs index 9bde523d2b..8223697aa8 100644 --- a/src/WebJobs.Script/ScriptConstants.cs +++ b/src/WebJobs.Script/ScriptConstants.cs @@ -64,6 +64,7 @@ public static class ScriptConstants public const string LogCategoryHost = "Host"; public const string LogCategoryFunction = "Function"; public const string LogCategoryWorker = "Worker"; + public const string LogCategoryWorkerConfig = "Host.LanguageWorkerConfig"; public const string SkipHostJsonConfigurationKey = "MS_SkipHostJsonConfiguration"; public const string SkipHostInitializationKey = "MS_SkipHostInitialization"; diff --git a/src/WebJobs.Script/ScriptHostBuilderExtensions.cs b/src/WebJobs.Script/ScriptHostBuilderExtensions.cs index 61ed8de519..f73d070afa 100644 --- a/src/WebJobs.Script/ScriptHostBuilderExtensions.cs +++ b/src/WebJobs.Script/ScriptHostBuilderExtensions.cs @@ -33,6 +33,7 @@ using Microsoft.Azure.WebJobs.Script.Workers.Http; using Microsoft.Azure.WebJobs.Script.Workers.Profiles; using Microsoft.Azure.WebJobs.Script.Workers.Rpc; +using Microsoft.Azure.WebJobs.Script.Workers.Rpc.Configuration; using Microsoft.Extensions.Configuration; using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.DependencyInjection.Extensions; @@ -338,7 +339,11 @@ public static IHostBuilder AddScriptHostCore(this IHostBuilder builder, ScriptAp if (applicationHostOptions.HasParentScope) { - // Forward the host LanguageWorkerOptions to the Job Host. + // Forward the host WorkerConfigurationResolverOptions and LanguageWorkerOptions to the Job Host. + var workerResolverOptions = applicationHostOptions.RootServiceProvider.GetService>(); + services.AddSingleton(workerResolverOptions); + services.AddSingleton>(s => new OptionsWrapper(workerResolverOptions.CurrentValue)); + var languageWorkerOptions = applicationHostOptions.RootServiceProvider.GetService>(); services.AddSingleton(languageWorkerOptions); services.AddSingleton>(s => new OptionsWrapper(languageWorkerOptions.CurrentValue)); @@ -346,6 +351,7 @@ public static IHostBuilder AddScriptHostCore(this IHostBuilder builder, ScriptAp } else { + services.ConfigureOptions(); services.ConfigureOptions(); AddCommonServices(services); } diff --git a/src/WebJobs.Script/Workers/Rpc/Configuration/DefaultWorkerConfigurationResolver.cs b/src/WebJobs.Script/Workers/Rpc/Configuration/DefaultWorkerConfigurationResolver.cs index 9abf62eb05..61be72fd94 100644 --- a/src/WebJobs.Script/Workers/Rpc/Configuration/DefaultWorkerConfigurationResolver.cs +++ b/src/WebJobs.Script/Workers/Rpc/Configuration/DefaultWorkerConfigurationResolver.cs @@ -4,8 +4,8 @@ using System; using System.Collections.Generic; using System.IO; -using Microsoft.Extensions.Configuration; using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; namespace Microsoft.Azure.WebJobs.Script.Workers.Rpc.Configuration { @@ -13,17 +13,22 @@ namespace Microsoft.Azure.WebJobs.Script.Workers.Rpc.Configuration internal sealed class DefaultWorkerConfigurationResolver : IWorkerConfigurationResolver { private readonly ILogger _logger; - private readonly IConfiguration _configuration; + private readonly IOptionsMonitor _workerConfigurationResolverOptions; - public DefaultWorkerConfigurationResolver(IConfiguration configuration, ILogger logger) + public DefaultWorkerConfigurationResolver(ILoggerFactory loggerFactory, IOptionsMonitor workerConfigurationResolverOptions) { - _logger = logger ?? throw new ArgumentNullException(nameof(logger)); - _configuration = configuration ?? throw new ArgumentNullException(nameof(configuration)); + if (loggerFactory is null) + { + throw new ArgumentNullException(nameof(loggerFactory)); + } + + _logger = _logger = loggerFactory.CreateLogger(ScriptConstants.LogCategoryWorkerConfig); + _workerConfigurationResolverOptions = workerConfigurationResolverOptions ?? throw new ArgumentNullException(nameof(workerConfigurationResolverOptions)); } public List GetWorkerConfigPaths() { - var workersDirPath = WorkerConfigurationHelper.GetWorkersDirPath(_configuration); + var workersDirPath = _workerConfigurationResolverOptions.CurrentValue.WorkersDirPath; _logger.LogDebug("Workers Directory set to: {workersDirPath}", workersDirPath); List workerConfigs = new(); diff --git a/src/WebJobs.Script/Workers/Rpc/Configuration/DynamicWorkerConfigurationResolver.cs b/src/WebJobs.Script/Workers/Rpc/Configuration/DynamicWorkerConfigurationResolver.cs index 24e3f4bdbb..52aeb85bd9 100644 --- a/src/WebJobs.Script/Workers/Rpc/Configuration/DynamicWorkerConfigurationResolver.cs +++ b/src/WebJobs.Script/Workers/Rpc/Configuration/DynamicWorkerConfigurationResolver.cs @@ -10,6 +10,7 @@ using Microsoft.Azure.WebJobs.Script.Workers.Profiles; using Microsoft.Extensions.Configuration; using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; namespace Microsoft.Azure.WebJobs.Script.Workers.Rpc.Configuration { @@ -17,30 +18,30 @@ namespace Microsoft.Azure.WebJobs.Script.Workers.Rpc.Configuration // It searches for worker configs in specified probing paths and the fallback path, and returns a list of worker configuration paths. internal sealed class DynamicWorkerConfigurationResolver : IWorkerConfigurationResolver { - private readonly IConfiguration _config; private readonly ILogger _logger; private readonly IWorkerProfileManager _profileManager; - private readonly IEnvironment _environment; private readonly IFileSystem _fileSystem; private readonly HashSet _workersAvailableForResolutionViaHostingConfig; private readonly List _workerProbingPaths; private readonly JsonSerializerOptions _jsonSerializerOptions = new() { PropertyNameCaseInsensitive = true }; + private readonly IOptionsMonitor _workerConfigurationResolverOptions; - public DynamicWorkerConfigurationResolver(IConfiguration config, - ILogger logger, - IEnvironment environment, + public DynamicWorkerConfigurationResolver(ILoggerFactory loggerFactory, IFileSystem fileSystem, IWorkerProfileManager workerProfileManager, - HashSet workersAvailableForResolutionViaHostingConfig, - List workerProbingPaths) + IOptionsMonitor workerConfigResolverOptions) { - _config = config ?? throw new ArgumentNullException(nameof(config)); - _logger = logger ?? throw new ArgumentNullException(nameof(logger)); - _environment = environment ?? throw new ArgumentNullException(nameof(environment)); + if (loggerFactory is null) + { + throw new ArgumentNullException(nameof(loggerFactory)); + } + + _logger = loggerFactory.CreateLogger(ScriptConstants.LogCategoryWorkerConfig); _fileSystem = fileSystem ?? throw new ArgumentNullException(nameof(fileSystem)); _profileManager = workerProfileManager ?? throw new ArgumentNullException(nameof(workerProfileManager)); - _workersAvailableForResolutionViaHostingConfig = workersAvailableForResolutionViaHostingConfig; - _workerProbingPaths = workerProbingPaths; + _workerConfigurationResolverOptions = workerConfigResolverOptions ?? throw new ArgumentNullException(nameof(workerConfigResolverOptions)); + _workerProbingPaths = workerConfigResolverOptions.CurrentValue.ProbingPaths; + _workersAvailableForResolutionViaHostingConfig = workerConfigResolverOptions.CurrentValue.WorkersAvailableForResolution ?? new HashSet(StringComparer.OrdinalIgnoreCase); } public List GetWorkerConfigPaths() @@ -50,13 +51,13 @@ public List GetWorkerConfigPaths() // Sample path: "///" var outputDict = new Dictionary(StringComparer.OrdinalIgnoreCase); - var workerRuntime = _environment.GetEnvironmentVariable(EnvironmentSettingNames.FunctionWorkerRuntime); + var workerRuntime = _workerConfigurationResolverOptions.CurrentValue.WorkerRuntime; // Search for worker configs in probing paths ResolveWorkerConfigsFromProbingPaths(workerRuntime, outputDict); - if (!_environment.IsMultiLanguageRuntimeEnvironment() && - !_environment.IsPlaceholderModeEnabled() && + if (!_workerConfigurationResolverOptions.CurrentValue.IsMultiLanguageWorkerEnvironment && + !_workerConfigurationResolverOptions.CurrentValue.IsPlaceholderModeEnabled && workerRuntime is not null && outputDict.ContainsKey(workerRuntime)) { @@ -78,7 +79,7 @@ private void ResolveWorkerConfigsFromProbingPaths(string workerRuntime, Dictiona return; } - string releaseChannel = EnvironmentExtensions.GetPlatformReleaseChannel(_environment); + string releaseChannel = _workerConfigurationResolverOptions.CurrentValue.ReleaseChannel; // probing path directory structure is: /// foreach (var probingPath in _workerProbingPaths) @@ -107,8 +108,8 @@ private void ResolveWorkerConfigsFromProbingPaths(string workerRuntime, Dictiona // Skip worker directories that don't match the current runtime or are not enabled via hosting config // Do not load all worker directories after the specialization is done and if it is not a multi-language runtime environment if (!_workersAvailableForResolutionViaHostingConfig.Contains(workerRuntimeDir) || - (!_environment.IsMultiLanguageRuntimeEnvironment() && - !_environment.IsPlaceholderModeEnabled() && + (!_workerConfigurationResolverOptions.CurrentValue.IsMultiLanguageWorkerEnvironment && + !_workerConfigurationResolverOptions.CurrentValue.IsPlaceholderModeEnabled && ShouldSkipWorkerDirectory(workerRuntime, workerRuntimeDir))) { continue; @@ -159,7 +160,7 @@ private void ResolveWorkerConfigsFromVersionsDirs(string languageWorkerPath, str private void ResolveWorkerConfigsFromWithinHost(string workerRuntime, Dictionary outputDict) { - var fallbackPath = WorkerConfigurationHelper.GetWorkersDirPath(_config); + var fallbackPath = _workerConfigurationResolverOptions.CurrentValue.WorkersDirPath; _logger.LogDebug("Searching for worker configs in the fallback directory: {fallbackPath}", fallbackPath); @@ -170,8 +171,8 @@ private void ResolveWorkerConfigsFromWithinHost(string workerRuntime, Dictionary string workerDir = Path.GetFileName(workerPath).ToLower(); if (outputDict.ContainsKey(workerDir) || - (!_environment.IsMultiLanguageRuntimeEnvironment() && - !_environment.IsPlaceholderModeEnabled() && + (!_workerConfigurationResolverOptions.CurrentValue.IsMultiLanguageWorkerEnvironment && + !_workerConfigurationResolverOptions.CurrentValue.IsPlaceholderModeEnabled && ShouldSkipWorkerDirectory(workerRuntime, workerDir))) { continue; @@ -183,8 +184,8 @@ private void ResolveWorkerConfigsFromWithinHost(string workerRuntime, Dictionary outputDict[workerDir] = workerPath; } - if (!_environment.IsMultiLanguageRuntimeEnvironment() && - !_environment.IsPlaceholderModeEnabled() && + if (!_workerConfigurationResolverOptions.CurrentValue.IsMultiLanguageWorkerEnvironment && + !_workerConfigurationResolverOptions.CurrentValue.IsPlaceholderModeEnabled && workerRuntime is not null && outputDict.ContainsKey(workerRuntime)) { @@ -252,7 +253,7 @@ private bool IsWorkerCompatibleWithHost(string workerDir) jsonSerializerOptions: _jsonSerializerOptions, workerDir: workerDir, profileManager: _profileManager, - config: _config, + languageWorkersSettings: _workerConfigurationResolverOptions.CurrentValue.LanguageWorkersSettings, logger: _logger); if (workerDescription.IsDisabled == true) diff --git a/src/WebJobs.Script/Workers/Rpc/Configuration/LanguageWorkerOptionsSetup.cs b/src/WebJobs.Script/Workers/Rpc/Configuration/LanguageWorkerOptionsSetup.cs index 67814c8c66..ffb39a70c5 100644 --- a/src/WebJobs.Script/Workers/Rpc/Configuration/LanguageWorkerOptionsSetup.cs +++ b/src/WebJobs.Script/Workers/Rpc/Configuration/LanguageWorkerOptionsSetup.cs @@ -23,7 +23,8 @@ internal class LanguageWorkerOptionsSetup : IConfigureOptions _functionsHostingConfigOptions; + private readonly IOptionsMonitor _workerConfigResolverOptions; + private readonly IWorkerConfigurationResolver _workerConfigurationResolver; public LanguageWorkerOptionsSetup(IConfiguration configuration, ILoggerFactory loggerFactory, @@ -31,7 +32,8 @@ public LanguageWorkerOptionsSetup(IConfiguration configuration, IMetricsLogger metricsLogger, IWorkerProfileManager workerProfileManager, IScriptHostManager scriptHostManager, - IOptions functionsHostingConfigOptions) + IOptionsMonitor workerConfigResolverOptions, + IWorkerConfigurationResolver workerConfigurationResolver) { if (loggerFactory is null) { @@ -43,9 +45,10 @@ public LanguageWorkerOptionsSetup(IConfiguration configuration, _environment = environment ?? throw new ArgumentNullException(nameof(environment)); _metricsLogger = metricsLogger ?? throw new ArgumentNullException(nameof(metricsLogger)); _workerProfileManager = workerProfileManager ?? throw new ArgumentNullException(nameof(workerProfileManager)); - _functionsHostingConfigOptions = functionsHostingConfigOptions ?? throw new ArgumentNullException(nameof(functionsHostingConfigOptions)); + _workerConfigResolverOptions = workerConfigResolverOptions ?? throw new ArgumentNullException(nameof(workerConfigResolverOptions)); + _workerConfigurationResolver = workerConfigurationResolver ?? throw new ArgumentNullException(nameof(workerConfigurationResolver)); - _logger = loggerFactory.CreateLogger("Host.LanguageWorkerConfig"); + _logger = loggerFactory.CreateLogger(ScriptConstants.LogCategoryWorkerConfig); } public void Configure(LanguageWorkerOptions options) @@ -77,11 +80,7 @@ public void Configure(LanguageWorkerOptions options) } } - var resolverFactory = new WorkerConfigurationResolverFactory(configuration, _logger, _environment, _workerProfileManager, _functionsHostingConfigOptions); - - IWorkerConfigurationResolver workerConfigurationResolver = resolverFactory.CreateResolver(); - - var configFactory = new RpcWorkerConfigFactory(configuration, _logger, SystemRuntimeInformation.Instance, _environment, _metricsLogger, _workerProfileManager, workerConfigurationResolver); + var configFactory = new RpcWorkerConfigFactory(configuration, _logger, SystemRuntimeInformation.Instance, _environment, _metricsLogger, _workerProfileManager, _workerConfigurationResolver, _workerConfigResolverOptions); options.WorkerConfigs = configFactory.GetConfigs(); } } diff --git a/src/WebJobs.Script/Workers/Rpc/Configuration/RpcWorkerConfigFactory.cs b/src/WebJobs.Script/Workers/Rpc/Configuration/RpcWorkerConfigFactory.cs index 3f4893971b..aa2b9a197e 100644 --- a/src/WebJobs.Script/Workers/Rpc/Configuration/RpcWorkerConfigFactory.cs +++ b/src/WebJobs.Script/Workers/Rpc/Configuration/RpcWorkerConfigFactory.cs @@ -13,6 +13,7 @@ using Microsoft.Azure.WebJobs.Script.Workers.Rpc.Configuration; using Microsoft.Extensions.Configuration; using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; namespace Microsoft.Azure.WebJobs.Script.Workers.Rpc { @@ -27,6 +28,7 @@ internal class RpcWorkerConfigFactory private readonly string _workerRuntime; private readonly IEnvironment _environment; private readonly IWorkerConfigurationResolver _workerConfigurationResolver; + private readonly IOptionsMonitor _workerConfigurationResolverOptions; private readonly JsonSerializerOptions _jsonSerializerOptions = new() { PropertyNameCaseInsensitive = true @@ -40,7 +42,8 @@ public RpcWorkerConfigFactory(IConfiguration config, IEnvironment environment, IMetricsLogger metricsLogger, IWorkerProfileManager workerProfileManager, - IWorkerConfigurationResolver workerConfigurationResolver) + IWorkerConfigurationResolver workerConfigurationResolver, + IOptionsMonitor workerConfigurationResolverOptions) { _config = config ?? throw new ArgumentNullException(nameof(config)); _logger = logger ?? throw new ArgumentNullException(nameof(logger)); @@ -50,6 +53,7 @@ public RpcWorkerConfigFactory(IConfiguration config, _profileManager = workerProfileManager ?? throw new ArgumentNullException(nameof(workerProfileManager)); _workerRuntime = _environment.GetEnvironmentVariable(RpcWorkerConstants.FunctionWorkerRuntimeSettingName); _workerConfigurationResolver = workerConfigurationResolver ?? throw new ArgumentNullException(nameof(workerConfigurationResolver)); + _workerConfigurationResolverOptions = workerConfigurationResolverOptions ?? throw new ArgumentNullException(nameof(workerConfigurationResolverOptions)); WorkersDirPath = WorkerConfigurationHelper.GetWorkersDirPath(config); } @@ -125,7 +129,7 @@ internal void AddProvider(string workerDir) var workerConfig = WorkerConfigurationHelper.GetWorkerConfigJsonElement(workerConfigPath); - RpcWorkerDescription workerDescription = WorkerConfigurationHelper.GetWorkerDescription(workerConfig, _jsonSerializerOptions, workerDir, _profileManager, _config, _logger); + RpcWorkerDescription workerDescription = WorkerConfigurationHelper.GetWorkerDescription(workerConfig, _jsonSerializerOptions, workerDir, _profileManager, _workerConfigurationResolverOptions.CurrentValue.LanguageWorkersSettings, _logger); if (workerDescription.IsDisabled == true) { diff --git a/src/WebJobs.Script/Workers/Rpc/Configuration/WorkerConfigurationHelper.cs b/src/WebJobs.Script/Workers/Rpc/Configuration/WorkerConfigurationHelper.cs index baf4e1a3a3..c2dfa623a3 100644 --- a/src/WebJobs.Script/Workers/Rpc/Configuration/WorkerConfigurationHelper.cs +++ b/src/WebJobs.Script/Workers/Rpc/Configuration/WorkerConfigurationHelper.cs @@ -19,7 +19,7 @@ internal static RpcWorkerDescription GetWorkerDescription( JsonSerializerOptions jsonSerializerOptions, string workerDir, IWorkerProfileManager profileManager, - IConfiguration config, + Dictionary languageWorkersSettings, ILogger logger) { var workerDescriptionElement = workerConfig.GetProperty(WorkerConstants.WorkerDescription); @@ -37,11 +37,14 @@ internal static RpcWorkerDescription GetWorkerDescription( } } - // Check if any app settings are provided for that language - var languageSection = config.GetSection($"{RpcWorkerConstants.LanguageWorkersSectionName}:{workerDescription.Language}"); workerDescription.Arguments ??= new List(); - GetWorkerDescriptionFromAppSettings(workerDescription, languageSection); - AddArgumentsFromAppSettings(workerDescription, languageSection); + + if (languageWorkersSettings is not null) + { + // Check if any app settings are provided for that language + GetWorkerDescriptionFromAppSettings(workerDescription, languageWorkersSettings); + AddArgumentsFromAppSettings(workerDescription, languageWorkersSettings); + } // Validate workerDescription workerDescription.ApplyDefaultsAndValidate(Directory.GetCurrentDirectory(), logger); @@ -112,21 +115,24 @@ private static List ReadWorkerDescriptionProfiles(Json return descriptionProfiles; } - private static void GetWorkerDescriptionFromAppSettings(RpcWorkerDescription workerDescription, IConfigurationSection languageSection) + private static void GetWorkerDescriptionFromAppSettings(RpcWorkerDescription workerDescription, Dictionary languageWorkersSettings) { - var defaultExecutablePathSetting = languageSection?.GetSection($"{WorkerConstants.WorkerDescriptionDefaultExecutablePath}"); - workerDescription.DefaultExecutablePath = defaultExecutablePathSetting?.Value != null ? defaultExecutablePathSetting.Value : workerDescription.DefaultExecutablePath; + if (languageWorkersSettings.TryGetValue($"{RpcWorkerConstants.LanguageWorkersSectionName}:{workerDescription.Language}:{WorkerConstants.WorkerDescriptionDefaultExecutablePath}", out string defaultExecutablePathSetting)) + { + workerDescription.DefaultExecutablePath = defaultExecutablePathSetting; + } - var defaultRuntimeVersionAppSetting = languageSection?.GetSection($"{WorkerConstants.WorkerDescriptionDefaultRuntimeVersion}"); - workerDescription.DefaultRuntimeVersion = defaultRuntimeVersionAppSetting?.Value != null ? defaultRuntimeVersionAppSetting.Value : workerDescription.DefaultRuntimeVersion; + if (languageWorkersSettings.TryGetValue($"{RpcWorkerConstants.LanguageWorkersSectionName}:{workerDescription.Language}:{WorkerConstants.WorkerDescriptionDefaultRuntimeVersion}", out string defaultRuntimeVersionAppSetting)) + { + workerDescription.DefaultRuntimeVersion = defaultRuntimeVersionAppSetting; + } } - internal static void AddArgumentsFromAppSettings(RpcWorkerDescription workerDescription, IConfigurationSection languageSection) + internal static void AddArgumentsFromAppSettings(RpcWorkerDescription workerDescription, Dictionary languageWorkersSettings) { - var argumentsSection = languageSection?.GetSection($"{WorkerConstants.WorkerDescriptionArguments}"); - if (argumentsSection?.Value != null) + if (languageWorkersSettings.TryGetValue($"{RpcWorkerConstants.LanguageWorkersSectionName}:{workerDescription.Language}:{WorkerConstants.WorkerDescriptionArguments}", out string argumentsValue)) { - ((List)workerDescription.Arguments).AddRange(Regex.Split(argumentsSection.Value, @"\s+")); + ((List)workerDescription.Arguments).AddRange(Regex.Split(argumentsValue, @"\s+")); } } diff --git a/src/WebJobs.Script/Workers/Rpc/Configuration/WorkerConfigurationResolverFactory.cs b/src/WebJobs.Script/Workers/Rpc/Configuration/WorkerConfigurationResolverFactory.cs deleted file mode 100644 index 03b593f642..0000000000 --- a/src/WebJobs.Script/Workers/Rpc/Configuration/WorkerConfigurationResolverFactory.cs +++ /dev/null @@ -1,105 +0,0 @@ -// Copyright (c) .NET Foundation. All rights reserved. -// Licensed under the MIT License. See License.txt in the project root for license information. - -using System; -using System.Collections.Generic; -using System.IO; -using System.Linq; -using System.Reflection; -using Microsoft.Azure.WebJobs.Script.Config; -using Microsoft.Azure.WebJobs.Script.Workers.Profiles; -using Microsoft.Extensions.Configuration; -using Microsoft.Extensions.Logging; -using Microsoft.Extensions.Options; - -namespace Microsoft.Azure.WebJobs.Script.Workers.Rpc.Configuration -{ - /// - /// Factory for creating worker configuration resolvers depending on if dynamic worker resolution is enabled or not. - /// - internal sealed class WorkerConfigurationResolverFactory : IWorkerConfigurationResolverFactory - { - private readonly IConfiguration _configuration; - private readonly ILogger _logger; - private readonly IEnvironment _environment; - private readonly IWorkerProfileManager _workerProfileManager; - private readonly IOptions _functionsHostingConfigOptions; - - public WorkerConfigurationResolverFactory( - IConfiguration configuration, - ILogger logger, IEnvironment environment, - IWorkerProfileManager workerProfileManager, - IOptions functionsHostingConfigOptions) - { - _configuration = configuration ?? throw new ArgumentNullException(nameof(configuration)); - _logger = logger ?? throw new ArgumentNullException(nameof(logger)); - _environment = environment ?? throw new ArgumentNullException(nameof(environment)); - _workerProfileManager = workerProfileManager ?? throw new ArgumentNullException(nameof(workerProfileManager)); - _functionsHostingConfigOptions = functionsHostingConfigOptions ?? throw new ArgumentNullException(nameof(functionsHostingConfigOptions)); - } - - public IWorkerConfigurationResolver CreateResolver() - { - HashSet workersAvailableForResolution = GetWorkersAvailableForResolutionViaHostingConfig(_functionsHostingConfigOptions); - List probingPaths = GetWorkerProbingPaths(); - bool dynamicWorkerResolutionEnabled = _environment.IsDynamicWorkerResolutionEnabled(workersAvailableForResolution); - - if (dynamicWorkerResolutionEnabled) - { - return new DynamicWorkerConfigurationResolver(_configuration, - _logger, - _environment, - FileUtility.Instance, - _workerProfileManager, - workersAvailableForResolution, - probingPaths); - } - - return new DefaultWorkerConfigurationResolver(_configuration, _logger); - } - - internal List GetWorkerProbingPaths() - { - var probingPaths = new List(); - - // If Environment variable is set, read probing paths from Environment - string probingPathsEnvValue = _environment.GetEnvironmentVariableOrDefault(EnvironmentSettingNames.WorkerProbingPaths, null); - - if (!string.IsNullOrEmpty(probingPathsEnvValue)) - { - probingPaths = probingPathsEnvValue.Split(';', StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries).ToList(); - } - else - { - if (_environment.IsWindowsEnvironment()) - { - // Harcoded site extensions path for Windows until Antares sets it as an Environment variable. - // Example probing path for Windows: "c:\\home\\SiteExtensions\\workers" - string windowsSiteExtensionsPath = GetWindowsSiteExtensionsPath(); - - if (!string.IsNullOrWhiteSpace(windowsSiteExtensionsPath)) - { - var windowsWorkerFullProbingPath = Path.Combine(windowsSiteExtensionsPath, RpcWorkerConstants.DefaultWorkersDirectoryName); - probingPaths.Add(windowsWorkerFullProbingPath); - } - } - } - - return probingPaths; - } - - internal static HashSet GetWorkersAvailableForResolutionViaHostingConfig(IOptions functionsHostingConfigOptions) => - (functionsHostingConfigOptions.Value?.WorkersAvailableForDynamicResolution ?? string.Empty) - .Split('|', StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries) - .ToHashSet(StringComparer.OrdinalIgnoreCase); - - internal static string GetWindowsSiteExtensionsPath() - { - var assemblyPath = Assembly.GetExecutingAssembly().Location; - var assemblyDir = Path.GetDirectoryName(assemblyPath); - - //Move 2 directories up to get to the SiteExtensions directory - return Directory.GetParent(assemblyDir)?.Parent?.FullName; - } - } -} \ No newline at end of file diff --git a/src/WebJobs.Script/Workers/Rpc/Configuration/WorkerConfigurationResolverOptions.cs b/src/WebJobs.Script/Workers/Rpc/Configuration/WorkerConfigurationResolverOptions.cs new file mode 100644 index 0000000000..5dd233f9d4 --- /dev/null +++ b/src/WebJobs.Script/Workers/Rpc/Configuration/WorkerConfigurationResolverOptions.cs @@ -0,0 +1,26 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the MIT License. See License.txt in the project root for license information. + +using System.Collections.Generic; + +namespace Microsoft.Azure.WebJobs.Script.Workers.Rpc.Configuration +{ + public class WorkerConfigurationResolverOptions + { + public string ReleaseChannel { get; set; } + + public string WorkerRuntime { get; set; } + + public bool IsPlaceholderModeEnabled { get; set; } + + public bool IsMultiLanguageWorkerEnvironment { get; set; } + + public string WorkersDirPath { get; set; } + + public List ProbingPaths { get; set; } + + public HashSet WorkersAvailableForResolution { get; set; } + + public Dictionary LanguageWorkersSettings { get; set; } + } +} \ No newline at end of file diff --git a/src/WebJobs.Script/Workers/Rpc/Configuration/WorkerConfigurationResolverOptionsSetup.cs b/src/WebJobs.Script/Workers/Rpc/Configuration/WorkerConfigurationResolverOptionsSetup.cs new file mode 100644 index 0000000000..1536e0861f --- /dev/null +++ b/src/WebJobs.Script/Workers/Rpc/Configuration/WorkerConfigurationResolverOptionsSetup.cs @@ -0,0 +1,129 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the MIT License. See License.txt in the project root for license information. + +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Reflection; +using Microsoft.Azure.WebJobs.Script.Config; +using Microsoft.Extensions.Configuration; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Options; + +namespace Microsoft.Azure.WebJobs.Script.Workers.Rpc.Configuration +{ + internal sealed class WorkerConfigurationResolverOptionsSetup : IConfigureOptions + { + private readonly IConfiguration _configuration; + private readonly IEnvironment _environment; + private readonly IScriptHostManager _scriptHostManager; + private readonly IOptions _functionsHostingConfigOptions; + + public WorkerConfigurationResolverOptionsSetup(IConfiguration configuration, + IEnvironment environment, + IScriptHostManager scriptHostManager, + IOptions functionsHostingConfigOptions) + { + _configuration = configuration ?? throw new ArgumentNullException(nameof(configuration)); + _environment = environment ?? throw new ArgumentNullException(nameof(environment)); + _scriptHostManager = scriptHostManager ?? throw new ArgumentNullException(nameof(scriptHostManager)); + _functionsHostingConfigOptions = functionsHostingConfigOptions ?? throw new ArgumentNullException(nameof(functionsHostingConfigOptions)); + } + + public void Configure(WorkerConfigurationResolverOptions options) + { + var configuration = _configuration; + if (_scriptHostManager is IServiceProvider scriptHostManagerServiceProvider) + { + var latestConfiguration = scriptHostManagerServiceProvider.GetService(); + if (latestConfiguration is not null) + { + configuration = new ConfigurationBuilder() + .AddConfiguration(_configuration) + .AddConfiguration(latestConfiguration) + .Build(); + } + } + + options.WorkerRuntime = _environment.GetEnvironmentVariable(EnvironmentSettingNames.FunctionWorkerRuntime); + options.ReleaseChannel = EnvironmentExtensions.GetPlatformReleaseChannel(_environment); + options.IsPlaceholderModeEnabled = _environment.IsPlaceholderModeEnabled(); + options.IsMultiLanguageWorkerEnvironment = _environment.IsMultiLanguageRuntimeEnvironment(); + options.WorkersDirPath = WorkerConfigurationHelper.GetWorkersDirPath(configuration); + options.ProbingPaths = GetWorkerProbingPaths(); + options.WorkersAvailableForResolution = GetWorkersAvailableForResolutionViaHostingConfig(_functionsHostingConfigOptions); + options.LanguageWorkersSettings = GetLanguageWorkersSettings(configuration); + } + + internal Dictionary GetLanguageWorkersSettings(IConfiguration configuration) + { + // Convert the required configuration sections to Dictionary + var languageWorkersSettings = new Dictionary(); + + foreach (var kvp in configuration.AsEnumerable()) + { + if (kvp.Key.StartsWith(RpcWorkerConstants.LanguageWorkersSectionName)) + { + languageWorkersSettings[kvp.Key] = kvp.Value; + } + } + + return languageWorkersSettings; + } + + internal List GetWorkerProbingPaths() + { + // If Configuration section is set, read probing paths from configuration. + IConfigurationSection probingPathsSection = _configuration.GetSection($"{RpcWorkerConstants.LanguageWorkersSectionName}") + ?.GetSection($"{RpcWorkerConstants.WorkerProbingPathsSectionName}"); + + var probingPathsList = probingPathsSection?.AsEnumerable(); + + List probingPaths = new List(); + + if (probingPathsList is not null) + { + for (int i = 0; i < probingPathsList.Count(); i++) + { + var path = probingPathsSection.GetSection($"{i}").Value; + if (!string.IsNullOrWhiteSpace(path)) + { + probingPaths.Add(path); + } + } + } + else + { + if (_environment.IsWindowsEnvironment()) + { + // Harcoded site extensions path for Windows until Antares sets it as an Environment variable. + // Example probing path for Windows: "c:\\home\\SiteExtensions\\workers" + string windowsSiteExtensionsPath = GetWindowsSiteExtensionsPath(); + + if (!string.IsNullOrWhiteSpace(windowsSiteExtensionsPath)) + { + var windowsWorkerFullProbingPath = Path.Combine(windowsSiteExtensionsPath, RpcWorkerConstants.DefaultWorkersDirectoryName); + probingPaths.Add(windowsWorkerFullProbingPath); + } + } + } + + return probingPaths; + } + + internal static HashSet GetWorkersAvailableForResolutionViaHostingConfig(IOptions functionsHostingConfigOptions) => + (functionsHostingConfigOptions.Value?.WorkersAvailableForDynamicResolution ?? string.Empty) + .Split('|', StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries) + .ToHashSet(StringComparer.OrdinalIgnoreCase); + + internal static string GetWindowsSiteExtensionsPath() + { + var assemblyPath = Assembly.GetExecutingAssembly().Location; + var assemblyDir = Path.GetDirectoryName(assemblyPath); + + //Move 2 directories up to get to the SiteExtensions directory + return Directory.GetParent(assemblyDir)?.Parent?.FullName; + } + } +} \ No newline at end of file diff --git a/src/WebJobs.Script/Workers/Rpc/RpcWorkerConstants.cs b/src/WebJobs.Script/Workers/Rpc/RpcWorkerConstants.cs index e192944241..3976c7f54c 100644 --- a/src/WebJobs.Script/Workers/Rpc/RpcWorkerConstants.cs +++ b/src/WebJobs.Script/Workers/Rpc/RpcWorkerConstants.cs @@ -26,6 +26,7 @@ public static class RpcWorkerConstants // Section names in host.json or AppSettings public const string LanguageWorkersSectionName = "languageWorkers"; + public const string WorkerProbingPathsSectionName = "probingPaths"; public const string HostRequirementsSectionName = "hostRequirements"; // Worker description constants diff --git a/test/WebJobs.Script.Tests.Shared/TestHostBuilderExtensions.cs b/test/WebJobs.Script.Tests.Shared/TestHostBuilderExtensions.cs index 9dc13e2eec..fc0268db67 100644 --- a/test/WebJobs.Script.Tests.Shared/TestHostBuilderExtensions.cs +++ b/test/WebJobs.Script.Tests.Shared/TestHostBuilderExtensions.cs @@ -20,6 +20,7 @@ using Microsoft.Azure.WebJobs.Script.Workers; using Microsoft.Azure.WebJobs.Script.Workers.Profiles; using Microsoft.Azure.WebJobs.Script.Workers.Rpc; +using Microsoft.Azure.WebJobs.Script.Workers.Rpc.Configuration; using Microsoft.Azure.WebJobs.Script.Workers.SharedMemoryDataTransfer; using Microsoft.Extensions.Configuration; using Microsoft.Extensions.DependencyInjection; @@ -83,6 +84,8 @@ public static IHostBuilder ConfigureDefaultTestWebScriptHost(this IHostBuilder b services.AddFunctionMetadataManager(); services.AddHostMetrics(); services.AddConfiguration(); + services.ConfigureOptions(); + services.AddSingleton(); services.ConfigureOptions(); configureRootServices?.Invoke(services); diff --git a/test/WebJobs.Script.Tests/Configuration/LanguageWorkerOptionsSetupTests.cs b/test/WebJobs.Script.Tests/Configuration/LanguageWorkerOptionsSetupTests.cs index 15703b5f72..f99e64eb5e 100644 --- a/test/WebJobs.Script.Tests/Configuration/LanguageWorkerOptionsSetupTests.cs +++ b/test/WebJobs.Script.Tests/Configuration/LanguageWorkerOptionsSetupTests.cs @@ -2,14 +2,17 @@ // Licensed under the MIT License. See License.txt in the project root for license information. using System; +using System.Collections.Generic; using System.IO; using System.Linq; +using System.Text; +using System.Text.Json; using Microsoft.Azure.WebJobs.Script.Config; using Microsoft.Azure.WebJobs.Script.Workers.Profiles; using Microsoft.Azure.WebJobs.Script.Workers.Rpc; +using Microsoft.Azure.WebJobs.Script.Workers.Rpc.Configuration; using Microsoft.Extensions.Configuration; using Microsoft.Extensions.Logging; -using Microsoft.Extensions.Logging.Abstractions; using Microsoft.Extensions.Options; using Microsoft.WebJobs.Script.Tests; using Moq; @@ -29,6 +32,9 @@ public class LanguageWorkerOptionsSetupTests [InlineData("node")] public void LanguageWorkerOptions_Expected_ListOfConfigs(string workerRuntime) { + var loggerProvider = new TestLoggerProvider(); + var loggerFactory = new LoggerFactory(); + loggerFactory.AddProvider(loggerProvider); var testEnvironment = new TestEnvironment(); var testMetricLogger = new TestMetricsLogger(); var configurationBuilder = new ConfigurationBuilder() @@ -62,9 +68,13 @@ public void LanguageWorkerOptions_Expected_ListOfConfigs(string workerRuntime) } }); - LanguageWorkerOptionsSetup setup = new LanguageWorkerOptionsSetup(configuration, NullLoggerFactory.Instance, testEnvironment, testMetricLogger, testProfileManager.Object, testScriptHostManager.Object, new OptionsWrapper(new FunctionsHostingConfigOptions())); - LanguageWorkerOptions options = new LanguageWorkerOptions(); + var hostingOptions = new FunctionsHostingConfigOptions(); + var optionsMonitor = WorkerConfigurationResolverTestsHelper.GetTestWorkerConfigurationResolverOptions(configuration, testEnvironment, testScriptHostManager.Object, new OptionsWrapper(hostingOptions)); + + var resolver = new DefaultWorkerConfigurationResolver(loggerFactory, optionsMonitor); + LanguageWorkerOptionsSetup setup = new LanguageWorkerOptionsSetup(configuration, loggerFactory, testEnvironment, testMetricLogger, testProfileManager.Object, testScriptHostManager.Object, optionsMonitor, resolver); + LanguageWorkerOptions options = new LanguageWorkerOptions(); setup.Configure(options); if (string.IsNullOrEmpty(workerRuntime)) @@ -96,21 +106,39 @@ public void LanguageWorkerOptions_EnabledWorkerResolution_Expected_ListOfConfigs var testEnvironment = new TestEnvironment(); var testMetricLogger = new TestMetricsLogger(); - var configurationBuilder = new ConfigurationBuilder() - .Add(new ScriptEnvironmentVariablesConfigurationSource()); - var configuration = configurationBuilder.Build(); var testProfileManager = new Mock(); var testScriptHostManager = new Mock(); string probingPathValue = string.Join(';', _probingPath1, string.Empty, "path-not-exists"); testEnvironment.SetEnvironmentVariable(RpcWorkerConstants.FunctionWorkerRuntimeSettingName, workerRuntime); testEnvironment.SetEnvironmentVariable(EnvironmentSettingNames.AntaresPlatformReleaseChannel, releaseChannel); - testEnvironment.SetEnvironmentVariable(EnvironmentSettingNames.WorkerProbingPaths, probingPathValue); + + var probingPaths = new List() { _probingPath1, string.Empty, "path-not-exists" }; + + var jsonObj = new + { + languageWorkers = new + { + probingPaths + } + }; + + var jsonString = JsonSerializer.Serialize(jsonObj, new JsonSerializerOptions { WriteIndented = true }); + var jsonStream = new MemoryStream(Encoding.UTF8.GetBytes(jsonString)); + + var configurationBuilder = new ConfigurationBuilder() + .Add(new ScriptEnvironmentVariablesConfigurationSource()) + .AddJsonStream(jsonStream); + + var configuration = configurationBuilder.Build(); var hostingOptions = new FunctionsHostingConfigOptions(); hostingOptions.Features.Add(RpcWorkerConstants.WorkersAvailableForDynamicResolution, hostingOptionsSetting); + var optionsMonitor = WorkerConfigurationResolverTestsHelper.GetTestWorkerConfigurationResolverOptions(configuration, testEnvironment, testScriptHostManager.Object, new OptionsWrapper(hostingOptions)); + + var resolver = new DynamicWorkerConfigurationResolver(loggerFactory, FileUtility.Instance, testProfileManager.Object, optionsMonitor); - LanguageWorkerOptionsSetup setup = new LanguageWorkerOptionsSetup(configuration, loggerFactory, testEnvironment, testMetricLogger, testProfileManager.Object, testScriptHostManager.Object, new OptionsWrapper(hostingOptions)); + LanguageWorkerOptionsSetup setup = new LanguageWorkerOptionsSetup(configuration, loggerFactory, testEnvironment, testMetricLogger, testProfileManager.Object, testScriptHostManager.Object, optionsMonitor, resolver); LanguageWorkerOptions options = new LanguageWorkerOptions(); setup.Configure(options); @@ -152,8 +180,11 @@ public void LanguageWorkerOptions_FallbackPath_Expected_ListOfConfigs(string wor var hostingOptions = new FunctionsHostingConfigOptions(); hostingOptions.Features.Add(RpcWorkerConstants.WorkersAvailableForDynamicResolution, hostingOptionsSetting); + var optionsMonitor = WorkerConfigurationResolverTestsHelper.GetTestWorkerConfigurationResolverOptions(configuration, testEnvironment, testScriptHostManager.Object, new OptionsWrapper(hostingOptions)); + + var resolver = new DynamicWorkerConfigurationResolver(loggerFactory, FileUtility.Instance, testProfileManager.Object, optionsMonitor); - LanguageWorkerOptionsSetup setup = new LanguageWorkerOptionsSetup(configuration, loggerFactory, testEnvironment, testMetricLogger, testProfileManager.Object, testScriptHostManager.Object, new OptionsWrapper(hostingOptions)); + LanguageWorkerOptionsSetup setup = new LanguageWorkerOptionsSetup(configuration, loggerFactory, testEnvironment, testMetricLogger, testProfileManager.Object, testScriptHostManager.Object, optionsMonitor, resolver); LanguageWorkerOptions options = new LanguageWorkerOptions(); setup.Configure(options); @@ -194,8 +225,11 @@ public void LanguageWorkerOptions_NullHostingConfig_FeatureDisabled_ListOfConfig var hostingOptions = new FunctionsHostingConfigOptions(); hostingOptions.Features.Add(RpcWorkerConstants.WorkersAvailableForDynamicResolution, hostingOptionsSetting); + var optionsMonitor = WorkerConfigurationResolverTestsHelper.GetTestWorkerConfigurationResolverOptions(configuration, testEnvironment, testScriptHostManager.Object, new OptionsWrapper(hostingOptions)); - LanguageWorkerOptionsSetup setup = new LanguageWorkerOptionsSetup(configuration, loggerFactory, testEnvironment, testMetricLogger, testProfileManager.Object, testScriptHostManager.Object, new OptionsWrapper(hostingOptions)); + var resolver = new DefaultWorkerConfigurationResolver(loggerFactory, optionsMonitor); + + LanguageWorkerOptionsSetup setup = new LanguageWorkerOptionsSetup(configuration, loggerFactory, testEnvironment, testMetricLogger, testProfileManager.Object, testScriptHostManager.Object, optionsMonitor, resolver); LanguageWorkerOptions options = new LanguageWorkerOptions(); setup.Configure(options); @@ -235,8 +269,11 @@ public void LanguageWorkerOptions_DisabledWorkerResolution_Expected_ListOfConfig testEnvironment.SetEnvironmentVariable(EnvironmentSettingNames.WorkerProbingPaths, probingPathValue); var hostingOptions = new FunctionsHostingConfigOptions(); + var optionsMonitor = WorkerConfigurationResolverTestsHelper.GetTestWorkerConfigurationResolverOptions(configuration, testEnvironment, testScriptHostManager.Object, new OptionsWrapper(hostingOptions)); + + var resolver = new DefaultWorkerConfigurationResolver(loggerFactory, optionsMonitor); - LanguageWorkerOptionsSetup setup = new LanguageWorkerOptionsSetup(configuration, loggerFactory, testEnvironment, testMetricLogger, testProfileManager.Object, testScriptHostManager.Object, new OptionsWrapper(hostingOptions)); + LanguageWorkerOptionsSetup setup = new LanguageWorkerOptionsSetup(configuration, loggerFactory, testEnvironment, testMetricLogger, testProfileManager.Object, testScriptHostManager.Object, optionsMonitor, resolver); LanguageWorkerOptions options = new LanguageWorkerOptions(); setup.Configure(options); @@ -274,8 +311,11 @@ public void LanguageWorkerOptions_FallbackPath_NullHostingConfig(string workerRu var hostingOptions = new FunctionsHostingConfigOptions(); hostingOptions.Features.Add(RpcWorkerConstants.WorkersAvailableForDynamicResolution, workerRuntime); + var optionsMonitor = WorkerConfigurationResolverTestsHelper.GetTestWorkerConfigurationResolverOptions(configuration, testEnvironment, testScriptHostManager.Object, new OptionsWrapper(hostingOptions)); + + var resolver = new DynamicWorkerConfigurationResolver(loggerFactory, FileUtility.Instance, testProfileManager.Object, optionsMonitor); - LanguageWorkerOptionsSetup setup = new LanguageWorkerOptionsSetup(configuration, loggerFactory, testEnvironment, testMetricLogger, testProfileManager.Object, testScriptHostManager.Object, new OptionsWrapper(hostingOptions)); + LanguageWorkerOptionsSetup setup = new LanguageWorkerOptionsSetup(configuration, loggerFactory, testEnvironment, testMetricLogger, testProfileManager.Object, testScriptHostManager.Object, optionsMonitor, resolver); LanguageWorkerOptions options = new LanguageWorkerOptions(); setup.Configure(options); diff --git a/test/WebJobs.Script.Tests/Extensions/EnvironmentExtensionsTests.cs b/test/WebJobs.Script.Tests/Extensions/EnvironmentExtensionsTests.cs index 085658e4b9..d40adfc3d6 100644 --- a/test/WebJobs.Script.Tests/Extensions/EnvironmentExtensionsTests.cs +++ b/test/WebJobs.Script.Tests/Extensions/EnvironmentExtensionsTests.cs @@ -3,10 +3,14 @@ using System; using System.Collections.Generic; -using System.Linq; +using Microsoft.Azure.WebJobs.Script.Config; using Microsoft.Azure.WebJobs.Script.Description; using Microsoft.Azure.WebJobs.Script.Workers.Rpc; +using Microsoft.Azure.WebJobs.Script.Workers.Rpc.Configuration; +using Microsoft.Extensions.Configuration; +using Microsoft.Extensions.Options; using Microsoft.WebJobs.Script.Tests; +using Moq; using Xunit; using static Microsoft.Azure.WebJobs.Script.EnvironmentSettingNames; @@ -562,12 +566,18 @@ public void IsInValidationMode_ReturnsExpectedResult(string functionGroup, bool public void IsDynamicWorkerResolutionEnabled_HostingConfigAndFeatureFlags_WorksAsExpected(string featureFlagValue, string hostingConfigSetting, bool expected) { - HashSet hostingConfigEnabledWorkers = hostingConfigSetting?.Split('|', StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries).ToHashSet(); + var mockConfiguration = new Mock(); + var mockScriptHostManager = new Mock(); + + var hostingOptions = new FunctionsHostingConfigOptions(); + hostingOptions.Features.Add(RpcWorkerConstants.WorkersAvailableForDynamicResolution, hostingConfigSetting); var testEnvironment = new TestEnvironment(); testEnvironment.SetEnvironmentVariable(AzureWebJobsFeatureFlags, featureFlagValue); - bool result = testEnvironment.IsDynamicWorkerResolutionEnabled(hostingConfigEnabledWorkers); + var optionsMonitor = WorkerConfigurationResolverTestsHelper.GetTestWorkerConfigurationResolverOptions(mockConfiguration.Object, testEnvironment, mockScriptHostManager.Object, new OptionsWrapper(hostingOptions)); + + bool result = testEnvironment.IsDynamicWorkerResolutionEnabled(optionsMonitor); Assert.Equal(expected, result); } @@ -581,13 +591,19 @@ public void IsDynamicWorkerResolutionEnabled_HostingConfigAndFeatureFlags_WorksA [InlineData("| ", null, "workflowapp", false)] public void IsDynamicWorkerResolutionEnabled_WorkerRuntimeAndMultiLanguage_WorksAsExpected(string hostingConfigSetting, string workerRuntime, string multilanguageApp, bool expected) { + var mockConfiguration = new Mock(); + var mockScriptHostManager = new Mock(); + + var hostingOptions = new FunctionsHostingConfigOptions(); + hostingOptions.Features.Add(RpcWorkerConstants.WorkersAvailableForDynamicResolution, hostingConfigSetting); + var testEnvironment = new TestEnvironment(); testEnvironment.SetEnvironmentVariable(AppKind, multilanguageApp); testEnvironment.SetEnvironmentVariable(FunctionWorkerRuntime, workerRuntime); - HashSet hostingConfigEnabledWorkers = hostingConfigSetting?.Split('|', StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries).ToHashSet(); + var optionsMonitor = WorkerConfigurationResolverTestsHelper.GetTestWorkerConfigurationResolverOptions(mockConfiguration.Object, testEnvironment, mockScriptHostManager.Object, new OptionsWrapper(hostingOptions)); - bool result = testEnvironment.IsDynamicWorkerResolutionEnabled(hostingConfigEnabledWorkers); + bool result = testEnvironment.IsDynamicWorkerResolutionEnabled(optionsMonitor); Assert.Equal(expected, result); } diff --git a/test/WebJobs.Script.Tests/WebJobsScriptHostServiceTests.cs b/test/WebJobs.Script.Tests/WebJobsScriptHostServiceTests.cs index 092beeabf5..f1bf71437d 100644 --- a/test/WebJobs.Script.Tests/WebJobsScriptHostServiceTests.cs +++ b/test/WebJobs.Script.Tests/WebJobsScriptHostServiceTests.cs @@ -16,6 +16,7 @@ using Microsoft.Azure.WebJobs.Script.WebHost.Configuration; using Microsoft.Azure.WebJobs.Script.WebHost.DependencyInjection; using Microsoft.Azure.WebJobs.Script.Workers.Rpc; +using Microsoft.Azure.WebJobs.Script.Workers.Rpc.Configuration; using Microsoft.Extensions.Configuration; using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Hosting; @@ -40,6 +41,7 @@ public class WebJobsScriptHostServiceTests private Mock _mockScriptWebHostEnvironment; private Mock _mockEnvironment; private HostBuiltChangeTokenSource _hostBuiltChangeTokenSource = new(); + private HostBuiltChangeTokenSource _hostBuiltChangeTokenSourceResolverOptions = new(); private IConfiguration _mockConfig; private OptionsWrapper _healthMonitorOptions; private HostPerformanceManager _hostPerformanceManager; @@ -119,7 +121,8 @@ public async Task StartAsync_Succeeds() _hostPerformanceManager, _healthMonitorOptions, metricsLogger, new Mock().Object, _mockConfig, mockEventManager.Object, _hostMetrics, _functionsHostingConfigOptions, - _hostBuiltChangeTokenSource); + _hostBuiltChangeTokenSource, + _hostBuiltChangeTokenSourceResolverOptions); await _hostService.StartAsync(CancellationToken.None); @@ -151,7 +154,8 @@ public async Task HostInitialization_OnInitializationException_MaintainsErrorInf _mockScriptWebHostEnvironment.Object, _mockEnvironment.Object, _hostPerformanceManager, _healthMonitorOptions, metricsLogger, new Mock().Object, _mockConfig, new TestScriptEventManager(), _hostMetrics, _functionsHostingConfigOptions, - _hostBuiltChangeTokenSource); + _hostBuiltChangeTokenSource, + _hostBuiltChangeTokenSourceResolverOptions); await _hostService.StartAsync(CancellationToken.None); Assert.True(AreRequiredMetricsGenerated(metricsLogger)); @@ -181,7 +185,7 @@ public async Task HostRestart_Specialization_Succeeds() _mockScriptWebHostEnvironment.Object, _mockEnvironment.Object, _hostPerformanceManager, _healthMonitorOptions, metricsLogger, new Mock().Object, - _mockConfig, new TestScriptEventManager(), _hostMetrics, _functionsHostingConfigOptions, _hostBuiltChangeTokenSource); + _mockConfig, new TestScriptEventManager(), _hostMetrics, _functionsHostingConfigOptions, _hostBuiltChangeTokenSource, _hostBuiltChangeTokenSourceResolverOptions); await _hostService.StartAsync(CancellationToken.None); Assert.True(AreRequiredMetricsGenerated(metricsLogger)); @@ -236,7 +240,7 @@ public async Task HostRestart_DuringInitializationWithError_Recovers() _mockScriptWebHostEnvironment.Object, _mockEnvironment.Object, _hostPerformanceManager, _healthMonitorOptions, metricsLogger, new Mock().Object, - _mockConfig, new TestScriptEventManager(), _hostMetrics, _functionsHostingConfigOptions, _hostBuiltChangeTokenSource); + _mockConfig, new TestScriptEventManager(), _hostMetrics, _functionsHostingConfigOptions, _hostBuiltChangeTokenSource, _hostBuiltChangeTokenSourceResolverOptions); TestLoggerProvider hostALogger = hostA.Object.GetTestLoggerProvider(); TestLoggerProvider hostBLogger = hostB.Object.GetTestLoggerProvider(); @@ -312,7 +316,7 @@ public async Task HostRestart_DuringInitialization_Cancels() _mockScriptWebHostEnvironment.Object, _mockEnvironment.Object, _hostPerformanceManager, _healthMonitorOptions, metricsLogger, new Mock().Object, - _mockConfig, new TestScriptEventManager(), _hostMetrics, _functionsHostingConfigOptions, _hostBuiltChangeTokenSource); + _mockConfig, new TestScriptEventManager(), _hostMetrics, _functionsHostingConfigOptions, _hostBuiltChangeTokenSource, _hostBuiltChangeTokenSourceResolverOptions); TestLoggerProvider hostALogger = hostA.Object.GetTestLoggerProvider(); @@ -382,7 +386,7 @@ public async Task DisposedHost_ServicesNotExposed() _mockScriptWebHostEnvironment.Object, _mockEnvironment.Object, _hostPerformanceManager, _healthMonitorOptions, metricsLogger, new Mock().Object, _mockConfig, new TestScriptEventManager(), - _hostMetrics, _functionsHostingConfigOptions, _hostBuiltChangeTokenSource); + _hostMetrics, _functionsHostingConfigOptions, _hostBuiltChangeTokenSource, _hostBuiltChangeTokenSourceResolverOptions); Task startTask = _hostService.StartAsync(CancellationToken.None); @@ -433,7 +437,7 @@ public async Task DisposesScriptHost() _mockScriptWebHostEnvironment.Object, _mockEnvironment.Object, _hostPerformanceManager, _healthMonitorOptions, metricsLogger, new Mock().Object, - _mockConfig, new TestScriptEventManager(), _hostMetrics, _functionsHostingConfigOptions, _hostBuiltChangeTokenSource); + _mockConfig, new TestScriptEventManager(), _hostMetrics, _functionsHostingConfigOptions, _hostBuiltChangeTokenSource, _hostBuiltChangeTokenSourceResolverOptions); var hostLogger = host.Object.GetTestLoggerProvider(); @@ -471,7 +475,7 @@ public async Task HostRestart_BeforeStart_WaitsForStartToContinue() _mockScriptWebHostEnvironment.Object, _mockEnvironment.Object, _hostPerformanceManager, _healthMonitorOptions, metricsLogger, new Mock().Object, _mockConfig, - new TestScriptEventManager(), _hostMetrics, _functionsHostingConfigOptions, _hostBuiltChangeTokenSource); + new TestScriptEventManager(), _hostMetrics, _functionsHostingConfigOptions, _hostBuiltChangeTokenSource, _hostBuiltChangeTokenSourceResolverOptions); // Simulate a call to specialize coming from the PlaceholderSpecializationMiddleware. This // can happen before we ever start the service, which could create invalid state. @@ -527,7 +531,7 @@ public void ShouldEnforceSequentialRestart_WithCorrectConfig(string value, bool _mockScriptWebHostEnvironment.Object, _mockEnvironment.Object, _hostPerformanceManager, _healthMonitorOptions, metricsLogger, new Mock().Object, config, new TestScriptEventManager(), - _hostMetrics, _functionsHostingConfigOptions, _hostBuiltChangeTokenSource); + _hostMetrics, _functionsHostingConfigOptions, _hostBuiltChangeTokenSource, _hostBuiltChangeTokenSourceResolverOptions); Assert.Equal(expectedResult, _hostService.ShouldEnforceSequentialRestart()); } @@ -565,7 +569,7 @@ public async Task DependencyTrackingTelemetryModule_Race() _mockScriptWebHostEnvironment.Object, _mockEnvironment.Object, _hostPerformanceManager, _healthMonitorOptions, new TestMetricsLogger(), new Mock().Object, _mockConfig, new TestScriptEventManager(), - _hostMetrics, _functionsHostingConfigOptions, _hostBuiltChangeTokenSource)) + _hostMetrics, _functionsHostingConfigOptions, _hostBuiltChangeTokenSource, _hostBuiltChangeTokenSourceResolverOptions)) { await _hostService.StartAsync(CancellationToken.None); diff --git a/test/WebJobs.Script.Tests/WorkerConfigurationResolverTestsHelper.cs b/test/WebJobs.Script.Tests/WorkerConfigurationResolverTestsHelper.cs new file mode 100644 index 0000000000..8592100b21 --- /dev/null +++ b/test/WebJobs.Script.Tests/WorkerConfigurationResolverTestsHelper.cs @@ -0,0 +1,30 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the MIT License. See License.txt in the project root for license information. + +using Microsoft.Azure.WebJobs.Script.Config; +using Microsoft.Azure.WebJobs.Script.Workers.Rpc.Configuration; +using Microsoft.Extensions.Configuration; +using Microsoft.Extensions.Options; + +namespace Microsoft.Azure.WebJobs.Script.Tests +{ + internal static class WorkerConfigurationResolverTestsHelper + { + internal static IOptionsMonitor GetTestWorkerConfigurationResolverOptions(IConfiguration configuration, + IEnvironment environment, + IScriptHostManager scriptHostManager, + IOptions functionsHostingConfigOptions) + { + var resolverOptionssetup = new WorkerConfigurationResolverOptionsSetup(configuration, environment, scriptHostManager, functionsHostingConfigOptions); + var resolverOptions = new WorkerConfigurationResolverOptions(); + resolverOptionssetup.Configure(resolverOptions); + + var factory = new TestOptionsFactory(resolverOptions); + var source = new TestChangeTokenSource(); + var changeTokens = new[] { source }; + var optionsMonitor = new OptionsMonitor(factory, changeTokens, factory); + + return optionsMonitor; + } + } +} diff --git a/test/WebJobs.Script.Tests/Workers/Rpc/RpcWorkerConfigFactoryTests.cs b/test/WebJobs.Script.Tests/Workers/Rpc/RpcWorkerConfigFactoryTests.cs index d78d930040..7b69c1c451 100644 --- a/test/WebJobs.Script.Tests/Workers/Rpc/RpcWorkerConfigFactoryTests.cs +++ b/test/WebJobs.Script.Tests/Workers/Rpc/RpcWorkerConfigFactoryTests.cs @@ -12,6 +12,9 @@ using Microsoft.Azure.WebJobs.Script.Workers.Rpc; using Microsoft.Azure.WebJobs.Script.Workers.Rpc.Configuration; using Microsoft.Extensions.Configuration; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; +using Microsoft.WebJobs.Script.Tests; using Moq; using Xunit; @@ -41,8 +44,15 @@ public void DefaultLanguageWorkersDir() var expectedWorkersDir = Path.Combine(Path.GetDirectoryName(new Uri(typeof(RpcWorkerConfigFactory).Assembly.Location).LocalPath), RpcWorkerConstants.DefaultWorkersDirectoryName); var config = new ConfigurationBuilder().Build(); var testLogger = new TestLogger("test"); - var workerConfigurationResolver = new DefaultWorkerConfigurationResolver(config, testLogger); - var configFactory = new RpcWorkerConfigFactory(config, testLogger, _testSysRuntimeInfo, _testEnvironment, new TestMetricsLogger(), _testWorkerProfileManager, workerConfigurationResolver); + + var testScriptHostManager = new Mock(); + var hostingOptions = new FunctionsHostingConfigOptions(); + + var optionsMonitor = WorkerConfigurationResolverTestsHelper.GetTestWorkerConfigurationResolverOptions(config, _testEnvironment, testScriptHostManager.Object, new OptionsWrapper(hostingOptions)); + var mockLogger = new Mock(); + + var workerConfigurationResolver = new DefaultWorkerConfigurationResolver(mockLogger.Object, optionsMonitor); + var configFactory = new RpcWorkerConfigFactory(config, testLogger, _testSysRuntimeInfo, _testEnvironment, new TestMetricsLogger(), _testWorkerProfileManager, workerConfigurationResolver, optionsMonitor); Assert.Equal(expectedWorkersDir, configFactory.WorkersDirPath); } @@ -75,8 +85,15 @@ public void LanguageWorker_WorkersDir_Set() }) .Build(); var testLogger = new TestLogger("test"); - var workerConfigurationResolver = new DefaultWorkerConfigurationResolver(config, testLogger); - var configFactory = new RpcWorkerConfigFactory(config, testLogger, _testSysRuntimeInfo, _testEnvironment, new TestMetricsLogger(), _testWorkerProfileManager, workerConfigurationResolver); + + var testScriptHostManager = new Mock(); + var hostingOptions = new FunctionsHostingConfigOptions(); + var mockLogger = new Mock(); + + var optionsMonitor = WorkerConfigurationResolverTestsHelper.GetTestWorkerConfigurationResolverOptions(config, _testEnvironment, testScriptHostManager.Object, new OptionsWrapper(hostingOptions)); + + var workerConfigurationResolver = new DefaultWorkerConfigurationResolver(mockLogger.Object, optionsMonitor); + var configFactory = new RpcWorkerConfigFactory(config, testLogger, _testSysRuntimeInfo, _testEnvironment, new TestMetricsLogger(), _testWorkerProfileManager, workerConfigurationResolver, optionsMonitor); Assert.Equal(expectedWorkersDir, configFactory.WorkersDirPath); } @@ -92,8 +109,14 @@ public void LanguageWorker_WorkersDir_NotSet() var config = configBuilder.Build(); var scriptSettingsManager = new ScriptSettingsManager(config); var testLogger = new TestLogger("test"); - var workerConfigurationResolver = new DefaultWorkerConfigurationResolver(config, testLogger); - var configFactory = new RpcWorkerConfigFactory(config, testLogger, _testSysRuntimeInfo, _testEnvironment, new TestMetricsLogger(), _testWorkerProfileManager, workerConfigurationResolver); + + var testScriptHostManager = new Mock(); + var hostingOptions = new FunctionsHostingConfigOptions(); + var optionsMonitor = WorkerConfigurationResolverTestsHelper.GetTestWorkerConfigurationResolverOptions(config, _testEnvironment, testScriptHostManager.Object, new OptionsWrapper(hostingOptions)); + var mockLogger = new Mock(); + + var workerConfigurationResolver = new DefaultWorkerConfigurationResolver(mockLogger.Object, optionsMonitor); + var configFactory = new RpcWorkerConfigFactory(config, testLogger, _testSysRuntimeInfo, _testEnvironment, new TestMetricsLogger(), _testWorkerProfileManager, workerConfigurationResolver, optionsMonitor); Assert.Equal(expectedWorkersDir, configFactory.WorkersDirPath); } @@ -117,8 +140,18 @@ public void WorkerDescription_Skipped_When_Profile_Disables_Worker() var scriptSettingsManager = new ScriptSettingsManager(config); var testLogger = new TestLogger("test"); _testEnvironment.SetEnvironmentVariable("ENV_VAR_BAR", "True"); - var workerConfigurationResolver = new DefaultWorkerConfigurationResolver(config, testLogger); - var configFactory = new RpcWorkerConfigFactory(config, testLogger, _testSysRuntimeInfo, _testEnvironment, new TestMetricsLogger(), _testWorkerProfileManager, workerConfigurationResolver); + + var loggerProvider = new TestLoggerProvider(); + var loggerFactory = new LoggerFactory(); + loggerFactory.AddProvider(loggerProvider); + + var testScriptHostManager = new Mock(); + + var hostingOptions = new FunctionsHostingConfigOptions(); + var optionsMonitor = WorkerConfigurationResolverTestsHelper.GetTestWorkerConfigurationResolverOptions(config, _testEnvironment, testScriptHostManager.Object, new OptionsWrapper(hostingOptions)); + + var workerConfigurationResolver = new DefaultWorkerConfigurationResolver(loggerFactory, optionsMonitor); + var configFactory = new RpcWorkerConfigFactory(config, testLogger, _testSysRuntimeInfo, _testEnvironment, new TestMetricsLogger(), _testWorkerProfileManager, workerConfigurationResolver, optionsMonitor); var workerConfigs = configFactory.GetConfigs(); var errors = testLogger.GetLogMessages().Where(m => m.Exception != null).ToList(); @@ -139,9 +172,19 @@ public void JavaPath_FromEnvVars() var configBuilder = ScriptSettingsManager.CreateDefaultConfigurationBuilder(); var config = configBuilder.Build(); var scriptSettingsManager = new ScriptSettingsManager(config); - var testLogger = new TestLogger("test"); - var workerConfigurationResolver = new DefaultWorkerConfigurationResolver(config, testLogger); - var configFactory = new RpcWorkerConfigFactory(config, testLogger, _testSysRuntimeInfo, _testEnvironment, new TestMetricsLogger(), _testWorkerProfileManager, workerConfigurationResolver); + + var loggerProvider = new TestLoggerProvider(); + var loggerFactory = new LoggerFactory(); + loggerFactory.AddProvider(loggerProvider); + var testLogger = loggerFactory.CreateLogger("test"); + + var testScriptHostManager = new Mock(); + + var hostingOptions = new FunctionsHostingConfigOptions(); + var optionsMonitor = WorkerConfigurationResolverTestsHelper.GetTestWorkerConfigurationResolverOptions(config, _testEnvironment, testScriptHostManager.Object, new OptionsWrapper(hostingOptions)); + + var workerConfigurationResolver = new DefaultWorkerConfigurationResolver(loggerFactory, optionsMonitor); + var configFactory = new RpcWorkerConfigFactory(config, testLogger, _testSysRuntimeInfo, _testEnvironment, new TestMetricsLogger(), _testWorkerProfileManager, workerConfigurationResolver, optionsMonitor); var workerConfigs = configFactory.GetConfigs(); var javaPath = workerConfigs.FirstOrDefault(c => c.Description.Language.Equals("java", StringComparison.OrdinalIgnoreCase)).Description.DefaultExecutablePath; Assert.DoesNotContain(@"%JAVA_HOME%", javaPath); @@ -159,12 +202,23 @@ public void DefaultWorkerConfigs_Overrides_DefaultWorkerRuntimeVersion_AppSettin .AddInMemoryCollection(testEnvVariables); var config = configBuilder.Build(); var scriptSettingsManager = new ScriptSettingsManager(config); - var testLogger = new TestLogger("test"); + + var loggerProvider = new TestLoggerProvider(); + var loggerFactory = new LoggerFactory(); + loggerFactory.AddProvider(loggerProvider); + var testLogger = loggerFactory.CreateLogger("test"); + _testEnvironment.SetEnvironmentVariable(EnvironmentSettingNames.AzureWebsitePlaceholderMode, "1"); using var variables = new TestScopedSettings(scriptSettingsManager, testEnvVariables); - var workerConfigurationResolver = new DefaultWorkerConfigurationResolver(config, testLogger); - var configFactory = new RpcWorkerConfigFactory(config, testLogger, _testSysRuntimeInfo, _testEnvironment, new TestMetricsLogger(), _testWorkerProfileManager, workerConfigurationResolver); + + var testScriptHostManager = new Mock(); + + var hostingOptions = new FunctionsHostingConfigOptions(); + var optionsMonitor = WorkerConfigurationResolverTestsHelper.GetTestWorkerConfigurationResolverOptions(config, _testEnvironment, testScriptHostManager.Object, new OptionsWrapper(hostingOptions)); + + var workerConfigurationResolver = new DefaultWorkerConfigurationResolver(loggerFactory, optionsMonitor); + var configFactory = new RpcWorkerConfigFactory(config, testLogger, _testSysRuntimeInfo, _testEnvironment, new TestMetricsLogger(), _testWorkerProfileManager, workerConfigurationResolver, optionsMonitor); var workerConfigs = configFactory.GetConfigs(); var pythonWorkerConfig = workerConfigs.FirstOrDefault(w => w.Description.Language.Equals("python", StringComparison.OrdinalIgnoreCase)); var powershellWorkerConfig = workerConfigs.FirstOrDefault(w => w.Description.Language.Equals("powershell", StringComparison.OrdinalIgnoreCase)); @@ -184,9 +238,19 @@ public void DefaultWorkerConfigs_Overrides_VersionAppSetting() var configBuilder = ScriptSettingsManager.CreateDefaultConfigurationBuilder(); var config = configBuilder.Build(); var scriptSettingsManager = new ScriptSettingsManager(config); - var testLogger = new TestLogger("test"); - var resolver = new DefaultWorkerConfigurationResolver(config, testLogger); - var configFactory = new RpcWorkerConfigFactory(config, testLogger, _testSysRuntimeInfo, testEnvironment, new TestMetricsLogger(), _testWorkerProfileManager, resolver); + + var loggerProvider = new TestLoggerProvider(); + var loggerFactory = new LoggerFactory(); + loggerFactory.AddProvider(loggerProvider); + var testLogger = loggerFactory.CreateLogger("test"); + + var testScriptHostManager = new Mock(); + + var hostingOptions = new FunctionsHostingConfigOptions(); + var optionsMonitor = WorkerConfigurationResolverTestsHelper.GetTestWorkerConfigurationResolverOptions(config, _testEnvironment, testScriptHostManager.Object, new OptionsWrapper(hostingOptions)); + + var resolver = new DefaultWorkerConfigurationResolver(loggerFactory, optionsMonitor); + var configFactory = new RpcWorkerConfigFactory(config, testLogger, _testSysRuntimeInfo, testEnvironment, new TestMetricsLogger(), _testWorkerProfileManager, resolver, optionsMonitor); var workerConfigs = configFactory.GetConfigs(); var powershellWorkerConfig = workerConfigs.FirstOrDefault(w => w.Description.Language.Equals("powershell", StringComparison.OrdinalIgnoreCase)); Assert.Equal(1, workerConfigs.Count); @@ -212,11 +276,20 @@ public void ShouldAddProvider_Returns_Expected(string workerLanguage, string wor _testEnvironment.SetEnvironmentVariable(RpcWorkerConstants.FunctionWorkerRuntimeSettingName, workerRuntime); } var config = new ConfigurationBuilder().Build(); - var testLogger = new TestLogger("test"); - var workerConfigurationResolver = new DefaultWorkerConfigurationResolver(config, testLogger); - var rpcWorkerConfigFactory = new RpcWorkerConfigFactory(config, testLogger, _testSysRuntimeInfo, _testEnvironment, new TestMetricsLogger(), _testWorkerProfileManager, workerConfigurationResolver); + + var loggerProvider = new TestLoggerProvider(); + var loggerFactory = new LoggerFactory(); + loggerFactory.AddProvider(loggerProvider); + var testLogger = loggerFactory.CreateLogger("test"); + + var testScriptHostManager = new Mock(); + var hostingOptions = new FunctionsHostingConfigOptions(); + var optionsMonitor = WorkerConfigurationResolverTestsHelper.GetTestWorkerConfigurationResolverOptions(config, _testEnvironment, testScriptHostManager.Object, new OptionsWrapper(hostingOptions)); + + var workerConfigurationResolver = new DefaultWorkerConfigurationResolver(loggerFactory, optionsMonitor); + var configFactory = new RpcWorkerConfigFactory(config, testLogger, _testSysRuntimeInfo, _testEnvironment, new TestMetricsLogger(), _testWorkerProfileManager, workerConfigurationResolver, optionsMonitor); _testEnvironment.SetEnvironmentVariable(RpcWorkerConstants.FunctionWorkerRuntimeSettingName, workerRuntime); - Assert.Equal(expectedResult, rpcWorkerConfigFactory.ShouldAddWorkerConfig(workerLanguage)); + Assert.Equal(expectedResult, configFactory.ShouldAddWorkerConfig(workerLanguage)); } [Theory] @@ -258,9 +331,14 @@ public void GetWorkerProcessCount_Tests(bool defaultWorkerConfig, bool setProces var config = new ConfigurationBuilder().Build(); var testLogger = new TestLogger("test"); - var workerConfigurationResolver = new DefaultWorkerConfigurationResolver(config, testLogger); - var rpcWorkerConfigFactory = new RpcWorkerConfigFactory(config, testLogger, _testSysRuntimeInfo, _testEnvironment, new TestMetricsLogger(), _testWorkerProfileManager, workerConfigurationResolver); - var result = rpcWorkerConfigFactory.GetWorkerProcessCount(workerConfig); + var testScriptHostManager = new Mock(); + var hostingOptions = new FunctionsHostingConfigOptions(); + var optionsMonitor = WorkerConfigurationResolverTestsHelper.GetTestWorkerConfigurationResolverOptions(config, _testEnvironment, testScriptHostManager.Object, new OptionsWrapper(hostingOptions)); + var mockLogger = new Mock(); + + var workerConfigurationResolver = new DefaultWorkerConfigurationResolver(mockLogger.Object, optionsMonitor); + var configFactory = new RpcWorkerConfigFactory(config, testLogger, _testSysRuntimeInfo, _testEnvironment, new TestMetricsLogger(), _testWorkerProfileManager, workerConfigurationResolver, optionsMonitor); + var result = configFactory.GetWorkerProcessCount(workerConfig); if (defaultWorkerConfig) { @@ -299,17 +377,23 @@ public void GetWorkerProcessCount_ThrowsException_Tests() var config = new ConfigurationBuilder().Build(); var testLogger = new TestLogger("test"); - var workerConfigurationResolver = new DefaultWorkerConfigurationResolver(config, testLogger); - var rpcWorkerConfigFactory = new RpcWorkerConfigFactory(config, testLogger, _testSysRuntimeInfo, _testEnvironment, new TestMetricsLogger(), _testWorkerProfileManager, workerConfigurationResolver); - var resultEx1 = Assert.Throws(() => rpcWorkerConfigFactory.GetWorkerProcessCount(workerConfig)); + + var testScriptHostManager = new Mock(); + var hostingOptions = new FunctionsHostingConfigOptions(); + var optionsMonitor = WorkerConfigurationResolverTestsHelper.GetTestWorkerConfigurationResolverOptions(config, _testEnvironment, testScriptHostManager.Object, new OptionsWrapper(hostingOptions)); + var mockLogger = new Mock(); + + var workerConfigurationResolver = new DefaultWorkerConfigurationResolver(mockLogger.Object, optionsMonitor); + var configFactory = new RpcWorkerConfigFactory(config, testLogger, _testSysRuntimeInfo, _testEnvironment, new TestMetricsLogger(), _testWorkerProfileManager, workerConfigurationResolver, optionsMonitor); + var resultEx1 = Assert.Throws(() => configFactory.GetWorkerProcessCount(workerConfig)); Assert.Contains("ProcessCount must be greater than 0", resultEx1.Message); workerConfig = CreateWorkerConfig(40, 10, "00:10:00", false); - var resultEx2 = Assert.Throws(() => rpcWorkerConfigFactory.GetWorkerProcessCount(workerConfig)); + var resultEx2 = Assert.Throws(() => configFactory.GetWorkerProcessCount(workerConfig)); Assert.Contains("ProcessCount must not be greater than MaxProcessCount", resultEx2.Message); workerConfig = CreateWorkerConfig(10, 10, "-800", false); - var resultEx3 = Assert.Throws(() => rpcWorkerConfigFactory.GetWorkerProcessCount(workerConfig)); + var resultEx3 = Assert.Throws(() => configFactory.GetWorkerProcessCount(workerConfig)); Assert.Contains("value could not be converted to System.TimeSpan", resultEx3.Message); } diff --git a/test/WebJobs.Script.Tests/Workers/Rpc/RpcWorkerConfigTests.cs b/test/WebJobs.Script.Tests/Workers/Rpc/RpcWorkerConfigTests.cs index 80317bfb9c..bdb78bdbcf 100644 --- a/test/WebJobs.Script.Tests/Workers/Rpc/RpcWorkerConfigTests.cs +++ b/test/WebJobs.Script.Tests/Workers/Rpc/RpcWorkerConfigTests.cs @@ -17,6 +17,7 @@ using Microsoft.Extensions.Configuration; using Microsoft.Extensions.Logging; using Microsoft.Extensions.Options; +using Microsoft.WebJobs.Script.Tests; using Moq; using Xunit; @@ -686,8 +687,18 @@ private IEnumerable TestReadWorkerProviderFromConfig(IEnumerabl var scriptHostOptions = new ScriptJobHostOptions(); var scriptSettingsManager = new ScriptSettingsManager(config); var workerProfileManager = new Mock(); - IWorkerConfigurationResolver workerConfigurationResolver = new DefaultWorkerConfigurationResolver(config, testLogger); - var configFactory = new RpcWorkerConfigFactory(config, testLogger, _testSysRuntimeInfo, _testEnvironment, new TestMetricsLogger(), workerProfileManager.Object, workerConfigurationResolver); + + var testScriptHostManager = new Mock(); + + var loggerProvider = new TestLoggerProvider(); + var loggerFactory = new LoggerFactory(); + loggerFactory.AddProvider(loggerProvider); + + var hostingOptions = new FunctionsHostingConfigOptions(); + var optionsMonitor = WorkerConfigurationResolverTestsHelper.GetTestWorkerConfigurationResolverOptions(config, _testEnvironment, testScriptHostManager.Object, new OptionsWrapper(hostingOptions)); + + IWorkerConfigurationResolver workerConfigurationResolver = new DefaultWorkerConfigurationResolver(loggerFactory, optionsMonitor); + var configFactory = new RpcWorkerConfigFactory(config, testLogger, _testSysRuntimeInfo, _testEnvironment, new TestMetricsLogger(), workerProfileManager.Object, workerConfigurationResolver, optionsMonitor); if (appSvcEnv) { diff --git a/test/WebJobs.Script.Tests/Workers/Rpc/WorkerConfigurationResolverOptionsSetupTests.cs b/test/WebJobs.Script.Tests/Workers/Rpc/WorkerConfigurationResolverOptionsSetupTests.cs new file mode 100644 index 0000000000..13731a2438 --- /dev/null +++ b/test/WebJobs.Script.Tests/Workers/Rpc/WorkerConfigurationResolverOptionsSetupTests.cs @@ -0,0 +1,63 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the MIT License. See License.txt in the project root for license information. + +using System.Collections.Generic; +using Microsoft.Azure.WebJobs.Script.Config; +using Microsoft.Azure.WebJobs.Script.Workers; +using Microsoft.Azure.WebJobs.Script.Workers.Rpc; +using Microsoft.Azure.WebJobs.Script.Workers.Rpc.Configuration; +using Microsoft.Extensions.Configuration; +using Microsoft.Extensions.Options; +using Moq; +using Xunit; + +namespace Microsoft.Azure.WebJobs.Script.Tests.Workers.Rpc +{ + public class WorkerConfigurationResolverOptionsSetupTests + { + private readonly Mock _mockConfiguration; + private readonly Mock _mockEnvironment; + private readonly Mock _mockScriptHostManager; + private readonly WorkerConfigurationResolverOptionsSetup _setup; + + public WorkerConfigurationResolverOptionsSetupTests() + { + _mockConfiguration = new Mock(); + _mockEnvironment = new Mock(); + _mockScriptHostManager = new Mock(); + var hostingOptions = new FunctionsHostingConfigOptions(); + + _setup = new WorkerConfigurationResolverOptionsSetup(_mockConfiguration.Object, _mockEnvironment.Object, _mockScriptHostManager.Object, new OptionsWrapper(hostingOptions)); + } + + [Fact] + public void Configure_WithRealEnvironmentValues_SetsCorrectDefaults() + { + // Arrange + var testEnvironment = new TestEnvironment(); + var configBuilder = new ConfigurationBuilder() + .AddInMemoryCollection(new Dictionary + { + [$"{RpcWorkerConstants.LanguageWorkersSectionName}:{WorkerConstants.WorkersDirectorySectionName}"] = "/default/workers" + }); + var configuration = configBuilder.Build(); + var mockScriptHostManager = new Mock(); + + var hostingOptions = new FunctionsHostingConfigOptions(); + + var setup = new WorkerConfigurationResolverOptionsSetup(configuration, testEnvironment, mockScriptHostManager.Object, new OptionsWrapper(hostingOptions)); + var options = new WorkerConfigurationResolverOptions(); + + // Act + setup.Configure(options); + + // Assert + Assert.Null(options.WorkerRuntime); // No worker runtime set + Assert.Equal(ScriptConstants.LatestPlatformChannelNameUpper, options.ReleaseChannel); // Default release channel + Assert.False(options.IsPlaceholderModeEnabled); // Default placeholder mode + Assert.False(options.IsMultiLanguageWorkerEnvironment); // Default multi-language mode + Assert.Equal("/default/workers", options.WorkersDirPath); + Assert.NotNull(options.LanguageWorkersSettings); + } + } +} \ No newline at end of file diff --git a/test/WebJobs.Script.Tests/Workers/Rpc/WorkerConfigurationResolverTests.cs b/test/WebJobs.Script.Tests/Workers/Rpc/WorkerConfigurationResolverTests.cs index 61a84ed45b..5bf3da45de 100644 --- a/test/WebJobs.Script.Tests/Workers/Rpc/WorkerConfigurationResolverTests.cs +++ b/test/WebJobs.Script.Tests/Workers/Rpc/WorkerConfigurationResolverTests.cs @@ -4,10 +4,16 @@ using System.Collections.Generic; using System.IO; using System.Linq; +using System.Text; +using System.Text.Json; +using Microsoft.Azure.WebJobs.Script.Config; using Microsoft.Azure.WebJobs.Script.Workers.Profiles; +using Microsoft.Azure.WebJobs.Script.Workers.Rpc; using Microsoft.Azure.WebJobs.Script.Workers.Rpc.Configuration; using Microsoft.Extensions.Configuration; using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; +using Microsoft.WebJobs.Script.Tests; using Moq; using Xunit; @@ -16,37 +22,46 @@ namespace Microsoft.Azure.WebJobs.Script.Tests.Workers.Rpc public class WorkerConfigurationResolverTests { private readonly Mock _mockProfileManager; - private readonly Mock _mockConfig; - private readonly Mock _mockLogger; private readonly string _probingPath1 = Path.GetFullPath("..\\..\\..\\..\\test\\TestWorkers\\ProbingPaths\\workers\\"); private readonly string _fallbackPath = Path.GetFullPath("workers"); - private List _probingPaths; public WorkerConfigurationResolverTests() { _mockProfileManager = new Mock(); - _mockConfig = new Mock(); - _mockLogger = new Mock(); - - _probingPaths = new List { _probingPath1, string.Empty, null, "path-not-exists" }; } [Theory] - [InlineData("LATEST", "java\\2.19.0", "node\\3.10.1", "powershell\\7.4", "dotnet-isolated", "python")] - [InlineData("STANDARD", "java\\2.18.0", "node\\3.10.1", "powershell\\7.4", "dotnet-isolated", "python")] - [InlineData("EXTENDED", "java\\2.18.0", "node\\3.10.1", "powershell\\7.4", "dotnet-isolated", "python")] - [InlineData("laTest", "java\\2.19.0", "node\\3.10.1", "powershell\\7.4", "dotnet-isolated", "python")] - [InlineData("abc", "java\\2.19.0", "node\\3.10.1", "powershell\\7.4", "dotnet-isolated", "python")] - [InlineData("Standard", "java\\2.18.0", "node\\3.10.1", "powershell\\7.4", "dotnet-isolated", "python")] + [InlineData("LATEST", "java\\2.19.0", "node\\3.10.1", "powershell", "dotnet-isolated", "python")] + [InlineData("STANDARD", "java\\2.18.0", "node\\3.10.1", "powershell", "dotnet-isolated", "python")] + [InlineData("EXTENDED", "java\\2.18.0", "node\\3.10.1", "powershell", "dotnet-isolated", "python")] + [InlineData("laTest", "java\\2.19.0", "node\\3.10.1", "powershell", "dotnet-isolated", "python")] + [InlineData("abc", "java\\2.19.0", "node\\3.10.1", "powershell", "dotnet-isolated", "python")] + [InlineData("Standard", "java\\2.18.0", "node\\3.10.1", "powershell", "dotnet-isolated", "python")] public void GetWorkerConfigs_MultiLanguageWorker_ReturnsExpectedConfigs(string releaseChannel, string java, string node, string powershell, string dotnetIsolated, string python) { // Arrange + var probingPaths = new List() { _probingPath1, string.Empty, "path-not-exists" }; + + var loggerProvider = new TestLoggerProvider(); + var loggerFactory = new LoggerFactory(); + loggerFactory.AddProvider(loggerProvider); + var mockEnvironment = new Mock(); mockEnvironment.Setup(p => p.GetEnvironmentVariable(EnvironmentSettingNames.AntaresPlatformReleaseChannel)).Returns(releaseChannel); mockEnvironment.Setup(p => p.GetEnvironmentVariable(EnvironmentSettingNames.AppKind)).Returns(ScriptConstants.WorkFlowAppKind); mockEnvironment.Setup(p => p.GetEnvironmentVariable(EnvironmentSettingNames.FunctionWorkerRuntime)).Returns((string)null); - var workerConfigurationResolver = new DynamicWorkerConfigurationResolver(_mockConfig.Object, _mockLogger.Object, mockEnvironment.Object, FileUtility.Instance, _mockProfileManager.Object, new HashSet() { "java", "node", "powershell" }, _probingPaths); + var config = GetConfiguration(probingPaths); + + var testScriptHostManager = new Mock(); + + var hostingOptions = new FunctionsHostingConfigOptions(); + hostingOptions.Features.Add(RpcWorkerConstants.WorkersAvailableForDynamicResolution, "java|node"); + + var optionsMonitor = WorkerConfigurationResolverTestsHelper.GetTestWorkerConfigurationResolverOptions(config, mockEnvironment.Object, testScriptHostManager.Object, new OptionsWrapper(hostingOptions)); + + // Act + var workerConfigurationResolver = new DynamicWorkerConfigurationResolver(loggerFactory, FileUtility.Instance, _mockProfileManager.Object, optionsMonitor); var result = workerConfigurationResolver.GetWorkerConfigPaths(); @@ -54,7 +69,7 @@ public void GetWorkerConfigs_MultiLanguageWorker_ReturnsExpectedConfigs(string r Assert.Equal(result.Count, 5); Assert.True(result.Any(r => r.Contains(Path.Combine(_probingPath1, java)))); Assert.True(result.Any(r => r.Contains(Path.Combine(_probingPath1, node)))); - Assert.True(result.Any(r => r.Contains(Path.Combine(_probingPath1, powershell)))); + Assert.True(result.Any(r => r.Contains(Path.Combine(_fallbackPath, powershell)))); Assert.True(result.Any(r => r.Contains(Path.Combine(_fallbackPath, dotnetIsolated)))); Assert.True(result.Any(r => r.Contains(Path.Combine(_fallbackPath, python)))); } @@ -67,6 +82,10 @@ public void GetWorkerConfigs_MultiLanguageWorker_ReturnsExpectedConfigs(string r public void GetWorkerConfigs_MultiLanguageWorker_NullOREmptyProbingPath_ReturnsExpectedConfigs(string probingPathValue, string releaseChannel) { // Arrange + var loggerProvider = new TestLoggerProvider(); + var loggerFactory = new LoggerFactory(); + loggerFactory.AddProvider(loggerProvider); + var mockEnvironment = new Mock(); mockEnvironment.Setup(p => p.GetEnvironmentVariable(EnvironmentSettingNames.AntaresPlatformReleaseChannel)).Returns(releaseChannel); mockEnvironment.Setup(p => p.GetEnvironmentVariable(EnvironmentSettingNames.AppKind)).Returns(ScriptConstants.WorkFlowAppKind); @@ -79,8 +98,16 @@ public void GetWorkerConfigs_MultiLanguageWorker_NullOREmptyProbingPath_ReturnsE probingPaths = new List(); } + var config = GetConfiguration(probingPaths); + + var testScriptHostManager = new Mock(); + + var hostingOptions = new FunctionsHostingConfigOptions(); + hostingOptions.Features.Add(RpcWorkerConstants.WorkersAvailableForDynamicResolution, "java|node|powershell"); + var optionsMonitor = WorkerConfigurationResolverTestsHelper.GetTestWorkerConfigurationResolverOptions(config, mockEnvironment.Object, testScriptHostManager.Object, new OptionsWrapper(hostingOptions)); + // Act - var workerConfigurationResolver = new DynamicWorkerConfigurationResolver(_mockConfig.Object, _mockLogger.Object, mockEnvironment.Object, FileUtility.Instance, _mockProfileManager.Object, new HashSet() { "java", "node", "powershell" }, probingPaths); + var workerConfigurationResolver = new DynamicWorkerConfigurationResolver(loggerFactory, FileUtility.Instance, _mockProfileManager.Object, optionsMonitor); var result = workerConfigurationResolver.GetWorkerConfigPaths(); @@ -120,12 +147,23 @@ public void GetWorkerConfigs_NullOREmptyProbingPath_ReturnsExpectedConfigs(strin probingPaths = new List(); } + var config = GetConfiguration(probingPaths); + var mockProfileManager = new Mock(); var mockConfig = new Mock(); - var mockLogger = new Mock(); + + var loggerProvider = new TestLoggerProvider(); + var loggerFactory = new LoggerFactory(); + loggerFactory.AddProvider(loggerProvider); + + var testScriptHostManager = new Mock(); + + var hostingOptions = new FunctionsHostingConfigOptions(); + hostingOptions.Features.Add(RpcWorkerConstants.WorkersAvailableForDynamicResolution, "java|node|powershell"); + var optionsMonitor = WorkerConfigurationResolverTestsHelper.GetTestWorkerConfigurationResolverOptions(config, mockEnv.Object, testScriptHostManager.Object, new OptionsWrapper(hostingOptions)); // Act - var workerConfigurationResolver = new DynamicWorkerConfigurationResolver(mockConfig.Object, mockLogger.Object, mockEnv.Object, FileUtility.Instance, mockProfileManager.Object, new HashSet() { "java", "node", "powershell" }, probingPaths); + var workerConfigurationResolver = new DynamicWorkerConfigurationResolver(loggerFactory, FileUtility.Instance, _mockProfileManager.Object, optionsMonitor); var result = workerConfigurationResolver.GetWorkerConfigPaths(); @@ -133,5 +171,25 @@ public void GetWorkerConfigs_NullOREmptyProbingPath_ReturnsExpectedConfigs(strin Assert.Equal(result.Count, 1); Assert.True(result.Any(r => r.Contains(Path.Combine(_fallbackPath, languageWorker)))); } + + private IConfiguration GetConfiguration(List probingPaths) + { + var jsonObj = new + { + languageWorkers = new + { + probingPaths + } + }; + + var jsonString = JsonSerializer.Serialize(jsonObj, new JsonSerializerOptions { WriteIndented = true }); + var jsonStream = new MemoryStream(Encoding.UTF8.GetBytes(jsonString)); + + var configurationBuilder = new ConfigurationBuilder() + .Add(new ScriptEnvironmentVariablesConfigurationSource()) + .AddJsonStream(jsonStream); + + return configurationBuilder.Build(); + } } } \ No newline at end of file From 2c2e8e3180aad57ec10445998975a6cdab770108 Mon Sep 17 00:00:00 2001 From: Surbhi Gupta Date: Mon, 28 Jul 2025 17:43:51 -0500 Subject: [PATCH 52/55] code cleanup and added comments in resolver options class --- .../Environment/EnvironmentExtensions.cs | 2 +- .../DefaultWorkerConfigurationResolver.cs | 10 ++--- .../DynamicWorkerConfigurationResolver.cs | 6 +-- .../LanguageWorkerOptionsSetup.cs | 1 - .../WorkerConfigurationResolverOptions.cs | 8 ++++ ...WorkerConfigurationResolverOptionsSetup.cs | 37 +++++++++---------- .../IWorkerConfigurationResolverFactory.cs | 19 ---------- 7 files changed, 32 insertions(+), 51 deletions(-) delete mode 100644 src/WebJobs.Script/Workers/Rpc/IWorkerConfigurationResolverFactory.cs diff --git a/src/WebJobs.Script/Environment/EnvironmentExtensions.cs b/src/WebJobs.Script/Environment/EnvironmentExtensions.cs index da51cdd059..14c417a786 100644 --- a/src/WebJobs.Script/Environment/EnvironmentExtensions.cs +++ b/src/WebJobs.Script/Environment/EnvironmentExtensions.cs @@ -285,7 +285,7 @@ public static bool IsWindowsElasticPremium(this IEnvironment environment) /// /// The environment to verify. /// if running in a Windows App Service app; otherwise, false. - public static bool IsWindowsEnvironment(this IEnvironment environment) + public static bool IsHostedWindowsEnvironment(this IEnvironment environment) { return environment.IsWindowsAzureManagedHosting() || environment.IsWindowsConsumption() || environment.IsWindowsElasticPremium(); } diff --git a/src/WebJobs.Script/Workers/Rpc/Configuration/DefaultWorkerConfigurationResolver.cs b/src/WebJobs.Script/Workers/Rpc/Configuration/DefaultWorkerConfigurationResolver.cs index 61be72fd94..46d89b0499 100644 --- a/src/WebJobs.Script/Workers/Rpc/Configuration/DefaultWorkerConfigurationResolver.cs +++ b/src/WebJobs.Script/Workers/Rpc/Configuration/DefaultWorkerConfigurationResolver.cs @@ -17,18 +17,16 @@ internal sealed class DefaultWorkerConfigurationResolver : IWorkerConfigurationR public DefaultWorkerConfigurationResolver(ILoggerFactory loggerFactory, IOptionsMonitor workerConfigurationResolverOptions) { - if (loggerFactory is null) - { - throw new ArgumentNullException(nameof(loggerFactory)); - } - - _logger = _logger = loggerFactory.CreateLogger(ScriptConstants.LogCategoryWorkerConfig); + _ = loggerFactory ?? throw new ArgumentNullException(nameof(loggerFactory)); + _logger = loggerFactory.CreateLogger(ScriptConstants.LogCategoryWorkerConfig); _workerConfigurationResolverOptions = workerConfigurationResolverOptions ?? throw new ArgumentNullException(nameof(workerConfigurationResolverOptions)); } public List GetWorkerConfigPaths() { + // "workers" directory path within the Host var workersDirPath = _workerConfigurationResolverOptions.CurrentValue.WorkersDirPath; + _logger.LogDebug("Workers Directory set to: {workersDirPath}", workersDirPath); List workerConfigs = new(); diff --git a/src/WebJobs.Script/Workers/Rpc/Configuration/DynamicWorkerConfigurationResolver.cs b/src/WebJobs.Script/Workers/Rpc/Configuration/DynamicWorkerConfigurationResolver.cs index 52aeb85bd9..e1ce2639a5 100644 --- a/src/WebJobs.Script/Workers/Rpc/Configuration/DynamicWorkerConfigurationResolver.cs +++ b/src/WebJobs.Script/Workers/Rpc/Configuration/DynamicWorkerConfigurationResolver.cs @@ -31,11 +31,7 @@ public DynamicWorkerConfigurationResolver(ILoggerFactory loggerFactory, IWorkerProfileManager workerProfileManager, IOptionsMonitor workerConfigResolverOptions) { - if (loggerFactory is null) - { - throw new ArgumentNullException(nameof(loggerFactory)); - } - + _ = loggerFactory ?? throw new ArgumentNullException(nameof(loggerFactory)); _logger = loggerFactory.CreateLogger(ScriptConstants.LogCategoryWorkerConfig); _fileSystem = fileSystem ?? throw new ArgumentNullException(nameof(fileSystem)); _profileManager = workerProfileManager ?? throw new ArgumentNullException(nameof(workerProfileManager)); diff --git a/src/WebJobs.Script/Workers/Rpc/Configuration/LanguageWorkerOptionsSetup.cs b/src/WebJobs.Script/Workers/Rpc/Configuration/LanguageWorkerOptionsSetup.cs index ffb39a70c5..f39b8b95ec 100644 --- a/src/WebJobs.Script/Workers/Rpc/Configuration/LanguageWorkerOptionsSetup.cs +++ b/src/WebJobs.Script/Workers/Rpc/Configuration/LanguageWorkerOptionsSetup.cs @@ -4,7 +4,6 @@ using System; using System.Collections.Generic; using System.Diagnostics; -using Microsoft.Azure.WebJobs.Script.Config; using Microsoft.Azure.WebJobs.Script.Diagnostics; using Microsoft.Azure.WebJobs.Script.Workers.Profiles; using Microsoft.Azure.WebJobs.Script.Workers.Rpc.Configuration; diff --git a/src/WebJobs.Script/Workers/Rpc/Configuration/WorkerConfigurationResolverOptions.cs b/src/WebJobs.Script/Workers/Rpc/Configuration/WorkerConfigurationResolverOptions.cs index 5dd233f9d4..31e34363e8 100644 --- a/src/WebJobs.Script/Workers/Rpc/Configuration/WorkerConfigurationResolverOptions.cs +++ b/src/WebJobs.Script/Workers/Rpc/Configuration/WorkerConfigurationResolverOptions.cs @@ -7,20 +7,28 @@ namespace Microsoft.Azure.WebJobs.Script.Workers.Rpc.Configuration { public class WorkerConfigurationResolverOptions { + // Gets or sets the value of platform release channel. public string ReleaseChannel { get; set; } + // Gets or sets the value of worker runtime. public string WorkerRuntime { get; set; } + // Gets or sets a value indicating whether placeholder mode is enabled. public bool IsPlaceholderModeEnabled { get; set; } + // Gets or sets a value indicating whether it is a multi-language worker environment. public bool IsMultiLanguageWorkerEnvironment { get; set; } + // Gets or sets the workers directory path within the Host. public string WorkersDirPath { get; set; } + // Gets or sets the list of probing paths for worker resolution. public List ProbingPaths { get; set; } + // Gets or sets the worker runtimes available for resolution via Hosting configuration. public HashSet WorkersAvailableForResolution { get; set; } + // Gets or sets the dictionary containing language workers related settings in configuration. public Dictionary LanguageWorkersSettings { get; set; } } } \ No newline at end of file diff --git a/src/WebJobs.Script/Workers/Rpc/Configuration/WorkerConfigurationResolverOptionsSetup.cs b/src/WebJobs.Script/Workers/Rpc/Configuration/WorkerConfigurationResolverOptionsSetup.cs index 1536e0861f..0a74684e3a 100644 --- a/src/WebJobs.Script/Workers/Rpc/Configuration/WorkerConfigurationResolverOptionsSetup.cs +++ b/src/WebJobs.Script/Workers/Rpc/Configuration/WorkerConfigurationResolverOptionsSetup.cs @@ -56,27 +56,10 @@ public void Configure(WorkerConfigurationResolverOptions options) options.LanguageWorkersSettings = GetLanguageWorkersSettings(configuration); } - internal Dictionary GetLanguageWorkersSettings(IConfiguration configuration) - { - // Convert the required configuration sections to Dictionary - var languageWorkersSettings = new Dictionary(); - - foreach (var kvp in configuration.AsEnumerable()) - { - if (kvp.Key.StartsWith(RpcWorkerConstants.LanguageWorkersSectionName)) - { - languageWorkersSettings[kvp.Key] = kvp.Value; - } - } - - return languageWorkersSettings; - } - internal List GetWorkerProbingPaths() { // If Configuration section is set, read probing paths from configuration. - IConfigurationSection probingPathsSection = _configuration.GetSection($"{RpcWorkerConstants.LanguageWorkersSectionName}") - ?.GetSection($"{RpcWorkerConstants.WorkerProbingPathsSectionName}"); + IConfigurationSection probingPathsSection = _configuration.GetSection($"{RpcWorkerConstants.LanguageWorkersSectionName}")?.GetSection($"{RpcWorkerConstants.WorkerProbingPathsSectionName}"); var probingPathsList = probingPathsSection?.AsEnumerable(); @@ -95,7 +78,7 @@ internal List GetWorkerProbingPaths() } else { - if (_environment.IsWindowsEnvironment()) + if (_environment.IsHostedWindowsEnvironment()) { // Harcoded site extensions path for Windows until Antares sets it as an Environment variable. // Example probing path for Windows: "c:\\home\\SiteExtensions\\workers" @@ -125,5 +108,21 @@ internal static string GetWindowsSiteExtensionsPath() //Move 2 directories up to get to the SiteExtensions directory return Directory.GetParent(assemblyDir)?.Parent?.FullName; } + + internal Dictionary GetLanguageWorkersSettings(IConfiguration configuration) + { + // Convert the required configuration sections to Dictionary + var languageWorkersSettings = new Dictionary(); + + foreach (var kvp in configuration.AsEnumerable()) + { + if (kvp.Key.StartsWith(RpcWorkerConstants.LanguageWorkersSectionName)) + { + languageWorkersSettings[kvp.Key] = kvp.Value; + } + } + + return languageWorkersSettings; + } } } \ No newline at end of file diff --git a/src/WebJobs.Script/Workers/Rpc/IWorkerConfigurationResolverFactory.cs b/src/WebJobs.Script/Workers/Rpc/IWorkerConfigurationResolverFactory.cs deleted file mode 100644 index 6a62fe0895..0000000000 --- a/src/WebJobs.Script/Workers/Rpc/IWorkerConfigurationResolverFactory.cs +++ /dev/null @@ -1,19 +0,0 @@ -// Copyright (c) .NET Foundation. All rights reserved. -// Licensed under the MIT License. See License.txt in the project root for license information. - -using Microsoft.Azure.WebJobs.Script.Workers.Rpc.Configuration; - -namespace Microsoft.Azure.WebJobs.Script.Workers.Rpc -{ - /// - /// Factory interface for creating worker configuration resolvers. - /// - internal interface IWorkerConfigurationResolverFactory - { - /// - /// Creates an appropriate worker configuration resolver based on the current environment and settings. - /// - /// An implementation of . - IWorkerConfigurationResolver CreateResolver(); - } -} \ No newline at end of file From 779fd2d9631f07f50c92bd69148fd0eee4356a52 Mon Sep 17 00:00:00 2001 From: Surbhi Gupta Date: Mon, 28 Jul 2025 22:07:11 -0500 Subject: [PATCH 53/55] Using Options to check if Dynamic worker resolution is enabled --- .../WebHostServiceCollectionExtensions.cs | 2 +- .../Environment/EnvironmentExtensions.cs | 33 ----------- src/WebJobs.Script/Utility.cs | 23 ++++++++ .../Extensions/EnvironmentExtensionsTests.cs | 54 ------------------ test/WebJobs.Script.Tests/UtilityTests.cs | 55 +++++++++++++++++++ 5 files changed, 79 insertions(+), 88 deletions(-) diff --git a/src/WebJobs.Script.WebHost/WebHostServiceCollectionExtensions.cs b/src/WebJobs.Script.WebHost/WebHostServiceCollectionExtensions.cs index 3e00032b3d..398de5b0d3 100644 --- a/src/WebJobs.Script.WebHost/WebHostServiceCollectionExtensions.cs +++ b/src/WebJobs.Script.WebHost/WebHostServiceCollectionExtensions.cs @@ -242,7 +242,7 @@ public static void AddWebJobsScriptHost(this IServiceCollection services, IConfi var workerProfileManager = p.GetService(); var loggerFactory = p.GetService(); - return environment.IsDynamicWorkerResolutionEnabled(workerConfigurationResolverOptions) ? + return Utility.IsDynamicWorkerResolutionEnabled(environment, workerConfigurationResolverOptions) ? new DynamicWorkerConfigurationResolver(loggerFactory, FileUtility.Instance, workerProfileManager, workerConfigurationResolverOptions) : new DefaultWorkerConfigurationResolver(loggerFactory, workerConfigurationResolverOptions); }); diff --git a/src/WebJobs.Script/Environment/EnvironmentExtensions.cs b/src/WebJobs.Script/Environment/EnvironmentExtensions.cs index 14c417a786..5728a97727 100644 --- a/src/WebJobs.Script/Environment/EnvironmentExtensions.cs +++ b/src/WebJobs.Script/Environment/EnvironmentExtensions.cs @@ -10,8 +10,6 @@ using Microsoft.Azure.WebJobs.Script.Config; using Microsoft.Azure.WebJobs.Script.Description; using Microsoft.Azure.WebJobs.Script.Workers.Rpc; -using Microsoft.Azure.WebJobs.Script.Workers.Rpc.Configuration; -using Microsoft.Extensions.Options; using static Microsoft.Azure.WebJobs.Script.EnvironmentSettingNames; using static Microsoft.Azure.WebJobs.Script.Utility; @@ -704,37 +702,6 @@ public static string GetPlatformReleaseChannel(this IEnvironment environment) return environment.GetEnvironmentVariable(AntaresPlatformReleaseChannel) ?? ScriptConstants.LatestPlatformChannelNameUpper; } - /// - /// Checks if the Dynamic Worker Resolution feature is disabled via feature flag. - /// Returns true if the feature is disabled, false otherwise. - /// - public static bool IsWorkerResolutionFeatureDisabled(this IEnvironment environment) - { - return FeatureFlags.IsEnabled(ScriptConstants.FeatureFlagDisableDynamicWorkerResolution, environment); - } - - // Users can disable dynamic worker resolution via setting the appropriate feature flag. - // Worker resolution can be enabled for specific workers at the stamp level via hosting config options. - // Feature flag takes precedence over hosting config options. - public static bool IsDynamicWorkerResolutionEnabled(this IEnvironment environment, IOptionsMonitor options) - { - if (environment.IsWorkerResolutionFeatureDisabled() || options.CurrentValue.WorkersAvailableForResolution is null) - { - return false; - } - - string workerRuntime = environment.GetEnvironmentVariable(RpcWorkerConstants.FunctionWorkerRuntimeSettingName); - - if (!environment.IsMultiLanguageRuntimeEnvironment() && - !string.IsNullOrWhiteSpace(workerRuntime) && - !environment.IsPlaceholderModeEnabled()) - { - return options.CurrentValue.WorkersAvailableForResolution.Contains(workerRuntime); - } - - return options.CurrentValue.WorkersAvailableForResolution.Any(); - } - public static bool IsApplicationInsightsAgentEnabled(this IEnvironment environment) { // cache the value of the environment variable diff --git a/src/WebJobs.Script/Utility.cs b/src/WebJobs.Script/Utility.cs index 30c343535d..c69717dae3 100644 --- a/src/WebJobs.Script/Utility.cs +++ b/src/WebJobs.Script/Utility.cs @@ -23,6 +23,7 @@ using Microsoft.Azure.WebJobs.Script.Diagnostics.Extensions; using Microsoft.Azure.WebJobs.Script.Models; using Microsoft.Azure.WebJobs.Script.Workers.Rpc; +using Microsoft.Azure.WebJobs.Script.Workers.Rpc.Configuration; using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Logging; using Microsoft.Extensions.Options; @@ -1065,6 +1066,28 @@ public static bool CanWorkerIndex(IEnumerable workerConfigs, IE return workerIndexingEnabled && workerIndexingAvailable; } + // Users can disable dynamic worker resolution via setting the appropriate feature flag. + // Worker resolution can be enabled for specific workers at the stamp level via hosting config options. + // Feature flag takes precedence over hosting config options. + public static bool IsDynamicWorkerResolutionEnabled(IEnvironment environment, IOptionsMonitor options) + { + if (FeatureFlags.IsEnabled(ScriptConstants.FeatureFlagDisableDynamicWorkerResolution, environment) || options.CurrentValue.WorkersAvailableForResolution is null) + { + return false; + } + + string workerRuntime = options.CurrentValue.WorkerRuntime; + + if (!options.CurrentValue.IsMultiLanguageWorkerEnvironment && + !string.IsNullOrWhiteSpace(workerRuntime) && + !options.CurrentValue.IsPlaceholderModeEnabled) + { + return options.CurrentValue.WorkersAvailableForResolution.Contains(workerRuntime); + } + + return options.CurrentValue.WorkersAvailableForResolution.Any(); + } + public static void LogAutorestGeneratedJsonIfExists(string rootScriptPath, ILogger logger) { string autorestGeneratedJsonPath = Path.Combine(rootScriptPath, ScriptConstants.AutorestGeenratedMetadataFileName); diff --git a/test/WebJobs.Script.Tests/Extensions/EnvironmentExtensionsTests.cs b/test/WebJobs.Script.Tests/Extensions/EnvironmentExtensionsTests.cs index d40adfc3d6..f48469e3ca 100644 --- a/test/WebJobs.Script.Tests/Extensions/EnvironmentExtensionsTests.cs +++ b/test/WebJobs.Script.Tests/Extensions/EnvironmentExtensionsTests.cs @@ -553,59 +553,5 @@ public void IsInValidationMode_ReturnsExpectedResult(string functionGroup, bool Assert.Equal(expected, env.IsInValidationMode()); } - - [Theory] - [InlineData(null, "node", true)] - [InlineData(null, "java|node", true)] - [InlineData(null, "", false)] - [InlineData(null, "| ", false)] - [InlineData(null, null, false)] - [InlineData(ScriptConstants.FeatureFlagDisableDynamicWorkerResolution, "node", false)] - [InlineData(ScriptConstants.FeatureFlagDisableDynamicWorkerResolution, "java|node", false)] - [InlineData(ScriptConstants.FeatureFlagDisableDynamicWorkerResolution, "| ", false)] - - public void IsDynamicWorkerResolutionEnabled_HostingConfigAndFeatureFlags_WorksAsExpected(string featureFlagValue, string hostingConfigSetting, bool expected) - { - var mockConfiguration = new Mock(); - var mockScriptHostManager = new Mock(); - - var hostingOptions = new FunctionsHostingConfigOptions(); - hostingOptions.Features.Add(RpcWorkerConstants.WorkersAvailableForDynamicResolution, hostingConfigSetting); - - var testEnvironment = new TestEnvironment(); - testEnvironment.SetEnvironmentVariable(AzureWebJobsFeatureFlags, featureFlagValue); - - var optionsMonitor = WorkerConfigurationResolverTestsHelper.GetTestWorkerConfigurationResolverOptions(mockConfiguration.Object, testEnvironment, mockScriptHostManager.Object, new OptionsWrapper(hostingOptions)); - - bool result = testEnvironment.IsDynamicWorkerResolutionEnabled(optionsMonitor); - - Assert.Equal(expected, result); - } - - [Theory] - [InlineData("node", "node", null, true)] - [InlineData("node", "java", null, false)] - [InlineData("java|node", null, null, true)] - [InlineData("node", "node", "workflowapp", true)] - [InlineData("java|node", null, "workflowapp", true)] - [InlineData("| ", null, "workflowapp", false)] - public void IsDynamicWorkerResolutionEnabled_WorkerRuntimeAndMultiLanguage_WorksAsExpected(string hostingConfigSetting, string workerRuntime, string multilanguageApp, bool expected) - { - var mockConfiguration = new Mock(); - var mockScriptHostManager = new Mock(); - - var hostingOptions = new FunctionsHostingConfigOptions(); - hostingOptions.Features.Add(RpcWorkerConstants.WorkersAvailableForDynamicResolution, hostingConfigSetting); - - var testEnvironment = new TestEnvironment(); - testEnvironment.SetEnvironmentVariable(AppKind, multilanguageApp); - testEnvironment.SetEnvironmentVariable(FunctionWorkerRuntime, workerRuntime); - - var optionsMonitor = WorkerConfigurationResolverTestsHelper.GetTestWorkerConfigurationResolverOptions(mockConfiguration.Object, testEnvironment, mockScriptHostManager.Object, new OptionsWrapper(hostingOptions)); - - bool result = testEnvironment.IsDynamicWorkerResolutionEnabled(optionsMonitor); - - Assert.Equal(expected, result); - } } } diff --git a/test/WebJobs.Script.Tests/UtilityTests.cs b/test/WebJobs.Script.Tests/UtilityTests.cs index 4a44afc296..849a2e4aab 100644 --- a/test/WebJobs.Script.Tests/UtilityTests.cs +++ b/test/WebJobs.Script.Tests/UtilityTests.cs @@ -16,6 +16,7 @@ using Microsoft.Azure.WebJobs.Script.Diagnostics; using Microsoft.Azure.WebJobs.Script.Models; using Microsoft.Azure.WebJobs.Script.Workers.Rpc; +using Microsoft.Extensions.Configuration; using Microsoft.Extensions.Logging; using Microsoft.Extensions.Options; using Microsoft.WebJobs.Script.Tests; @@ -1016,6 +1017,60 @@ public void WorkerIndexingDecisionLogic_NullWorkerIndexingProperty(bool workerIn Assert.Equal(expected, workerShouldIndex); } + [Theory] + [InlineData(null, "node", true)] + [InlineData(null, "java|node", true)] + [InlineData(null, "", false)] + [InlineData(null, "| ", false)] + [InlineData(null, null, false)] + [InlineData(ScriptConstants.FeatureFlagDisableDynamicWorkerResolution, "node", false)] + [InlineData(ScriptConstants.FeatureFlagDisableDynamicWorkerResolution, "java|node", false)] + [InlineData(ScriptConstants.FeatureFlagDisableDynamicWorkerResolution, "| ", false)] + + public void IsDynamicWorkerResolutionEnabled_HostingConfigAndFeatureFlags_WorksAsExpected(string featureFlagValue, string hostingConfigSetting, bool expected) + { + var mockConfiguration = new Mock(); + var mockScriptHostManager = new Mock(); + + var hostingOptions = new FunctionsHostingConfigOptions(); + hostingOptions.Features.Add(RpcWorkerConstants.WorkersAvailableForDynamicResolution, hostingConfigSetting); + + var testEnvironment = new TestEnvironment(); + testEnvironment.SetEnvironmentVariable(EnvironmentSettingNames.AzureWebJobsFeatureFlags, featureFlagValue); + + var optionsMonitor = WorkerConfigurationResolverTestsHelper.GetTestWorkerConfigurationResolverOptions(mockConfiguration.Object, testEnvironment, mockScriptHostManager.Object, new OptionsWrapper(hostingOptions)); + + bool result = Utility.IsDynamicWorkerResolutionEnabled(testEnvironment, optionsMonitor); + + Assert.Equal(expected, result); + } + + [Theory] + [InlineData("node", "node", null, true)] + [InlineData("node", "java", null, false)] + [InlineData("java|node", null, null, true)] + [InlineData("node", "node", "workflowapp", true)] + [InlineData("java|node", null, "workflowapp", true)] + [InlineData("| ", null, "workflowapp", false)] + public void IsDynamicWorkerResolutionEnabled_WorkerRuntimeAndMultiLanguage_WorksAsExpected(string hostingConfigSetting, string workerRuntime, string multilanguageApp, bool expected) + { + var mockConfiguration = new Mock(); + var mockScriptHostManager = new Mock(); + + var hostingOptions = new FunctionsHostingConfigOptions(); + hostingOptions.Features.Add(RpcWorkerConstants.WorkersAvailableForDynamicResolution, hostingConfigSetting); + + var testEnvironment = new TestEnvironment(); + testEnvironment.SetEnvironmentVariable(EnvironmentSettingNames.AppKind, multilanguageApp); + testEnvironment.SetEnvironmentVariable(EnvironmentSettingNames.FunctionWorkerRuntime, workerRuntime); + + var optionsMonitor = WorkerConfigurationResolverTestsHelper.GetTestWorkerConfigurationResolverOptions(mockConfiguration.Object, testEnvironment, mockScriptHostManager.Object, new OptionsWrapper(hostingOptions)); + + bool result = Utility.IsDynamicWorkerResolutionEnabled(testEnvironment, optionsMonitor); + + Assert.Equal(expected, result); + } + [Theory] [InlineData("True", true, true)] [InlineData("False", false, true)] From 4e03f808100729832dd2397c2ac60b30111ff038 Mon Sep 17 00:00:00 2001 From: Surbhi Gupta Date: Tue, 29 Jul 2025 10:23:52 -0500 Subject: [PATCH 54/55] Tests cleanup --- .../LanguageWorkerOptionsSetupTests.cs | 31 ++--------- .../Extensions/EnvironmentExtensionsTests.cs | 5 -- .../WorkerConfigurationResolverTestsHelper.cs | 43 ++++++++++++++- .../Rpc/RpcWorkerConfigFactoryTests.cs | 55 +++++-------------- .../Workers/Rpc/RpcWorkerConfigTests.cs | 9 +-- .../Rpc/WorkerConfigurationResolverTests.cs | 26 +-------- 6 files changed, 67 insertions(+), 102 deletions(-) diff --git a/test/WebJobs.Script.Tests/Configuration/LanguageWorkerOptionsSetupTests.cs b/test/WebJobs.Script.Tests/Configuration/LanguageWorkerOptionsSetupTests.cs index f99e64eb5e..b1a582cd50 100644 --- a/test/WebJobs.Script.Tests/Configuration/LanguageWorkerOptionsSetupTests.cs +++ b/test/WebJobs.Script.Tests/Configuration/LanguageWorkerOptionsSetupTests.cs @@ -32,9 +32,7 @@ public class LanguageWorkerOptionsSetupTests [InlineData("node")] public void LanguageWorkerOptions_Expected_ListOfConfigs(string workerRuntime) { - var loggerProvider = new TestLoggerProvider(); - var loggerFactory = new LoggerFactory(); - loggerFactory.AddProvider(loggerProvider); + var loggerFactory = WorkerConfigurationResolverTestsHelper.GetTestLoggerFactory(); var testEnvironment = new TestEnvironment(); var testMetricLogger = new TestMetricsLogger(); var configurationBuilder = new ConfigurationBuilder() @@ -68,13 +66,13 @@ public void LanguageWorkerOptions_Expected_ListOfConfigs(string workerRuntime) } }); - var hostingOptions = new FunctionsHostingConfigOptions(); - var optionsMonitor = WorkerConfigurationResolverTestsHelper.GetTestWorkerConfigurationResolverOptions(configuration, testEnvironment, testScriptHostManager.Object, new OptionsWrapper(hostingOptions)); + var optionsMonitor = WorkerConfigurationResolverTestsHelper.GetTestWorkerConfigurationResolverOptions(configuration, testEnvironment, testScriptHostManager.Object, null); var resolver = new DefaultWorkerConfigurationResolver(loggerFactory, optionsMonitor); - LanguageWorkerOptionsSetup setup = new LanguageWorkerOptionsSetup(configuration, loggerFactory, testEnvironment, testMetricLogger, testProfileManager.Object, testScriptHostManager.Object, optionsMonitor, resolver); - LanguageWorkerOptions options = new LanguageWorkerOptions(); + var setup = new LanguageWorkerOptionsSetup(configuration, loggerFactory, testEnvironment, testMetricLogger, testProfileManager.Object, testScriptHostManager.Object, optionsMonitor, resolver); + var options = new LanguageWorkerOptions(); + setup.Configure(options); if (string.IsNullOrEmpty(workerRuntime)) @@ -108,29 +106,12 @@ public void LanguageWorkerOptions_EnabledWorkerResolution_Expected_ListOfConfigs var testMetricLogger = new TestMetricsLogger(); var testProfileManager = new Mock(); var testScriptHostManager = new Mock(); - string probingPathValue = string.Join(';', _probingPath1, string.Empty, "path-not-exists"); testEnvironment.SetEnvironmentVariable(RpcWorkerConstants.FunctionWorkerRuntimeSettingName, workerRuntime); testEnvironment.SetEnvironmentVariable(EnvironmentSettingNames.AntaresPlatformReleaseChannel, releaseChannel); var probingPaths = new List() { _probingPath1, string.Empty, "path-not-exists" }; - - var jsonObj = new - { - languageWorkers = new - { - probingPaths - } - }; - - var jsonString = JsonSerializer.Serialize(jsonObj, new JsonSerializerOptions { WriteIndented = true }); - var jsonStream = new MemoryStream(Encoding.UTF8.GetBytes(jsonString)); - - var configurationBuilder = new ConfigurationBuilder() - .Add(new ScriptEnvironmentVariablesConfigurationSource()) - .AddJsonStream(jsonStream); - - var configuration = configurationBuilder.Build(); + var configuration = WorkerConfigurationResolverTestsHelper.GetConfigurationWithProbingPaths(probingPaths); var hostingOptions = new FunctionsHostingConfigOptions(); hostingOptions.Features.Add(RpcWorkerConstants.WorkersAvailableForDynamicResolution, hostingOptionsSetting); diff --git a/test/WebJobs.Script.Tests/Extensions/EnvironmentExtensionsTests.cs b/test/WebJobs.Script.Tests/Extensions/EnvironmentExtensionsTests.cs index f48469e3ca..9ea8bcf5d5 100644 --- a/test/WebJobs.Script.Tests/Extensions/EnvironmentExtensionsTests.cs +++ b/test/WebJobs.Script.Tests/Extensions/EnvironmentExtensionsTests.cs @@ -3,14 +3,9 @@ using System; using System.Collections.Generic; -using Microsoft.Azure.WebJobs.Script.Config; using Microsoft.Azure.WebJobs.Script.Description; using Microsoft.Azure.WebJobs.Script.Workers.Rpc; -using Microsoft.Azure.WebJobs.Script.Workers.Rpc.Configuration; -using Microsoft.Extensions.Configuration; -using Microsoft.Extensions.Options; using Microsoft.WebJobs.Script.Tests; -using Moq; using Xunit; using static Microsoft.Azure.WebJobs.Script.EnvironmentSettingNames; diff --git a/test/WebJobs.Script.Tests/WorkerConfigurationResolverTestsHelper.cs b/test/WebJobs.Script.Tests/WorkerConfigurationResolverTestsHelper.cs index 8592100b21..82005bc090 100644 --- a/test/WebJobs.Script.Tests/WorkerConfigurationResolverTestsHelper.cs +++ b/test/WebJobs.Script.Tests/WorkerConfigurationResolverTestsHelper.cs @@ -1,10 +1,16 @@ // Copyright (c) .NET Foundation. All rights reserved. // Licensed under the MIT License. See License.txt in the project root for license information. +using System.Collections.Generic; +using System.IO; +using System.Text; +using System.Text.Json; using Microsoft.Azure.WebJobs.Script.Config; using Microsoft.Azure.WebJobs.Script.Workers.Rpc.Configuration; using Microsoft.Extensions.Configuration; +using Microsoft.Extensions.Logging; using Microsoft.Extensions.Options; +using Microsoft.WebJobs.Script.Tests; namespace Microsoft.Azure.WebJobs.Script.Tests { @@ -13,8 +19,14 @@ internal static class WorkerConfigurationResolverTestsHelper internal static IOptionsMonitor GetTestWorkerConfigurationResolverOptions(IConfiguration configuration, IEnvironment environment, IScriptHostManager scriptHostManager, - IOptions functionsHostingConfigOptions) + IOptions functionsHostingConfigOptions = null) { + if (functionsHostingConfigOptions is null) + { + var hostingOptions = new FunctionsHostingConfigOptions(); + functionsHostingConfigOptions = new OptionsWrapper(hostingOptions); + } + var resolverOptionssetup = new WorkerConfigurationResolverOptionsSetup(configuration, environment, scriptHostManager, functionsHostingConfigOptions); var resolverOptions = new WorkerConfigurationResolverOptions(); resolverOptionssetup.Configure(resolverOptions); @@ -26,5 +38,34 @@ internal static IOptionsMonitor GetTestWorke return optionsMonitor; } + + internal static IConfiguration GetConfigurationWithProbingPaths(List probingPaths) + { + var jsonObj = new + { + languageWorkers = new + { + probingPaths + } + }; + + var jsonString = JsonSerializer.Serialize(jsonObj, new JsonSerializerOptions { WriteIndented = true }); + var jsonStream = new MemoryStream(Encoding.UTF8.GetBytes(jsonString)); + + var configurationBuilder = new ConfigurationBuilder() + .Add(new ScriptEnvironmentVariablesConfigurationSource()) + .AddJsonStream(jsonStream); + + return configurationBuilder.Build(); + } + + internal static LoggerFactory GetTestLoggerFactory() + { + var loggerProvider = new TestLoggerProvider(); + var loggerFactory = new LoggerFactory(); + loggerFactory.AddProvider(loggerProvider); + + return loggerFactory; + } } } diff --git a/test/WebJobs.Script.Tests/Workers/Rpc/RpcWorkerConfigFactoryTests.cs b/test/WebJobs.Script.Tests/Workers/Rpc/RpcWorkerConfigFactoryTests.cs index 7b69c1c451..aedaa019ed 100644 --- a/test/WebJobs.Script.Tests/Workers/Rpc/RpcWorkerConfigFactoryTests.cs +++ b/test/WebJobs.Script.Tests/Workers/Rpc/RpcWorkerConfigFactoryTests.cs @@ -46,9 +46,7 @@ public void DefaultLanguageWorkersDir() var testLogger = new TestLogger("test"); var testScriptHostManager = new Mock(); - var hostingOptions = new FunctionsHostingConfigOptions(); - - var optionsMonitor = WorkerConfigurationResolverTestsHelper.GetTestWorkerConfigurationResolverOptions(config, _testEnvironment, testScriptHostManager.Object, new OptionsWrapper(hostingOptions)); + var optionsMonitor = WorkerConfigurationResolverTestsHelper.GetTestWorkerConfigurationResolverOptions(config, _testEnvironment, testScriptHostManager.Object, null); var mockLogger = new Mock(); var workerConfigurationResolver = new DefaultWorkerConfigurationResolver(mockLogger.Object, optionsMonitor); @@ -87,10 +85,8 @@ public void LanguageWorker_WorkersDir_Set() var testLogger = new TestLogger("test"); var testScriptHostManager = new Mock(); - var hostingOptions = new FunctionsHostingConfigOptions(); var mockLogger = new Mock(); - - var optionsMonitor = WorkerConfigurationResolverTestsHelper.GetTestWorkerConfigurationResolverOptions(config, _testEnvironment, testScriptHostManager.Object, new OptionsWrapper(hostingOptions)); + var optionsMonitor = WorkerConfigurationResolverTestsHelper.GetTestWorkerConfigurationResolverOptions(config, _testEnvironment, testScriptHostManager.Object, null); var workerConfigurationResolver = new DefaultWorkerConfigurationResolver(mockLogger.Object, optionsMonitor); var configFactory = new RpcWorkerConfigFactory(config, testLogger, _testSysRuntimeInfo, _testEnvironment, new TestMetricsLogger(), _testWorkerProfileManager, workerConfigurationResolver, optionsMonitor); @@ -111,8 +107,7 @@ public void LanguageWorker_WorkersDir_NotSet() var testLogger = new TestLogger("test"); var testScriptHostManager = new Mock(); - var hostingOptions = new FunctionsHostingConfigOptions(); - var optionsMonitor = WorkerConfigurationResolverTestsHelper.GetTestWorkerConfigurationResolverOptions(config, _testEnvironment, testScriptHostManager.Object, new OptionsWrapper(hostingOptions)); + var optionsMonitor = WorkerConfigurationResolverTestsHelper.GetTestWorkerConfigurationResolverOptions(config, _testEnvironment, testScriptHostManager.Object, null); var mockLogger = new Mock(); var workerConfigurationResolver = new DefaultWorkerConfigurationResolver(mockLogger.Object, optionsMonitor); @@ -141,14 +136,9 @@ public void WorkerDescription_Skipped_When_Profile_Disables_Worker() var testLogger = new TestLogger("test"); _testEnvironment.SetEnvironmentVariable("ENV_VAR_BAR", "True"); - var loggerProvider = new TestLoggerProvider(); - var loggerFactory = new LoggerFactory(); - loggerFactory.AddProvider(loggerProvider); - + var loggerFactory = WorkerConfigurationResolverTestsHelper.GetTestLoggerFactory(); var testScriptHostManager = new Mock(); - - var hostingOptions = new FunctionsHostingConfigOptions(); - var optionsMonitor = WorkerConfigurationResolverTestsHelper.GetTestWorkerConfigurationResolverOptions(config, _testEnvironment, testScriptHostManager.Object, new OptionsWrapper(hostingOptions)); + var optionsMonitor = WorkerConfigurationResolverTestsHelper.GetTestWorkerConfigurationResolverOptions(config, _testEnvironment, testScriptHostManager.Object); var workerConfigurationResolver = new DefaultWorkerConfigurationResolver(loggerFactory, optionsMonitor); var configFactory = new RpcWorkerConfigFactory(config, testLogger, _testSysRuntimeInfo, _testEnvironment, new TestMetricsLogger(), _testWorkerProfileManager, workerConfigurationResolver, optionsMonitor); @@ -173,15 +163,10 @@ public void JavaPath_FromEnvVars() var config = configBuilder.Build(); var scriptSettingsManager = new ScriptSettingsManager(config); - var loggerProvider = new TestLoggerProvider(); - var loggerFactory = new LoggerFactory(); - loggerFactory.AddProvider(loggerProvider); + var loggerFactory = WorkerConfigurationResolverTestsHelper.GetTestLoggerFactory(); var testLogger = loggerFactory.CreateLogger("test"); - var testScriptHostManager = new Mock(); - - var hostingOptions = new FunctionsHostingConfigOptions(); - var optionsMonitor = WorkerConfigurationResolverTestsHelper.GetTestWorkerConfigurationResolverOptions(config, _testEnvironment, testScriptHostManager.Object, new OptionsWrapper(hostingOptions)); + var optionsMonitor = WorkerConfigurationResolverTestsHelper.GetTestWorkerConfigurationResolverOptions(config, _testEnvironment, testScriptHostManager.Object); var workerConfigurationResolver = new DefaultWorkerConfigurationResolver(loggerFactory, optionsMonitor); var configFactory = new RpcWorkerConfigFactory(config, testLogger, _testSysRuntimeInfo, _testEnvironment, new TestMetricsLogger(), _testWorkerProfileManager, workerConfigurationResolver, optionsMonitor); @@ -213,9 +198,7 @@ public void DefaultWorkerConfigs_Overrides_DefaultWorkerRuntimeVersion_AppSettin using var variables = new TestScopedSettings(scriptSettingsManager, testEnvVariables); var testScriptHostManager = new Mock(); - - var hostingOptions = new FunctionsHostingConfigOptions(); - var optionsMonitor = WorkerConfigurationResolverTestsHelper.GetTestWorkerConfigurationResolverOptions(config, _testEnvironment, testScriptHostManager.Object, new OptionsWrapper(hostingOptions)); + var optionsMonitor = WorkerConfigurationResolverTestsHelper.GetTestWorkerConfigurationResolverOptions(config, _testEnvironment, testScriptHostManager.Object, null); var workerConfigurationResolver = new DefaultWorkerConfigurationResolver(loggerFactory, optionsMonitor); var configFactory = new RpcWorkerConfigFactory(config, testLogger, _testSysRuntimeInfo, _testEnvironment, new TestMetricsLogger(), _testWorkerProfileManager, workerConfigurationResolver, optionsMonitor); @@ -239,15 +222,10 @@ public void DefaultWorkerConfigs_Overrides_VersionAppSetting() var config = configBuilder.Build(); var scriptSettingsManager = new ScriptSettingsManager(config); - var loggerProvider = new TestLoggerProvider(); - var loggerFactory = new LoggerFactory(); - loggerFactory.AddProvider(loggerProvider); + var loggerFactory = WorkerConfigurationResolverTestsHelper.GetTestLoggerFactory(); var testLogger = loggerFactory.CreateLogger("test"); - var testScriptHostManager = new Mock(); - - var hostingOptions = new FunctionsHostingConfigOptions(); - var optionsMonitor = WorkerConfigurationResolverTestsHelper.GetTestWorkerConfigurationResolverOptions(config, _testEnvironment, testScriptHostManager.Object, new OptionsWrapper(hostingOptions)); + var optionsMonitor = WorkerConfigurationResolverTestsHelper.GetTestWorkerConfigurationResolverOptions(config, _testEnvironment, testScriptHostManager.Object, null); var resolver = new DefaultWorkerConfigurationResolver(loggerFactory, optionsMonitor); var configFactory = new RpcWorkerConfigFactory(config, testLogger, _testSysRuntimeInfo, testEnvironment, new TestMetricsLogger(), _testWorkerProfileManager, resolver, optionsMonitor); @@ -277,14 +255,11 @@ public void ShouldAddProvider_Returns_Expected(string workerLanguage, string wor } var config = new ConfigurationBuilder().Build(); - var loggerProvider = new TestLoggerProvider(); - var loggerFactory = new LoggerFactory(); - loggerFactory.AddProvider(loggerProvider); + var loggerFactory = WorkerConfigurationResolverTestsHelper.GetTestLoggerFactory(); var testLogger = loggerFactory.CreateLogger("test"); var testScriptHostManager = new Mock(); - var hostingOptions = new FunctionsHostingConfigOptions(); - var optionsMonitor = WorkerConfigurationResolverTestsHelper.GetTestWorkerConfigurationResolverOptions(config, _testEnvironment, testScriptHostManager.Object, new OptionsWrapper(hostingOptions)); + var optionsMonitor = WorkerConfigurationResolverTestsHelper.GetTestWorkerConfigurationResolverOptions(config, _testEnvironment, testScriptHostManager.Object, null); var workerConfigurationResolver = new DefaultWorkerConfigurationResolver(loggerFactory, optionsMonitor); var configFactory = new RpcWorkerConfigFactory(config, testLogger, _testSysRuntimeInfo, _testEnvironment, new TestMetricsLogger(), _testWorkerProfileManager, workerConfigurationResolver, optionsMonitor); @@ -332,8 +307,7 @@ public void GetWorkerProcessCount_Tests(bool defaultWorkerConfig, bool setProces var testLogger = new TestLogger("test"); var testScriptHostManager = new Mock(); - var hostingOptions = new FunctionsHostingConfigOptions(); - var optionsMonitor = WorkerConfigurationResolverTestsHelper.GetTestWorkerConfigurationResolverOptions(config, _testEnvironment, testScriptHostManager.Object, new OptionsWrapper(hostingOptions)); + var optionsMonitor = WorkerConfigurationResolverTestsHelper.GetTestWorkerConfigurationResolverOptions(config, _testEnvironment, testScriptHostManager.Object, null); var mockLogger = new Mock(); var workerConfigurationResolver = new DefaultWorkerConfigurationResolver(mockLogger.Object, optionsMonitor); @@ -379,8 +353,7 @@ public void GetWorkerProcessCount_ThrowsException_Tests() var testLogger = new TestLogger("test"); var testScriptHostManager = new Mock(); - var hostingOptions = new FunctionsHostingConfigOptions(); - var optionsMonitor = WorkerConfigurationResolverTestsHelper.GetTestWorkerConfigurationResolverOptions(config, _testEnvironment, testScriptHostManager.Object, new OptionsWrapper(hostingOptions)); + var optionsMonitor = WorkerConfigurationResolverTestsHelper.GetTestWorkerConfigurationResolverOptions(config, _testEnvironment, testScriptHostManager.Object, null); var mockLogger = new Mock(); var workerConfigurationResolver = new DefaultWorkerConfigurationResolver(mockLogger.Object, optionsMonitor); diff --git a/test/WebJobs.Script.Tests/Workers/Rpc/RpcWorkerConfigTests.cs b/test/WebJobs.Script.Tests/Workers/Rpc/RpcWorkerConfigTests.cs index bdb78bdbcf..37ed5508cf 100644 --- a/test/WebJobs.Script.Tests/Workers/Rpc/RpcWorkerConfigTests.cs +++ b/test/WebJobs.Script.Tests/Workers/Rpc/RpcWorkerConfigTests.cs @@ -687,15 +687,10 @@ private IEnumerable TestReadWorkerProviderFromConfig(IEnumerabl var scriptHostOptions = new ScriptJobHostOptions(); var scriptSettingsManager = new ScriptSettingsManager(config); var workerProfileManager = new Mock(); - var testScriptHostManager = new Mock(); + var loggerFactory = WorkerConfigurationResolverTestsHelper.GetTestLoggerFactory(); - var loggerProvider = new TestLoggerProvider(); - var loggerFactory = new LoggerFactory(); - loggerFactory.AddProvider(loggerProvider); - - var hostingOptions = new FunctionsHostingConfigOptions(); - var optionsMonitor = WorkerConfigurationResolverTestsHelper.GetTestWorkerConfigurationResolverOptions(config, _testEnvironment, testScriptHostManager.Object, new OptionsWrapper(hostingOptions)); + var optionsMonitor = WorkerConfigurationResolverTestsHelper.GetTestWorkerConfigurationResolverOptions(config, _testEnvironment, testScriptHostManager.Object, null); IWorkerConfigurationResolver workerConfigurationResolver = new DefaultWorkerConfigurationResolver(loggerFactory, optionsMonitor); var configFactory = new RpcWorkerConfigFactory(config, testLogger, _testSysRuntimeInfo, _testEnvironment, new TestMetricsLogger(), workerProfileManager.Object, workerConfigurationResolver, optionsMonitor); diff --git a/test/WebJobs.Script.Tests/Workers/Rpc/WorkerConfigurationResolverTests.cs b/test/WebJobs.Script.Tests/Workers/Rpc/WorkerConfigurationResolverTests.cs index 5bf3da45de..51b8f071ab 100644 --- a/test/WebJobs.Script.Tests/Workers/Rpc/WorkerConfigurationResolverTests.cs +++ b/test/WebJobs.Script.Tests/Workers/Rpc/WorkerConfigurationResolverTests.cs @@ -51,7 +51,7 @@ public void GetWorkerConfigs_MultiLanguageWorker_ReturnsExpectedConfigs(string r mockEnvironment.Setup(p => p.GetEnvironmentVariable(EnvironmentSettingNames.AppKind)).Returns(ScriptConstants.WorkFlowAppKind); mockEnvironment.Setup(p => p.GetEnvironmentVariable(EnvironmentSettingNames.FunctionWorkerRuntime)).Returns((string)null); - var config = GetConfiguration(probingPaths); + var config = WorkerConfigurationResolverTestsHelper.GetConfigurationWithProbingPaths(probingPaths); var testScriptHostManager = new Mock(); @@ -98,7 +98,7 @@ public void GetWorkerConfigs_MultiLanguageWorker_NullOREmptyProbingPath_ReturnsE probingPaths = new List(); } - var config = GetConfiguration(probingPaths); + var config = WorkerConfigurationResolverTestsHelper.GetConfigurationWithProbingPaths(probingPaths); var testScriptHostManager = new Mock(); @@ -147,7 +147,7 @@ public void GetWorkerConfigs_NullOREmptyProbingPath_ReturnsExpectedConfigs(strin probingPaths = new List(); } - var config = GetConfiguration(probingPaths); + var config = WorkerConfigurationResolverTestsHelper.GetConfigurationWithProbingPaths(probingPaths); var mockProfileManager = new Mock(); var mockConfig = new Mock(); @@ -171,25 +171,5 @@ public void GetWorkerConfigs_NullOREmptyProbingPath_ReturnsExpectedConfigs(strin Assert.Equal(result.Count, 1); Assert.True(result.Any(r => r.Contains(Path.Combine(_fallbackPath, languageWorker)))); } - - private IConfiguration GetConfiguration(List probingPaths) - { - var jsonObj = new - { - languageWorkers = new - { - probingPaths - } - }; - - var jsonString = JsonSerializer.Serialize(jsonObj, new JsonSerializerOptions { WriteIndented = true }); - var jsonStream = new MemoryStream(Encoding.UTF8.GetBytes(jsonString)); - - var configurationBuilder = new ConfigurationBuilder() - .Add(new ScriptEnvironmentVariablesConfigurationSource()) - .AddJsonStream(jsonStream); - - return configurationBuilder.Build(); - } } } \ No newline at end of file From e184c4f306ec1704b9560c5fc6b43f2b5061d022 Mon Sep 17 00:00:00 2001 From: Surbhi Gupta Date: Wed, 30 Jul 2025 00:44:02 -0500 Subject: [PATCH 55/55] Added WorkerConfigurationResolverOptionsSetupTests --- .../DefaultWorkerConfigurationResolver.cs | 3 +- .../DynamicWorkerConfigurationResolver.cs | 3 +- .../LanguageWorkerOptionsSetupTests.cs | 29 ++------ .../WorkerConfigurationResolverTestsHelper.cs | 2 +- .../Workers/Rpc/RpcWorkerConfigTests.cs | 2 +- ...rConfigurationResolverOptionsSetupTests.cs | 69 ++++++++++++++----- 6 files changed, 63 insertions(+), 45 deletions(-) diff --git a/src/WebJobs.Script/Workers/Rpc/Configuration/DefaultWorkerConfigurationResolver.cs b/src/WebJobs.Script/Workers/Rpc/Configuration/DefaultWorkerConfigurationResolver.cs index 46d89b0499..55dc64a7df 100644 --- a/src/WebJobs.Script/Workers/Rpc/Configuration/DefaultWorkerConfigurationResolver.cs +++ b/src/WebJobs.Script/Workers/Rpc/Configuration/DefaultWorkerConfigurationResolver.cs @@ -17,8 +17,7 @@ internal sealed class DefaultWorkerConfigurationResolver : IWorkerConfigurationR public DefaultWorkerConfigurationResolver(ILoggerFactory loggerFactory, IOptionsMonitor workerConfigurationResolverOptions) { - _ = loggerFactory ?? throw new ArgumentNullException(nameof(loggerFactory)); - _logger = loggerFactory.CreateLogger(ScriptConstants.LogCategoryWorkerConfig); + _logger = loggerFactory is not null ? loggerFactory.CreateLogger(ScriptConstants.LogCategoryWorkerConfig) : throw new ArgumentNullException(nameof(loggerFactory)); _workerConfigurationResolverOptions = workerConfigurationResolverOptions ?? throw new ArgumentNullException(nameof(workerConfigurationResolverOptions)); } diff --git a/src/WebJobs.Script/Workers/Rpc/Configuration/DynamicWorkerConfigurationResolver.cs b/src/WebJobs.Script/Workers/Rpc/Configuration/DynamicWorkerConfigurationResolver.cs index e1ce2639a5..e0464b9238 100644 --- a/src/WebJobs.Script/Workers/Rpc/Configuration/DynamicWorkerConfigurationResolver.cs +++ b/src/WebJobs.Script/Workers/Rpc/Configuration/DynamicWorkerConfigurationResolver.cs @@ -31,8 +31,7 @@ public DynamicWorkerConfigurationResolver(ILoggerFactory loggerFactory, IWorkerProfileManager workerProfileManager, IOptionsMonitor workerConfigResolverOptions) { - _ = loggerFactory ?? throw new ArgumentNullException(nameof(loggerFactory)); - _logger = loggerFactory.CreateLogger(ScriptConstants.LogCategoryWorkerConfig); + _logger = loggerFactory is not null ? loggerFactory.CreateLogger(ScriptConstants.LogCategoryWorkerConfig) : throw new ArgumentNullException(nameof(loggerFactory)); _fileSystem = fileSystem ?? throw new ArgumentNullException(nameof(fileSystem)); _profileManager = workerProfileManager ?? throw new ArgumentNullException(nameof(workerProfileManager)); _workerConfigurationResolverOptions = workerConfigResolverOptions ?? throw new ArgumentNullException(nameof(workerConfigResolverOptions)); diff --git a/test/WebJobs.Script.Tests/Configuration/LanguageWorkerOptionsSetupTests.cs b/test/WebJobs.Script.Tests/Configuration/LanguageWorkerOptionsSetupTests.cs index b1a582cd50..39b61e099e 100644 --- a/test/WebJobs.Script.Tests/Configuration/LanguageWorkerOptionsSetupTests.cs +++ b/test/WebJobs.Script.Tests/Configuration/LanguageWorkerOptionsSetupTests.cs @@ -115,6 +115,7 @@ public void LanguageWorkerOptions_EnabledWorkerResolution_Expected_ListOfConfigs var hostingOptions = new FunctionsHostingConfigOptions(); hostingOptions.Features.Add(RpcWorkerConstants.WorkersAvailableForDynamicResolution, hostingOptionsSetting); + var optionsMonitor = WorkerConfigurationResolverTestsHelper.GetTestWorkerConfigurationResolverOptions(configuration, testEnvironment, testScriptHostManager.Object, new OptionsWrapper(hostingOptions)); var resolver = new DynamicWorkerConfigurationResolver(loggerFactory, FileUtility.Instance, testProfileManager.Object, optionsMonitor); @@ -148,16 +149,12 @@ public void LanguageWorkerOptions_FallbackPath_Expected_ListOfConfigs(string wor var testEnvironment = new TestEnvironment(); var testMetricLogger = new TestMetricsLogger(); - var configurationBuilder = new ConfigurationBuilder() - .Add(new ScriptEnvironmentVariablesConfigurationSource()); - var configuration = configurationBuilder.Build(); + var configuration = new ConfigurationBuilder().Add(new ScriptEnvironmentVariablesConfigurationSource()).Build(); var testProfileManager = new Mock(); var testScriptHostManager = new Mock(); - string probingPathValue = null; testEnvironment.SetEnvironmentVariable(RpcWorkerConstants.FunctionWorkerRuntimeSettingName, workerRuntime); testEnvironment.SetEnvironmentVariable(EnvironmentSettingNames.AntaresPlatformReleaseChannel, releaseChannel); - testEnvironment.SetEnvironmentVariable(EnvironmentSettingNames.WorkerProbingPaths, probingPathValue); var hostingOptions = new FunctionsHostingConfigOptions(); hostingOptions.Features.Add(RpcWorkerConstants.WorkersAvailableForDynamicResolution, hostingOptionsSetting); @@ -193,16 +190,12 @@ public void LanguageWorkerOptions_NullHostingConfig_FeatureDisabled_ListOfConfig var testEnvironment = new TestEnvironment(); var testMetricLogger = new TestMetricsLogger(); - var configurationBuilder = new ConfigurationBuilder() - .Add(new ScriptEnvironmentVariablesConfigurationSource()); - var configuration = configurationBuilder.Build(); + var configuration = new ConfigurationBuilder().Add(new ScriptEnvironmentVariablesConfigurationSource()).Build(); var testProfileManager = new Mock(); var testScriptHostManager = new Mock(); - string probingPathValue = string.Join(';', _probingPath1, string.Empty, "path-not-exists"); testEnvironment.SetEnvironmentVariable(RpcWorkerConstants.FunctionWorkerRuntimeSettingName, workerRuntime); testEnvironment.SetEnvironmentVariable(EnvironmentSettingNames.AntaresPlatformReleaseChannel, releaseChannel); - testEnvironment.SetEnvironmentVariable(EnvironmentSettingNames.WorkerProbingPaths, probingPathValue); var hostingOptions = new FunctionsHostingConfigOptions(); hostingOptions.Features.Add(RpcWorkerConstants.WorkersAvailableForDynamicResolution, hostingOptionsSetting); @@ -238,20 +231,14 @@ public void LanguageWorkerOptions_DisabledWorkerResolution_Expected_ListOfConfig var testEnvironment = new TestEnvironment(); var testMetricLogger = new TestMetricsLogger(); - var configurationBuilder = new ConfigurationBuilder() - .Add(new ScriptEnvironmentVariablesConfigurationSource()); - var configuration = configurationBuilder.Build(); + var configuration = new ConfigurationBuilder().Add(new ScriptEnvironmentVariablesConfigurationSource()).Build(); var testProfileManager = new Mock(); var testScriptHostManager = new Mock(); - string probingPathValue = null; testEnvironment.SetEnvironmentVariable(RpcWorkerConstants.FunctionWorkerRuntimeSettingName, workerRuntime); testEnvironment.SetEnvironmentVariable(EnvironmentSettingNames.AntaresPlatformReleaseChannel, releaseChannel); - testEnvironment.SetEnvironmentVariable(EnvironmentSettingNames.WorkerProbingPaths, probingPathValue); - - var hostingOptions = new FunctionsHostingConfigOptions(); - var optionsMonitor = WorkerConfigurationResolverTestsHelper.GetTestWorkerConfigurationResolverOptions(configuration, testEnvironment, testScriptHostManager.Object, new OptionsWrapper(hostingOptions)); + var optionsMonitor = WorkerConfigurationResolverTestsHelper.GetTestWorkerConfigurationResolverOptions(configuration, testEnvironment, testScriptHostManager.Object, null); var resolver = new DefaultWorkerConfigurationResolver(loggerFactory, optionsMonitor); LanguageWorkerOptionsSetup setup = new LanguageWorkerOptionsSetup(configuration, loggerFactory, testEnvironment, testMetricLogger, testProfileManager.Object, testScriptHostManager.Object, optionsMonitor, resolver); @@ -280,15 +267,11 @@ public void LanguageWorkerOptions_FallbackPath_NullHostingConfig(string workerRu var testEnvironment = new TestEnvironment(); var testMetricLogger = new TestMetricsLogger(); - var configurationBuilder = new ConfigurationBuilder() - .Add(new ScriptEnvironmentVariablesConfigurationSource()); - var configuration = configurationBuilder.Build(); + var configuration = new ConfigurationBuilder().Add(new ScriptEnvironmentVariablesConfigurationSource()).Build(); var testProfileManager = new Mock(); var testScriptHostManager = new Mock(); - string probingPathValue = null; testEnvironment.SetEnvironmentVariable(RpcWorkerConstants.FunctionWorkerRuntimeSettingName, workerRuntime); - testEnvironment.SetEnvironmentVariable(EnvironmentSettingNames.WorkerProbingPaths, probingPathValue); var hostingOptions = new FunctionsHostingConfigOptions(); hostingOptions.Features.Add(RpcWorkerConstants.WorkersAvailableForDynamicResolution, workerRuntime); diff --git a/test/WebJobs.Script.Tests/WorkerConfigurationResolverTestsHelper.cs b/test/WebJobs.Script.Tests/WorkerConfigurationResolverTestsHelper.cs index 82005bc090..8e9a6f77c8 100644 --- a/test/WebJobs.Script.Tests/WorkerConfigurationResolverTestsHelper.cs +++ b/test/WebJobs.Script.Tests/WorkerConfigurationResolverTestsHelper.cs @@ -24,7 +24,7 @@ internal static IOptionsMonitor GetTestWorke if (functionsHostingConfigOptions is null) { var hostingOptions = new FunctionsHostingConfigOptions(); - functionsHostingConfigOptions = new OptionsWrapper(hostingOptions); + functionsHostingConfigOptions = new OptionsWrapper(new FunctionsHostingConfigOptions()); } var resolverOptionssetup = new WorkerConfigurationResolverOptionsSetup(configuration, environment, scriptHostManager, functionsHostingConfigOptions); diff --git a/test/WebJobs.Script.Tests/Workers/Rpc/RpcWorkerConfigTests.cs b/test/WebJobs.Script.Tests/Workers/Rpc/RpcWorkerConfigTests.cs index 37ed5508cf..165da137a6 100644 --- a/test/WebJobs.Script.Tests/Workers/Rpc/RpcWorkerConfigTests.cs +++ b/test/WebJobs.Script.Tests/Workers/Rpc/RpcWorkerConfigTests.cs @@ -692,7 +692,7 @@ private IEnumerable TestReadWorkerProviderFromConfig(IEnumerabl var optionsMonitor = WorkerConfigurationResolverTestsHelper.GetTestWorkerConfigurationResolverOptions(config, _testEnvironment, testScriptHostManager.Object, null); - IWorkerConfigurationResolver workerConfigurationResolver = new DefaultWorkerConfigurationResolver(loggerFactory, optionsMonitor); + var workerConfigurationResolver = new DefaultWorkerConfigurationResolver(loggerFactory, optionsMonitor); var configFactory = new RpcWorkerConfigFactory(config, testLogger, _testSysRuntimeInfo, _testEnvironment, new TestMetricsLogger(), workerProfileManager.Object, workerConfigurationResolver, optionsMonitor); if (appSvcEnv) diff --git a/test/WebJobs.Script.Tests/Workers/Rpc/WorkerConfigurationResolverOptionsSetupTests.cs b/test/WebJobs.Script.Tests/Workers/Rpc/WorkerConfigurationResolverOptionsSetupTests.cs index 13731a2438..c955cde698 100644 --- a/test/WebJobs.Script.Tests/Workers/Rpc/WorkerConfigurationResolverOptionsSetupTests.cs +++ b/test/WebJobs.Script.Tests/Workers/Rpc/WorkerConfigurationResolverOptionsSetupTests.cs @@ -2,6 +2,7 @@ // Licensed under the MIT License. See License.txt in the project root for license information. using System.Collections.Generic; +using System.Linq; using Microsoft.Azure.WebJobs.Script.Config; using Microsoft.Azure.WebJobs.Script.Workers; using Microsoft.Azure.WebJobs.Script.Workers.Rpc; @@ -15,35 +16,64 @@ namespace Microsoft.Azure.WebJobs.Script.Tests.Workers.Rpc { public class WorkerConfigurationResolverOptionsSetupTests { - private readonly Mock _mockConfiguration; - private readonly Mock _mockEnvironment; - private readonly Mock _mockScriptHostManager; - private readonly WorkerConfigurationResolverOptionsSetup _setup; - - public WorkerConfigurationResolverOptionsSetupTests() + [Fact] + public void Configure_WithRealEnvironmentValues_SetsCorrectDefaults() { - _mockConfiguration = new Mock(); - _mockEnvironment = new Mock(); - _mockScriptHostManager = new Mock(); + // Arrange + var testEnvironment = new TestEnvironment(); + var configBuilder = new ConfigurationBuilder() + .AddInMemoryCollection(new Dictionary + { + [$"{RpcWorkerConstants.LanguageWorkersSectionName}:{WorkerConstants.WorkersDirectorySectionName}"] = "/default/workers", + [$"{RpcWorkerConstants.LanguageWorkersSectionName}:{RpcWorkerConstants.WorkerProbingPathsSectionName}:0"] = "testPath1", + [$"{RpcWorkerConstants.LanguageWorkersSectionName}:{RpcWorkerConstants.WorkerProbingPathsSectionName}:1"] = "testPath2", + [$"{RpcWorkerConstants.LanguageWorkersSectionName}:{RpcWorkerConstants.WorkerProbingPathsSectionName}:2"] = " ", + }); + var configuration = configBuilder.Build(); + var mockScriptHostManager = new Mock(); + var hostingOptions = new FunctionsHostingConfigOptions(); - _setup = new WorkerConfigurationResolverOptionsSetup(_mockConfiguration.Object, _mockEnvironment.Object, _mockScriptHostManager.Object, new OptionsWrapper(hostingOptions)); + var setup = new WorkerConfigurationResolverOptionsSetup(configuration, testEnvironment, mockScriptHostManager.Object, new OptionsWrapper(hostingOptions)); + var options = new WorkerConfigurationResolverOptions(); + + // Act + setup.Configure(options); + + // Assert + Assert.Null(options.WorkerRuntime); + Assert.Equal(ScriptConstants.LatestPlatformChannelNameUpper, options.ReleaseChannel); + Assert.False(options.IsPlaceholderModeEnabled); + Assert.False(options.IsMultiLanguageWorkerEnvironment); + Assert.Equal("/default/workers", options.WorkersDirPath); + Assert.NotNull(options.LanguageWorkersSettings); + + Assert.Equal(2, options.ProbingPaths.Count); + Assert.True(options.ProbingPaths.Contains("testPath1")); + Assert.True(options.ProbingPaths.Contains("testPath2")); + + Assert.False(options.WorkersAvailableForResolution.Any()); } [Fact] - public void Configure_WithRealEnvironmentValues_SetsCorrectDefaults() + public void Configure_WithRealEnvironmentValues_SetsCorrectDefaults1() { // Arrange var testEnvironment = new TestEnvironment(); var configBuilder = new ConfigurationBuilder() .AddInMemoryCollection(new Dictionary { - [$"{RpcWorkerConstants.LanguageWorkersSectionName}:{WorkerConstants.WorkersDirectorySectionName}"] = "/default/workers" + [$"{RpcWorkerConstants.LanguageWorkersSectionName}:{WorkerConstants.WorkersDirectorySectionName}"] = "/default/workers", }); var configuration = configBuilder.Build(); var mockScriptHostManager = new Mock(); + testEnvironment.SetEnvironmentVariable(RpcWorkerConstants.FunctionWorkerRuntimeSettingName, "java"); + testEnvironment.SetEnvironmentVariable(EnvironmentSettingNames.AntaresPlatformReleaseChannel, "standard"); + testEnvironment.SetEnvironmentVariable(EnvironmentSettingNames.AppKind, "workflowapp"); + var hostingOptions = new FunctionsHostingConfigOptions(); + hostingOptions.Features.Add(RpcWorkerConstants.WorkersAvailableForDynamicResolution, "java|node"); var setup = new WorkerConfigurationResolverOptionsSetup(configuration, testEnvironment, mockScriptHostManager.Object, new OptionsWrapper(hostingOptions)); var options = new WorkerConfigurationResolverOptions(); @@ -52,12 +82,19 @@ public void Configure_WithRealEnvironmentValues_SetsCorrectDefaults() setup.Configure(options); // Assert - Assert.Null(options.WorkerRuntime); // No worker runtime set - Assert.Equal(ScriptConstants.LatestPlatformChannelNameUpper, options.ReleaseChannel); // Default release channel - Assert.False(options.IsPlaceholderModeEnabled); // Default placeholder mode - Assert.False(options.IsMultiLanguageWorkerEnvironment); // Default multi-language mode + Assert.Equal("java", options.WorkerRuntime); + Assert.Equal("standard", options.ReleaseChannel); + Assert.False(options.IsPlaceholderModeEnabled); + Assert.False(options.IsMultiLanguageWorkerEnvironment); Assert.Equal("/default/workers", options.WorkersDirPath); Assert.NotNull(options.LanguageWorkersSettings); + + Assert.NotNull(options.ProbingPaths); + Assert.False(options.ProbingPaths.Any()); + + Assert.True(options.WorkersAvailableForResolution.Count == 2); + Assert.True(options.WorkersAvailableForResolution.Contains("java")); + Assert.True(options.WorkersAvailableForResolution.Contains("node")); } } } \ No newline at end of file