[pack] Adding Multi language Support in Azure Function v3.x (#8449)

* Adding Multiple Languages Support in Functions

* Addressing PR Comments

* Adding Test Cases

* Fixing Failing Tests

* Resolving PR Comments

* Resolving PR Comments

* Resolving PR Comments

* Resolving PR Comments

* Removing IsMultiLanguageRuntime Cache

* Resolving PR Comments

* Caching IsMultiLanguageEnabled

Author: Siddharth

src/WebJobs.Script/Description/Workers/Rpc/MultiLanguageFunctionDescriptorProvider.cs

@@ -0,0 +1,53 @@
+// Copyright (c) .NET Foundation. All rights reserved.
+// Licensed under the MIT License. See License.txt in the project root for license information.
+
+using System;
+using System.Collections.Generic;
+using System.Threading.Tasks;
+using Microsoft.AspNetCore.Hosting;
+using Microsoft.Azure.WebJobs.Script.Extensibility;
+using Microsoft.Azure.WebJobs.Script.Workers;
+using Microsoft.Azure.WebJobs.Script.Workers.Rpc;
+using Microsoft.Extensions.Logging;
+
+namespace Microsoft.Azure.WebJobs.Script.Description
+{
+    internal class MultiLanguageFunctionDescriptorProvider : WorkerFunctionDescriptorProvider
+    {
+        private readonly IList<RpcWorkerConfig> _workerConfig;
+
+        /// <summary>
+        /// Initializes a new instance of the <see cref="MultiLanguageFunctionDescriptorProvider"/> class.
+        /// </summary>
+        /// <param name="host"><see cref="ScriptHost"/> instance.</param>
+        /// <param name="workerConfig">All supported <see cref="RpcWorkerConfig"/>.</param>
+        /// <param name="config"><see cref="ScriptJobHostOptions"/> instance.</param>
+        /// <param name="bindingProviders">List of <see cref="IScriptBindingProvider"/> instances.</param>
+        /// <param name="dispatcher"><see cref="IFunctionInvocationDispatcher"/> instance.</param>
+        /// <param name="loggerFactory"><see cref="ILoggerFactory"/> instance.</param>
+        /// <param name="applicationLifetime"><see cref="IApplicationLifetime"/> instance.</param>
+        /// <param name="workerInitializationTimeout">Worker initialization timeout.</param>
+        public MultiLanguageFunctionDescriptorProvider(ScriptHost host, IList<RpcWorkerConfig> workerConfig, ScriptJobHostOptions config, ICollection<IScriptBindingProvider> bindingProviders,
+            IFunctionInvocationDispatcher dispatcher, ILoggerFactory loggerFactory, IApplicationLifetime applicationLifetime, TimeSpan workerInitializationTimeout)
+            : base(host, config, bindingProviders, dispatcher, loggerFactory, applicationLifetime, workerInitializationTimeout)
+        {
+            _workerConfig = workerConfig ?? throw new ArgumentNullException(nameof(workerConfig));
+        }
+
+        /// <inheritdoc/>
+        public override async Task<(bool, FunctionDescriptor)> TryCreate(FunctionMetadata functionMetadata)
+        {
+            if (functionMetadata == null)
+            {
+                throw new ArgumentNullException(nameof(functionMetadata));
+            }
+
+            if (!Utility.IsFunctionMetadataLanguageSupportedByWorkerRuntime(functionMetadata, _workerConfig))
+            {
+                return (false, null);
+            }
+
+            return await base.TryCreate(functionMetadata);
+        }
+    }
+}

src/WebJobs.Script/Environment/EnvironmentExtensions.cs

@@ -7,6 +7,7 @@
 using System.IO;
 using System.Linq;
 using System.Runtime.InteropServices;
+using Microsoft.Azure.WebJobs.Script.Config;
 using Microsoft.Azure.WebJobs.Script.Models;
 using Microsoft.Azure.WebJobs.Script.Workers.Rpc;
 using Microsoft.Extensions.Options;
@@ -16,6 +17,8 @@ namespace Microsoft.Azure.WebJobs.Script
 {
     internal static class EnvironmentExtensions
     {
+        private static bool? isMultiLanguageEnabled;
+
         // For testing
         internal static string BaseDirectory { get; set; }
 
@@ -304,14 +307,26 @@ public static string GetAntaresComputerName(this IEnvironment environment)
         }
 
         /// <summary>
-        /// Gets the computer name.
+        /// Gets if runtime enviromnent is logic apps.
         /// </summary>
         public static bool IsLogicApp(this IEnvironment environment)
         {
             string appKind = environment.GetEnvironmentVariable(AppKind)?.ToLower();
             return !string.IsNullOrEmpty(appKind) && appKind.Contains(ScriptConstants.WorkFlowAppKind);
         }
 
+        /// <summary>
+        /// Gets if runtime environment needs multi language
+        /// </summary>
+        public static bool IsMultiLanguageRuntimeEnvironment(this IEnvironment environment)
+        {
+            if (!isMultiLanguageEnabled.HasValue)
+            {
+                isMultiLanguageEnabled = environment.IsLogicApp() && FeatureFlags.IsEnabled(ScriptConstants.FeatureFlagEnableMultiLanguageWorker, environment);
+            }
+            return isMultiLanguageEnabled.Value;
+        }
+
         /// <summary>
         /// Gets the Antares version.
         /// </summary>
@@ -507,5 +522,14 @@ public static HashSet<string> GetLanguageWorkerListToStartInPlaceholder(this IEn
             }
             return placeholderRuntimeSet;
         }
+
+        /// <summary>
+        /// Clears all cached static flags in <see cref="EnvironmentExtensions"/>.
+        /// Currently we only use it to purge static initialisations in across tests.
+        /// </summary>
+        public static void ClearCache()
+        {
+            isMultiLanguageEnabled = null;
+        }
     }
 }

src/WebJobs.Script/Host/ScriptHost.cs

@@ -304,7 +304,8 @@ public async Task InitializeAsync(CancellationToken cancellationToken = default)
                         // Windows Consumption as well.
                         string runtimeVersion = _environment.GetEnvironmentVariable(RpcWorkerConstants.FunctionWorkerRuntimeVersionSettingName);
 
-                        if (!string.IsNullOrEmpty(runtimeVersion))
+                        // If the environemt is multi language, we are running multiple language worker, and thus not honoring 'FUNCTIONS_WORKER_RUNTIME_VERSION'
+                        if (!string.IsNullOrEmpty(runtimeVersion) && !_environment.IsMultiLanguageRuntimeEnvironment())
                         {
                             runtimeStack = string.Concat(runtimeStack, "-", runtimeVersion);
                         }
@@ -540,6 +541,13 @@ private void AddFunctionDescriptors(IEnumerable<FunctionMetadata> functionMetada
                 _logger.AddingDescriptorProviderForLanguage(RpcWorkerConstants.DotNetLanguageWorkerName);
                 _descriptorProviders.Add(new DotNetFunctionDescriptorProvider(this, ScriptOptions, _bindingProviders, _metricsLogger, _loggerFactory));
             }
+            else if (_environment.IsMultiLanguageRuntimeEnvironment())
+            {
+                _logger.AddingDescriptorProviderForLanguage("All (Multi Language)");
+
+                _descriptorProviders.Add(new MultiLanguageFunctionDescriptorProvider(this, _languageWorkerOptions.Value.WorkerConfigs, ScriptOptions, _bindingProviders,
+                    _functionDispatcher, _loggerFactory, _applicationLifetime, _languageWorkerOptions.Value.WorkerConfigs.Max(wc => wc.CountOptions.InitializationTimeout)));
+            }
             else if (_isHttpWorker)
             {
                 _logger.AddingDescriptorProviderForHttpWorker();
@@ -557,7 +565,7 @@ private void AddFunctionDescriptors(IEnumerable<FunctionMetadata> functionMetada
                 var workerConfig = _languageWorkerOptions.Value.WorkerConfigs?.FirstOrDefault(c => c.Description.Language.Equals(_workerRuntime, StringComparison.OrdinalIgnoreCase));
 
                 // If there's no worker config, use the default (for legacy behavior; mostly for tests).
-                TimeSpan initializationTimeout = workerConfig?.CountOptions?.InitializationTimeout ?? WorkerProcessCountOptions.DefaultInitializationTimeout;
+                var initializationTimeout = workerConfig?.CountOptions?.InitializationTimeout ?? WorkerProcessCountOptions.DefaultInitializationTimeout;
 
                 _descriptorProviders.Add(new RpcFunctionDescriptorProvider(this, _workerRuntime, ScriptOptions, _bindingProviders,
                     _functionDispatcher, _loggerFactory, _applicationLifetime, initializationTimeout));

src/WebJobs.Script/ScriptConstants.cs

@@ -119,6 +119,7 @@ public static class ScriptConstants
         public const string FeatureFlagDisableMergedWebHostScriptHostConfiguration = "DisableMergedConfiguration";
         public const string FeatureFlagDisableAspNetCoreGrpc = "DisableAspNetCoreGrpc";
         public const string FeatureFlagEnableWorkerIndexing = "EnableWorkerIndexing";
+        public const string FeatureFlagEnableMultiLanguageWorker = "EnableMultiLanguageWorker";
 
         public const string AdminJwtValidAudienceFormat = "https://{0}.azurewebsites.net/azurefunctions";
         public const string AdminJwtValidIssuerFormat = "https://{0}.scm.azurewebsites.net";

src/WebJobs.Script/Utility.cs

@@ -673,9 +673,15 @@ internal static bool IsFunctionMetadataLanguageSupportedByWorkerRuntime(Function
             {
                 return true;
             }
+
             return !string.IsNullOrEmpty(functionMetadata.Language) && functionMetadata.Language.Equals(workerRuntime, StringComparison.OrdinalIgnoreCase);
         }
 
+        internal static bool IsFunctionMetadataLanguageSupportedByWorkerRuntime(FunctionMetadata functionMetadata, IList<RpcWorkerConfig> workerConfigs)
+        {
+            return !string.IsNullOrEmpty(functionMetadata.Language) && workerConfigs.Select(wc => wc.Description.Language).Contains(functionMetadata.Language);
+        }
+
         public static bool IsDotNetLanguageFunction(string functionLanguage)
         {
             return dotNetLanguages.Any(lang => string.Equals(lang, functionLanguage, StringComparison.OrdinalIgnoreCase));
@@ -906,9 +912,9 @@ public static void ValidateRetryOptions(RetryOptions
 
         public static bool CanWorkerIndex(IEnumerable<RpcWorkerConfig> workerConfigs, IEnvironment environment)
         {
-            var workerRuntime = environment.GetEnvironmentVariable(EnvironmentSettingNames.FunctionWorkerRuntime);
-            if (workerConfigs != null)
+            if (workerConfigs != null && !environment.IsMultiLanguageRuntimeEnvironment())
             {
+                var workerRuntime = environment.GetEnvironmentVariable(EnvironmentSettingNames.FunctionWorkerRuntime);
                 var workerConfig = workerConfigs.Where(c => c.Description != null && c.Description.Language != null && c.Description.Language.Equals(workerRuntime, StringComparison.InvariantCultureIgnoreCase)).FirstOrDefault();
 
                 // if feature flag is enabled and workerConfig.WorkerIndexing == true, then return true

src/WebJobs.Script/Workers/Rpc/Configuration/RpcWorkerConfigFactory.cs

@@ -284,6 +284,12 @@ internal bool ShouldAddWorkerConfig(string workerDescriptionLanguage)
                 return true;
             }
 
+            if (_environment.IsMultiLanguageRuntimeEnvironment())
+            {
+                _logger.LogInformation($"Found multi-language runtime environment. Starting WorkerConfig for language: {workerDescriptionLanguage}");
+                return true;
+            }
+
             if (!string.IsNullOrEmpty(_workerRuntime))
             {
                 _logger.LogDebug($"EnvironmentVariable {RpcWorkerConstants.FunctionWorkerRuntimeSettingName}: {_workerRuntime}");