4.21.1 hotfix (#9268)

* Switching the worker indexing to use hosting config feature flag (#9245)

* Initial commit.

* fixing issue with worker indexing, dotnet-isolated, and placeholders (#9253)

* fixing issue with worker indexing, dotnet-isolated, and placeholders (#9253)

* Update Python Worker Version to 4.14.0 (#9261)

* Update Python Worker Version to 4.14.0

* Update release_notes.md

---------

Co-authored-by: AzureFunctionsPython <[email protected]>

* Setting up workerConfig with ConfigureOptionsWithChangeTokenSource (#9264)

* Sending metadata request once per scriptHost instance (#9248)

Co-authored-by: Brett Samblanet <[email protected]>

* Restart workers when we need to refresh the worker metadata (#9244)

* Incrementing patch version to 4.21.1

* Removing duplicate mention of #9253

---------

Co-authored-by: Naren Soni <[email protected]>
Co-authored-by: Yogesh <[email protected]>
Co-authored-by: Brett Samblanet <[email protected]>
Co-authored-by: gavin-aguiar <[email protected]>
Co-authored-by: AzureFunctionsPython <[email protected]>
Co-authored-by: azfuncgh <[email protected]>

Author: 7 people

NuGet.config

@@ -9,6 +9,7 @@
     <add key="AzureFunctions@staging" value="https://azfunc.pkgs.visualstudio.com/e6a70c92-4128-439f-8012-382fe78d6396/_packaging/AzureFunctions%40staging/nuget/v3/index.json" />
     <add key="AzureFunctionsRelease" value="https://azfunc.pkgs.visualstudio.com/e6a70c92-4128-439f-8012-382fe78d6396/_packaging/AzureFunctionsRelease/nuget/v3/index.json" />
     <add key="AzureFunctionsPreRelease" value="https://azfunc.pkgs.visualstudio.com/e6a70c92-4128-439f-8012-382fe78d6396/_packaging/AzureFunctionsPreRelease/nuget/v3/index.json" />
+    <add key="AzureFunctionsTempStaging" value="https://azfunc.pkgs.visualstudio.com/e6a70c92-4128-439f-8012-382fe78d6396/_packaging/AzureFunctionsTempStaging/nuget/v3/index.json" />
     <!-- dotnet-tools is required for Microsoft.CodeAnalysis.CSharp.Analyzer.Testing.XUnit -->
     <add key="dotnet-tools" value="https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-tools/nuget/v3/index.json"/>
   </packageSources>

build/common.props

@@ -5,7 +5,7 @@
     <LangVersion>latest</LangVersion>
     <MajorVersion>4</MajorVersion>
     <MinorVersion>21</MinorVersion>
-    <PatchVersion>0</PatchVersion>
+    <PatchVersion>1</PatchVersion>
     <BuildNumber Condition="'$(BuildNumber)' == '' ">0</BuildNumber>
     <PreviewVersion></PreviewVersion>        
 

build/python.props

@@ -1,5 +1,5 @@
 <Project>
   <ItemGroup>
-    <PackageReference Include="Microsoft.Azure.Functions.PythonWorker" Version="4.11.0" />
+    <PackageReference Include="Microsoft.Azure.Functions.PythonWorker" Version="4.14.0" />
   </ItemGroup>
 </Project>

release_notes.md

@@ -3,7 +3,10 @@
 <!-- Please add your release notes in the following format:
 - My change description (#PR)
 -->
+- Update Python Worker Version to [4.14.0](https://github.com/Azure/azure-functions-python-worker/releases/tag/4.14.0)
 - Update Java Worker Version to [2.11.0](https://github.com/Azure/azure-functions-java-worker/releases/tag/2.11.0)
 
 **Release sprint:** Sprint 142
 [ [bugs](https://github.com/Azure/azure-functions-host/issues?q=is%3Aissue+milestone%3A%22Functions+Sprint+143%22+label%3Abug+is%3Aclosed) | [features](https://github.com/Azure/azure-functions-host/issues?q=is%3Aissue+milestone%3A%22Functions+Sprint+143%22+label%3Afeature+is%3Aclosed) ]
+- Fixing bug with placeholder misses in dotnet-isolated #9253
+- (Update Python Worker Version to 4.14.0 (#9261))

src/WebJobs.Script.Grpc/Channel/GrpcWorkerChannel.cs

@@ -52,6 +52,7 @@ internal class GrpcWorkerChannel : IRpcWorkerChannel, IDisposable
         private readonly IOptions<WorkerConcurrencyOptions> _workerConcurrencyOptions;
         private readonly WaitCallback _processInbound;
         private readonly object _syncLock = new object();
+        private readonly object _metadataLock = new object();
         private readonly Dictionary<MsgType, Queue<PendingItem>> _pendingActions = new();
         private readonly ChannelWriter<OutboundGrpcEvent> _outbound;
         private readonly ChannelReader<InboundGrpcEvent> _inbound;
@@ -85,6 +86,7 @@ internal class GrpcWorkerChannel : IRpcWorkerChannel, IDisposable
         private IHttpProxyService _httpProxyService;
         private Uri _httpProxyEndpoint;
         private System.Timers.Timer _timer;
+        private bool _functionMetadataRequestSent = false;
 
         internal GrpcWorkerChannel(
             string workerId,
@@ -540,6 +542,8 @@ internal FunctionLoadRequestCollection GetFunctionLoadRequestCollection(IEnumera
         public Task SendFunctionEnvironmentReloadRequest()
         {
             _functionsIndexingTask = new TaskCompletionSource<List<RawFunctionMetadata>>(TaskCreationOptions.RunContinuationsAsynchronously);
+            _functionMetadataRequestSent = false;
+
             _workerChannelLogger.LogDebug("Sending FunctionEnvironmentReloadRequest to WorkerProcess with Pid: '{0}'", _rpcWorkerProcess.Id);
             IDisposable latencyEvent = _metricsLogger.LatencyEvent(MetricEventNames.SpecializationEnvironmentReloadRequestResponse);
 
@@ -795,22 +799,32 @@ public Task<List<RawFunctionMetadata>> GetFunctionMetadata()
 
         internal Task<List<RawFunctionMetadata>> SendFunctionMetadataRequest()
         {
-            // reset indexing task when in case we need to send another request
-            _functionsIndexingTask = new TaskCompletionSource<List<RawFunctionMetadata>>(TaskCreationOptions.RunContinuationsAsynchronously);
+            _workerChannelLogger.LogDebug("Fetching worker metadata, FunctionMetadataReceived set to: {functionMetadataReceived}", _functionMetadataRequestSent);
+            if (!_functionMetadataRequestSent)
+            {
+                lock (_metadataLock)
+                {
+                    if (!_functionMetadataRequestSent)
+                    {
+                        RegisterCallbackForNextGrpcMessage(MsgType.FunctionMetadataResponse, _functionLoadTimeout, 1,
+                    msg => ProcessFunctionMetadataResponses(msg.Message.FunctionMetadataResponse), HandleWorkerMetadataRequestError);
 
-            RegisterCallbackForNextGrpcMessage(MsgType.FunctionMetadataResponse, _functionLoadTimeout, 1,
-                msg => ProcessFunctionMetadataResponses(msg.Message.FunctionMetadataResponse), HandleWorkerMetadataRequestError);
+                        _workerChannelLogger.LogDebug("Sending WorkerMetadataRequest to {language} worker with worker ID {workerID}", _runtime, _workerId);
 
-            _workerChannelLogger.LogDebug("Sending WorkerMetadataRequest to {language} worker with worker ID {workerID}", _runtime, _workerId);
+                        // sends the function app directory path to worker for indexing
+                        SendStreamingMessage(new StreamingMessage
+                        {
+                            FunctionsMetadataRequest = new FunctionsMetadataRequest()
+                            {
+                                FunctionAppDirectory = _applicationHostOptions.CurrentValue.ScriptPath
+                            }
+                        });
 
-            // sends the function app directory path to worker for indexing
-            SendStreamingMessage(new StreamingMessage
-            {
-                FunctionsMetadataRequest = new FunctionsMetadataRequest()
-                {
-                    FunctionAppDirectory = _applicationHostOptions.CurrentValue.ScriptPath
+                        _functionMetadataRequestSent = true;
+                    }
                 }
-            });
+            }
+
             return _functionsIndexingTask.Task;
         }
 
@@ -845,7 +859,8 @@ internal void ProcessFunctionMetadataResponses(FunctionMetadataResponse function
                         FunctionDirectory = metadata.Directory,
                         ScriptFile = metadata.ScriptFile,
                         EntryPoint = metadata.EntryPoint,
-                        Name = metadata.Name
+                        Name = metadata.Name,
+                        Language = metadata.Language
                     };
 
                     functionMetadata.SetFunctionId(metadata.FunctionId);

src/WebJobs.Script.WebHost/Models/AzureStorageInfoValue.cs

@@ -83,7 +83,12 @@ public static AzureStorageInfoValue FromEnvironmentVariable(KeyValuePair<string,
             }
 
             var parts = environmentVariable.Value?.Split(new[] { '|' }, StringSplitOptions.RemoveEmptyEntries);
-            if (parts == null || parts.Length != 4)
+            if (parts == null)
+            {
+                return null;
+            }
+
+            if (parts.Length != 4 && parts.Length != 5)
             {
                 return null;
             }

src/WebJobs.Script.WebHost/WebHostServiceCollectionExtensions.cs

@@ -175,13 +175,9 @@ public static void AddWebJobsScriptHost(this IServiceCollection services, IConfi
             // Language Worker Hosted Services need to be intialized before WebJobsScriptHostService
             ScriptHostBuilderExtensions.AddCommonServices(services);
 
-            services.AddSingleton<IFunctionMetadataProvider>(sp =>
-            {
-                return new FunctionMetadataProvider(
-                    sp.GetRequiredService<ILogger<FunctionMetadataProvider>>(),
-                    ActivatorUtilities.CreateInstance<WorkerFunctionMetadataProvider>(sp),
-                    ActivatorUtilities.CreateInstance<HostFunctionMetadataProvider>(sp));
-            });
+            services.AddSingleton<IWorkerFunctionMetadataProvider, WorkerFunctionMetadataProvider>();
+            services.AddSingleton<IHostFunctionMetadataProvider, HostFunctionMetadataProvider>();
+            services.AddSingleton<IFunctionMetadataProvider, FunctionMetadataProvider>();
 
             // Core script host services
             services.AddSingleton<WebJobsScriptHostService>();
@@ -208,7 +204,7 @@ public static void AddWebJobsScriptHost(this IServiceCollection services, IConfi
             services.AddSingleton<IOptionsChangeTokenSource<ScriptApplicationHostOptions>, ScriptApplicationHostOptionsChangeTokenSource>();
 
             services.ConfigureOptions<StandbyOptionsSetup>();
-            services.ConfigureOptions<LanguageWorkerOptionsSetup>();
+            services.ConfigureOptionsWithChangeTokenSource<LanguageWorkerOptions, LanguageWorkerOptionsSetup, SpecializationChangeTokenSource<LanguageWorkerOptions>>();
             services.ConfigureOptionsWithChangeTokenSource<AppServiceOptions, AppServiceOptionsSetup, SpecializationChangeTokenSource<AppServiceOptions>>();
             services.ConfigureOptionsWithChangeTokenSource<HttpBodyControlOptions, HttpBodyControlOptionsSetup, SpecializationChangeTokenSource<HttpBodyControlOptions>>();
             services.ConfigureOptions<FunctionsHostingConfigOptionsSetup>();

src/WebJobs.Script/Config/FunctionsHostingConfigOptions.cs

@@ -43,6 +43,28 @@ public bool WorkerWarmupEnabled
             }
         }
 
+        /// <summary>
+        /// Gets a value indicating whether worker indexing feature is enabled in the hosting config.
+        /// </summary>
+        public bool WorkerIndexingEnabled
+        {
+            get
+            {
+                return GetFeature(RpcWorkerConstants.WorkerIndexingEnabled) == "1";
+            }
+        }
+
+        /// <summary>
+        /// Gets a string delimited by '|' that contains the name of the apps with worker indexing disabled.
+        /// </summary>
+        public string WorkerIndexingDisabledApps
+        {
+            get
+            {
+                return GetFeature(RpcWorkerConstants.WorkerIndexingDisabledApps) ?? string.Empty;
+            }
+        }
+
         /// <summary>
         /// Gets a value indicating whether Linux Log Backoff is disabled in the hosting config.
         /// </summary>

src/WebJobs.Script/DependencyInjection/ScriptStartupTypeLocator.cs

@@ -89,7 +89,7 @@ public async Task<IEnumerable<Type>> GetExtensionsStartupTypesAsync()
                 ExtensionBundleDetails bundleDetails = await _extensionBundleManager.GetExtensionBundleDetails();
                 ValidateBundleRequirements(bundleDetails);
 
-                var functionMetadataCollection = _functionMetadataManager.GetFunctionMetadata(forceRefresh: true, includeCustomProviders: false);
+                var functionMetadataCollection = _functionMetadataManager.GetFunctionMetadata(forceRefresh: true, includeCustomProviders: false, workerConfigs: workerConfigs);
                 bindingsSet = new HashSet<string>(StringComparer.OrdinalIgnoreCase);
 
                 // Generate a Hashset of all the binding types used in the function app

src/WebJobs.Script/Host/FunctionMetadataManager.cs

@@ -30,17 +30,15 @@ public class FunctionMetadataManager : IFunctionMetadataManager
         private bool _servicesReset = false;
         private ILogger _logger;
         private IOptions<ScriptJobHostOptions> _scriptOptions;
-        private IOptionsMonitor<LanguageWorkerOptions> _languageWorkerOptions;
         private ImmutableArray<FunctionMetadata> _functionMetadataArray;
         private Dictionary<string, ICollection<string>> _functionErrors = new Dictionary<string, ICollection<string>>();
         private ConcurrentDictionary<string, FunctionMetadata> _functionMetadataMap = new ConcurrentDictionary<string, FunctionMetadata>(StringComparer.OrdinalIgnoreCase);
 
         public FunctionMetadataManager(IOptions<ScriptJobHostOptions> scriptOptions, IFunctionMetadataProvider functionMetadataProvider,
             IOptions<HttpWorkerOptions> httpWorkerOptions, IScriptHostManager scriptHostManager, ILoggerFactory loggerFactory,
-            IOptionsMonitor<LanguageWorkerOptions> languageWorkerOptions, IEnvironment environment)
+            IEnvironment environment)
         {
             _scriptOptions = scriptOptions;
-            _languageWorkerOptions = languageWorkerOptions;
             _serviceProvider = scriptHostManager as IServiceProvider;
             _functionMetadataProvider = functionMetadataProvider;
             _loggerFactory = loggerFactory;
@@ -84,11 +82,11 @@ public bool TryGetFunctionMetadata(string functionName, out FunctionMetadata fun
         /// <param name="applyAllowList">Apply functions allow list filter.</param>
         /// <param name="includeCustomProviders">Include any metadata provided by IFunctionProvider when loading the metadata</param>
         /// <returns> An Immmutable array of FunctionMetadata.</returns>
-        public ImmutableArray<FunctionMetadata> GetFunctionMetadata(bool forceRefresh, bool applyAllowList = true, bool includeCustomProviders = true)
+        public ImmutableArray<FunctionMetadata> GetFunctionMetadata(bool forceRefresh, bool applyAllowList = true, bool includeCustomProviders = true, IList<RpcWorkerConfig> workerConfigs = null)
         {
             if (forceRefresh || _servicesReset || _functionMetadataArray.IsDefaultOrEmpty)
             {
-                _functionMetadataArray = LoadFunctionMetadata(forceRefresh, includeCustomProviders);
+                _functionMetadataArray = LoadFunctionMetadata(forceRefresh, includeCustomProviders, workerConfigs: workerConfigs);
                 _logger.FunctionMetadataManagerFunctionsLoaded(ApplyAllowList(_functionMetadataArray).Count());
                 _servicesReset = false;
             }
@@ -114,28 +112,37 @@ private void InitializeServices()
 
             _isHttpWorker = _serviceProvider.GetService<IOptions<HttpWorkerOptions>>()?.Value?.Description != null;
             _scriptOptions = _serviceProvider.GetService<IOptions<ScriptJobHostOptions>>();
-            _languageWorkerOptions = _serviceProvider.GetService<IOptionsMonitor<LanguageWorkerOptions>>();
 
             // Resetting the logger switches the logger scope to Script Host level,
             // also making the logs available to Application Insights
             _logger = _serviceProvider?.GetService<ILoggerFactory>().CreateLogger(LogCategories.Startup);
             _servicesReset = true;
         }
 
+        /// <summary>
+        /// This is the worker configuration created in the jobhost scope during placeholder initialization
+        /// This is used as a fallback incase the config is not passed down from previous method call.
+        /// </summary>
+        private IList<RpcWorkerConfig> GetFallbackWorkerConfig()
+        {
+            return _serviceProvider.GetService<IOptionsMonitor<LanguageWorkerOptions>>().CurrentValue.WorkerConfigs;
+        }
+
         /// <summary>
         /// Read all functions and populate function metadata.
         /// </summary>
-        internal ImmutableArray<FunctionMetadata> LoadFunctionMetadata(bool forceRefresh = false, bool includeCustomProviders = true, IFunctionInvocationDispatcher dispatcher = null)
+        internal ImmutableArray<FunctionMetadata> LoadFunctionMetadata(bool forceRefresh = false, bool includeCustomProviders = true, IFunctionInvocationDispatcher dispatcher = null, IList<RpcWorkerConfig> workerConfigs = null)
         {
+            workerConfigs ??= GetFallbackWorkerConfig();
+
             _functionMetadataMap.Clear();
 
             ICollection<string> functionsAllowList = _scriptOptions?.Value?.Functions;
             _logger.FunctionMetadataManagerLoadingFunctionsMetadata();
 
             ImmutableArray<FunctionMetadata> immutableFunctionMetadata;
-            var workerConfigs = _languageWorkerOptions.CurrentValue.WorkerConfigs;
 
-            immutableFunctionMetadata = _functionMetadataProvider.GetFunctionMetadataAsync(workerConfigs, SystemEnvironment.Instance, forceRefresh).GetAwaiter().GetResult();
+            immutableFunctionMetadata = _functionMetadataProvider.GetFunctionMetadataAsync(workerConfigs, _environment, forceRefresh).GetAwaiter().GetResult();
 
             var functionMetadataList = new List<FunctionMetadata>();
             _functionErrors = new Dictionary<string, ICollection<string>>();