[pack] Reacting to ScriptHost configuration changes for AzureStorageProvider

Author: karshinlin

release_notes.md

@@ -1,16 +1,2 @@
 ### Release notes
-<!-- Please add your release notes in the following format:
-- My change description (#PR)
--->
-- Update Python Worker Version to [1.2.1](https://github.com/Azure/azure-functions-python-worker/releases/tag/1.2.1)
-
-- Added a feature flag to opt out of the default behavior where the host sets the environment name to `Development` when running in debug mode. To disable the behavior, set the app setting: `AzureWebJobsFeatureFlags` to `DisableDevModeInDebug`
-- Reorder CORS and EasyAuth middleware to prevent EasyAuth from blocking CORS requests (#7315)
-- Updated PowerShell Worker (PS7) to [3.0.833](https://github.com/Azure/azure-functions-powershell-worker/releases/tag/v3.0.833)
-- Add IDistributedLockManager for Kubernetes environment (#7327)
-- Call EnableDrainModeAsync on ApplicationStopping (#7262)
-
-**Release sprint:** Sprint 100
-[ [bugs](https://github.com/Azure/azure-functions-host/issues?q=is%3Aissue+milestone%3A%22Functions+Sprint+100%22+label%3Abug+is%3Aclosed) | [features](https://github.com/Azure/azure-functions-host/issues?q=is%3Aissue+milestone%3A%22Functions+Sprint+100%22+label%3Afeature+is%3Aclosed) ]
-- Update App Service Authentication/Authorization on Linux Consumption from 1.4.0 to 1.4.5. Release notes for this feature captured at https://github.com/Azure/app-service-announcements/issues. (#7205)
-- Add v12 Storage-dependent implementations to Functions Host to support Managed Identity integration
+This is a hotfix addressing issue with AzureStorageProvider using the wrong configuration in custom startup scenarios (#7424) [ [bug](https://github.com/Azure/azure-functions-host/issues/7440) ]

src/WebJobs.Script/ScriptHostBuilderExtensions.cs

@@ -464,10 +464,15 @@ private static void AddProcessRegistry(IServiceCollection services)
         private static IDistributedLockManager GetBlobLockManager(IServiceProvider provider)
         {
             var azureStorageProvider = provider.GetRequiredService<IAzureStorageProvider>();
+            var configuration = provider.GetRequiredService<IConfiguration>();
             var loggerFactory = provider.GetRequiredService<ILoggerFactory>();
             try
             {
-                var container = azureStorageProvider.GetBlobContainerClient();
+                // BlobLeaseDistributedLockManager is created during ScriptHost service registration since it is
+                // conditioned on having a valid storage connection. The configuration should be from the IServiceProvider
+                // that registers the IDistributedLockManager.
+                // The configuration used by AzureStorageProvider will be updated through the IScriptHostManager.HostInitializing event handler
+                var container = azureStorageProvider.GetBlobContainerClient(configuration);
                 return new BlobLeaseDistributedLockManager(loggerFactory, azureStorageProvider);
             }
             catch (InvalidOperationException)

src/WebJobs.Script/StorageProvider/AzureStorageProvider.cs

@@ -5,6 +5,7 @@
 using Azure.Storage.Blobs;
 using Microsoft.Azure.WebJobs.Script.StorageProvider;
 using Microsoft.Extensions.Configuration;
+using Microsoft.Extensions.DependencyInjection;
 using Microsoft.Extensions.Logging;
 using Microsoft.Extensions.Options;
 
@@ -18,16 +19,23 @@ namespace Microsoft.Azure.WebJobs.Script
     internal class AzureStorageProvider : IAzureStorageProvider
     {
         private readonly BlobServiceClientProvider _blobServiceClientProvider;
-        private readonly IConfiguration _configuration;
-        private readonly ILogger _logger;
+        private readonly IServiceProvider _serviceProvider;
         private IOptionsMonitor<JobHostInternalStorageOptions> _storageOptions;
+        private IConfiguration _configuration;
 
-        public AzureStorageProvider(IConfiguration configuration, BlobServiceClientProvider blobServiceClientProvider, IOptionsMonitor<JobHostInternalStorageOptions> options, ILogger<AzureStorageProvider> logger)
+        public AzureStorageProvider(IScriptHostManager scriptHostManager, IConfiguration configuration, BlobServiceClientProvider blobServiceClientProvider, IOptionsMonitor<JobHostInternalStorageOptions> options)
         {
             _blobServiceClientProvider = blobServiceClientProvider;
             _configuration = configuration;
+            _serviceProvider = scriptHostManager as IServiceProvider;
             _storageOptions = options;
-            _logger = logger;
+
+            // Every time script host is re-intializing, we also need to re-initialize
+            // services that change with the scope of the script host.
+            scriptHostManager.HostInitializing += (s, e) =>
+            {
+                InitializeServices();
+            };
         }
 
         /// <summary>
@@ -49,25 +57,30 @@ public virtual bool TryGetBlobServiceClientFromConnectionString(out BlobServiceC
         /// <param name="client">client to instantiate</param>
         /// <param name="connection">Name of the connection to use</param>
         /// <returns>successful client creation</returns>
-        public virtual bool TryGetBlobServiceClientFromConnection(out BlobServiceClient client, string connection = null)
+        public virtual bool TryGetBlobServiceClientFromConnection(out BlobServiceClient client, string connection = null, IConfiguration configurationOverride = null)
         {
             var connectionToUse = connection ?? ConnectionStringNames.Storage;
-            return _blobServiceClientProvider.TryGet(connectionToUse, out client);
+            return _blobServiceClientProvider.TryGet(connectionToUse, configurationOverride ?? _configuration, out client);
         }
 
-        public virtual BlobContainerClient GetBlobContainerClient()
+        public virtual BlobContainerClient GetBlobContainerClient(IConfiguration configurationOverride = null)
         {
             if (_storageOptions?.CurrentValue.InternalSasBlobContainer != null)
             {
                 return new BlobContainerClient(new Uri(_storageOptions.CurrentValue.InternalSasBlobContainer));
             }
 
-            if (!TryGetBlobServiceClientFromConnection(out BlobServiceClient blobServiceClient, ConnectionStringNames.Storage))
+            if (!TryGetBlobServiceClientFromConnection(out BlobServiceClient blobServiceClient, ConnectionStringNames.Storage, configurationOverride))
             {
                 throw new InvalidOperationException($"Could not create BlobServiceClient to obtain the BlobContainerClient using Connection: {ConnectionStringNames.Storage}");
             }
 
             return blobServiceClient.GetBlobContainerClient(ScriptConstants.AzureWebJobsHostsContainerName);
         }
+
+        private void InitializeServices()
+        {
+            _configuration = _serviceProvider.GetService<IConfiguration>();
+        }
     }
 }

src/WebJobs.Script/StorageProvider/BlobServiceClientProvider.cs

@@ -16,8 +16,8 @@ namespace Microsoft.Azure.WebJobs.Script.StorageProvider
     /// </summary>
     internal class BlobServiceClientProvider : StorageClientProvider<BlobServiceClient, BlobClientOptions>
     {
-        public BlobServiceClientProvider(IConfiguration configuration, AzureComponentFactory componentFactory, AzureEventSourceLogForwarder logForwarder, ILogger<BlobServiceClient> logger)
-            : base(configuration, componentFactory, logForwarder, logger) { }
+        public BlobServiceClientProvider(AzureComponentFactory componentFactory, AzureEventSourceLogForwarder logForwarder, ILogger<BlobServiceClient> logger)
+            : base(componentFactory, logForwarder, logger) { }
 
         /// <inheritdoc/>
         protected override string ServiceUriSubDomain

src/WebJobs.Script/StorageProvider/IAzureStorageProvider.cs

@@ -2,6 +2,7 @@
 // Licensed under the MIT License. See License.txt in the project root for license information.
 
 using Azure.Storage.Blobs;
+using Microsoft.Extensions.Configuration;
 
 namespace Microsoft.Azure.WebJobs.Script
 {
@@ -13,8 +14,8 @@ public interface IAzureStorageProvider
     {
         bool TryGetBlobServiceClientFromConnectionString(out BlobServiceClient client, string connectionString);
 
-        bool TryGetBlobServiceClientFromConnection(out BlobServiceClient client, string connection);
+        bool TryGetBlobServiceClientFromConnection(out BlobServiceClient client, string connection, IConfiguration configurationOverride = null);
 
-        BlobContainerClient GetBlobContainerClient();
+        BlobContainerClient GetBlobContainerClient(IConfiguration configurationOverride = null);
     }
 }

src/WebJobs.Script/StorageProvider/StorageClientProvider.cs

@@ -21,7 +21,6 @@ namespace Microsoft.Azure.WebJobs.Script.StorageProvider
     /// </summary>
     internal abstract class StorageClientProvider<TClient, TClientOptions> where TClientOptions : ClientOptions
     {
-        private readonly IConfiguration _configuration;
         private readonly AzureComponentFactory _componentFactory;
         private readonly AzureEventSourceLogForwarder _logForwarder;
         private readonly ILogger _logger;
@@ -32,9 +31,8 @@ internal abstract class StorageClientProvider<TClient, TClientOptions> where TCl
         /// <param name="configuration">The configuration to use when creating Client-specific objects. <see cref="IConfiguration"/></param>
         /// <param name="componentFactory">The Azure factory responsible for creating clients. <see cref="AzureComponentFactory"/></param>
         /// <param name="logForwarder">Log forwarder that forwards events to ILogger. <see cref="AzureEventSourceLogForwarder"/></param>
-        public StorageClientProvider(IConfiguration configuration, AzureComponentFactory componentFactory, AzureEventSourceLogForwarder logForwarder, ILogger<TClient> logger)
+        public StorageClientProvider(AzureComponentFactory componentFactory, AzureEventSourceLogForwarder logForwarder, ILogger<TClient> logger)
         {
-            _configuration = configuration;
             _componentFactory = componentFactory;
             _logForwarder = logForwarder;
             _logger = logger;
@@ -53,13 +51,14 @@ public StorageClientProvider(IConfiguration configuration, AzureComponentFactory
         /// Gets the storage client specified by <paramref name="name"/>
         /// </summary>
         /// <param name="name">Name of the connection to use</param>
+        /// <param name="configuration">Configuration to use</param>
         /// <param name="client">Client that was created if the connection was valid; otherwise, the default value for the type of <paramref name="client"/>.</param>
         /// <returns>true if the client could be created; false otherwise</returns>
-        public virtual bool TryGet(string name, out TClient client)
+        public virtual bool TryGet(string name, IConfiguration configuration, out TClient client)
         {
             try
             {
-                client = Get(name);
+                client = Get(name, configuration);
                 return true;
             }
             catch (Exception ex)
@@ -96,27 +95,29 @@ public virtual bool TryGetFromConnectionString(string connectionString, out TCli
         /// </summary>
         /// <param name="name">Name of the connection to use</param>
         /// <param name="resolver">A resolver to interpret the provided connection <paramref name="name"/>.</param>
+        /// <param name="configuration">Configuration to use</param>
         /// <returns>Client that was created.</returns>
-        public virtual TClient Get(string name, INameResolver resolver)
+        public virtual TClient Get(string name, INameResolver resolver, IConfiguration configuration)
         {
             var resolvedName = resolver.ResolveWholeString(name);
-            return this.Get(resolvedName);
+            return this.Get(resolvedName, configuration);
         }
 
         /// <summary>
         /// Gets the storage client specified by <paramref name="name"/>
         /// </summary>
         /// <param name="name">Name of the connection to use</param>
+        /// <param name="configuration">Configuration to use</param>
         /// <returns>Client that was created.</returns>
-        public virtual TClient Get(string name)
+        public virtual TClient Get(string name, IConfiguration configuration)
         {
             if (string.IsNullOrWhiteSpace(name))
             {
                 name = ConnectionStringNames.Storage; // default
             }
 
             // $$$ Where does validation happen?
-            IConfigurationSection connectionSection = _configuration.GetWebJobsConnectionStringSection(name);
+            IConfigurationSection connectionSection = configuration.GetWebJobsConnectionStringSection(name);
             if (!connectionSection.Exists())
             {
                 // Not found
@@ -148,7 +149,7 @@ protected virtual TClient CreateClient(IConfiguration configuration, TokenCreden
         /// <returns>Storage client</returns>
         public virtual TClient GetHost()
         {
-            return this.Get(null);
+            return this.Get(null, null);
         }
 
         /// <summary>

test/WebJobs.Script.Tests.Integration/Host/SingletonTests/SingletonEndToEndTests.cs

@@ -25,6 +25,7 @@
 using Newtonsoft.Json.Linq;
 using Xunit;
 using Microsoft.Azure.WebJobs.Host;
+using Microsoft.WebJobs.Script.Tests;
 
 namespace Microsoft.Azure.WebJobs.Script.Tests.Integration.Host.SingletonTests
 {
@@ -655,6 +656,7 @@ private IHost CreateTestJobHost<TProg>(int hostId, Action<IHostBuilder> extraCon
                     });
 
                     services.AddSingleton<IDistributedLockManager, BlobLeaseDistributedLockManager>();
+                    TestHostBuilderExtensions.AddMockedSingleton<IScriptHostManager>(services);
                     services.AddAzureStorageProvider();
                 });
 

test/WebJobs.Script.Tests.Integration/ScriptHostEndToEnd/FunctionGeneratorEndToEndTests.cs

@@ -12,6 +12,7 @@
 using Microsoft.Extensions.DependencyInjection;
 using Microsoft.Extensions.Hosting;
 using Microsoft.Extensions.Logging;
+using Microsoft.WebJobs.Script.Tests;
 using Xunit;
 
 namespace Microsoft.Azure.WebJobs.Script.Tests
@@ -70,7 +71,7 @@ public async Task Generate_EndToEnd()
                 {
                     s.AddSingleton<ITypeLocator>(new TestTypeLocator(functionType));
                     s.AddSingleton<ILoggerFactory>(new LoggerFactory());
-
+                    TestHostBuilderExtensions.AddMockedSingleton<IScriptHostManager>(s);
                     s.AddAzureStorageProvider();
                 });
 

test/WebJobs.Script.Tests.Integration/Storage/BlobStorageProviderTests.cs

@@ -36,14 +36,14 @@ public BlobStorageProviderTests()
         [Fact]
         public async Task TestBlobStorageProvider_TryConnectionName()
         {
-            Assert.True(_blobServiceClientProvider.TryGet(StorageConnection, out BlobServiceClient client));
+            Assert.True(_blobServiceClientProvider.TryGet(StorageConnection, _configuration, out BlobServiceClient client));
             await VerifyServiceAvailable(client);
         }
 
         [Fact]
         public async Task TestBlobStorageProvider_ConnectionName()
         {
-            BlobServiceClient client = _blobServiceClientProvider.Get(StorageConnection);
+            BlobServiceClient client = _blobServiceClientProvider.Get(StorageConnection, _configuration);
             await VerifyServiceAvailable(client);
         }
 
@@ -52,7 +52,7 @@ public async Task TestBlobStorageProvider_ConnectionNameWithResolver()
         {
             var resolver = new DefaultNameResolver(_configuration);
 
-            BlobServiceClient client = _blobServiceClientProvider.Get(StorageConnection, resolver);
+            BlobServiceClient client = _blobServiceClientProvider.Get(StorageConnection, resolver, _configuration);
             await VerifyServiceAvailable(client);
         }
 

test/WebJobs.Script.Tests.Shared/TestHelpers.cs

@@ -20,6 +20,7 @@
 using Microsoft.Extensions.DependencyInjection;
 using Microsoft.Extensions.Hosting;
 using Microsoft.Extensions.Options;
+using Microsoft.WebJobs.Script.Tests;
 
 namespace Microsoft.Azure.WebJobs.Script.Tests
 {
@@ -407,6 +408,7 @@ public static IAzureStorageProvider GetAzureStorageProvider(IConfiguration confi
                 {
                     // Override configuration
                     services.AddSingleton(configuration);
+                    TestHostBuilderExtensions.AddMockedSingleton<IScriptHostManager>(services);
                     services.AddAzureStorageProvider();
                     if (storageOptions != null)
                     {