Adds support to create TableServiceClient from JobHost configuration (#10891)

Author: cjaliaga

src/WebJobs.Script/StorageProvider/HostAzureTableStorageProvider.cs

@@ -3,6 +3,8 @@
 
 using System;
 using Azure.Data.Tables;
+using Microsoft.Azure.WebJobs.Script.Config;
+using Microsoft.Azure.WebJobs.Script.Configuration;
 using Microsoft.Extensions.Azure;
 using Microsoft.Extensions.Configuration;
 using Microsoft.Extensions.Logging;
@@ -15,11 +17,32 @@ internal sealed class HostAzureTableStorageProvider : IAzureTableStorageProvider
         private readonly TableServiceClientProvider _tableServiceClientProvider;
         private readonly IConfiguration _configuration;
 
-        public HostAzureTableStorageProvider(IConfiguration configuration, ILogger<HostAzureTableStorageProvider> logger, AzureComponentFactory componentFactory, AzureEventSourceLogForwarder logForwarder)
+        public HostAzureTableStorageProvider(IScriptHostManager scriptHostManager, IConfiguration configuration, ILogger<HostAzureTableStorageProvider> logger, AzureComponentFactory componentFactory, AzureEventSourceLogForwarder logForwarder)
         {
             _logger = logger ?? throw new ArgumentNullException(nameof(logger));
             _tableServiceClientProvider = new TableServiceClientProvider(componentFactory, logForwarder);
-            _configuration = configuration ?? throw new ArgumentNullException(nameof(configuration));
+
+            if (configuration is null)
+            {
+                throw new ArgumentNullException(nameof(configuration));
+            }
+
+            if (FeatureFlags.IsEnabled(ScriptConstants.FeatureFlagDisableMergedWebHostScriptHostConfiguration))
+            {
+                _configuration = configuration;
+            }
+            else
+            {
+                if (scriptHostManager == null)
+                {
+                    throw new ArgumentNullException(nameof(scriptHostManager));
+                }
+
+                _configuration = new ConfigurationBuilder()
+                    .Add(new ActiveHostConfigurationSource(scriptHostManager))
+                    .AddConfiguration(configuration)
+                    .Build();
+            }
         }
 
         public bool TryCreateTableServiceClient(string connection, out TableServiceClient client)

test/WebJobs.Script.Tests.Integration/Scale/TableStorageScaleMetricsRepositoryTests.cs

@@ -43,7 +43,7 @@ public TableStorageScaleMetricsRepositoryTests()
             _loggerProvider = new TestLoggerProvider();
             ILoggerFactory loggerFactory = new LoggerFactory();
             loggerFactory.AddProvider(_loggerProvider);
-            _azureTableStorageProvider = TestHelpers.GetAzureTableStorageProvider(configuration, new TestEnvironment());
+            _azureTableStorageProvider = TestHelpers.GetAzureTableStorageProvider(configuration, environment: new TestEnvironment());
             // Allow for up to 30 seconds of creation retries for tests due to slow table deletes
             _repository = new TableStorageScaleMetricsRepository(_hostIdProviderMock.Object, new OptionsWrapper<ScaleOptions>(_scaleOptions), loggerFactory, _azureTableStorageProvider, 60);
 

test/WebJobs.Script.Tests.Integration/Storage/AzureTableStorageProviderTests.cs

@@ -3,6 +3,7 @@
 
 using System;
 using System.Collections.Generic;
+using System.Text;
 using System.Threading.Tasks;
 using Azure.Data.Tables;
 using Microsoft.Extensions.Configuration;
@@ -33,6 +34,48 @@ public async Task TryCreateHostingTableServiceClient_ConnectionInWebHostConfigur
             await VerifyTableServiceClientAvailable(client);
         }
 
+        [Fact]
+        public async Task TryCreateHostingTableServiceClient_ConnectionInJobHostConfiguration()
+        {
+            var testConfiguration = TestHelpers.GetTestConfiguration();
+            var testData = new Dictionary<string, string>(StringComparer.Ordinal)
+            {
+                { "ConnectionStrings:AzureWebJobsStorage", testConfiguration.GetWebJobsConnectionString(StorageConnection) },
+                { "AzureWebJobsStorage", "" }
+            };
+
+            var webHostConfiguration = new ConfigurationBuilder().Build();
+            var jobHostConfiguration = new ConfigurationBuilder()
+                .AddInMemoryCollection(testData)
+                .Build();
+
+            var azureTableStorageProvider = TestHelpers.GetAzureTableStorageProvider(webHostConfiguration, jobHostConfiguration);
+            azureTableStorageProvider.TryCreateHostingTableServiceClient(out TableServiceClient client);
+            await VerifyTableServiceClientAvailable(client);
+        }
+
+        [Theory]
+        [InlineData("AzureWebJobsStorage:accountName", "storage")]
+        [InlineData("AzureWebJobsStorage:AccountName", "storage")]
+        [InlineData("AzureWebJobsStorage:tableServiceUri", "https://mytable.functions")]
+        [InlineData("AzureWebJobsStorage:TableServiceUri", "https://mytable.functions")]
+        public void TryCreateHostingTableServiceClient_IdentityBasedConnections(string key, string value)
+        {
+            var testData = new Dictionary<string, string>(StringComparer.Ordinal)
+            {
+                { key, value }
+            };
+
+            // Using a case sensitive data source to match the ScriptEnvironmentVariablesConfigurationSource behavior on Linux (case-sensitive env vars)
+            var webHostConfiguration = new ConfigurationBuilder().Build();
+            var jobHostConfiguration = new ConfigurationBuilder()
+                .Add(new CaseSensitiveConfigurationSource(testData))
+                .Build();
+
+            var azureTableStorageProvider = TestHelpers.GetAzureTableStorageProvider(webHostConfiguration, jobHostConfiguration);
+            Assert.True(azureTableStorageProvider.TryCreateHostingTableServiceClient(out _));
+        }
+
         [Fact]
         public void TryCreateHostingTableServiceClient_NoConnectionThrowsException()
         {
@@ -45,6 +88,36 @@ public void TryCreateHostingTableServiceClient_NoConnectionThrowsException()
             Assert.False(azureTableStorageProvider.TryCreateTableServiceClient(ConnectionStringNames.Storage, out TableServiceClient blobServiceClient));
         }
 
+        [Theory]
+        [InlineData("ConnectionStrings:AzureWebJobsStorage1")]
+        [InlineData("AzureWebJobsStorage1")]
+        [InlineData("Storage1")]
+        public void TestAzureBlobStorageProvider_JobHostConfigurationWinsConflict(string connectionName)
+        {
+            var bytes = Encoding.UTF8.GetBytes("someKey");
+            var encodedString = Convert.ToBase64String(bytes);
+
+            var webHostConfigData = new Dictionary<string, string>(StringComparer.OrdinalIgnoreCase)
+            {
+                { connectionName, $"DefaultEndpointsProtocol=https;AccountName=webHostAccount;AccountKey={encodedString};EndpointSuffix=core.windows.net" },
+            };
+            var webHostConfiguration = new ConfigurationBuilder()
+                .AddInMemoryCollection(webHostConfigData)
+                .Build();
+
+            var jobHostConfigData = new Dictionary<string, string>(StringComparer.OrdinalIgnoreCase)
+            {
+                { connectionName, $"DefaultEndpointsProtocol=https;AccountName=jobHostAccount;AccountKey={encodedString};EndpointSuffix=core.windows.net" },
+            };
+            var jobHostConfiguration = new ConfigurationBuilder()
+                .AddInMemoryCollection(jobHostConfigData)
+                .Build();
+
+            var azureTableStorageProvider = TestHelpers.GetAzureTableStorageProvider(webHostConfiguration, jobHostConfiguration);
+            Assert.True(azureTableStorageProvider.TryCreateTableServiceClient("Storage1", out TableServiceClient client));
+            Assert.Equal("webHostAccount", client.AccountName, ignoreCase: true);
+        }
+
         private async Task VerifyTableServiceClientAvailable(TableServiceClient client)
         {
             try
@@ -57,5 +130,32 @@ private async Task VerifyTableServiceClientAvailable(TableServiceClient client)
                 Assert.False(true, $"Could not establish connection to TableService. {e}");
             }
         }
+
+        private class CaseSensitiveConfigurationSource : IConfigurationSource
+        {
+            private readonly Dictionary<string, string> _data;
+            public CaseSensitiveConfigurationSource(Dictionary<string, string> data)
+            {
+                _data = data;
+            }
+            public IConfigurationProvider Build(IConfigurationBuilder builder)
+            {
+                return new CaseSensitiveConfigurationProvider(_data);
+            }
+        }
+
+        private class CaseSensitiveConfigurationProvider : ConfigurationProvider
+        {
+            private readonly Dictionary<string, string> _data;
+            public CaseSensitiveConfigurationProvider(Dictionary<string, string> data)
+            {
+                _data = data;
+            }
+            public override void Load()
+            {
+                // _data might be already case-insensitive but we want to ensure it's case-sensitive
+                Data = new Dictionary<string, string>(_data, StringComparer.Ordinal);
+            }
+        }
     }
 }

test/WebJobs.Script.Tests.Shared/TestHelpers.cs

@@ -504,29 +504,37 @@ public static IServiceCollection AddTestAzureBlobStorageProvider(IServiceCollect
             return services;
         }
 
-        public static IAzureTableStorageProvider GetAzureTableStorageProvider(IConfiguration configuration, IEnvironment environment = default)
+        public static IAzureTableStorageProvider GetAzureTableStorageProvider(IConfiguration webHostConfiguration, IConfiguration jobHostConfiguration = null, IEnvironment environment = default)
         {
             environment ??= new TestEnvironment();
 
             IHost tempHost = new HostBuilder()
                 .ConfigureServices(services =>
                 {
-                    AddTestAzureTableStorageProvider(services, configuration, environment);
+                    AddTestAzureTableStorageProvider(services, jobHostConfiguration ?? webHostConfiguration, environment);
                 })
                 .ConfigureAppConfiguration(c =>
                 {
-                    c.AddConfiguration(configuration);
+                    c.AddConfiguration(webHostConfiguration);
                 })
                 .Build();
 
             var azureTableStorageProvider = tempHost.Services.GetRequiredService<IAzureTableStorageProvider>();
             return azureTableStorageProvider;
         }
 
-        public static IServiceCollection AddTestAzureTableStorageProvider(IServiceCollection services, IConfiguration configuration, IEnvironment environment)
+        public static IServiceCollection AddTestAzureTableStorageProvider(IServiceCollection services, IConfiguration configuration, IEnvironment environment, IScriptHostManager scriptHostManager = null)
         {
             // Adds necessary Azure services to create clients
             services.AddAzureClientsCore();
+
+            if (scriptHostManager == null)
+            {
+                scriptHostManager = new TestScriptHostService(configuration);
+            }
+
+            services.AddSingleton<IScriptHostManager>(scriptHostManager);
+
             services.AddSingleton<IAzureTableStorageProvider, HostAzureTableStorageProvider>();
 
             return services;