Adding support for "secretless" storage (updating host to Storage SDK 12.x)

Author: karshinlin

azure-pipelines.yml

@@ -349,6 +349,7 @@ jobs:
         **\WebJobs.Script.Tests.Integration.csproj
     env:
       AzureWebJobsStorage: $(Storage)
+      AzureWebJobsSecondaryStorage: $(SecondaryStorage)
       ConnectionStrings__CosmosDB: $(CosmosDB)
       AzureWebJobsEventHubSender: $(EventHub)
       AzureWebJobsEventHubReceiver: $(EventHub)
@@ -405,11 +406,13 @@ jobs:
       targetType: 'inline'
       script: |
         Write-Host "##vso[task.setvariable variable=AzureWebJobsStorage]$env:AzureWebJobsStorageSecretMap"
+        Write-Host "##vso[task.setvariable variable=AzureWebJobsSeconaryStorage]$env:AzureWebJobsSecondaryStorageSecretMap"
         Write-Host "##vso[task.setvariable variable=ConnectionStrings__CosmosDB]$env:CosmosDbSecretMap"
         Write-Host "##vso[task.setvariable variable=AzureWebJobsEventHubSender]$env:AzureWebJobsEventHubSenderSecretMap"
         Write-Host "##vso[task.setvariable variable=AzureWebJobsEventHubReceiver]$env:AzureWebJobsEventHubReceiverSecretMap"
     env:
       AzureWebJobsStorageSecretMap: $(Storage)
+      AzureWebJobsSecondaryStorageSecretMap: $(SecondaryStorage)
       CosmosDbSecretMap: $(CosmosDb)
       AzureWebJobsEventHubSenderSecretMap: $(EventHub)
       AzureWebJobsEventHubReceiverSecretMap: $(EventHub)

release_notes.md

@@ -11,4 +11,5 @@
 
 **Release sprint:** Sprint 100
 [ [bugs](https://github.com/Azure/azure-functions-host/issues?q=is%3Aissue+milestone%3A%22Functions+Sprint+100%22+label%3Abug+is%3Aclosed) | [features](https://github.com/Azure/azure-functions-host/issues?q=is%3Aissue+milestone%3A%22Functions+Sprint+100%22+label%3Afeature+is%3Aclosed) ]
-- Update App Service Authentication/Authorization on Linux Consumption from 1.4.0 to 1.4.5. Release notes for this feature captured at https://github.com/Azure/app-service-announcements/issues. (#7205)
+- Update App Service Authentication/Authorization on Linux Consumption from 1.4.0 to 1.4.5. Release notes for this feature captured at https://github.com/Azure/app-service-announcements/issues. (#7205)
+- Add v12 Storage-dependent implementations to Functions Host to support Managed Identity integration

src/WebJobs.Script.WebHost/BreakingChangeAnalysis/BlobChangeAnalysisStateProvider.cs

@@ -2,13 +2,14 @@
 // Licensed under the MIT License. See License.txt in the project root for license information.
 
 using System;
+using System.IO;
 using System.Threading;
 using System.Threading.Tasks;
+using Azure;
+using Azure.Storage.Blobs;
 using Microsoft.Azure.WebJobs.Host.Executors;
 using Microsoft.Extensions.Configuration;
 using Microsoft.Extensions.Logging;
-using Microsoft.WindowsAzure.Storage;
-using Microsoft.WindowsAzure.Storage.Blob;
 
 namespace Microsoft.Azure.WebJobs.Script.ChangeAnalysis
 {
@@ -19,63 +20,84 @@ internal sealed class BlobChangeAnalysisStateProvider : IChangeAnalysisStateProv
         private readonly ILogger _logger;
         private readonly IConfiguration _configuration;
         private readonly IHostIdProvider _hostIdProvider;
+        private readonly IAzureStorageProvider _azureStorageProvider;
 
-        public BlobChangeAnalysisStateProvider(IConfiguration configuration, IHostIdProvider hostIdProvider, ILogger<BlobChangeAnalysisStateProvider> logger)
+        public BlobChangeAnalysisStateProvider(IConfiguration configuration, IHostIdProvider hostIdProvider, ILogger<BlobChangeAnalysisStateProvider> logger, IAzureStorageProvider azureStorageProvider)
         {
             _configuration = configuration;
             _hostIdProvider = hostIdProvider;
             _logger = logger;
+            _azureStorageProvider = azureStorageProvider;
         }
 
         public async Task<ChangeAnalysisState> GetCurrentAsync(CancellationToken cancellationToken)
         {
-            string storageConnectionString = _configuration.GetWebJobsConnectionString(ConnectionStringNames.Storage);
-
-            if (string.IsNullOrEmpty(storageConnectionString) ||
-                !CloudStorageAccount.TryParse(storageConnectionString, out CloudStorageAccount account))
+            if (!_azureStorageProvider.TryGetBlobServiceClientFromConnection(out BlobServiceClient blobServiceClient, ConnectionStringNames.Storage))
             {
                 throw new InvalidOperationException($"The {nameof(BlobChangeAnalysisStateProvider)} requires the default storage account '{ConnectionStringNames.Storage}', which is not defined.");
             }
 
             string hostId = await _hostIdProvider.GetHostIdAsync(cancellationToken);
             string analysisBlobPath = $"changeanalysis/{hostId}/sentinel";
 
-            CloudBlobClient blobClient = account.CreateCloudBlobClient();
-            var blobContainer = blobClient.GetContainerReference(ScriptConstants.AzureWebJobsHostsContainerName);
-
             DateTimeOffset lastAnalysisTime = DateTimeOffset.MinValue;
-
-            var reference = blobContainer.GetBlockBlobReference(analysisBlobPath);
-            bool blobExists = await reference.ExistsAsync(null, null, cancellationToken);
-            if (blobExists)
+            BlobClient blobClient = default;
+            try
             {
-                reference.Metadata.TryGetValue(AnalysisTimestampMetadataName, out string lastAnalysisMetadata);
+                var blobContainerClient = blobServiceClient.GetBlobContainerClient(ScriptConstants.AzureWebJobsHostsContainerName);
+                blobClient = blobContainerClient.GetBlobClient(analysisBlobPath);
+                if (await blobClient.ExistsAsync(cancellationToken: cancellationToken))
+                {
+                    var blobPropertiesResponse = await blobClient.GetPropertiesAsync(cancellationToken: cancellationToken);
+                    blobPropertiesResponse.Value.Metadata.TryGetValue(AnalysisTimestampMetadataName, out string lastAnalysisMetadata);
 
-                _logger.LogInformation("Last analysis flag value '{flag}'.", lastAnalysisMetadata ?? "(null)");
+                    _logger.LogInformation("Last analysis flag value '{flag}'.", lastAnalysisMetadata ?? "(null)");
 
-                if (!DateTimeOffset.TryParse(lastAnalysisMetadata, out lastAnalysisTime))
-                {
-                    _logger.LogInformation("Unable to parse last analysis timestamp flag. Default (MinValue) will be used.");
+                    if (!DateTimeOffset.TryParse(lastAnalysisMetadata, out lastAnalysisTime))
+                    {
+                        _logger.LogInformation("Unable to parse last analysis timestamp flag. Default (MinValue) will be used.");
+                    }
                 }
             }
+            catch (RequestFailedException ex)
+            {
+                _logger.LogError(ex, "Unable to get current change analysis state.");
+            }
 
-            return new ChangeAnalysisState(lastAnalysisTime, reference);
+            return new ChangeAnalysisState(lastAnalysisTime, blobClient);
         }
 
         public async Task SetTimestampAsync(DateTimeOffset timestamp, object handle, CancellationToken cancellationToken)
         {
-            if (handle is CloudBlockBlob blob)
+            if (handle is BlobClient blobClient)
             {
-                if (!await blob.ExistsAsync())
+                try
                 {
-                    await blob.UploadTextAsync(string.Empty);
-                }
+                    if (!await blobClient.ExistsAsync())
+                    {
+                        using (Stream stream = new MemoryStream())
+                        {
+                            await blobClient.UploadAsync(stream, cancellationToken: cancellationToken);
+                        }
+                    }
 
-                string timestampValue = timestamp.ToString("O");
-                blob.Metadata[AnalysisTimestampMetadataName] = timestampValue;
-                await blob.SetMetadataAsync(null, null, null, cancellationToken);
+                    string timestampValue = timestamp.ToString("O");
 
-                _logger.LogInformation("Analysis blob metadata updated with analysis timestamp '{timestamp}'.", timestampValue);
+                    var blobPropertiesResponse = await blobClient.GetPropertiesAsync(cancellationToken: cancellationToken);
+                    blobPropertiesResponse.Value.Metadata[AnalysisTimestampMetadataName] = timestampValue;
+
+                    await blobClient.SetMetadataAsync(blobPropertiesResponse.Value.Metadata, cancellationToken: cancellationToken);
+
+                    _logger.LogInformation("Analysis blob metadata updated with analysis timestamp '{timestamp}'.", timestampValue);
+                }
+                catch (RequestFailedException ex)
+                {
+                    _logger.LogError(ex, "Unable to set change analysis state metadata.");
+                }
+            }
+            else
+            {
+                _logger.LogError("Analysis blob SetTimestampAsync was given a null BlobClient");
             }
         }
     }

src/WebJobs.Script.WebHost/Management/FunctionsSyncManager.cs

@@ -11,8 +11,7 @@
 using System.Text.RegularExpressions;
 using System.Threading;
 using System.Threading.Tasks;
-using Microsoft.Azure.Storage;
-using Microsoft.Azure.Storage.Blob;
+using Azure.Storage.Blobs;
 using Microsoft.Azure.WebJobs.Host.Executors;
 using Microsoft.Azure.WebJobs.Script.Description;
 using Microsoft.Azure.WebJobs.Script.Models;
@@ -60,10 +59,11 @@ public class FunctionsSyncManager : IFunctionsSyncManager, IDisposable
         private readonly HostNameProvider _hostNameProvider;
         private readonly IFunctionMetadataManager _functionMetadataManager;
         private readonly SemaphoreSlim _syncSemaphore = new SemaphoreSlim(1, 1);
+        private readonly IAzureStorageProvider _azureStorageProvider;
 
-        private CloudBlockBlob _hashBlob;
+        private BlobClient _hashBlobClient;
 
-        public FunctionsSyncManager(IConfiguration configuration, IHostIdProvider hostIdProvider, IOptionsMonitor<ScriptApplicationHostOptions> applicationHostOptions, ILogger<FunctionsSyncManager> logger, HttpClient httpClient, ISecretManagerProvider secretManagerProvider, IScriptWebHostEnvironment webHostEnvironment, IEnvironment environment, HostNameProvider hostNameProvider, IFunctionMetadataManager functionMetadataManager)
+        public FunctionsSyncManager(IConfiguration configuration, IHostIdProvider hostIdProvider, IOptionsMonitor<ScriptApplicationHostOptions> applicationHostOptions, ILogger<FunctionsSyncManager> logger, HttpClient httpClient, ISecretManagerProvider secretManagerProvider, IScriptWebHostEnvironment webHostEnvironment, IEnvironment environment, HostNameProvider hostNameProvider, IFunctionMetadataManager functionMetadataManager, IAzureStorageProvider azureStorageProvider)
         {
             _applicationHostOptions = applicationHostOptions;
             _logger = logger;
@@ -75,6 +75,7 @@ public FunctionsSyncManager(IConfiguration configuration, IHostIdProvider hostId
             _environment = environment;
             _hostNameProvider = hostNameProvider;
             _functionMetadataManager = functionMetadataManager;
+            _azureStorageProvider = azureStorageProvider;
         }
 
         internal bool ArmCacheEnabled
@@ -104,8 +105,8 @@ public async Task<SyncTriggersResult> TrySyncTriggersAsync(bool isBackgroundSync
             {
                 await _syncSemaphore.WaitAsync();
 
-                var hashBlob = await GetHashBlobAsync();
-                if (isBackgroundSync && hashBlob == null)
+                var hashBlobClient = await GetHashBlobAsync();
+                if (isBackgroundSync && hashBlobClient == null)
                 {
                     // short circuit before doing any work in background sync
                     // cases where we need to check/update hash but don't have
@@ -128,7 +129,7 @@ public async Task<SyncTriggersResult> TrySyncTriggersAsync(bool isBackgroundSync
                 string newHash = null;
                 if (isBackgroundSync)
                 {
-                    newHash = await CheckHashAsync(hashBlob, payload.Content);
+                    newHash = await CheckHashAsync(hashBlobClient, payload.Content);
                     shouldSyncTriggers = newHash != null;
                 }
 
@@ -137,7 +138,7 @@ public async Task<SyncTriggersResult> TrySyncTriggersAsync(bool isBackgroundSync
                     var (success, error) = await SetTriggersAsync(payload.Content);
                     if (success && newHash != null)
                     {
-                        await UpdateHashAsync(hashBlob, newHash);
+                        await UpdateHashAsync(hashBlobClient, newHash);
                     }
                     result.Success = success;
                     result.Error = error;
@@ -193,7 +194,7 @@ internal static bool IsSyncTriggersEnvironment(IScriptWebHostEnvironment webHost
             return true;
         }
 
-        internal async Task<string> CheckHashAsync(CloudBlockBlob hashBlob, string content)
+        internal async Task<string> CheckHashAsync(BlobClient hashBlobClient, string content)
         {
             try
             {
@@ -210,9 +211,13 @@ internal async Task<string> CheckHashAsync(CloudBlockBlob hashBlob, string conte
 
                 // get the last hash value if present
                 string lastHash = null;
-                if (await hashBlob.ExistsAsync())
+                if (await hashBlobClient.ExistsAsync())
                 {
-                    lastHash = await hashBlob.DownloadTextAsync();
+                    var downloadResponse = await hashBlobClient.DownloadAsync();
+                    using (StreamReader reader = new StreamReader(downloadResponse.Value.Content))
+                    {
+                        lastHash = reader.ReadToEnd();
+                    }
                     _logger.LogDebug($"SyncTriggers hash (Last='{lastHash}', Current='{currentHash}')");
                 }
 
@@ -234,13 +239,16 @@ internal async Task<string> CheckHashAsync(CloudBlockBlob hashBlob, string conte
             return null;
         }
 
-        internal async Task UpdateHashAsync(CloudBlockBlob hashBlob, string hash)
+        internal async Task UpdateHashAsync(BlobClient hashBlobClient, string hash)
         {
             try
             {
                 // hash value has changed or was not yet stored
                 // update the last hash value in storage
-                await hashBlob.UploadTextAsync(hash);
+                using (Stream stream = new MemoryStream(Encoding.UTF8.GetBytes(hash)))
+                {
+                    await hashBlobClient.UploadAsync(stream);
+                }
                 _logger.LogDebug($"SyncTriggers hash updated to '{hash}'");
             }
             catch (Exception ex)
@@ -250,23 +258,19 @@ internal async Task UpdateHashAsync(CloudBlockBlob hashBlob, string hash)
             }
         }
 
-        internal async Task<CloudBlockBlob> GetHashBlobAsync()
+        internal async Task<BlobClient> GetHashBlobAsync()
         {
-            if (_hashBlob == null)
+            if (_hashBlobClient == null)
             {
-                string storageConnectionString = _configuration.GetWebJobsConnectionString(ConnectionStringNames.Storage);
-                CloudStorageAccount account = null;
-                if (!string.IsNullOrEmpty(storageConnectionString) &&
-                    CloudStorageAccount.TryParse(storageConnectionString, out account))
+                if (_azureStorageProvider.TryGetBlobServiceClientFromConnection(out BlobServiceClient blobClient, ConnectionStringNames.Storage))
                 {
                     string hostId = await _hostIdProvider.GetHostIdAsync(CancellationToken.None);
-                    CloudBlobClient blobClient = account.CreateCloudBlobClient();
-                    var blobContainer = blobClient.GetContainerReference(ScriptConstants.AzureWebJobsHostsContainerName);
+                    var blobContainerClient = blobClient.GetBlobContainerClient(ScriptConstants.AzureWebJobsHostsContainerName);
                     string hashBlobPath = $"synctriggers/{hostId}/last";
-                    _hashBlob = blobContainer.GetBlockBlobReference(hashBlobPath);
+                    _hashBlobClient = blobContainerClient.GetBlobClient(hashBlobPath);
                 }
             }
-            return _hashBlob;
+            return _hashBlobClient;
         }
 
         public async Task<SyncTriggersPayload> GetSyncTriggersPayload()

src/WebJobs.Script.WebHost/Security/KeyManagement/BlobStorageSasSecretsRepository.cs

@@ -2,7 +2,7 @@
 // Licensed under the MIT License. See License.txt in the project root for license information.
 
 using System;
-using Microsoft.Azure.Storage.Blob;
+using Azure.Storage.Blobs;
 using Microsoft.Azure.WebJobs.Script.WebHost.Diagnostics.Extensions;
 using Microsoft.Extensions.Logging;
 
@@ -13,14 +13,14 @@ namespace Microsoft.Azure.WebJobs.Script.WebHost
     /// </summary>
     public sealed class BlobStorageSasSecretsRepository : BlobStorageSecretsRepository
     {
-        public BlobStorageSasSecretsRepository(string secretSentinelDirectoryPath, string containerSasUri, string siteSlotName, ILogger logger, IEnvironment environment)
-            : base(secretSentinelDirectoryPath, containerSasUri, siteSlotName, logger, environment)
+        public BlobStorageSasSecretsRepository(string secretSentinelDirectoryPath, string containerSasUri, string siteSlotName, ILogger logger, IEnvironment environment, IAzureStorageProvider azureStorageProvider)
+            : base(secretSentinelDirectoryPath, containerSasUri, siteSlotName, logger, environment, azureStorageProvider)
         {
         }
 
-        protected override CloudBlobContainer CreateBlobContainer(string containerSasUri)
+        protected override BlobContainerClient CreateBlobContainerClient(string containerSasUri)
         {
-            return new CloudBlobContainer(new Uri(containerSasUri));
+            return new BlobContainerClient(new Uri(containerSasUri));
         }
 
         protected override void LogErrorMessage(string operation)