Added feature to connect to blob secret storage with sas and supporting tests.

gzuber · fabiocav · commit 46da4bf1a9e1 · 2019-07-26T13:01:22.000-07:00
diff --git a/src/WebJobs.Script.WebHost/Security/KeyManagement/BlobStorageSasSecretsRepository.cs b/src/WebJobs.Script.WebHost/Security/KeyManagement/BlobStorageSasSecretsRepository.cs
@@ -0,0 +1,23 @@
+﻿// Copyright (c) .NET Foundation. All rights reserved.
+// Licensed under the MIT License. See License.txt in the project root for license information.
+
+using System;
+using Microsoft.WindowsAzure.Storage.Blob;
+
+namespace Microsoft.Azure.WebJobs.Script.WebHost
+{
+    /// <summary>
+    /// An <see cref="BlobStorageSecretsRepository"/> implementation that uses a SAS connection string to connect to Azure blob storage.
+    /// </summary>
+    public sealed class BlobStorageSasSecretsRepository : BlobStorageSecretsRepository
+    {
+        public BlobStorageSasSecretsRepository(string secretSentinelDirectoryPath, string containerSasUri, string siteSlotName)
+            : base(secretSentinelDirectoryPath, containerSasUri, siteSlotName)
+        { }
+
+        protected override CloudBlobContainer CreateBlobContainer(string containerSasUri)
+        {
+            return new CloudBlobContainer(new Uri(containerSasUri));
+        }
+    }
+}
diff --git a/src/WebJobs.Script.WebHost/Security/KeyManagement/BlobStorageSecretsRepository.cs b/src/WebJobs.Script.WebHost/Security/KeyManagement/BlobStorageSecretsRepository.cs
@@ -17,7 +17,7 @@ namespace Microsoft.Azure.WebJobs.Script.WebHost
     /// <summary>
     /// An <see cref="ISecretsRepository"/> implementation that uses Azure blob storage as the backing store.
     /// </summary>
-    public sealed class BlobStorageSecretsRepository : BaseSecretsRepository
+    public class BlobStorageSecretsRepository : BaseSecretsRepository
     {
         private readonly string _secretsBlobPath;
         private readonly string _hostSecretsBlobPath;
@@ -44,13 +44,7 @@ public BlobStorageSecretsRepository(string secretSentinelDirectoryPath, string a
             _hostSecretsBlobPath = string.Format("{0}/{1}", _secretsBlobPath, ScriptConstants.HostMetadataFileName);
 
             _accountConnectionString = accountConnectionString;
-            CloudStorageAccount account = CloudStorageAccount.Parse(_accountConnectionString);
-            CloudBlobClient client = account.CreateCloudBlobClient();
-
-            _blobContainer = client.GetContainerReference(_secretsContainerName);
-
-            // TODO: Remove this (it is already slated to be removed)
-            _blobContainer.CreateIfNotExistsAsync().GetAwaiter().GetResult();
+            _blobContainer = CreateBlobContainer(_accountConnectionString);
         }
 
         public override bool IsEncryptionSupported
@@ -61,6 +55,18 @@ public override bool IsEncryptionSupported
             }
         }
 
+        protected virtual CloudBlobContainer CreateBlobContainer(string connectionString)
+        {
+            CloudStorageAccount account = CloudStorageAccount.Parse(_accountConnectionString);
+            CloudBlobClient client = account.CreateCloudBlobClient();
+            CloudBlobContainer container = client.GetContainerReference(_secretsContainerName);
+
+            // TODO: Remove this (it is already slated to be removed)
+            container.CreateIfNotExistsAsync().GetAwaiter().GetResult();
+
+            return container;
+        }
+
         public override async Task<ScriptSecrets> ReadAsync(ScriptSecretsType type, string functionName)
         {
             string secretsContent = null;
diff --git a/src/WebJobs.Script.WebHost/Security/KeyManagement/DefaultSecretManagerProvider.cs b/src/WebJobs.Script.WebHost/Security/KeyManagement/DefaultSecretManagerProvider.cs
@@ -15,6 +15,7 @@ namespace Microsoft.Azure.WebJobs.Script.WebHost
     public sealed class DefaultSecretManagerProvider : ISecretManagerProvider
     {
         private const string FileStorage = "Files";
+        private const string BlobStorage = "Blob";
         private readonly ILogger _logger;
         private readonly IMetricsLogger _metricsLogger;
         private readonly IOptionsMonitor<ScriptApplicationHostOptions> _options;
@@ -56,6 +57,7 @@ internal ISecretsRepository CreateSecretsRepository()
         {
             string secretStorageType = Environment.GetEnvironmentVariable(EnvironmentSettingNames.AzureWebJobsSecretStorageType);
             string storageString = _configuration.GetWebJobsConnectionString(ConnectionStringNames.Storage);
+            string secretStorageSas = _environment.GetEnvironmentVariable(EnvironmentSettingNames.AzureWebJobsSecretStorageSas);
             if (secretStorageType != null && secretStorageType.Equals(FileStorage, StringComparison.OrdinalIgnoreCase))
             {
                 return new FileSystemSecretsRepository(_options.CurrentValue.SecretsPath);
@@ -70,16 +72,21 @@ internal ISecretsRepository CreateSecretsRepository()
             {
                 return new KubernetesSecretsRepository(_environment, new SimpleKubernetesClient(_environment));
             }
-            else if (storageString == null)
+            else if (secretStorageSas != null)
             {
-                throw new InvalidOperationException($"Secret initialization from Blob storage failed due to a missing Azure Storage connection string. If you intend to use files for secrets, add an App Setting key '{EnvironmentSettingNames.AzureWebJobsSecretStorageType}' with value '{FileStorage}'.");
+                string siteSlotName = _environment.GetAzureWebsiteUniqueSlotName() ?? _hostIdProvider.GetHostIdAsync(CancellationToken.None).GetAwaiter().GetResult();
+                return new BlobStorageSasSecretsRepository(Path.Combine(_options.CurrentValue.SecretsPath, "Sentinels"), secretStorageSas, siteSlotName);
             }
-            else
+            else if (storageString != null)
             {
                 string siteSlotName = _environment.GetAzureWebsiteUniqueSlotName() ?? _hostIdProvider.GetHostIdAsync(CancellationToken.None).GetAwaiter().GetResult();
-
                 return new BlobStorageSecretsRepository(Path.Combine(_options.CurrentValue.SecretsPath, "Sentinels"), storageString, siteSlotName);
             }
+            else
+            {
+                throw new InvalidOperationException($"Secret initialization from Blob storage failed due to missing both an Azure Storage connection string and a SAS connection uri. " +
+                    $"For Blob Storage, please provide at least one of these. If you intend to use files for secrets, add an App Setting key '{EnvironmentSettingNames.AzureWebJobsSecretStorageType}' with value '{FileStorage}'.");
+            }
         }
     }
 }
diff --git a/src/WebJobs.Script/Environment/EnvironmentSettingNames.cs b/src/WebJobs.Script/Environment/EnvironmentSettingNames.cs
@@ -20,6 +20,7 @@ public static class EnvironmentSettingNames
         public const string TypeScriptCompilerPath = "AzureWebJobs_TypeScriptPath";
         public const string AzureWebsiteAppCountersName = "WEBSITE_COUNTERS_APP";
         public const string AzureWebJobsSecretStorageType = "AzureWebJobsSecretStorageType";
+        public const string AzureWebJobsSecretStorageSas = "AzureWebJobsSecretStorageSas";
         public const string AppInsightsInstrumentationKey = "APPINSIGHTS_INSTRUMENTATIONKEY";
         public const string AppInsightsQuickPulseAuthApiKey = "APPINSIGHTS_QUICKPULSEAUTHAPIKEY";
         public const string FunctionsExtensionVersion = "FUNCTIONS_EXTENSION_VERSION";
diff --git a/test/WebJobs.Script.Tests.Integration/Host/SecretsRepositoryTests.cs b/test/WebJobs.Script.Tests.Integration/Host/SecretsRepositoryTests.cs
@@ -36,6 +36,7 @@ public enum SecretsRepositoryType
         {
             FileSystem,
             BlobStorage,
+            BlobStorageSas,
             KeyVault
         }
 
@@ -45,10 +46,12 @@ public async Task FileSystemRepo_Constructor_CreatesSecretPathIfNotExists()
             await Constructor_CreatesSecretPathIfNotExists(SecretsRepositoryType.FileSystem);
         }
 
-        [Fact]
-        public async Task BlobStorageRepo_Constructor_CreatesSecretPathIfNotExists()
+        [Theory]
+        [InlineData(SecretsRepositoryType.BlobStorage)]
+        [InlineData(SecretsRepositoryType.BlobStorageSas)]
+        public async Task BlobStorageRepo_Constructor_CreatesSecretPathIfNotExists(SecretsRepositoryType repositoryType)
         {
-            await Constructor_CreatesSecretPathIfNotExists(SecretsRepositoryType.BlobStorage);
+            await Constructor_CreatesSecretPathIfNotExists(repositoryType);
         }
 
         [Fact]
@@ -80,11 +83,13 @@ private async Task Constructor_CreatesSecretPathIfNotExists(SecretsRepositoryTyp
         }
 
         [Theory]
-        [InlineData(ScriptSecretsType.Host)]
-        [InlineData(ScriptSecretsType.Function)]
-        public async Task BlobStorageRepo_ReadAsync_ReadsExpectedFile(ScriptSecretsType secretsType)
+        [InlineData(SecretsRepositoryType.BlobStorage, ScriptSecretsType.Host)]
+        [InlineData(SecretsRepositoryType.BlobStorage, ScriptSecretsType.Function)]
+        [InlineData(SecretsRepositoryType.BlobStorageSas, ScriptSecretsType.Host)]
+        [InlineData(SecretsRepositoryType.BlobStorageSas, ScriptSecretsType.Function)]
+        public async Task BlobStorageRepo_ReadAsync_ReadsExpectedFile(SecretsRepositoryType repositoryType, ScriptSecretsType secretsType)
         {
-            await ReadAsync_ReadsExpectedFile(SecretsRepositoryType.BlobStorage, secretsType);
+            await ReadAsync_ReadsExpectedFile(repositoryType, secretsType);
         }
 
         [Theory]
@@ -152,11 +157,13 @@ private async Task ReadAsync_ReadsExpectedFile(SecretsRepositoryType repositoryT
         }
 
         [Theory]
-        [InlineData(ScriptSecretsType.Host)]
-        [InlineData(ScriptSecretsType.Function)]
-        public async Task BlobStorageRepo_WriteAsync_CreatesExpectedFile(ScriptSecretsType secretsType)
+        [InlineData(SecretsRepositoryType.BlobStorage, ScriptSecretsType.Host)]
+        [InlineData(SecretsRepositoryType.BlobStorage, ScriptSecretsType.Function)]
+        [InlineData(SecretsRepositoryType.BlobStorageSas, ScriptSecretsType.Host)]
+        [InlineData(SecretsRepositoryType.BlobStorageSas, ScriptSecretsType.Function)]
+        public async Task BlobStorageRepo_WriteAsync_CreatesExpectedFile(SecretsRepositoryType repositoryType, ScriptSecretsType secretsType)
         {
-            await WriteAsync_CreatesExpectedFile(SecretsRepositoryType.BlobStorage, secretsType);
+            await WriteAsync_CreatesExpectedFile(repositoryType, secretsType);
         }
 
         [Theory]
@@ -204,7 +211,7 @@ private async Task WriteAsync_CreatesExpectedFile(SecretsRepositoryType reposito
 
                 string filePath = Path.Combine(directory.Path, $"{testFunctionName ?? "host"}.json");
 
-                if (repositoryType == SecretsRepositoryType.BlobStorage)
+                if (repositoryType == SecretsRepositoryType.BlobStorage || repositoryType == SecretsRepositoryType.BlobStorageSas)
                 {
                     Assert.True(_fixture.MarkerFileExists(testFunctionName ?? "host"));
                 }
@@ -235,10 +242,12 @@ public async Task FileSystemRepo_WriteAsync_ChangeNotificationUpdatesExistingSec
             await WriteAsync_ChangeNotificationUpdatesExistingSecret(SecretsRepositoryType.FileSystem);
         }
 
-        [Fact]
-        public async Task BlobStorageRepo_WriteAsync_ChangeNotificationUpdatesExistingSecret()
+        [Theory]
+        [InlineData(SecretsRepositoryType.BlobStorage)]
+        [InlineData(SecretsRepositoryType.BlobStorageSas)]
+        public async Task BlobStorageRepo_WriteAsync_ChangeNotificationUpdatesExistingSecret(SecretsRepositoryType repositoryType)
         {
-            await WriteAsync_ChangeNotificationUpdatesExistingSecret(SecretsRepositoryType.BlobStorage);
+            await WriteAsync_ChangeNotificationUpdatesExistingSecret(repositoryType);
         }
 
         private async Task WriteAsync_ChangeNotificationUpdatesExistingSecret(SecretsRepositoryType repositoryType)
@@ -300,6 +309,8 @@ public async Task FileSystemRepo_PurgeOldSecrets_RemovesOldAndKeepsCurrentSecret
         [InlineData(SecretsRepositoryType.FileSystem, ScriptSecretsType.Function)]
         [InlineData(SecretsRepositoryType.BlobStorage, ScriptSecretsType.Host)]
         [InlineData(SecretsRepositoryType.BlobStorage, ScriptSecretsType.Function)]
+        [InlineData(SecretsRepositoryType.BlobStorageSas, ScriptSecretsType.Host)]
+        [InlineData(SecretsRepositoryType.BlobStorageSas, ScriptSecretsType.Function)]
         public async Task GetSecretSnapshots_ReturnsExpected(SecretsRepositoryType repositoryType, ScriptSecretsType secretsType)
         {
             using (var directory = new TempDirectory())
@@ -341,7 +352,6 @@ public Fixture()
                 TestSiteName = "Test_test";
                 var configuration = TestHelpers.GetTestConfiguration();
                 BlobConnectionString = configuration.GetWebJobsConnectionString(ConnectionStringNames.Storage);
-                BlobContainer = CloudStorageAccount.Parse(BlobConnectionString).CreateCloudBlobClient().GetContainerReference("azure-webjobs-secrets");
                 KeyVaultConnectionString = configuration.GetWebJobsConnectionString(EnvironmentSettingNames.AzureWebJobsSecretStorageKeyVaultConnectionString);
                 KeyVaultName = configuration.GetWebJobsConnectionString(EnvironmentSettingNames.AzureWebJobsSecretStorageKeyVaultName);
                 AzureServiceTokenProvider azureServiceTokenProvider = new AzureServiceTokenProvider(KeyVaultConnectionString);
@@ -354,6 +364,8 @@ public Fixture()
 
             public string BlobConnectionString { get; private set; }
 
+            public Uri BlobSasConnectionUri { get; private set; }
+
             public CloudBlobContainer BlobContainer { get; private set; }
 
             public KeyVaultClient KeyVaultClient { get; private set; }
@@ -373,6 +385,16 @@ public async Task TestInitialize(SecretsRepositoryType repositoryType, string se
                     TestSiteName = testSiteName;
                 }
 
+                if (RepositoryType == SecretsRepositoryType.BlobStorageSas)
+                {
+                    BlobSasConnectionUri = await TestHelpers.CreateBlobContainerSas(BlobConnectionString, "azure-webjobs-secrets-sas");
+                    BlobContainer = new CloudBlobContainer(BlobSasConnectionUri);
+                }
+                else
+                {
+                    BlobContainer = CloudStorageAccount.Parse(BlobConnectionString).CreateCloudBlobClient().GetContainerReference("azure-webjobs-secrets");
+                }
+
                 await ClearAllBlobSecrets();
                 ClearAllFileSecrets();
                 await ClearAllKeyVaultSecrets();
@@ -384,6 +406,10 @@ public ISecretsRepository GetNewSecretRepository()
                 {
                     return new BlobStorageSecretsRepository(SecretsDirectory, BlobConnectionString, TestSiteName);
                 }
+                else if (RepositoryType == SecretsRepositoryType.BlobStorageSas)
+                {
+                    return new BlobStorageSasSecretsRepository(SecretsDirectory, BlobSasConnectionUri.ToString(), TestSiteName);
+                }
                 else if (RepositoryType == SecretsRepositoryType.FileSystem)
                 {
                     return new FileSystemSecretsRepository(SecretsDirectory);
@@ -435,6 +461,7 @@ public async Task WriteSecret(string functionNameOrHost, ScriptSecrets scriptSec
                         WriteSecretsToFile(functionNameOrHost, ScriptSecretSerializer.SerializeSecrets(scriptSecret));
                         break;
                     case SecretsRepositoryType.BlobStorage:
+                    case SecretsRepositoryType.BlobStorageSas:
                         await WriteSecretsBlobAndUpdateSentinelFile(functionNameOrHost, ScriptSecretSerializer.SerializeSecrets(scriptSecret));
                         break;
                     case SecretsRepositoryType.KeyVault:
@@ -485,6 +512,7 @@ public async Task<ScriptSecrets> GetSecretText(string functionNameOrHost, Script
                         secrets = ScriptSecretSerializer.DeserializeSecrets(type, secretText);
                         break;
                     case SecretsRepositoryType.BlobStorage:
+                    case SecretsRepositoryType.BlobStorageSas:
                         secrets = await GetSecretBlobText(functionNameOrHost, type);
                         break;
                     case SecretsRepositoryType.KeyVault:
@@ -559,7 +587,13 @@ private string GetSecretName(string secretName)
 
             private async Task ClearAllBlobSecrets()
             {
-                await BlobContainer.CreateIfNotExistsAsync();
+                // A sas connection requires the container to already exist, it
+                // doesn't have permission to create it
+                if (RepositoryType != SecretsRepositoryType.BlobStorageSas)
+                {
+                    await BlobContainer.CreateIfNotExistsAsync();
+                }
+
                 var blobs = await BlobContainer.ListBlobsSegmentedAsync(prefix: TestSiteName.ToLowerInvariant(), useFlatBlobListing: true,
                     blobListingDetails: BlobListingDetails.None, maxResults: 100, currentToken: null, options: null, operationContext: null);
                 foreach (IListBlobItem blob in blobs.Results)
diff --git a/test/WebJobs.Script.Tests.Shared/TestHelpers.cs b/test/WebJobs.Script.Tests.Shared/TestHelpers.cs
@@ -350,5 +350,23 @@ public static async Task<Uri> CreateBlobSas(string connectionString, string file
 
             return sasUri;
         }
+
+        public static async Task<Uri> CreateBlobContainerSas(string connectionString, string blobContainer)
+        {
+            CloudStorageAccount storageAccount = CloudStorageAccount.Parse(connectionString);
+            var blobClient = storageAccount.CreateCloudBlobClient();
+            var container = blobClient.GetContainerReference(blobContainer);
+            await container.CreateIfNotExistsAsync();
+
+            var policy = new SharedAccessBlobPolicy
+            {
+                SharedAccessStartTime = DateTime.UtcNow,
+                SharedAccessExpiryTime = DateTime.UtcNow.AddHours(1),
+                Permissions = SharedAccessBlobPermissions.Read | SharedAccessBlobPermissions.Write | SharedAccessBlobPermissions.List | SharedAccessBlobPermissions.Delete
+            };
+            var sas = container.GetSharedAccessSignature(policy);
+
+            return new Uri(container.StorageUri.PrimaryUri, sas);
+        }
     }
 }

Original file line number	Diff line number	Diff line change
`@@ -15,6 +15,7 @@ namespace Microsoft.Azure.WebJobs.Script.WebHost`
`15`	`15`	`public sealed class DefaultSecretManagerProvider : ISecretManagerProvider`
`16`	`16`	`{`
`17`	`17`	`private const string FileStorage = "Files";`
	`18`	`+ private const string BlobStorage = "Blob";`
`18`	`19`	`private readonly ILogger _logger;`
`19`	`20`	`private readonly IMetricsLogger _metricsLogger;`
`20`	`21`	`private readonly IOptionsMonitor<ScriptApplicationHostOptions> _options;`
`@@ -56,6 +57,7 @@ internal ISecretsRepository CreateSecretsRepository()`
`56`	`57`	`{`
`57`	`58`	`string secretStorageType = Environment.GetEnvironmentVariable(EnvironmentSettingNames.AzureWebJobsSecretStorageType);`
`58`	`59`	`string storageString = _configuration.GetWebJobsConnectionString(ConnectionStringNames.Storage);`
	`60`	`+ string secretStorageSas = _environment.GetEnvironmentVariable(EnvironmentSettingNames.AzureWebJobsSecretStorageSas);`
`59`	`61`	`if (secretStorageType != null && secretStorageType.Equals(FileStorage, StringComparison.OrdinalIgnoreCase))`
`60`	`62`	`{`
`61`	`63`	`return new FileSystemSecretsRepository(_options.CurrentValue.SecretsPath);`
`@@ -70,16 +72,21 @@ internal ISecretsRepository CreateSecretsRepository()`
`70`	`72`	`{`
`71`	`73`	`return new KubernetesSecretsRepository(_environment, new SimpleKubernetesClient(_environment));`
`72`	`74`	`}`
`73`		`- else if (storageString == null)`
	`75`	`+ else if (secretStorageSas != null)`
`74`	`76`	`{`
`75`		`- throw new InvalidOperationException($"Secret initialization from Blob storage failed due to a missing Azure Storage connection string. If you intend to use files for secrets, add an App Setting key '{EnvironmentSettingNames.AzureWebJobsSecretStorageType}' with value '{FileStorage}'.");`
	`77`	`+ string siteSlotName = _environment.GetAzureWebsiteUniqueSlotName() ?? _hostIdProvider.GetHostIdAsync(CancellationToken.None).GetAwaiter().GetResult();`
	`78`	`+ return new BlobStorageSasSecretsRepository(Path.Combine(_options.CurrentValue.SecretsPath, "Sentinels"), secretStorageSas, siteSlotName);`
`76`	`79`	`}`
`77`		`- else`
	`80`	`+ else if (storageString != null)`
`78`	`81`	`{`
`79`	`82`	`string siteSlotName = _environment.GetAzureWebsiteUniqueSlotName() ?? _hostIdProvider.GetHostIdAsync(CancellationToken.None).GetAwaiter().GetResult();`
`80`		`-`
`81`	`83`	`return new BlobStorageSecretsRepository(Path.Combine(_options.CurrentValue.SecretsPath, "Sentinels"), storageString, siteSlotName);`
`82`	`84`	`}`
	`85`	`+ else`
	`86`	`+ {`
	`87`	`+ throw new InvalidOperationException($"Secret initialization from Blob storage failed due to missing both an Azure Storage connection string and a SAS connection uri. " +`
	`88`	`+ $"For Blob Storage, please provide at least one of these. If you intend to use files for secrets, add an App Setting key '{EnvironmentSettingNames.AzureWebJobsSecretStorageType}' with value '{FileStorage}'.");`
	`89`	`+ }`
`83`	`90`	`}`
`84`	`91`	`}`
`85`	`92`	`}`