Skip to content

Commit d1cf72e

Browse files
committed
Enabling saving secrets to blob instead of file storage
1 parent 5344f80 commit d1cf72e

File tree

11 files changed

+588
-121
lines changed

11 files changed

+588
-121
lines changed

src/WebJobs.Script.WebHost/GlobalSuppressions.cs

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -100,4 +100,5 @@
100100
[assembly: System.Diagnostics.CodeAnalysis.SuppressMessage("Microsoft.Reliability", "CA2000:Dispose objects before losing scope", Scope = "member", Target = "Microsoft.Azure.WebJobs.Script.WebHost.Controllers.ExceptionProcessingHandler.#Handle(System.Web.Http.ExceptionHandling.ExceptionHandlerContext)")]
101101
[assembly: System.Diagnostics.CodeAnalysis.SuppressMessage("Microsoft.Performance", "CA1813:AvoidUnsealedAttributes", Scope = "type", Target = "Microsoft.Azure.WebJobs.Script.WebHost.Filters.AuthorizationLevelAttribute")]
102102
[assembly: System.Diagnostics.CodeAnalysis.SuppressMessage("Microsoft.Usage", "CA2227:CollectionPropertiesShouldBeReadOnly", Scope = "member", Target = "Microsoft.Azure.WebJobs.Script.WebHost.Models.Swagger.HttpOperationInfo.#InputParameters")]
103-
[assembly: System.Diagnostics.CodeAnalysis.SuppressMessage("Microsoft.Usage", "CA2227:CollectionPropertiesShouldBeReadOnly", Scope = "member", Target = "Microsoft.Azure.WebJobs.Script.WebHost.Models.Swagger.SwaggerDocument.#ApiEndpoints")]
103+
[assembly: System.Diagnostics.CodeAnalysis.SuppressMessage("Microsoft.Usage", "CA2227:CollectionPropertiesShouldBeReadOnly", Scope = "member", Target = "Microsoft.Azure.WebJobs.Script.WebHost.Models.Swagger.SwaggerDocument.#ApiEndpoints")]
104+
[assembly: System.Diagnostics.CodeAnalysis.SuppressMessage("Microsoft.Reliability", "CA2000:Dispose objects before losing scope", Scope = "member", Target = "Microsoft.Azure.WebJobs.Script.WebHost.DefaultSecretsRepositoryFactory.#Create(Microsoft.Azure.WebJobs.Script.Config.ScriptSettingsManager,Microsoft.Azure.WebJobs.Script.WebHost.WebHostSettings,Microsoft.Azure.WebJobs.Script.ScriptHostConfiguration)")]
Lines changed: 157 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,157 @@
1+
// Copyright (c) .NET Foundation. All rights reserved.
2+
// Licensed under the MIT License. See License.txt in the project root for license information.
3+
4+
using System;
5+
using System.Collections.Generic;
6+
using System.Globalization;
7+
using System.IO;
8+
using System.Linq;
9+
using System.Threading.Tasks;
10+
using Microsoft.Azure.WebJobs.Host;
11+
//using Microsoft.Azure.WebJobs.Script.Config;
12+
using Microsoft.Azure.WebJobs.Script.IO;
13+
using Microsoft.WindowsAzure.Storage;
14+
using Microsoft.WindowsAzure.Storage.Blob;
15+
16+
namespace Microsoft.Azure.WebJobs.Script.WebHost
17+
{
18+
/// <summary>
19+
/// An <see cref="ISecretsRepository"/> implementation that uses Azure blob storage as the backing store.
20+
/// </summary>
21+
public sealed class BlobStorageSecretsRepository : ISecretsRepository, IDisposable
22+
{
23+
private readonly string _secretsSentinelFilePath;
24+
private readonly string _secretsBlobPath;
25+
private readonly string _hostSecretsSentinelFilePath;
26+
private readonly string _hostSecretsBlobPath;
27+
private readonly AutoRecoveringFileSystemWatcher _sentinelFileWatcher;
28+
private readonly CloudBlobContainer _blobContainer;
29+
private readonly string _secretsContainerName = "azure-webjobs-secrets";
30+
private readonly string _accountConnectionString;
31+
private bool _disposed = false;
32+
33+
public BlobStorageSecretsRepository(string secretSentinelDirectoryPath, string accountConnectionString, string siteHostName)
34+
{
35+
if (secretSentinelDirectoryPath == null)
36+
{
37+
throw new ArgumentNullException(nameof(secretSentinelDirectoryPath));
38+
}
39+
if (accountConnectionString == null)
40+
{
41+
throw new ArgumentNullException(nameof(accountConnectionString));
42+
}
43+
44+
_secretsSentinelFilePath = secretSentinelDirectoryPath;
45+
_hostSecretsSentinelFilePath = Path.Combine(_secretsSentinelFilePath, ScriptConstants.HostMetadataFileName);
46+
47+
Directory.CreateDirectory(_secretsSentinelFilePath);
48+
49+
_sentinelFileWatcher = new AutoRecoveringFileSystemWatcher(_secretsSentinelFilePath, "*.json");
50+
_sentinelFileWatcher.Changed += OnChanged;
51+
52+
_secretsBlobPath = siteHostName.ToLowerInvariant();
53+
_hostSecretsBlobPath = string.Format("{0}/{1}", _secretsBlobPath, ScriptConstants.HostMetadataFileName);
54+
55+
_accountConnectionString = accountConnectionString;
56+
CloudStorageAccount account = CloudStorageAccount.Parse(_accountConnectionString);
57+
CloudBlobClient client = account.CreateCloudBlobClient();
58+
59+
_blobContainer = client.GetContainerReference(_secretsContainerName);
60+
_blobContainer.CreateIfNotExists();
61+
}
62+
63+
public event EventHandler<SecretsChangedEventArgs> SecretsChanged;
64+
65+
private string GetSecretsBlobPath(ScriptSecretsType secretsType, string functionName = null)
66+
{
67+
return secretsType == ScriptSecretsType.Host
68+
? _hostSecretsBlobPath
69+
: string.Format("{0}/{1}", _secretsBlobPath, GetSecretFileName(functionName));
70+
}
71+
72+
private string GetSecretsSentinelFilePath(ScriptSecretsType secretsType, string functionName = null)
73+
{
74+
return secretsType == ScriptSecretsType.Host
75+
? _hostSecretsSentinelFilePath
76+
: Path.Combine(_secretsSentinelFilePath, GetSecretFileName(functionName));
77+
}
78+
79+
private static string GetSecretFileName(string functionName)
80+
{
81+
return string.Format(CultureInfo.InvariantCulture, "{0}.json", functionName.ToLowerInvariant());
82+
}
83+
84+
private void OnChanged(object sender, FileSystemEventArgs e)
85+
{
86+
var changeHandler = SecretsChanged;
87+
if (changeHandler != null)
88+
{
89+
var args = new SecretsChangedEventArgs { SecretsType = ScriptSecretsType.Host };
90+
91+
if (string.Compare(Path.GetFileName(e.FullPath), ScriptConstants.HostMetadataFileName, StringComparison.OrdinalIgnoreCase) != 0)
92+
{
93+
args.SecretsType = ScriptSecretsType.Function;
94+
args.Name = Path.GetFileNameWithoutExtension(e.FullPath).ToLowerInvariant();
95+
}
96+
97+
changeHandler(this, args);
98+
}
99+
}
100+
101+
public async Task<string> ReadAsync(ScriptSecretsType type, string functionName)
102+
{
103+
string secretsContent = null;
104+
string blobPath = GetSecretsBlobPath(type, functionName);
105+
CloudBlockBlob secretBlob = _blobContainer.GetBlockBlobReference(blobPath);
106+
107+
if (await secretBlob.ExistsAsync())
108+
{
109+
secretsContent = await secretBlob.DownloadTextAsync();
110+
}
111+
112+
return secretsContent;
113+
}
114+
115+
public async Task WriteAsync(ScriptSecretsType type, string functionName, string secretsContent)
116+
{
117+
if (secretsContent == null)
118+
{
119+
throw new ArgumentNullException(nameof(secretsContent));
120+
}
121+
122+
string blobPath = GetSecretsBlobPath(type, functionName);
123+
CloudBlockBlob secretBlob = _blobContainer.GetBlockBlobReference(blobPath);
124+
using (StreamWriter writer = new StreamWriter(await secretBlob.OpenWriteAsync()))
125+
{
126+
await writer.WriteAsync(secretsContent);
127+
}
128+
129+
string filePath = GetSecretsSentinelFilePath(type, functionName);
130+
await FileUtility.WriteAsync(filePath, DateTime.UtcNow.ToString());
131+
}
132+
133+
public async Task PurgeOldSecretsAsync(IList<string> currentFunctions, TraceWriter traceWriter)
134+
{
135+
// no-op - allow stale secrets to remain
136+
await Task.Yield();
137+
}
138+
139+
private void Dispose(bool disposing)
140+
{
141+
if (!_disposed)
142+
{
143+
if (disposing)
144+
{
145+
_sentinelFileWatcher.Dispose();
146+
}
147+
148+
_disposed = true;
149+
}
150+
}
151+
152+
public void Dispose()
153+
{
154+
Dispose(true);
155+
}
156+
}
157+
}
Lines changed: 31 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,31 @@
1+
// Copyright (c) .NET Foundation. All rights reserved.
2+
// Licensed under the MIT License. See License.txt in the project root for license information.
3+
4+
using System;
5+
using System.Collections.Generic;
6+
using System.IO;
7+
using System.Linq;
8+
using System.Web;
9+
using Microsoft.Azure.WebJobs.Host;
10+
using Microsoft.Azure.WebJobs.Script.Config;
11+
12+
namespace Microsoft.Azure.WebJobs.Script.WebHost
13+
{
14+
public sealed class DefaultSecretsRepositoryFactory : ISecretsRepositoryFactory
15+
{
16+
public ISecretsRepository Create(ScriptSettingsManager settingsManager, WebHostSettings webHostSettings, ScriptHostConfiguration config)
17+
{
18+
string secretStorageType = settingsManager.GetSetting(EnvironmentSettingNames.AzureWebJobsSecretStorageType);
19+
string storageString = AmbientConnectionStringProvider.Instance.GetConnectionString(ConnectionStringNames.Storage);
20+
if (secretStorageType != null && secretStorageType.Equals("Blob", StringComparison.OrdinalIgnoreCase) && storageString != null)
21+
{
22+
string siteHostId = settingsManager.AzureWebsiteDefaultSubdomain ?? config.HostConfig.HostId;
23+
return new BlobStorageSecretsRepository(Path.Combine(webHostSettings.SecretsPath, "Sentinels"), storageString, siteHostId);
24+
}
25+
else
26+
{
27+
return new FileSystemSecretsRepository(webHostSettings.SecretsPath);
28+
}
29+
}
30+
}
31+
}
Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,14 @@
1+
// Copyright (c) .NET Foundation. All rights reserved.
2+
// Licensed under the MIT License. See License.txt in the project root for license information.
3+
4+
using System;
5+
using Microsoft.Azure.WebJobs.Host;
6+
using Microsoft.Azure.WebJobs.Script.Config;
7+
8+
namespace Microsoft.Azure.WebJobs.Script.WebHost
9+
{
10+
public interface ISecretsRepositoryFactory
11+
{
12+
ISecretsRepository Create(ScriptSettingsManager settingsManager, WebHostSettings webHostSettings, ScriptHostConfiguration config);
13+
}
14+
}

src/WebJobs.Script.WebHost/WebJobs.Script.WebHost.csproj

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -429,7 +429,9 @@
429429
<DesignTime>True</DesignTime>
430430
<DependentUpon>Resources.resx</DependentUpon>
431431
</Compile>
432+
<Compile Include="Security\BlobStorageSecretsRepository.cs" />
432433
<Compile Include="Security\DefaultSecretManagerFactory.cs" />
434+
<Compile Include="Security\DefaultSecretsRepositoryFactory.cs" />
433435
<Compile Include="Security\FileSystemSecretsRepository.cs" />
434436
<Compile Include="Security\FunctionSecrets.cs" />
435437
<Compile Include="Security\DataProtectionKeyValueConverter.cs" />
@@ -438,6 +440,7 @@
438440
<Compile Include="Security\IKeyValueWriter.cs" />
439441
<Compile Include="Security\ISecretManager.cs" />
440442
<Compile Include="Security\ISecretManagerFactory.cs" />
443+
<Compile Include="Security\ISecretRepositoryFactory.cs" />
441444
<Compile Include="Security\ISecretsRepository.cs" />
442445
<Compile Include="Security\KeyOperationResult.cs" />
443446
<Compile Include="Security\ScriptSecretsType.cs" />

src/WebJobs.Script.WebHost/WebScriptHostManager.cs

Lines changed: 10 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -43,7 +43,7 @@ public class WebScriptHostManager : ScriptHostManager
4343
private IDictionary<IHttpRoute, FunctionDescriptor> _httpFunctions;
4444
private HttpRouteCollection _httpRoutes;
4545

46-
public WebScriptHostManager(ScriptHostConfiguration config, ISecretManagerFactory secretManagerFactory, ScriptSettingsManager settingsManager, WebHostSettings webHostSettings, IScriptHostFactory scriptHostFactory = null)
46+
public WebScriptHostManager(ScriptHostConfiguration config, ISecretManagerFactory secretManagerFactory, ScriptSettingsManager settingsManager, WebHostSettings webHostSettings, IScriptHostFactory scriptHostFactory = null, ISecretsRepositoryFactory secretsRepositoryFactory = null)
4747
: base(config, settingsManager, scriptHostFactory)
4848
{
4949
_config = config;
@@ -63,10 +63,17 @@ public WebScriptHostManager(ScriptHostConfiguration config, ISecretManagerFactor
6363
}
6464

6565
config.IsSelfHost = webHostSettings.IsSelfHost;
66-
67-
_secretManager = secretManagerFactory.Create(settingsManager, config.TraceWriter, new FileSystemSecretsRepository(webHostSettings.SecretsPath));
66+
6867
_performanceManager = new HostPerformanceManager(settingsManager);
6968
_swaggerDocumentManager = new SwaggerDocumentManager(config);
69+
70+
var secretsRepository = secretsRepositoryFactory.Create(settingsManager, webHostSettings, config);
71+
_secretManager = secretManagerFactory.Create(settingsManager, config.TraceWriter, secretsRepository);
72+
}
73+
74+
public WebScriptHostManager(ScriptHostConfiguration config, ISecretManagerFactory secretManagerFactory, ScriptSettingsManager settingsManager, WebHostSettings webHostSettings, IScriptHostFactory scriptHostFactory)
75+
: this(config, secretManagerFactory, settingsManager, webHostSettings, scriptHostFactory, new DefaultSecretsRepositoryFactory())
76+
{
7077
}
7178

7279
public WebScriptHostManager(ScriptHostConfiguration config, ISecretManagerFactory secretManagerFactory, ScriptSettingsManager settingsManager, WebHostSettings webHostSettings)

src/WebJobs.Script/EnvironmentSettingNames.cs

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -19,5 +19,6 @@ public static class EnvironmentSettingNames
1919
public const string CompilationReleaseMode = "AzureWebJobsDotNetReleaseCompilation";
2020
public const string AzureWebJobsDisableHomepage = "AzureWebJobsDisableHomepage";
2121
public const string AzureWebsiteAppCountersName = "WEBSITE_COUNTERS_APP";
22+
public const string AzureWebJobsSecretStorageType = "AzureWebJobsSecretStorageType";
2223
}
2324
}

test/WebJobs.Script.Tests.Integration/Host/WebScriptHostManagerTests.cs

Lines changed: 10 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -27,12 +27,12 @@ public class WebScriptHostManagerTests : IClassFixture<WebScriptHostManagerTests
2727
{
2828
private readonly ScriptSettingsManager _settingsManager;
2929
private readonly TempDirectory _secretsDirectory = new TempDirectory();
30-
private Fixture _fixture;
30+
private WebScriptHostManagerTests.Fixture _fixture;
3131

3232
// Some tests need their own manager that differs from the fixture.
3333
private WebScriptHostManager _manager;
3434

35-
public WebScriptHostManagerTests(Fixture fixture)
35+
public WebScriptHostManagerTests(WebScriptHostManagerTests.Fixture fixture)
3636
{
3737
_fixture = fixture;
3838
_settingsManager = ScriptSettingsManager.Instance;
@@ -109,10 +109,11 @@ public async Task EmptyHost_StartsSuccessfully()
109109
RootLogPath = logDir,
110110
FileLoggingMode = FileLoggingMode.Always
111111
};
112-
ISecretsRepository repository = new FileSystemSecretsRepository(secretsDir);
112+
string connectionString = AmbientConnectionStringProvider.Instance.GetConnectionString(ConnectionStringNames.Storage);
113+
ISecretsRepository repository = new BlobStorageSecretsRepository(secretsDir, connectionString, "EmptyHost_StartsSuccessfully");
113114
ISecretManager secretManager = new SecretManager(_settingsManager, repository, NullTraceWriter.Instance);
114115
WebHostSettings webHostSettings = new WebHostSettings();
115-
webHostSettings.SecretsPath = _secretsDirectory.Path;
116+
webHostSettings.SecretsPath = _secretsDirectory.Path;
116117

117118
ScriptHostManager hostManager = new WebScriptHostManager(config, new TestSecretManagerFactory(secretManager), _settingsManager, webHostSettings);
118119

@@ -149,8 +150,8 @@ public async Task MultipleHostRestarts()
149150
RootScriptPath = functionTestDir,
150151
FileLoggingMode = FileLoggingMode.Always,
151152
};
152-
153-
ISecretsRepository repository = new FileSystemSecretsRepository(secretsDir);
153+
154+
ISecretsRepository repository = new FileSystemSecretsRepository(_secretsDirectory.Path);
154155
SecretManager secretManager = new SecretManager(_settingsManager, repository, NullTraceWriter.Instance);
155156
WebHostSettings webHostSettings = new WebHostSettings();
156157
webHostSettings.SecretsPath = _secretsDirectory.Path;
@@ -331,11 +332,11 @@ public Fixture()
331332
RootLogPath = logRoot,
332333
FileLoggingMode = FileLoggingMode.Always
333334
};
334-
335+
335336
ISecretsRepository repository = new FileSystemSecretsRepository(SecretsPath);
336337
ISecretManager secretManager = new SecretManager(_settingsManager, repository, NullTraceWriter.Instance);
337338
WebHostSettings webHostSettings = new WebHostSettings();
338-
webHostSettings.SecretsPath = _secretsDirectory.Path;
339+
webHostSettings.SecretsPath = SecretsPath;
339340

340341
var hostConfig = config.HostConfig;
341342
var testEventGenerator = new TestSystemEventGenerator();
@@ -360,7 +361,7 @@ public Fixture()
360361
"Info WebJobs.Indexing Found the following functions:",
361362
"Info The next 5 occurrences of the schedule will be:",
362363
"Info WebJobs.Host Job host started",
363-
"Error The following 1 functions are in error:"
364+
"Error The following 1 functions are in error:"
364365
};
365366
foreach (string pattern in expectedPatterns)
366367
{

0 commit comments

Comments
 (0)