Skip to content

Commit 2cff983

Browse files
alrodalrod
committed
Migrate secrets if storage type is blob
1 parent b964dfc commit 2cff983

File tree

7 files changed

+311
-4
lines changed

7 files changed

+311
-4
lines changed

‎src/WebJobs.Script.WebHost/Security/KeyManagement/BlobStorageSecretsMigrationRepository.cs‎

Lines changed: 144 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,144 @@
1+
// Copyright (c) .NET Foundation. All rights reserved.
2+
// Licensed under the MIT License. See License.txt in the project root for license information.
3+
4+
using System;
5+
using System.Collections.Generic;
6+
using System.IO;
7+
using System.Linq;
8+
using System.Threading.Tasks;
9+
using Microsoft.Extensions.Logging;
10+
using Microsoft.WindowsAzure.Storage.Blob;
11+
12+
namespace Microsoft.Azure.WebJobs.Script.WebHost
13+
{
14+
public class BlobStorageSecretsMigrationRepository : ISecretsRepository, IDisposable
15+
{
16+
private readonly string _secretSentinelDirectoryPath;
17+
private readonly string _accountConnectionString;
18+
private readonly string _siteSlotName;
19+
private readonly ILogger _logger;
20+
private readonly Lazy<Task<BlobStorageSecretsRepository>> _blobStorageSecretsRepositoryTask;
21+
22+
public BlobStorageSecretsMigrationRepository(string secretSentinelDirectoryPath, string accountConnectionString, string siteSlotName, ILogger logger)
23+
{
24+
_secretSentinelDirectoryPath = secretSentinelDirectoryPath;
25+
_accountConnectionString = accountConnectionString;
26+
_siteSlotName = siteSlotName;
27+
_logger = logger;
28+
29+
_blobStorageSecretsRepositoryTask = new Lazy<Task<BlobStorageSecretsRepository>>(BlobStorageSecretsRepositoryFactory);
30+
}
31+
32+
public event EventHandler<SecretsChangedEventArgs> SecretsChanged;
33+
34+
private BlobStorageSecretsRepository BlobStorageSecretsRepository => _blobStorageSecretsRepositoryTask.Value.Result;
35+
36+
private void BlobStorageSecretsMigrationRepository_SecretsChanged(object sender, SecretsChangedEventArgs e)
37+
{
38+
SecretsChanged?.Invoke(this, e);
39+
}
40+
41+
private async Task<BlobStorageSecretsRepository> BlobStorageSecretsRepositoryFactory()
42+
{
43+
BlobStorageSecretsRepository blobStorageSecretsRepository = new BlobStorageSecretsRepository(_secretSentinelDirectoryPath, _accountConnectionString, _siteSlotName);
44+
blobStorageSecretsRepository.SecretsChanged += BlobStorageSecretsMigrationRepository_SecretsChanged;
45+
try
46+
{
47+
await CopyKeysFromFileSystemToBlobStorage(blobStorageSecretsRepository);
48+
}
49+
catch (Exception ex)
50+
{
51+
_logger?.LogTrace("{0}", ex.ToString());
52+
}
53+
return blobStorageSecretsRepository;
54+
}
55+
56+
public async Task<string> ReadAsync(ScriptSecretsType type, string functionName)
57+
{
58+
return await BlobStorageSecretsRepository.ReadAsync(type, functionName);
59+
}
60+
61+
public async Task WriteAsync(ScriptSecretsType type, string functionName, string secretsContent)
62+
{
63+
await BlobStorageSecretsRepository.WriteAsync(type, functionName, secretsContent);
64+
}
65+
66+
public async Task WriteSnapshotAsync(ScriptSecretsType type, string functionName, string secretsContent)
67+
{
68+
await BlobStorageSecretsRepository.WriteSnapshotAsync(type, functionName, secretsContent);
69+
}
70+
71+
public async Task PurgeOldSecretsAsync(IList<string> currentFunctions, ILogger logger)
72+
{
73+
await BlobStorageSecretsRepository.PurgeOldSecretsAsync(currentFunctions, logger);
74+
}
75+
76+
public async Task<string[]> GetSecretSnapshots(ScriptSecretsType type, string functionName)
77+
{
78+
return await BlobStorageSecretsRepository.GetSecretSnapshots(type, functionName);
79+
}
80+
81+
public void Dispose()
82+
{
83+
BlobStorageSecretsRepository.Dispose();
84+
}
85+
86+
private async Task CopyKeysFromFileSystemToBlobStorage(BlobStorageSecretsRepository blobStorageSecretsRepository)
87+
{
88+
string migrateSentinelPath = Path.Combine(blobStorageSecretsRepository.SecretsSentinelFilePath, "migrate-sentinel.json");
89+
if (File.Exists(migrateSentinelPath))
90+
{
91+
// Migration is already done
92+
_logger?.LogTrace("Sentinel file is detected.");
93+
return;
94+
}
95+
96+
try
97+
{
98+
using (var stream = new FileStream(migrateSentinelPath, FileMode.CreateNew))
99+
using (var writer = new StreamWriter(stream))
100+
{
101+
//write file
102+
_logger?.LogTrace("Sentinel file created.");
103+
}
104+
}
105+
catch (IOException)
106+
{
107+
_logger?.LogTrace("Sentinel file is already created by another instance.");
108+
return;
109+
}
110+
111+
string[] files = Directory.GetFiles(blobStorageSecretsRepository.SecretsSentinelFilePath.Replace("Sentinels", string.Empty));
112+
if (await blobStorageSecretsRepository.BlobContainer.ExistsAsync())
113+
{
114+
BlobResultSegment resultSegment = await blobStorageSecretsRepository.BlobContainer.ListBlobsSegmentedAsync(blobStorageSecretsRepository.SecretsBlobPath + "/", null);
115+
116+
// Check for conflicts
117+
if (resultSegment.Results.ToArray().Length > 0)
118+
{
119+
_logger?.LogTrace("Conflict detected. Secrets container is not empty.");
120+
return;
121+
}
122+
}
123+
else
124+
{
125+
await blobStorageSecretsRepository.BlobContainer.CreateIfNotExistsAsync();
126+
}
127+
128+
if (files.Length > 0)
129+
{
130+
List<Task> copyTasks = new List<Task>();
131+
foreach (string file in files)
132+
{
133+
string blobName = Path.GetFileName(file);
134+
CloudBlockBlob cloudBlockBlob = blobStorageSecretsRepository.BlobContainer.GetBlockBlobReference(blobStorageSecretsRepository.SecretsBlobPath + "/" + blobName);
135+
Task copyTask = cloudBlockBlob.UploadFromFileAsync(file);
136+
copyTasks.Add(copyTask);
137+
_logger?.LogTrace("'{0}' was migrated.", cloudBlockBlob.StorageUri.PrimaryUri.AbsoluteUri.ToString());
138+
}
139+
await Task.WhenAll(copyTasks);
140+
}
141+
_logger?.LogTrace("Finished successfully.");
142+
}
143+
}
144+
}

‎src/WebJobs.Script.WebHost/Security/KeyManagement/BlobStorageSecretsRepository.cs‎

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -67,6 +67,30 @@ public BlobStorageSecretsRepository(string secretSentinelDirectoryPath, string a
6767

6868
public event EventHandler<SecretsChangedEventArgs> SecretsChanged;
6969

70+
public CloudBlobContainer BlobContainer
71+
{
72+
get
73+
{
74+
return _blobContainer;
75+
}
76+
}
77+
78+
public string SecretsSentinelFilePath
79+
{
80+
get
81+
{
82+
return _secretsSentinelFilePath;
83+
}
84+
}
85+
86+
public string SecretsBlobPath
87+
{
88+
get
89+
{
90+
return _secretsBlobPath;
91+
}
92+
}
93+
7094
private string GetSecretsBlobPath(ScriptSecretsType secretsType, string functionName = null)
7195
{
7296
return secretsType == ScriptSecretsType.Host

‎src/WebJobs.Script.WebHost/Security/KeyManagement/DefaultSecretsRepositoryFactory.cs‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -8,19 +8,20 @@
88
using System.Web;
99
using Microsoft.Azure.WebJobs.Host;
1010
using Microsoft.Azure.WebJobs.Script.Config;
11+
using Microsoft.Extensions.Logging;
1112

1213
namespace Microsoft.Azure.WebJobs.Script.WebHost
1314
{
1415
public sealed class DefaultSecretsRepositoryFactory : ISecretsRepositoryFactory
1516
{
16-
public ISecretsRepository Create(ScriptSettingsManager settingsManager, WebHostSettings webHostSettings, ScriptHostConfiguration config)
17+
public ISecretsRepository Create(ScriptSettingsManager settingsManager, WebHostSettings webHostSettings, ScriptHostConfiguration config, ILogger logger)
1718
{
1819
string secretStorageType = settingsManager.GetSetting(EnvironmentSettingNames.AzureWebJobsSecretStorageType);
1920
string storageString = AmbientConnectionStringProvider.Instance.GetConnectionString(ConnectionStringNames.Storage);
2021
if (secretStorageType != null && secretStorageType.Equals("Blob", StringComparison.OrdinalIgnoreCase) && storageString != null)
2122
{
2223
string siteSlotName = settingsManager.AzureWebsiteUniqueSlotName ?? config.HostConfig.HostId;
23-
return new BlobStorageSecretsRepository(Path.Combine(webHostSettings.SecretsPath, "Sentinels"), storageString, siteSlotName);
24+
return new BlobStorageSecretsMigrationRepository(Path.Combine(webHostSettings.SecretsPath, "Sentinels"), storageString, siteSlotName, logger);
2425
}
2526
else
2627
{

‎src/WebJobs.Script.WebHost/Security/KeyManagement/ISecretsRepositoryFactory.cs‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -3,11 +3,12 @@
33

44
using System;
55
using Microsoft.Azure.WebJobs.Script.Config;
6+
using Microsoft.Extensions.Logging;
67

78
namespace Microsoft.Azure.WebJobs.Script.WebHost
89
{
910
public interface ISecretsRepositoryFactory
1011
{
11-
ISecretsRepository Create(ScriptSettingsManager settingsManager, WebHostSettings webHostSettings, ScriptHostConfiguration config);
12+
ISecretsRepository Create(ScriptSettingsManager settingsManager, WebHostSettings webHostSettings, ScriptHostConfiguration config, ILogger logger);
1213
}
1314
}

‎src/WebJobs.Script.WebHost/WebScriptHostManager.cs‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -75,7 +75,7 @@ public WebScriptHostManager(ScriptHostConfiguration config,
7575
config.IsSelfHost = webHostSettings.IsSelfHost;
7676

7777
secretsRepositoryFactory = secretsRepositoryFactory ?? new DefaultSecretsRepositoryFactory();
78-
var secretsRepository = secretsRepositoryFactory.Create(settingsManager, webHostSettings, config);
78+
ISecretsRepository secretsRepository = secretsRepositoryFactory.Create(settingsManager, webHostSettings, config, loggerFactory.CreateLogger(ScriptConstants.LogCategoryMigration));
7979
_secretManager = secretManagerFactory.Create(settingsManager, loggerFactory.CreateLogger(ScriptConstants.LogCategoryHostGeneral), secretsRepository);
8080
eventGenerator = eventGenerator ?? new EtwEventGenerator();
8181

‎src/WebJobs.Script/ScriptConstants.cs‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -48,6 +48,7 @@ public static class ScriptConstants
4848
public const string LogCategoryHost = "Host";
4949
public const string LogCategoryFunction = "Function";
5050
public const string LogCategoryWorker = "Worker";
51+
public const string LogCategoryMigration = "Host.Migration";
5152
public const string ConsoleLoggingMode = "consoleLoggingMode";
5253

5354
// Define all system parameters we inject with a prefix to avoid collisions

‎test/WebJobs.Script.Tests.Integration/Host/SecretsRepositoryMigrationTests.cs‎

Lines changed: 136 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,136 @@
1+
// Copyright (c) .NET Foundation. All rights reserved.
2+
// Licensed under the MIT License. See License.txt in the project root for license information.
3+
4+
using Microsoft.Azure.WebJobs.Script.Config;
5+
using Microsoft.Azure.WebJobs.Script.WebHost;
6+
using Microsoft.Extensions.Logging;
7+
using Microsoft.WebJobs.Script.Tests;
8+
using System;
9+
using System.Collections.Generic;
10+
using System.IO;
11+
using System.Linq;
12+
using System.Text;
13+
using System.Threading.Tasks;
14+
using WebJobs.Script.Tests;
15+
using Xunit;
16+
using static Microsoft.Azure.WebJobs.Script.Tests.SecretsRepositoryTests;
17+
18+
namespace Microsoft.Azure.WebJobs.Script.Tests.Integration.Host
19+
{
20+
public class SecretsRepositoryMigrationTests : IClassFixture<SecretsRepositoryMigrationTests.Fixture>
21+
{
22+
private readonly SecretsRepositoryMigrationTests.Fixture _fixture;
23+
private readonly ScriptSettingsManager _settingsManager;
24+
25+
public SecretsRepositoryMigrationTests(SecretsRepositoryMigrationTests.Fixture fixture)
26+
{
27+
_fixture = fixture;
28+
_settingsManager = ScriptSettingsManager.Instance;
29+
}
30+
31+
[Fact]
32+
public async Task SecretMigrate_Successful()
33+
{
34+
using (var directory = new TempDirectory())
35+
{
36+
try
37+
{
38+
39+
await _fixture.TestInitialize(SecretsRepositoryType.FileSystem, directory.Path);
40+
var loggerProvider = new TestLoggerProvider();
41+
42+
var fileRepo = _fixture.GetFileSystemSecretsRepository();
43+
string hostContent = Guid.NewGuid().ToString();
44+
string functionContent = Guid.NewGuid().ToString();
45+
await fileRepo.WriteAsync(ScriptSecretsType.Host, "host", hostContent);
46+
await fileRepo.WriteAsync(ScriptSecretsType.Function, "test1", functionContent);
47+
48+
_settingsManager.SetSetting(EnvironmentSettingNames.AzureWebsiteSlotName, "Production");
49+
_settingsManager.SetSetting(EnvironmentSettingNames.AzureWebsiteName, "test-app");
50+
51+
var blobRepoMigration = _fixture.GetBlobStorageSecretsMigrationRepository(loggerProvider.CreateLogger(ScriptConstants.LogCategoryMigration));
52+
53+
await blobRepoMigration.ReadAsync(ScriptSecretsType.Host, "host");
54+
var logs = loggerProvider.GetAllLogMessages().ToArray();
55+
Assert.Contains(logs[logs.Length - 1].FormattedMessage, "Finished successfully.");
56+
string hostContentFromBlob = await blobRepoMigration.ReadAsync(ScriptSecretsType.Host, "");
57+
Assert.Contains(hostContent, hostContentFromBlob);
58+
string hostContentFromFunction = await blobRepoMigration.ReadAsync(ScriptSecretsType.Function, "test1");
59+
Assert.Contains(functionContent, hostContentFromFunction);
60+
61+
var blobRepoMigration2 = _fixture.GetBlobStorageSecretsMigrationRepository(loggerProvider.CreateLogger(""));
62+
await blobRepoMigration2.ReadAsync(ScriptSecretsType.Host, "host");
63+
logs = loggerProvider.GetAllLogMessages().ToArray();
64+
Assert.Contains(logs[logs.Length - 1].FormattedMessage, "Sentinel file is detected.");
65+
}
66+
finally
67+
{
68+
_settingsManager.SetSetting(EnvironmentSettingNames.AzureWebsiteSlotName, null);
69+
_settingsManager.SetSetting(EnvironmentSettingNames.AzureWebsiteName, null);
70+
}
71+
}
72+
}
73+
74+
[Fact]
75+
public async Task SecretMigrate_Conflict()
76+
{
77+
using (var directory = new TempDirectory())
78+
{
79+
try
80+
{
81+
await _fixture.TestInitialize(SecretsRepositoryType.FileSystem, directory.Path);
82+
var loggerProvider = new TestLoggerProvider();
83+
84+
var fileRepo = _fixture.GetFileSystemSecretsRepository();
85+
string hostContent = Guid.NewGuid().ToString();
86+
string functionContent = Guid.NewGuid().ToString();
87+
await fileRepo.WriteAsync(ScriptSecretsType.Host, "host", hostContent);
88+
await fileRepo.WriteAsync(ScriptSecretsType.Function, "test1", functionContent);
89+
90+
_settingsManager.SetSetting(EnvironmentSettingNames.AzureWebsiteSlotName, "Production");
91+
_settingsManager.SetSetting(EnvironmentSettingNames.AzureWebsiteName, "test-app");
92+
93+
var blobRepo = _fixture.GetBlobStorageRepository();
94+
await blobRepo.WriteAsync(ScriptSecretsType.Host, "host", hostContent);
95+
await blobRepo.WriteAsync(ScriptSecretsType.Function, "test1", functionContent);
96+
97+
98+
var blobRepoMigration = _fixture.GetBlobStorageSecretsMigrationRepository(loggerProvider.CreateLogger(ScriptConstants.LogCategoryMigration));
99+
await blobRepoMigration.ReadAsync(ScriptSecretsType.Host, "host");
100+
101+
var logs = loggerProvider.GetAllLogMessages().ToArray();
102+
Assert.Contains("Conflict detected", loggerProvider.GetAllLogMessages().ToList().Last().FormattedMessage);
103+
}
104+
finally
105+
{
106+
_settingsManager.SetSetting(EnvironmentSettingNames.AzureWebsiteSlotName, null);
107+
_settingsManager.SetSetting(EnvironmentSettingNames.AzureWebsiteName, null);
108+
109+
}
110+
}
111+
}
112+
113+
public class Fixture : SecretsRepositoryTests.Fixture
114+
{
115+
public Fixture() : base()
116+
{
117+
}
118+
119+
public ISecretsRepository GetFileSystemSecretsRepository()
120+
{
121+
return new FileSystemSecretsRepository(SecretsDirectory);
122+
}
123+
124+
public ISecretsRepository GetBlobStorageRepository()
125+
{
126+
return new BlobStorageSecretsRepository(Path.Combine(SecretsDirectory, "Sentinels"), BlobConnectionString, TestSiteName);
127+
}
128+
129+
public BlobStorageSecretsMigrationRepository GetBlobStorageSecretsMigrationRepository(ILogger logger)
130+
{
131+
var repo = new BlobStorageSecretsMigrationRepository(Path.Combine(SecretsDirectory, "Sentinels"), BlobConnectionString, TestSiteName, logger);
132+
return repo;
133+
}
134+
}
135+
}
136+
}

0 commit comments

Comments
 (0)