Skip to content

Commit c09e93a

Browse files
mathewcpragnagopa
mathewc
authored and
pragnagopa
committed
Reset cached secrets on SyncTriggers (#7698) (#7947)
1 parent 9fb0710 commit c09e93a

File tree

6 files changed

+234
-67
lines changed

6 files changed

+234
-67
lines changed

‎src/WebJobs.Script.WebHost/Management/FunctionsSyncManager.cs‎

Lines changed: 61 additions & 33 deletions
Original file line numberDiff line numberDiff line change
@@ -183,6 +183,31 @@ private void PrepareSyncTriggers()
183183
_secretManagerProvider.Current.ClearCache();
184184
}
185185

186+
/// <summary>
187+
/// SyncTriggers is performed whenever deployments or other changes are made to the application.
188+
/// There are some operations we want to perform whenever this happens.
189+
/// </summary>
190+
private void PrepareSyncTriggers()
191+
{
192+
// We clear cache to ensure that secrets are reloaded. This is important because secrets are part
193+
// of the StartupContext payload (see StartupContextProvider) and that payload comes from the
194+
// SyncTriggers operation. So there's a chicken and egg situation here. Consider the following scenario:
195+
// - app is using blob storage for keys
196+
// - a SyncTriggers operation has happened previously and the StartupContext has key info
197+
// - app instances initialize keys from StartupContext (keys aren't loaded from storage)
198+
// - user updates the app to use a new storage account
199+
// - a SyncTriggers operation is performed
200+
// - the app initializes from StartupContext, and **previous old key info is loaded**
201+
// - the SyncTriggers operation uses this old key info, so trigger cache is never updated with new key info
202+
// - Portal/ARM APIs will continue to show old key info.
203+
// By clearing cache, we ensure that this host instance reloads keys when they're requested, and the SyncTriggers
204+
// operation will contain current keys.
205+
if (_secretManagerProvider.SecretsEnabled)
206+
{
207+
_secretManagerProvider.Current.ClearCache();
208+
}
209+
}
210+
186211
internal static bool IsSyncTriggersEnvironment(IScriptWebHostEnvironment webHostEnvironment, IEnvironment environment)
187212
{
188213
if (environment.IsCoreTools())
@@ -339,45 +364,48 @@ public async Task<SyncTriggersPayload> GetSyncTriggersPayload()
339364
}
340365
}
341366

342-
// Add functions secrets to the payload
343-
// Only secret types we own/control can we cache directly
344-
// Encryption is handled by Antares before storage
345-
var secretsStorageType = _environment.GetEnvironmentVariable(EnvironmentSettingNames.AzureWebJobsSecretStorageType);
346-
if (string.IsNullOrEmpty(secretsStorageType) ||
347-
string.Compare(secretsStorageType, "files", StringComparison.OrdinalIgnoreCase) == 0 ||
348-
string.Compare(secretsStorageType, "blob", StringComparison.OrdinalIgnoreCase) == 0)
367+
if (_secretManagerProvider.SecretsEnabled)
349368
{
350-
var functionAppSecrets = new FunctionAppSecrets();
351-
352-
// add host secrets
353-
var hostSecretsInfo = await _secretManagerProvider.Current.GetHostSecretsAsync();
354-
functionAppSecrets.Host = new FunctionAppSecrets.HostSecrets
369+
// Add functions secrets to the payload
370+
// Only secret types we own/control can we cache directly
371+
// Encryption is handled by Antares before storage
372+
var secretsStorageType = _environment.GetEnvironmentVariable(EnvironmentSettingNames.AzureWebJobsSecretStorageType);
373+
if (string.IsNullOrEmpty(secretsStorageType) ||
374+
string.Compare(secretsStorageType, "files", StringComparison.OrdinalIgnoreCase) == 0 ||
375+
string.Compare(secretsStorageType, "blob", StringComparison.OrdinalIgnoreCase) == 0)
355376
{
356-
Master = hostSecretsInfo.MasterKey,
357-
Function = hostSecretsInfo.FunctionKeys,
358-
System = hostSecretsInfo.SystemKeys
359-
};
377+
var functionAppSecrets = new FunctionAppSecrets();
360378

361-
// add function secrets
362-
var httpFunctions = functionsMetadata.Where(p => !p.IsProxy() && p.InputBindings.Any(q => q.IsTrigger && string.Compare(q.Type, "httptrigger", StringComparison.OrdinalIgnoreCase) == 0)).Select(p => p.Name).ToArray();
363-
functionAppSecrets.Function = new FunctionAppSecrets.FunctionSecrets[httpFunctions.Length];
364-
for (int i = 0; i < httpFunctions.Length; i++)
365-
{
366-
var currFunctionName = httpFunctions[i];
367-
var currSecrets = await _secretManagerProvider.Current.GetFunctionSecretsAsync(currFunctionName);
368-
functionAppSecrets.Function[i] = new FunctionAppSecrets.FunctionSecrets
379+
// add host secrets
380+
var hostSecretsInfo = await _secretManagerProvider.Current.GetHostSecretsAsync();
381+
functionAppSecrets.Host = new FunctionAppSecrets.HostSecrets
369382
{
370-
Name = currFunctionName,
371-
Secrets = currSecrets
383+
Master = hostSecretsInfo.MasterKey,
384+
Function = hostSecretsInfo.FunctionKeys,
385+
System = hostSecretsInfo.SystemKeys
372386
};
373-
}
374387

375-
result.Add("secrets", JObject.FromObject(functionAppSecrets));
376-
}
377-
else
378-
{
379-
// TODO: handle other external key storage types
380-
// like KeyVault when the feature comes online
388+
// add function secrets
389+
var httpFunctions = functionsMetadata.Where(p => !p.IsProxy() && p.InputBindings.Any(q => q.IsTrigger && string.Compare(q.Type, "httptrigger", StringComparison.OrdinalIgnoreCase) == 0)).Select(p => p.Name).ToArray();
390+
functionAppSecrets.Function = new FunctionAppSecrets.FunctionSecrets[httpFunctions.Length];
391+
for (int i = 0; i < httpFunctions.Length; i++)
392+
{
393+
var currFunctionName = httpFunctions[i];
394+
var currSecrets = await _secretManagerProvider.Current.GetFunctionSecretsAsync(currFunctionName);
395+
functionAppSecrets.Function[i] = new FunctionAppSecrets.FunctionSecrets
396+
{
397+
Name = currFunctionName,
398+
Secrets = currSecrets
399+
};
400+
}
401+
402+
result.Add("secrets", JObject.FromObject(functionAppSecrets));
403+
}
404+
else
405+
{
406+
// TODO: handle other external key storage types
407+
// like KeyVault when the feature comes online
408+
}
381409
}
382410

383411
string json = JsonConvert.SerializeObject(result);

‎src/WebJobs.Script.WebHost/Security/KeyManagement/DefaultSecretManagerProvider.cs‎

Lines changed: 104 additions & 31 deletions
Original file line numberDiff line numberDiff line change
@@ -26,6 +26,7 @@ public sealed class DefaultSecretManagerProvider : ISecretManagerProvider
2626
private readonly StartupContextProvider _startupContextProvider;
2727
private readonly IAzureStorageProvider _azureStorageProvider;
2828
private Lazy<ISecretManager> _secretManagerLazy;
29+
private Lazy<bool> _secretsEnabledLazy;
2930

3031
public DefaultSecretManagerProvider(IOptionsMonitor<ScriptApplicationHostOptions> options, IHostIdProvider hostIdProvider,
3132
IConfiguration configuration, IEnvironment environment, ILoggerFactory loggerFactory, IMetricsLogger metricsLogger, HostNameProvider hostNameProvider, StartupContextProvider startupContextProvider, IAzureStorageProvider azureStorageProvider)
@@ -45,74 +46,146 @@ public DefaultSecretManagerProvider(IOptionsMonitor<ScriptApplicationHostOptions
4546
_loggerFactory = loggerFactory;
4647
_metricsLogger = metricsLogger ?? throw new ArgumentNullException(nameof(metricsLogger));
4748
_secretManagerLazy = new Lazy<ISecretManager>(Create);
49+
_secretsEnabledLazy = new Lazy<bool>(GetSecretsEnabled);
4850

4951
// When these options change (due to specialization), we need to reset the secret manager.
5052
options.OnChange(_ => ResetSecretManager());
5153

5254
_azureStorageProvider = azureStorageProvider;
5355
}
5456

57+
public bool SecretsEnabled
58+
{
59+
get
60+
{
61+
if (_secretManagerLazy.IsValueCreated)
62+
{
63+
return true;
64+
}
65+
return _secretsEnabledLazy.Value;
66+
}
67+
}
68+
5569
public ISecretManager Current => _secretManagerLazy.Value;
5670

57-
private void ResetSecretManager() => Interlocked.Exchange(ref _secretManagerLazy, new Lazy<ISecretManager>(Create));
71+
private void ResetSecretManager()
72+
{
73+
Interlocked.Exchange(ref _secretsEnabledLazy, new Lazy<bool>(GetSecretsEnabled));
74+
Interlocked.Exchange(ref _secretManagerLazy, new Lazy<ISecretManager>(Create));
75+
}
5876

5977
private ISecretManager Create() => new SecretManager(CreateSecretsRepository(), _loggerFactory.CreateLogger<SecretManager>(), _metricsLogger, _hostNameProvider, _startupContextProvider);
6078

6179
internal ISecretsRepository CreateSecretsRepository()
6280
{
63-
ISecretsRepository repository;
81+
ISecretsRepository repository = null;
82+
83+
if (TryGetSecretsRepositoryType(out Type repositoryType))
84+
{
85+
if (repositoryType == typeof(FileSystemSecretsRepository))
86+
{
87+
repository = new FileSystemSecretsRepository(_options.CurrentValue.SecretsPath, _loggerFactory.CreateLogger<FileSystemSecretsRepository>(), _environment);
88+
}
89+
else if (repositoryType == typeof(KeyVaultSecretsRepository))
90+
{
91+
string azureWebJobsSecretStorageKeyVaultName = Environment.GetEnvironmentVariable(EnvironmentSettingNames.AzureWebJobsSecretStorageKeyVaultName);
92+
string azureWebJobsSecretStorageKeyVaultConnectionString = Environment.GetEnvironmentVariable(EnvironmentSettingNames.AzureWebJobsSecretStorageKeyVaultConnectionString);
93+
94+
repository = new KeyVaultSecretsRepository(Path.Combine(_options.CurrentValue.SecretsPath, "Sentinels"),
95+
azureWebJobsSecretStorageKeyVaultName,
96+
azureWebJobsSecretStorageKeyVaultConnectionString,
97+
_loggerFactory.CreateLogger<KeyVaultSecretsRepository>(),
98+
_environment);
99+
}
100+
else if (repositoryType == typeof(KubernetesSecretsRepository))
101+
{
102+
repository = new KubernetesSecretsRepository(_environment, new SimpleKubernetesClient(_environment, _loggerFactory.CreateLogger<SimpleKubernetesClient>()));
103+
}
104+
else if (repositoryType == typeof(BlobStorageSasSecretsRepository))
105+
{
106+
string secretStorageSas = _environment.GetEnvironmentVariable(EnvironmentSettingNames.AzureWebJobsSecretStorageSas);
107+
string siteSlotName = _environment.GetAzureWebsiteUniqueSlotName() ?? _hostIdProvider.GetHostIdAsync(CancellationToken.None).GetAwaiter().GetResult();
108+
repository = new BlobStorageSasSecretsRepository(Path.Combine(_options.CurrentValue.SecretsPath, "Sentinels"),
109+
secretStorageSas,
110+
siteSlotName,
111+
_loggerFactory.CreateLogger<BlobStorageSasSecretsRepository>(),
112+
_environment,
113+
_azureStorageProvider);
114+
}
115+
else if (repositoryType == typeof(BlobStorageSecretsRepository))
116+
{
117+
string siteSlotName = _environment.GetAzureWebsiteUniqueSlotName() ?? _hostIdProvider.GetHostIdAsync(CancellationToken.None).GetAwaiter().GetResult();
118+
repository = new BlobStorageSecretsRepository(Path.Combine(_options.CurrentValue.SecretsPath, "Sentinels"),
119+
ConnectionStringNames.Storage,
120+
siteSlotName,
121+
_loggerFactory.CreateLogger<BlobStorageSecretsRepository>(),
122+
_environment,
123+
_azureStorageProvider);
124+
}
125+
}
126+
127+
if (repository == null)
128+
{
129+
throw new InvalidOperationException($"Secret initialization from Blob storage failed due to missing both an Azure Storage connection string and a SAS connection uri. " +
130+
$"For Blob Storage, please provide at least one of these. If you intend to use files for secrets, add an App Setting key '{EnvironmentSettingNames.AzureWebJobsSecretStorageType}' with value '{FileStorage}'.");
131+
}
132+
133+
ILogger logger = _loggerFactory.CreateLogger<DefaultSecretManagerProvider>();
134+
logger.LogInformation("Resolved secret storage provider {provider}", repository.Name);
64135

136+
return repository;
137+
}
138+
139+
/// <summary>
140+
/// Determines the repository Type to use based on configured settings.
141+
/// </summary>
142+
/// <remarks>
143+
/// For scenarios where the app isn't configured for key storage (e.g. no AzureWebJobsSecretStorageType explicitly configured,
144+
/// no storage connection string for default blob storage, etc.). Note that it's still possible for the creation of the repository
145+
/// to fail due to invalid values. This method just does preliminary config checks to determine the Type.
146+
/// </remarks>
147+
/// <param name="repositoryType">The repository Type or null.</param>
148+
/// <returns>True if a Type was determined, false otherwise.</returns>
149+
internal bool TryGetSecretsRepositoryType(out Type repositoryType)
150+
{
65151
string secretStorageType = Environment.GetEnvironmentVariable(EnvironmentSettingNames.AzureWebJobsSecretStorageType);
66152
string secretStorageSas = _environment.GetEnvironmentVariable(EnvironmentSettingNames.AzureWebJobsSecretStorageSas);
67153

68154
if (secretStorageType != null && secretStorageType.Equals(FileStorage, StringComparison.OrdinalIgnoreCase))
69155
{
70-
repository = new FileSystemSecretsRepository(_options.CurrentValue.SecretsPath, _loggerFactory.CreateLogger<FileSystemSecretsRepository>(), _environment);
156+
repositoryType = typeof(FileSystemSecretsRepository);
157+
return true;
71158
}
72159
else if (secretStorageType != null && secretStorageType.Equals("keyvault", StringComparison.OrdinalIgnoreCase))
73160
{
74-
string azureWebJobsSecretStorageKeyVaultName = Environment.GetEnvironmentVariable(EnvironmentSettingNames.AzureWebJobsSecretStorageKeyVaultName);
75-
string azureWebJobsSecretStorageKeyVaultConnectionString = Environment.GetEnvironmentVariable(EnvironmentSettingNames.AzureWebJobsSecretStorageKeyVaultConnectionString);
76-
77-
repository = new KeyVaultSecretsRepository(Path.Combine(_options.CurrentValue.SecretsPath, "Sentinels"),
78-
azureWebJobsSecretStorageKeyVaultName,
79-
azureWebJobsSecretStorageKeyVaultConnectionString,
80-
_loggerFactory.CreateLogger<KeyVaultSecretsRepository>(),
81-
_environment);
161+
repositoryType = typeof(KeyVaultSecretsRepository);
162+
return true;
82163
}
83164
else if (secretStorageType != null && secretStorageType.Equals("kubernetes", StringComparison.OrdinalIgnoreCase))
84165
{
85-
repository = new KubernetesSecretsRepository(_environment, new SimpleKubernetesClient(_environment, _loggerFactory.CreateLogger<SimpleKubernetesClient>()));
166+
repositoryType = typeof(KubernetesSecretsRepository);
167+
return true;
86168
}
87169
else if (secretStorageSas != null)
88170
{
89-
string siteSlotName = _environment.GetAzureWebsiteUniqueSlotName() ?? _hostIdProvider.GetHostIdAsync(CancellationToken.None).GetAwaiter().GetResult();
90-
repository = new BlobStorageSasSecretsRepository(Path.Combine(_options.CurrentValue.SecretsPath, "Sentinels"),
91-
secretStorageSas,
92-
siteSlotName,
93-
_loggerFactory.CreateLogger<BlobStorageSasSecretsRepository>(),
94-
_environment,
95-
_azureStorageProvider);
171+
repositoryType = typeof(BlobStorageSasSecretsRepository);
172+
return true;
96173
}
97174
else if (_azureStorageProvider.ConnectionExists(ConnectionStringNames.Storage))
98175
{
99-
string siteSlotName = _environment.GetAzureWebsiteUniqueSlotName() ?? _hostIdProvider.GetHostIdAsync(CancellationToken.None).GetAwaiter().GetResult();
100-
repository = new BlobStorageSecretsRepository(Path.Combine(_options.CurrentValue.SecretsPath, "Sentinels"),
101-
ConnectionStringNames.Storage,
102-
siteSlotName,
103-
_loggerFactory.CreateLogger<BlobStorageSecretsRepository>(),
104-
_environment,
105-
_azureStorageProvider);
176+
repositoryType = typeof(BlobStorageSecretsRepository);
177+
return true;
106178
}
107179
else
108180
{
109-
throw new InvalidOperationException($"Secret initialization from Blob storage failed due to missing both an Azure Storage connection string and a SAS connection uri. " +
110-
$"For Blob Storage, please provide at least one of these. If you intend to use files for secrets, add an App Setting key '{EnvironmentSettingNames.AzureWebJobsSecretStorageType}' with value '{FileStorage}'.");
181+
repositoryType = null;
182+
return false;
111183
}
184+
}
112185

113-
ILogger logger = _loggerFactory.CreateLogger<DefaultSecretManagerProvider>();
114-
logger.LogInformation("Resolved secret storage provider {provider}", repository.Name);
115-
return repository;
186+
internal bool GetSecretsEnabled()
187+
{
188+
return TryGetSecretsRepositoryType(out _);
116189
}
117190
}
118191
}

‎src/WebJobs.Script.WebHost/Security/KeyManagement/ISecretManagerProvider.cs‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,14 @@ namespace Microsoft.Azure.WebJobs.Script.WebHost
55
{
66
public interface ISecretManagerProvider
77
{
8+
/// <summary>
9+
/// Gets a value indicating whether we're configured to use secrets.
10+
/// </summary>
11+
bool SecretsEnabled { get; }
12+
13+
/// <summary>
14+
/// Gets or creates the <see cref="ISecretManager"./>
15+
/// </summary>
816
ISecretManager Current { get; }
917
}
1018
}

0 commit comments

Comments
 (0)