Host keys race fix

Author: alrod

src/WebJobs.Script.WebHost/Security/SecretManager.cs

@@ -8,6 +8,7 @@
 using System.IO;
 using System.Linq;
 using System.Security.Cryptography;
+using System.Threading;
 using System.Threading.Tasks;
 using Microsoft.Azure.WebJobs.Host;
 using Microsoft.Azure.WebJobs.Script.Config;
@@ -24,6 +25,7 @@ public class SecretManager : IDisposable, ISecretManager
         private readonly ILogger _logger;
         private readonly ISecretsRepository _repository;
         private HostSecretsInfo _hostSecrets;
+        private SemaphoreSlim _hostSecretsLock = new SemaphoreSlim(1, 1);
 
         // for testing
         public SecretManager()
@@ -62,53 +64,64 @@ protected virtual void Dispose(bool disposing)
             if (disposing)
             {
                 (_repository as IDisposable)?.Dispose();
+                _hostSecretsLock.Dispose();
             }
         }
 
         public async virtual Task<HostSecretsInfo> GetHostSecretsAsync()
         {
             if (_hostSecrets == null)
             {
-                HostSecrets hostSecrets = await LoadSecretsAsync<HostSecrets>();
-
-                if (hostSecrets == null)
-                {
-                    // host secrets do not yet exist so generate them
-                    _traceWriter.Verbose(Resources.TraceHostSecretGeneration);
-                    _logger?.LogDebug(Resources.TraceHostSecretGeneration);
-                    hostSecrets = GenerateHostSecrets();
-                    await PersistSecretsAsync(hostSecrets);
-                }
-
+                HostSecrets hostSecrets;
+                // Allow only one thread to modify the secrets
+                await _hostSecretsLock.WaitAsync();
                 try
                 {
-                    // Host secrets will be in the original persisted state at this point (e.g. encrypted),
-                    // so we read the secrets running them through the appropriate readers
-                    hostSecrets = ReadHostSecrets(hostSecrets);
-                }
-                catch (CryptographicException)
-                {
-                    _traceWriter.Verbose(Resources.TraceNonDecryptedHostSecretRefresh);
-                    _logger?.LogDebug(Resources.TraceNonDecryptedHostSecretRefresh);
-                    await PersistSecretsAsync(hostSecrets, null, true);
-                    await RefreshSecretsAsync(hostSecrets);
-                }
+                    hostSecrets = await LoadSecretsAsync<HostSecrets>();
 
-                // If the persistence state of any of our secrets is stale (e.g. the encryption key has been rotated), update
-                // the state and persist the secrets
-                if (hostSecrets.HasStaleKeys)
-                {
-                    _traceWriter.Verbose(Resources.TraceStaleHostSecretRefresh);
-                    _logger?.LogDebug(Resources.TraceStaleHostSecretRefresh);
-                    await RefreshSecretsAsync(hostSecrets);
-                }
+                    if (hostSecrets == null)
+                    {
+                        // host secrets do not yet exist so generate them
+                        _traceWriter.Verbose(Resources.TraceHostSecretGeneration);
+                        _logger?.LogDebug(Resources.TraceHostSecretGeneration);
+                        hostSecrets = GenerateHostSecrets();
+                        await PersistSecretsAsync(hostSecrets);
+                    }
+
+                    try
+                    {
+                        // Host secrets will be in the original persisted state at this point (e.g. encrypted),
+                        // so we read the secrets running them through the appropriate readers
+                        hostSecrets = ReadHostSecrets(hostSecrets);
+                    }
+                    catch (CryptographicException)
+                    {
+                        _traceWriter.Verbose(Resources.TraceNonDecryptedHostSecretRefresh);
+                        _logger?.LogDebug(Resources.TraceNonDecryptedHostSecretRefresh);
+                        await PersistSecretsAsync(hostSecrets, null, true);
+                        await RefreshSecretsAsync(hostSecrets);
+                    }
+
+                    // If the persistence state of any of our secrets is stale (e.g. the encryption key has been rotated), update
+                    // the state and persist the secrets
+                    if (hostSecrets.HasStaleKeys)
+                    {
+                        _traceWriter.Verbose(Resources.TraceStaleHostSecretRefresh);
+                        _logger?.LogDebug(Resources.TraceStaleHostSecretRefresh);
+                        await RefreshSecretsAsync(hostSecrets);
+                    }
 
-                _hostSecrets = new HostSecretsInfo
+                    _hostSecrets = new HostSecretsInfo
+                    {
+                        MasterKey = hostSecrets.MasterKey.Value,
+                        FunctionKeys = hostSecrets.FunctionKeys.ToDictionary(s => s.Name, s => s.Value),
+                        SystemKeys = hostSecrets.SystemKeys.ToDictionary(s => s.Name, s => s.Value)
+                    };
+                }
+                finally
                 {
-                    MasterKey = hostSecrets.MasterKey.Value,
-                    FunctionKeys = hostSecrets.FunctionKeys.ToDictionary(s => s.Name, s => s.Value),
-                    SystemKeys = hostSecrets.SystemKeys.ToDictionary(s => s.Name, s => s.Value)
-                };
+                    _hostSecretsLock.Release();
+                }
             }
 
             return _hostSecrets;