Reset cached secrets on SyncTriggers (#7698) (#7931)

Author: mathewc

src/WebJobs.Script.WebHost/Management/FunctionsSyncManager.cs

@@ -106,6 +106,8 @@ public async Task<SyncTriggersResult> TrySyncTriggersAsync(bool isBackgroundSync
             {
                 await _syncSemaphore.WaitAsync();
 
+                PrepareSyncTriggers();
+
                 var hashBlobClient = await GetHashBlobAsync();
                 if (isBackgroundSync && hashBlobClient == null && !_environment.IsKubernetesManagedHosting())
                 {
@@ -160,6 +162,31 @@ public async Task<SyncTriggersResult> TrySyncTriggersAsync(bool isBackgroundSync
             return result;
         }
 
+        /// <summary>
+        /// SyncTriggers is performed whenever deployments or other changes are made to the application.
+        /// There are some operations we want to perform whenever this happens.
+        /// </summary>
+        private void PrepareSyncTriggers()
+        {
+            // We clear cache to ensure that secrets are reloaded. This is important because secrets are part
+            // of the StartupContext payload (see StartupContextProvider) and that payload comes from the
+            // SyncTriggers operation. So there's a chicken and egg situation here. Consider the following scenario:
+            //   - app is using blob storage for keys
+            //   - a SyncTriggers operation has happened previously and the StartupContext has key info
+            //   - app instances initialize keys from StartupContext (keys aren't loaded from storage)
+            //   - user updates the app to use a new storage account
+            //   - a SyncTriggers operation is performed
+            //   - the app initializes from StartupContext, and **previous old key info is loaded**
+            //   - the SyncTriggers operation uses this old key info, so trigger cache is never updated with new key info
+            //   - Portal/ARM APIs will continue to show old key info.
+            // By clearing cache, we ensure that this host instance reloads keys when they're requested, and the SyncTriggers
+            // operation will contain current keys.
+            if (_secretManagerProvider.SecretsEnabled)
+            {
+                _secretManagerProvider.Current.ClearCache();
+            }
+        }
+
         internal static bool IsSyncTriggersEnvironment(IScriptWebHostEnvironment webHostEnvironment, IEnvironment environment)
         {
             if (environment.IsCoreTools())
@@ -316,45 +343,48 @@ public async Task<SyncTriggersPayload> GetSyncTriggersPayload()
                 }
             }
 
-            // Add functions secrets to the payload
-            // Only secret types we own/control can we cache directly
-            // Encryption is handled by Antares before storage
-            var secretsStorageType = _environment.GetEnvironmentVariable(EnvironmentSettingNames.AzureWebJobsSecretStorageType);
-            if (string.IsNullOrEmpty(secretsStorageType) ||
-                string.Compare(secretsStorageType, "files", StringComparison.OrdinalIgnoreCase) == 0 ||
-                string.Compare(secretsStorageType, "blob", StringComparison.OrdinalIgnoreCase) == 0)
+            if (_secretManagerProvider.SecretsEnabled)
             {
-                var functionAppSecrets = new FunctionAppSecrets();
-
-                // add host secrets
-                var hostSecretsInfo = await _secretManagerProvider.Current.GetHostSecretsAsync();
-                functionAppSecrets.Host = new FunctionAppSecrets.HostSecrets
+                // Add functions secrets to the payload
+                // Only secret types we own/control can we cache directly
+                // Encryption is handled by Antares before storage
+                var secretsStorageType = _environment.GetEnvironmentVariable(EnvironmentSettingNames.AzureWebJobsSecretStorageType);
+                if (string.IsNullOrEmpty(secretsStorageType) ||
+                    string.Compare(secretsStorageType, "files", StringComparison.OrdinalIgnoreCase) == 0 ||
+                    string.Compare(secretsStorageType, "blob", StringComparison.OrdinalIgnoreCase) == 0)
                 {
-                    Master = hostSecretsInfo.MasterKey,
-                    Function = hostSecretsInfo.FunctionKeys,
-                    System = hostSecretsInfo.SystemKeys
-                };
+                    var functionAppSecrets = new FunctionAppSecrets();
 
-                // add function secrets
-                var httpFunctions = functionsMetadata.Where(p => p.InputBindings.Any(q => q.IsTrigger && string.Compare(q.Type, "httptrigger", StringComparison.OrdinalIgnoreCase) == 0)).Select(p => p.Name).ToArray();
-                functionAppSecrets.Function = new FunctionAppSecrets.FunctionSecrets[httpFunctions.Length];
-                for (int i = 0; i < httpFunctions.Length; i++)
-                {
-                    var currFunctionName = httpFunctions[i];
-                    var currSecrets = await _secretManagerProvider.Current.GetFunctionSecretsAsync(currFunctionName);
-                    functionAppSecrets.Function[i] = new FunctionAppSecrets.FunctionSecrets
+                    // add host secrets
+                    var hostSecretsInfo = await _secretManagerProvider.Current.GetHostSecretsAsync();
+                    functionAppSecrets.Host = new FunctionAppSecrets.HostSecrets
                     {
-                        Name = currFunctionName,
-                        Secrets = currSecrets
+                        Master = hostSecretsInfo.MasterKey,
+                        Function = hostSecretsInfo.FunctionKeys,
+                        System = hostSecretsInfo.SystemKeys
                     };
-                }
 
-                result.Add("secrets", JObject.FromObject(functionAppSecrets));
-            }
-            else
-            {
-                // TODO: handle other external key storage types
-                // like KeyVault when the feature comes online
+                    // add function secrets
+                    var httpFunctions = functionsMetadata.Where(p => p.InputBindings.Any(q => q.IsTrigger && string.Compare(q.Type, "httptrigger", StringComparison.OrdinalIgnoreCase) == 0)).Select(p => p.Name).ToArray();
+                    functionAppSecrets.Function = new FunctionAppSecrets.FunctionSecrets[httpFunctions.Length];
+                    for (int i = 0; i < httpFunctions.Length; i++)
+                    {
+                        var currFunctionName = httpFunctions[i];
+                        var currSecrets = await _secretManagerProvider.Current.GetFunctionSecretsAsync(currFunctionName);
+                        functionAppSecrets.Function[i] = new FunctionAppSecrets.FunctionSecrets
+                        {
+                            Name = currFunctionName,
+                            Secrets = currSecrets
+                        };
+                    }
+
+                    result.Add("secrets", JObject.FromObject(functionAppSecrets));
+                }
+                else
+                {
+                    // TODO: handle other external key storage types
+                    // like KeyVault when the feature comes online
+                }
             }
 
             string json = JsonConvert.SerializeObject(result);