[v3x] Include minimum durable payload to sync trigger (#8292)

TsuyoshiUshio · web-flow · commit 2d6b315d3ef4 · 2022-04-08T16:54:55.000-07:00
* Include minimum durable payload to synctrigger

* update release note

* added missed change on v4
diff --git a/release_notes.md b/release_notes.md
@@ -4,6 +4,7 @@
 -->
 - Read FuncitonsHostingConfigurations to enable worker concurrency feature (#8125)
 - Updated Java Worker Version to [1.11.0](https://github.com/Azure/azure-functions-java-worker/releases/tag/1.11.0)
+- Add Include minimum durable payload to sync trigger (#8292)
 
 **Release sprint:** Sprint 118
 [ [bugs](https://github.com/Azure/azure-functions-host/issues?q=is%3Aissue+milestone%3A%22Functions+Sprint+118%22+label%3Abug+is%3Aclosed) | [features](https://github.com/Azure/azure-functions-host/issues?q=is%3Aissue+milestone%3A%22Functions+Sprint+118%22+label%3Afeature+is%3Aclosed) ]
diff --git a/src/WebJobs.Script.WebHost/Management/FunctionsSyncManager.cs b/src/WebJobs.Script.WebHost/Management/FunctionsSyncManager.cs
@@ -34,6 +34,8 @@ public class FunctionsSyncManager : IFunctionsSyncManager, IDisposable
         private const string DurableTaskV1StorageConnectionName = "azureStorageConnectionStringName";
         private const string DurableTaskV2StorageOptions = "storageProvider";
         private const string DurableTaskV2StorageConnectionName = "connectionStringName";
+        private const string DurableTaskV2MaxConcurrentActivityFunctions = "maxConcurrentActivityFunctions";
+        private const string DurableTaskV2MaxConcurrentOrchestratorFunctions = "maxConcurrentOrchestratorFunctions";
         private const string DurableTask = "durableTask";
 
         // 45 alphanumeric characters gives us a buffer in our table/queue/blob container names.
@@ -483,6 +485,21 @@ private static JObject UpdateDurableFunctionConfig(JObject trigger, DurableConfi
                 {
                     trigger[Connection] = durableTaskConfig.Connection;
                 }
+
+                if (durableTaskConfig.StorageProvider != null)
+                {
+                    trigger[DurableTaskV2StorageOptions] = durableTaskConfig.StorageProvider;
+                }
+
+                if (durableTaskConfig.MaxConcurrentOrchestratorFunctions != 0)
+                {
+                    trigger[DurableTaskV2MaxConcurrentOrchestratorFunctions] = durableTaskConfig.MaxConcurrentOrchestratorFunctions;
+                }
+
+                if (durableTaskConfig.MaxConcurrentActivityFunctions != 0)
+                {
+                    trigger[DurableTaskV2MaxConcurrentActivityFunctions] = durableTaskConfig.MaxConcurrentActivityFunctions;
+                }
             }
             return trigger;
         }
@@ -597,6 +614,18 @@ private DurableConfig GetDurableV2Config(JObject durableHostConfig)
                     {
                         config.Connection = nameValue.ToString();
                     }
+
+                    config.StorageProvider = storageOptions;
+                }
+
+                if (durableHostConfig.TryGetValue(DurableTaskV2MaxConcurrentOrchestratorFunctions, StringComparison.OrdinalIgnoreCase, out JToken maxConcurrentOrchestratorFunctions) && maxConcurrentOrchestratorFunctions != null)
+                {
+                    config.MaxConcurrentOrchestratorFunctions = int.Parse(maxConcurrentOrchestratorFunctions.ToString());
+                }
+
+                if (durableHostConfig.TryGetValue(DurableTaskV2MaxConcurrentActivityFunctions, StringComparison.OrdinalIgnoreCase, out JToken maxConcurrentActivityFunctions) && maxConcurrentActivityFunctions != null)
+                {
+                    config.MaxConcurrentActivityFunctions = int.Parse(maxConcurrentActivityFunctions.ToString());
                 }
             }
 
@@ -723,9 +752,15 @@ private class DurableConfig
 
             public string Connection { get; set; }
 
+            public JToken StorageProvider { get; set; }
+
+            public int MaxConcurrentActivityFunctions { get; set; }
+
+            public int MaxConcurrentOrchestratorFunctions { get; set; }
+
             public bool HasValues()
             {
-                return this.HubName != null || this.Connection != null;
+                return this.HubName != null || this.Connection != null || this.StorageProvider != null || this.MaxConcurrentOrchestratorFunctions != 0 || this.MaxConcurrentOrchestratorFunctions != 0;
             }
         }
     }
diff --git a/test/WebJobs.Script.Tests.Integration/Management/FunctionsSyncManagerTests.cs b/test/WebJobs.Script.Tests.Integration/Management/FunctionsSyncManagerTests.cs
@@ -143,12 +143,13 @@ public FunctionsSyncManagerTests()
             _functionsSyncManager = new FunctionsSyncManager(configuration, hostIdProviderMock.Object, optionsMonitor, loggerFactory.CreateLogger<FunctionsSyncManager>(), httpClient, _secretManagerProviderMock.Object, _mockWebHostEnvironment.Object, _mockEnvironment.Object, _hostNameProvider, functionMetadataManager, azureStorageProvider);
         }
 
-        private string GetExpectedSyncTriggersPayload(string postedConnection = DefaultTestConnection, string postedTaskHub = DefaultTestTaskHub)
+        private string GetExpectedSyncTriggersPayload(string postedConnection = DefaultTestConnection, string postedTaskHub = DefaultTestTaskHub, string durableVersion = "V2")
         {
             string taskHubSegment = postedTaskHub != null ? $",\"taskHubName\":\"{postedTaskHub}\"" : "";
+            string storageProviderSegment = postedConnection != null && durableVersion == "V2" ? $",\"storageProvider\":{{\"connectionStringName\":\"DurableConnection\"}}" : "";
             return "[{\"authLevel\":\"anonymous\",\"type\":\"httpTrigger\",\"direction\":\"in\",\"name\":\"req\",\"functionName\":\"function1\"}," +
-                $"{{\"name\":\"myQueueItem\",\"type\":\"orchestrationTrigger\",\"direction\":\"in\",\"queueName\":\"myqueue-items\",\"connection\":\"{postedConnection}\",\"functionName\":\"function2\"{taskHubSegment}}}," +
-                $"{{\"name\":\"myQueueItem\",\"type\":\"activityTrigger\",\"direction\":\"in\",\"queueName\":\"myqueue-items\",\"connection\":\"{postedConnection}\",\"functionName\":\"function3\"{taskHubSegment}}}]";
+                $"{{\"name\":\"myQueueItem\",\"type\":\"orchestrationTrigger\",\"direction\":\"in\",\"queueName\":\"myqueue-items\",\"connection\":\"{postedConnection}\",\"functionName\":\"function2\"{taskHubSegment}{storageProviderSegment}}}," +
+                $"{{\"name\":\"myQueueItem\",\"type\":\"activityTrigger\",\"direction\":\"in\",\"queueName\":\"myqueue-items\",\"connection\":\"{postedConnection}\",\"functionName\":\"function3\"{taskHubSegment}{storageProviderSegment}}}]";
         }
 
         private void ResetMockFileSystem(string hostJsonContent = null, string extensionsJsonContent = null)
@@ -235,18 +236,18 @@ public async Task TrySyncTriggers_PostsExpectedContent(bool cacheEnabled)
 
                 if (cacheEnabled)
                 {
-                    VerifyResultWithCacheOn();
+                    VerifyResultWithCacheOn(durableVersion: "V1");
                 }
                 else
                 {
-                    VerifyResultWithCacheOff();
+                    VerifyResultWithCacheOff(durableVersion: "V1");
                 }
             }
         }
 
-        private void VerifyResultWithCacheOn(string connection = DefaultTestConnection, string expectedTaskHub = "TestHubValue")
+        private void VerifyResultWithCacheOn(string connection = DefaultTestConnection, string expectedTaskHub = "TestHubValue", string durableVersion = "V2")
         {
-            string expectedSyncTriggersPayload = GetExpectedSyncTriggersPayload(postedConnection: connection, postedTaskHub: expectedTaskHub);
+            string expectedSyncTriggersPayload = GetExpectedSyncTriggersPayload(postedConnection: connection, postedTaskHub: expectedTaskHub, durableVersion);
             // verify triggers
             var result = JObject.Parse(_contentBuilder.ToString());
             var triggers = result["triggers"];
@@ -286,9 +287,9 @@ private void VerifyResultWithCacheOn(string connection = DefaultTestConnection,
             Assert.False(triggersLog.Contains("secrets"));
         }
 
-        private void VerifyResultWithCacheOff()
+        private void VerifyResultWithCacheOff(string durableVersion)
         {
-            string expectedSyncTriggersPayload = GetExpectedSyncTriggersPayload();
+            string expectedSyncTriggersPayload = GetExpectedSyncTriggersPayload(durableVersion: durableVersion);
             var triggers = JArray.Parse(_contentBuilder.ToString());
             Assert.Equal(expectedSyncTriggersPayload, triggers.ToString(Formatting.None));
 
@@ -363,7 +364,7 @@ public async Task TrySyncTriggers_BackgroundSync_PostsExpectedContent()
                 Assert.Equal(1, _mockHttpHandler.RequestCount);
                 var result = JObject.Parse(_contentBuilder.ToString());
                 var triggers = result["triggers"];
-                Assert.Equal(GetExpectedSyncTriggersPayload(), triggers.ToString(Formatting.None));
+                Assert.Equal(GetExpectedSyncTriggersPayload(durableVersion: "V1"), triggers.ToString(Formatting.None));
 
                 string hash = string.Empty;
                 var downloadResponse = await hashBlob.DownloadAsync();
@@ -429,7 +430,7 @@ public async Task TrySyncTriggers_BackgroundSync_SetTriggersFailure_HashNotUpdat
                 Assert.Equal(1, _mockHttpHandler.RequestCount);
                 var result = JObject.Parse(_contentBuilder.ToString());
                 var triggers = result["triggers"];
-                Assert.Equal(GetExpectedSyncTriggersPayload(), triggers.ToString(Formatting.None));
+                Assert.Equal(GetExpectedSyncTriggersPayload(durableVersion: "V1"), triggers.ToString(Formatting.None));
                 bool hashBlobExists = await hashBlob.ExistsAsync();
                 Assert.False(hashBlobExists);
 
@@ -488,7 +489,7 @@ public async Task TrySyncTriggers_NoDurableTaskHub_DurableV1ExtensionJson_V1Defa
                 Assert.True(syncResult.Success, "SyncTriggers should return success true");
                 Assert.True(string.IsNullOrEmpty(syncResult.Error), "Error should be null or empty");
 
-                VerifyResultWithCacheOn(expectedTaskHub: null, connection: null);
+                VerifyResultWithCacheOn(expectedTaskHub: null, connection: null, durableVersion: "V1");
             }
         }
 
@@ -567,7 +568,7 @@ public async Task TrySyncTriggers_DurableV1ExtensionJson_V1ConfigPosted()
                 Assert.True(syncResult.Success, "SyncTriggers should return success true");
                 Assert.True(string.IsNullOrEmpty(syncResult.Error), "Error should be null or empty");
 
-                VerifyResultWithCacheOn(expectedTaskHub: "DurableTask", connection: "DurableConnection");
+                VerifyResultWithCacheOn(expectedTaskHub: "DurableTask", connection: "DurableConnection", durableVersion: "V1");
             }
         }
 
@@ -602,6 +603,92 @@ public async Task TrySyncTriggers_DurableV2ExtensionJson_V2ConfigPosted()
             }
         }
 
+        [Theory]
+        [InlineData("DurableMsSQLProviderPayload", "DurableMsSQLProviderPayload")]
+        [InlineData("DurableAdditionalPayload", "DurableMsSQLProviderPayload")] // Payload trimed to the minimum payload
+        [InlineData("DurableHasHubNameAndOrchestrationConfig", "DurableHasHubNameAndOrchestrationConfigPayload")]
+        [InlineData("DurableHasStorageAndActivityConfig", "DurableHasStorageAndActivityConfigPayload")]
+        [InlineData("Empty", "EmptyDurablePayload")]
+        public async Task TrySyncTriggers_DurableV2ExtensionJson_StorageProviderPosted(string scenarioName, string expectedPayload)
+        {
+            _mockEnvironment.Setup(p => p.GetEnvironmentVariable(EnvironmentSettingNames.AzureWebsiteName)).Returns("TestHubValue");
+            using (var env = new TestScopedEnvironmentVariable(_vars))
+            {
+                var durableConfig = GetDurableConfig(scenarioName);
+
+                var hostConfig = GetHostConfig(durableConfig, useBundles: false);
+
+                // See what happens when extension.json is not present but bundles are used.
+                SetupDurableExtension(hostConfig.ToString(), durableExtensionJsonVersion: "2.0.0.0");
+
+                // Act
+                var syncResult = await _functionsSyncManager.TrySyncTriggersAsync();
+
+                // Assert
+                Assert.True(syncResult.Success, "SyncTriggers should return success true");
+                Assert.True(string.IsNullOrEmpty(syncResult.Error), "Error should be null or empty");
+
+                //Verify Result
+                var result = JObject.Parse(_contentBuilder.ToString());
+                var triggers = result["triggers"];
+                var triggersString = triggers.ToString();
+                Assert.Equal(GetPayloadFromFile($"{expectedPayload}.json"), triggers.ToString());
+            }
+        }
+
+        private JObject GetDurableConfig(string scenarioName)
+        {
+            var durableConfig = new JObject();
+            durableConfig["hubName"] = "DurableTask";
+
+            var azureStorageConfig = new JObject();
+            azureStorageConfig["type"] = "mssql";
+            azureStorageConfig["connectionStringName"] = "SQLDB_Connection";
+            azureStorageConfig["taskEventLockTimeout"] = "00:02:00";
+            azureStorageConfig["createDatabaseIfNotExists"] = true;
+            durableConfig["storageProvider"] = azureStorageConfig;
+            durableConfig["maxConcurrentActivityFunctions"] = 12;
+            durableConfig["maxConcurrentOrchestratorFunctions"] = 10;
+
+            switch (scenarioName)
+            {
+                case "DurableMsSQLProviderPayload":
+                    return durableConfig;
+                case "DurableAdditionalPayload":
+                    // These additional parameters are exptected to be ignored.
+                    durableConfig["extendedSessionsEnabled"] = true;
+                    durableConfig["extendedSessionIdleTimeoutInSeconds"] = 30;
+                    return durableConfig;
+                case "DurableHasHubNameAndOrchestrationConfig":
+                    durableConfig = new JObject();
+                    durableConfig["hubName"] = "DurableTask";
+                    durableConfig["maxConcurrentOrchestratorFunctions"] = 10;
+                    return durableConfig;
+                case "DurableHasStorageAndActivityConfig":
+                    durableConfig = new JObject();
+                    azureStorageConfig = new JObject();
+                    azureStorageConfig["type"] = "mssql";
+                    azureStorageConfig["connectionStringName"] = "SQLDB_Connection";
+                    azureStorageConfig["taskEventLockTimeout"] = "00:02:00";
+                    azureStorageConfig["createDatabaseIfNotExists"] = true;
+                    durableConfig["storageProvider"] = azureStorageConfig;
+                    durableConfig["maxConcurrentActivityFunctions"] = 12;
+                    return durableConfig;
+                case "Empty":
+                    return new JObject();
+                default: return new JObject();
+            }
+
+        }
+
+
+        private string GetPayloadFromFile(string fileName)
+        {
+            var fullPath = Path.Combine("Management", "Payload", fileName);
+            return File.ReadAllText(fullPath);
+        }
+
+
         [Fact]
         public async Task UpdateHashAsync_Succeeds()
         {
diff --git a/test/WebJobs.Script.Tests.Integration/Management/Payload/DurableHasHubNameAndOrchestrationConfigPayload.json b/test/WebJobs.Script.Tests.Integration/Management/Payload/DurableHasHubNameAndOrchestrationConfigPayload.json
@@ -0,0 +1,29 @@
+[
+  {
+    "authLevel": "anonymous",
+    "type": "httpTrigger",
+    "direction": "in",
+    "name": "req",
+    "functionName": "function1"
+  },
+  {
+    "name": "myQueueItem",
+    "type": "orchestrationTrigger",
+    "direction": "in",
+    "queueName": "myqueue-items",
+    "connection": "",
+    "functionName": "function2",
+    "taskHubName": "DurableTask",
+    "maxConcurrentOrchestratorFunctions": 10
+  },
+  {
+    "name": "myQueueItem",
+    "type": "activityTrigger",
+    "direction": "in",
+    "queueName": "myqueue-items",
+    "connection": "",
+    "functionName": "function3",
+    "taskHubName": "DurableTask",
+    "maxConcurrentOrchestratorFunctions": 10
+  }
+]
diff --git a/test/WebJobs.Script.Tests.Integration/Management/Payload/DurableHasStorageAndActivityConfigPayload.json b/test/WebJobs.Script.Tests.Integration/Management/Payload/DurableHasStorageAndActivityConfigPayload.json
@@ -0,0 +1,41 @@
+[
+  {
+    "authLevel": "anonymous",
+    "type": "httpTrigger",
+    "direction": "in",
+    "name": "req",
+    "functionName": "function1"
+  },
+  {
+    "name": "myQueueItem",
+    "type": "orchestrationTrigger",
+    "direction": "in",
+    "queueName": "myqueue-items",
+    "connection": "SQLDB_Connection",
+    "functionName": "function2",
+    "taskHubName": "TestHubValue",
+    "storageProvider": {
+      "type": "mssql",
+      "connectionStringName": "SQLDB_Connection",
+      "taskEventLockTimeout": "00:02:00",
+      "createDatabaseIfNotExists": true
+    },
+    "maxConcurrentActivityFunctions": 12
+  },
+  {
+    "name": "myQueueItem",
+    "type": "activityTrigger",
+    "direction": "in",
+    "queueName": "myqueue-items",
+    "connection": "SQLDB_Connection",
+    "functionName": "function3",
+    "taskHubName": "TestHubValue",
+    "storageProvider": {
+      "type": "mssql",
+      "connectionStringName": "SQLDB_Connection",
+      "taskEventLockTimeout": "00:02:00",
+      "createDatabaseIfNotExists": true
+    },
+    "maxConcurrentActivityFunctions": 12
+  }
+]
diff --git a/test/WebJobs.Script.Tests.Integration/Management/Payload/DurableMsSQLProviderPayload.json b/test/WebJobs.Script.Tests.Integration/Management/Payload/DurableMsSQLProviderPayload.json
@@ -0,0 +1,43 @@
+[
+  {
+    "authLevel": "anonymous",
+    "type": "httpTrigger",
+    "direction": "in",
+    "name": "req",
+    "functionName": "function1"
+  },
+  {
+    "name": "myQueueItem",
+    "type": "orchestrationTrigger",
+    "direction": "in",
+    "queueName": "myqueue-items",
+    "connection": "SQLDB_Connection",
+    "functionName": "function2",
+    "taskHubName": "DurableTask",
+    "storageProvider": {
+      "type": "mssql",
+      "connectionStringName": "SQLDB_Connection",
+      "taskEventLockTimeout": "00:02:00",
+      "createDatabaseIfNotExists": true
+    },
+    "maxConcurrentOrchestratorFunctions": 10,
+    "maxConcurrentActivityFunctions": 12
+  },
+  {
+    "name": "myQueueItem",
+    "type": "activityTrigger",
+    "direction": "in",
+    "queueName": "myqueue-items",
+    "connection": "SQLDB_Connection",
+    "functionName": "function3",
+    "taskHubName": "DurableTask",
+    "storageProvider": {
+      "type": "mssql",
+      "connectionStringName": "SQLDB_Connection",
+      "taskEventLockTimeout": "00:02:00",
+      "createDatabaseIfNotExists": true
+    },
+    "maxConcurrentOrchestratorFunctions": 10,
+    "maxConcurrentActivityFunctions": 12
+  }
+]
diff --git a/test/WebJobs.Script.Tests.Integration/Management/Payload/EmptyDurablePayload.json b/test/WebJobs.Script.Tests.Integration/Management/Payload/EmptyDurablePayload.json
diff --git a/test/WebJobs.Script.Tests.Integration/WebJobs.Script.Tests.Integration.csproj b/test/WebJobs.Script.Tests.Integration/WebJobs.Script.Tests.Integration.csproj

Original file line number	Diff line number	Diff line change
`@@ -34,6 +34,8 @@ public class FunctionsSyncManager : IFunctionsSyncManager, IDisposable`
`34`	`34`	`private const string DurableTaskV1StorageConnectionName = "azureStorageConnectionStringName";`
`35`	`35`	`private const string DurableTaskV2StorageOptions = "storageProvider";`
`36`	`36`	`private const string DurableTaskV2StorageConnectionName = "connectionStringName";`
	`37`	`+ private const string DurableTaskV2MaxConcurrentActivityFunctions = "maxConcurrentActivityFunctions";`
	`38`	`+ private const string DurableTaskV2MaxConcurrentOrchestratorFunctions = "maxConcurrentOrchestratorFunctions";`
`37`	`39`	`private const string DurableTask = "durableTask";`
`38`	`40`
`39`	`41`	`// 45 alphanumeric characters gives us a buffer in our table/queue/blob container names.`
`@@ -483,6 +485,21 @@ private static JObject UpdateDurableFunctionConfig(JObject trigger, DurableConfi`
`483`	`485`	`{`
`484`	`486`	`trigger[Connection] = durableTaskConfig.Connection;`
`485`	`487`	`}`
	`488`	`+`
	`489`	`+ if (durableTaskConfig.StorageProvider != null)`
	`490`	`+ {`
	`491`	`+ trigger[DurableTaskV2StorageOptions] = durableTaskConfig.StorageProvider;`
	`492`	`+ }`
	`493`	`+`
	`494`	`+ if (durableTaskConfig.MaxConcurrentOrchestratorFunctions != 0)`
	`495`	`+ {`
	`496`	`+ trigger[DurableTaskV2MaxConcurrentOrchestratorFunctions] = durableTaskConfig.MaxConcurrentOrchestratorFunctions;`
	`497`	`+ }`
	`498`	`+`
	`499`	`+ if (durableTaskConfig.MaxConcurrentActivityFunctions != 0)`
	`500`	`+ {`
	`501`	`+ trigger[DurableTaskV2MaxConcurrentActivityFunctions] = durableTaskConfig.MaxConcurrentActivityFunctions;`
	`502`	`+ }`
`486`	`503`	`}`
`487`	`504`	`return trigger;`
`488`	`505`	`}`
`@@ -597,6 +614,18 @@ private DurableConfig GetDurableV2Config(JObject durableHostConfig)`
`597`	`614`	`{`
`598`	`615`	`config.Connection = nameValue.ToString();`
`599`	`616`	`}`
	`617`	`+`
	`618`	`+ config.StorageProvider = storageOptions;`
	`619`	`+ }`
	`620`	`+`
	`621`	`+ if (durableHostConfig.TryGetValue(DurableTaskV2MaxConcurrentOrchestratorFunctions, StringComparison.OrdinalIgnoreCase, out JToken maxConcurrentOrchestratorFunctions) && maxConcurrentOrchestratorFunctions != null)`
	`622`	`+ {`
	`623`	`+ config.MaxConcurrentOrchestratorFunctions = int.Parse(maxConcurrentOrchestratorFunctions.ToString());`
	`624`	`+ }`
	`625`	`+`
	`626`	`+ if (durableHostConfig.TryGetValue(DurableTaskV2MaxConcurrentActivityFunctions, StringComparison.OrdinalIgnoreCase, out JToken maxConcurrentActivityFunctions) && maxConcurrentActivityFunctions != null)`
	`627`	`+ {`
	`628`	`+ config.MaxConcurrentActivityFunctions = int.Parse(maxConcurrentActivityFunctions.ToString());`
`600`	`629`	`}`
`601`	`630`	`}`
`602`	`631`
`@@ -723,9 +752,15 @@ private class DurableConfig`
`723`	`752`
`724`	`753`	`public string Connection { get; set; }`
`725`	`754`
	`755`	`+ public JToken StorageProvider { get; set; }`
	`756`	`+`
	`757`	`+ public int MaxConcurrentActivityFunctions { get; set; }`
	`758`	`+`
	`759`	`+ public int MaxConcurrentOrchestratorFunctions { get; set; }`
	`760`	`+`
`726`	`761`	`public bool HasValues()`
`727`	`762`	`{`
`728`		`- return this.HubName != null \|\| this.Connection != null;`
	`763`	`+ return this.HubName != null \|\| this.Connection != null \|\| this.StorageProvider != null \|\| this.MaxConcurrentOrchestratorFunctions != 0 \|\| this.MaxConcurrentOrchestratorFunctions != 0;`
`729`	`764`	`}`
`730`	`765`	`}`
`731`	`766`	`}`