Skip to content

Commit 8cf3270

Browse files
Include minimum durable payload to sync trigger (#8229)
* Include minimum durable payload to sync trigger * update releaseNote * fix CI fail * fix issue of release note
1 parent 931a4e4 commit 8cf3270

File tree

8 files changed

+283
-14
lines changed

8 files changed

+283
-14
lines changed

release_notes.md

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@
44
-->
55
- Updated Java Worker Version to [2.2.3](https://github.com/Azure/azure-functions-java-worker/releases/tag/2.2.3)
66
- Updated Node.js Worker Version to [3.3.0](https://github.com/Azure/azure-functions-nodejs-worker/releases/tag/v3.3.0)
7+
- Add Include minimum durable payload to sync trigger (#8292)
78

89
**Release sprint:** Sprint 118
910
[ [bugs](https://github.com/Azure/azure-functions-host/issues?q=is%3Aissue+milestone%3A%22Functions+Sprint+118%22+label%3Abug+is%3Aclosed) | [features](https://github.com/Azure/azure-functions-host/issues?q=is%3Aissue+milestone%3A%22Functions+Sprint+118%22+label%3Afeature+is%3Aclosed) ]

src/WebJobs.Script.WebHost/Management/FunctionsSyncManager.cs

Lines changed: 36 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -35,6 +35,8 @@ public class FunctionsSyncManager : IFunctionsSyncManager, IDisposable
3535
private const string DurableTaskV1StorageConnectionName = "azureStorageConnectionStringName";
3636
private const string DurableTaskV2StorageOptions = "storageProvider";
3737
private const string DurableTaskV2StorageConnectionName = "connectionStringName";
38+
private const string DurableTaskV2MaxConcurrentActivityFunctions = "maxConcurrentActivityFunctions";
39+
private const string DurableTaskV2MaxConcurrentOrchestratorFunctions = "maxConcurrentOrchestratorFunctions";
3840
private const string DurableTask = "durableTask";
3941

4042
// 45 alphanumeric characters gives us a buffer in our table/queue/blob container names.
@@ -477,6 +479,21 @@ private static JObject UpdateDurableFunctionConfig(JObject trigger, DurableConfi
477479
{
478480
trigger[Connection] = durableTaskConfig.Connection;
479481
}
482+
483+
if (durableTaskConfig.StorageProvider != null)
484+
{
485+
trigger[DurableTaskV2StorageOptions] = durableTaskConfig.StorageProvider;
486+
}
487+
488+
if (durableTaskConfig.MaxConcurrentOrchestratorFunctions != 0)
489+
{
490+
trigger[DurableTaskV2MaxConcurrentOrchestratorFunctions] = durableTaskConfig.MaxConcurrentOrchestratorFunctions;
491+
}
492+
493+
if (durableTaskConfig.MaxConcurrentActivityFunctions != 0)
494+
{
495+
trigger[DurableTaskV2MaxConcurrentActivityFunctions] = durableTaskConfig.MaxConcurrentActivityFunctions;
496+
}
480497
}
481498
return trigger;
482499
}
@@ -591,6 +608,18 @@ private DurableConfig GetDurableV2Config(JObject durableHostConfig)
591608
{
592609
config.Connection = nameValue.ToString();
593610
}
611+
612+
config.StorageProvider = storageOptions;
613+
}
614+
615+
if (durableHostConfig.TryGetValue(DurableTaskV2MaxConcurrentOrchestratorFunctions, StringComparison.OrdinalIgnoreCase, out JToken maxConcurrentOrchestratorFunctions) && maxConcurrentOrchestratorFunctions != null)
616+
{
617+
config.MaxConcurrentOrchestratorFunctions = int.Parse(maxConcurrentOrchestratorFunctions.ToString());
618+
}
619+
620+
if (durableHostConfig.TryGetValue(DurableTaskV2MaxConcurrentActivityFunctions, StringComparison.OrdinalIgnoreCase, out JToken maxConcurrentActivityFunctions) && maxConcurrentActivityFunctions != null)
621+
{
622+
config.MaxConcurrentActivityFunctions = int.Parse(maxConcurrentActivityFunctions.ToString());
594623
}
595624
}
596625

@@ -717,9 +746,15 @@ private class DurableConfig
717746

718747
public string Connection { get; set; }
719748

749+
public JToken StorageProvider { get; set; }
750+
751+
public int MaxConcurrentActivityFunctions { get; set; }
752+
753+
public int MaxConcurrentOrchestratorFunctions { get; set; }
754+
720755
public bool HasValues()
721756
{
722-
return this.HubName != null || this.Connection != null;
757+
return this.HubName != null || this.Connection != null || this.StorageProvider != null || this.MaxConcurrentOrchestratorFunctions != 0 || this.MaxConcurrentOrchestratorFunctions != 0;
723758
}
724759
}
725760
}

test/WebJobs.Script.Tests.Integration/Management/FunctionsSyncManagerTests.cs

Lines changed: 100 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -147,12 +147,13 @@ public FunctionsSyncManagerTests()
147147
_functionsSyncManager = new FunctionsSyncManager(configuration, hostIdProviderMock.Object, optionsMonitor, loggerFactory.CreateLogger<FunctionsSyncManager>(), httpClientFactory, _secretManagerProviderMock.Object, _mockWebHostEnvironment.Object, _mockEnvironment.Object, _hostNameProvider, functionMetadataManager, azureBlobStorageProvider);
148148
}
149149

150-
private string GetExpectedSyncTriggersPayload(string postedConnection = DefaultTestConnection, string postedTaskHub = DefaultTestTaskHub)
150+
private string GetExpectedSyncTriggersPayload(string postedConnection = DefaultTestConnection, string postedTaskHub = DefaultTestTaskHub, string durableVersion = "V2")
151151
{
152152
string taskHubSegment = postedTaskHub != null ? $",\"taskHubName\":\"{postedTaskHub}\"" : "";
153+
string storageProviderSegment = postedConnection != null && durableVersion == "V2" ? $",\"storageProvider\":{{\"connectionStringName\":\"DurableConnection\"}}" : "";
153154
return "[{\"authLevel\":\"anonymous\",\"type\":\"httpTrigger\",\"direction\":\"in\",\"name\":\"req\",\"functionName\":\"function1\"}," +
154-
$"{{\"name\":\"myQueueItem\",\"type\":\"orchestrationTrigger\",\"direction\":\"in\",\"queueName\":\"myqueue-items\",\"connection\":\"{postedConnection}\",\"functionName\":\"function2\"{taskHubSegment}}}," +
155-
$"{{\"name\":\"myQueueItem\",\"type\":\"activityTrigger\",\"direction\":\"in\",\"queueName\":\"myqueue-items\",\"connection\":\"{postedConnection}\",\"functionName\":\"function3\"{taskHubSegment}}}]";
155+
$"{{\"name\":\"myQueueItem\",\"type\":\"orchestrationTrigger\",\"direction\":\"in\",\"queueName\":\"myqueue-items\",\"connection\":\"{postedConnection}\",\"functionName\":\"function2\"{taskHubSegment}{storageProviderSegment}}}," +
156+
$"{{\"name\":\"myQueueItem\",\"type\":\"activityTrigger\",\"direction\":\"in\",\"queueName\":\"myqueue-items\",\"connection\":\"{postedConnection}\",\"functionName\":\"function3\"{taskHubSegment}{storageProviderSegment}}}]";
156157
}
157158

158159
private void ResetMockFileSystem(string hostJsonContent = null, string extensionsJsonContent = null)
@@ -239,18 +240,18 @@ public async Task TrySyncTriggers_PostsExpectedContent(bool cacheEnabled)
239240

240241
if (cacheEnabled)
241242
{
242-
VerifyResultWithCacheOn();
243+
VerifyResultWithCacheOn(durableVersion: "V1");
243244
}
244245
else
245246
{
246-
VerifyResultWithCacheOff();
247+
VerifyResultWithCacheOff(durableVersion: "V1");
247248
}
248249
}
249250
}
250251

251-
private void VerifyResultWithCacheOn(string connection = DefaultTestConnection, string expectedTaskHub = "TestHubValue")
252+
private void VerifyResultWithCacheOn(string connection = DefaultTestConnection, string expectedTaskHub = "TestHubValue", string durableVersion = "V2")
252253
{
253-
string expectedSyncTriggersPayload = GetExpectedSyncTriggersPayload(postedConnection: connection, postedTaskHub: expectedTaskHub);
254+
string expectedSyncTriggersPayload = GetExpectedSyncTriggersPayload(postedConnection: connection, postedTaskHub: expectedTaskHub, durableVersion);
254255
// verify triggers
255256
var result = JObject.Parse(_contentBuilder.ToString());
256257
var triggers = result["triggers"];
@@ -290,9 +291,9 @@ private void VerifyResultWithCacheOn(string connection = DefaultTestConnection,
290291
Assert.False(triggersLog.Contains("secrets"));
291292
}
292293

293-
private void VerifyResultWithCacheOff()
294+
private void VerifyResultWithCacheOff(string durableVersion)
294295
{
295-
string expectedSyncTriggersPayload = GetExpectedSyncTriggersPayload();
296+
string expectedSyncTriggersPayload = GetExpectedSyncTriggersPayload(durableVersion: durableVersion);
296297
var triggers = JArray.Parse(_contentBuilder.ToString());
297298
Assert.Equal(expectedSyncTriggersPayload, triggers.ToString(Formatting.None));
298299

@@ -367,7 +368,7 @@ public async Task TrySyncTriggers_BackgroundSync_PostsExpectedContent()
367368
Assert.Equal(1, _mockHttpHandler.RequestCount);
368369
var result = JObject.Parse(_contentBuilder.ToString());
369370
var triggers = result["triggers"];
370-
Assert.Equal(GetExpectedSyncTriggersPayload(), triggers.ToString(Formatting.None));
371+
Assert.Equal(GetExpectedSyncTriggersPayload(durableVersion: "V1"), triggers.ToString(Formatting.None));
371372

372373
string hash = string.Empty;
373374
var downloadResponse = await hashBlob.DownloadAsync();
@@ -433,7 +434,7 @@ public async Task TrySyncTriggers_BackgroundSync_SetTriggersFailure_HashNotUpdat
433434
Assert.Equal(1, _mockHttpHandler.RequestCount);
434435
var result = JObject.Parse(_contentBuilder.ToString());
435436
var triggers = result["triggers"];
436-
Assert.Equal(GetExpectedSyncTriggersPayload(), triggers.ToString(Formatting.None));
437+
Assert.Equal(GetExpectedSyncTriggersPayload(durableVersion: "V1"), triggers.ToString(Formatting.None));
437438
bool hashBlobExists = await hashBlob.ExistsAsync();
438439
Assert.False(hashBlobExists);
439440

@@ -492,7 +493,7 @@ public async Task TrySyncTriggers_NoDurableTaskHub_DurableV1ExtensionJson_V1Defa
492493
Assert.True(syncResult.Success, "SyncTriggers should return success true");
493494
Assert.True(string.IsNullOrEmpty(syncResult.Error), "Error should be null or empty");
494495

495-
VerifyResultWithCacheOn(expectedTaskHub: null, connection: null);
496+
VerifyResultWithCacheOn(expectedTaskHub: null, connection: null, durableVersion: "V1");
496497
}
497498
}
498499

@@ -571,7 +572,7 @@ public async Task TrySyncTriggers_DurableV1ExtensionJson_V1ConfigPosted()
571572
Assert.True(syncResult.Success, "SyncTriggers should return success true");
572573
Assert.True(string.IsNullOrEmpty(syncResult.Error), "Error should be null or empty");
573574

574-
VerifyResultWithCacheOn(expectedTaskHub: "DurableTask", connection: "DurableConnection");
575+
VerifyResultWithCacheOn(expectedTaskHub: "DurableTask", connection: "DurableConnection", durableVersion: "V1");
575576
}
576577
}
577578

@@ -606,6 +607,92 @@ public async Task TrySyncTriggers_DurableV2ExtensionJson_V2ConfigPosted()
606607
}
607608
}
608609

610+
[Theory]
611+
[InlineData("DurableMsSQLProviderPayload", "DurableMsSQLProviderPayload")]
612+
[InlineData("DurableAdditionalPayload", "DurableMsSQLProviderPayload")] // Payload trimed to the minimum payload
613+
[InlineData("DurableHasHubNameAndOrchestrationConfig", "DurableHasHubNameAndOrchestrationConfigPayload")]
614+
[InlineData("DurableHasStorageAndActivityConfig", "DurableHasStorageAndActivityConfigPayload")]
615+
[InlineData("Empty", "EmptyDurablePayload")]
616+
public async Task TrySyncTriggers_DurableV2ExtensionJson_StorageProviderPosted(string scenarioName, string expectedPayload)
617+
{
618+
_mockEnvironment.Setup(p => p.GetEnvironmentVariable(EnvironmentSettingNames.AzureWebsiteName)).Returns("TestHubValue");
619+
using (var env = new TestScopedEnvironmentVariable(_vars))
620+
{
621+
var durableConfig = GetDurableConfig(scenarioName);
622+
623+
var hostConfig = GetHostConfig(durableConfig, useBundles: false);
624+
625+
// See what happens when extension.json is not present but bundles are used.
626+
SetupDurableExtension(hostConfig.ToString(), durableExtensionJsonVersion: "2.0.0.0");
627+
628+
// Act
629+
var syncResult = await _functionsSyncManager.TrySyncTriggersAsync();
630+
631+
// Assert
632+
Assert.True(syncResult.Success, "SyncTriggers should return success true");
633+
Assert.True(string.IsNullOrEmpty(syncResult.Error), "Error should be null or empty");
634+
635+
//Verify Result
636+
var result = JObject.Parse(_contentBuilder.ToString());
637+
var triggers = result["triggers"];
638+
var triggersString = triggers.ToString();
639+
Assert.Equal(GetPayloadFromFile($"{expectedPayload}.json"), triggers.ToString());
640+
}
641+
}
642+
643+
private JObject GetDurableConfig(string scenarioName)
644+
{
645+
var durableConfig = new JObject();
646+
durableConfig["hubName"] = "DurableTask";
647+
648+
var azureStorageConfig = new JObject();
649+
azureStorageConfig["type"] = "mssql";
650+
azureStorageConfig["connectionStringName"] = "SQLDB_Connection";
651+
azureStorageConfig["taskEventLockTimeout"] = "00:02:00";
652+
azureStorageConfig["createDatabaseIfNotExists"] = true;
653+
durableConfig["storageProvider"] = azureStorageConfig;
654+
durableConfig["maxConcurrentActivityFunctions"] = 12;
655+
durableConfig["maxConcurrentOrchestratorFunctions"] = 10;
656+
657+
switch (scenarioName)
658+
{
659+
case "DurableMsSQLProviderPayload":
660+
return durableConfig;
661+
case "DurableAdditionalPayload":
662+
// These additional parameters are exptected to be ignored.
663+
durableConfig["extendedSessionsEnabled"] = true;
664+
durableConfig["extendedSessionIdleTimeoutInSeconds"] = 30;
665+
return durableConfig;
666+
case "DurableHasHubNameAndOrchestrationConfig":
667+
durableConfig = new JObject();
668+
durableConfig["hubName"] = "DurableTask";
669+
durableConfig["maxConcurrentOrchestratorFunctions"] = 10;
670+
return durableConfig;
671+
case "DurableHasStorageAndActivityConfig":
672+
durableConfig = new JObject();
673+
azureStorageConfig = new JObject();
674+
azureStorageConfig["type"] = "mssql";
675+
azureStorageConfig["connectionStringName"] = "SQLDB_Connection";
676+
azureStorageConfig["taskEventLockTimeout"] = "00:02:00";
677+
azureStorageConfig["createDatabaseIfNotExists"] = true;
678+
durableConfig["storageProvider"] = azureStorageConfig;
679+
durableConfig["maxConcurrentActivityFunctions"] = 12;
680+
return durableConfig;
681+
case "Empty":
682+
return new JObject();
683+
default: return new JObject();
684+
}
685+
686+
}
687+
688+
689+
private string GetPayloadFromFile(string fileName)
690+
{
691+
var fullPath = Path.Combine("Management", "Payload", fileName);
692+
return File.ReadAllText(fullPath);
693+
}
694+
695+
609696
[Fact]
610697
public async Task UpdateHashAsync_Succeeds()
611698
{
Lines changed: 29 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,29 @@
1+
[
2+
{
3+
"authLevel": "anonymous",
4+
"type": "httpTrigger",
5+
"direction": "in",
6+
"name": "req",
7+
"functionName": "function1"
8+
},
9+
{
10+
"name": "myQueueItem",
11+
"type": "orchestrationTrigger",
12+
"direction": "in",
13+
"queueName": "myqueue-items",
14+
"connection": "",
15+
"functionName": "function2",
16+
"taskHubName": "DurableTask",
17+
"maxConcurrentOrchestratorFunctions": 10
18+
},
19+
{
20+
"name": "myQueueItem",
21+
"type": "activityTrigger",
22+
"direction": "in",
23+
"queueName": "myqueue-items",
24+
"connection": "",
25+
"functionName": "function3",
26+
"taskHubName": "DurableTask",
27+
"maxConcurrentOrchestratorFunctions": 10
28+
}
29+
]
Lines changed: 41 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,41 @@
1+
[
2+
{
3+
"authLevel": "anonymous",
4+
"type": "httpTrigger",
5+
"direction": "in",
6+
"name": "req",
7+
"functionName": "function1"
8+
},
9+
{
10+
"name": "myQueueItem",
11+
"type": "orchestrationTrigger",
12+
"direction": "in",
13+
"queueName": "myqueue-items",
14+
"connection": "SQLDB_Connection",
15+
"functionName": "function2",
16+
"taskHubName": "TestHubValue",
17+
"storageProvider": {
18+
"type": "mssql",
19+
"connectionStringName": "SQLDB_Connection",
20+
"taskEventLockTimeout": "00:02:00",
21+
"createDatabaseIfNotExists": true
22+
},
23+
"maxConcurrentActivityFunctions": 12
24+
},
25+
{
26+
"name": "myQueueItem",
27+
"type": "activityTrigger",
28+
"direction": "in",
29+
"queueName": "myqueue-items",
30+
"connection": "SQLDB_Connection",
31+
"functionName": "function3",
32+
"taskHubName": "TestHubValue",
33+
"storageProvider": {
34+
"type": "mssql",
35+
"connectionStringName": "SQLDB_Connection",
36+
"taskEventLockTimeout": "00:02:00",
37+
"createDatabaseIfNotExists": true
38+
},
39+
"maxConcurrentActivityFunctions": 12
40+
}
41+
]
Lines changed: 43 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,43 @@
1+
[
2+
{
3+
"authLevel": "anonymous",
4+
"type": "httpTrigger",
5+
"direction": "in",
6+
"name": "req",
7+
"functionName": "function1"
8+
},
9+
{
10+
"name": "myQueueItem",
11+
"type": "orchestrationTrigger",
12+
"direction": "in",
13+
"queueName": "myqueue-items",
14+
"connection": "SQLDB_Connection",
15+
"functionName": "function2",
16+
"taskHubName": "DurableTask",
17+
"storageProvider": {
18+
"type": "mssql",
19+
"connectionStringName": "SQLDB_Connection",
20+
"taskEventLockTimeout": "00:02:00",
21+
"createDatabaseIfNotExists": true
22+
},
23+
"maxConcurrentOrchestratorFunctions": 10,
24+
"maxConcurrentActivityFunctions": 12
25+
},
26+
{
27+
"name": "myQueueItem",
28+
"type": "activityTrigger",
29+
"direction": "in",
30+
"queueName": "myqueue-items",
31+
"connection": "SQLDB_Connection",
32+
"functionName": "function3",
33+
"taskHubName": "DurableTask",
34+
"storageProvider": {
35+
"type": "mssql",
36+
"connectionStringName": "SQLDB_Connection",
37+
"taskEventLockTimeout": "00:02:00",
38+
"createDatabaseIfNotExists": true
39+
},
40+
"maxConcurrentOrchestratorFunctions": 10,
41+
"maxConcurrentActivityFunctions": 12
42+
}
43+
]

0 commit comments

Comments
 (0)