Skip to content

Commit 20d6ab5

Browse files
mathewcmathewc
authored
Adding new HostId generation logic for CV2 (#9157)
1 parent 22cb215 commit 20d6ab5

File tree

4 files changed

+160
-44
lines changed

4 files changed

+160
-44
lines changed

‎src/WebJobs.Script.WebHost/Management/FunctionsSyncManager.cs‎

Lines changed: 29 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,7 @@
2222
using Microsoft.Azure.WebJobs.Script.WebHost.Models;
2323
using Microsoft.Azure.WebJobs.Script.WebHost.Security;
2424
using Microsoft.Extensions.Configuration;
25+
using Microsoft.Extensions.Hosting;
2526
using Microsoft.Extensions.Logging;
2627
using Microsoft.Extensions.Options;
2728
using Newtonsoft.Json;
@@ -315,23 +316,22 @@ public async Task<SyncTriggersPayload> GetSyncTriggersPayload()
315316
var triggersArray = new JArray(triggers);
316317
int count = triggersArray.Count;
317318

319+
// Form the base minimal result
320+
string hostId = await _hostIdProvider.GetHostIdAsync(CancellationToken.None);
321+
JObject result = GetMinimalPayload(hostId, triggersArray);
322+
318323
if (!ArmCacheEnabled)
319324
{
320-
// extended format is disabled - just return triggers
325+
// extended format is disabled - just return minimal results
321326
return new SyncTriggersPayload
322327
{
323-
Content = JsonConvert.SerializeObject(triggersArray),
328+
Content = JsonConvert.SerializeObject(result),
324329
Count = count
325330
};
326331
}
327332

328-
// Add triggers to the payload
329-
JObject result = new JObject();
330-
result.Add("triggers", triggersArray);
331-
332333
// Add all listable functions details to the payload
333334
JObject functions = new JObject();
334-
string routePrefix = await WebFunctionsManager.GetRoutePrefix(hostOptions.RootScriptPath);
335335
var listableFunctions = _functionMetadataManager.GetFunctionMetadata().Where(m => !m.IsCodeless());
336336
var functionDetails = await WebFunctionsManager.GetFunctionMetadataResponse(listableFunctions, hostOptions, _hostNameProvider);
337337
result.Add("functions", new JArray(functionDetails.Select(p => JObject.FromObject(p))));
@@ -395,15 +395,12 @@ public async Task<SyncTriggersPayload> GetSyncTriggersPayload()
395395

396396
if (json.Length > ScriptConstants.MaxTriggersStringLength && !_environment.IsKubernetesManagedHosting())
397397
{
398-
// The settriggers call to the FE enforces a max request size
399-
// limit. If we're over limit, revert to the minimal triggers
400-
// format.
398+
// The settriggers call to the FE enforces a max request size limit.
399+
// If we're over limit, revert to the minimal triggers format.
401400
_logger.LogWarning($"SyncTriggers payload of length '{json.Length}' exceeds max length of '{ScriptConstants.MaxTriggersStringLength}'. Reverting to minimal format.");
402-
return new SyncTriggersPayload
403-
{
404-
Content = JsonConvert.SerializeObject(triggersArray),
405-
Count = count
406-
};
401+
402+
var minimalResult = GetMinimalPayload(hostId, triggersArray);
403+
json = JsonConvert.SerializeObject(minimalResult);
407404
}
408405

409406
return new SyncTriggersPayload
@@ -413,6 +410,23 @@ public async Task<SyncTriggersPayload> GetSyncTriggersPayload()
413410
};
414411
}
415412

413+
private JObject GetMinimalPayload(string hostId, JArray triggersArray)
414+
{
415+
JObject result = new JObject
416+
{
417+
{ "triggers", triggersArray }
418+
};
419+
420+
if (_environment.IsFlexConsumptionSku())
421+
{
422+
// Currently we're only sending the HostId for Flex Consumption. Eventually we'll do this for all SKUs.
423+
// When the HostId is sent, ScaleController will use it directly rather than compute it itself.
424+
result["hostId"] = hostId;
425+
}
426+
427+
return result;
428+
}
429+
416430
internal static async Task<JObject> GetHostJsonExtensionsAsync(IOptionsMonitor<ScriptApplicationHostOptions> applicationHostOptions, ILogger logger)
417431
{
418432
var hostOptions = applicationHostOptions.CurrentValue.ToHostOptions();

‎src/WebJobs.Script/Host/ScriptHostIdProvider.cs‎

Lines changed: 18 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@
33

44
using System;
55
using System.Linq;
6+
using System.Security.Cryptography;
67
using System.Text;
78
using System.Threading;
89
using System.Threading.Tasks;
@@ -33,6 +34,7 @@ public Task<string> GetHostIdAsync(CancellationToken cancellationToken)
3334
string hostId = _config[ConfigurationSectionNames.HostIdPath];
3435
if (hostId == null)
3536
{
37+
// if the user hasn't configured an explicit ID, we generate the default ID.
3638
HostIdResult result = GetDefaultHostId(_environment, _options.CurrentValue);
3739
hostId = result.HostId;
3840
if (result.IsTruncated && !result.IsLocal)
@@ -48,9 +50,22 @@ internal static HostIdResult GetDefaultHostId(IEnvironment environment, ScriptAp
4850
{
4951
HostIdResult result = new HostIdResult();
5052

51-
// We're setting the default here on the newly created configuration
52-
// If the user has explicitly set the HostID via host.json, it will overwrite
53-
// what we set here
53+
if (environment.IsFlexConsumptionSku())
54+
{
55+
// in Flex Consumption, we use a Guid based host ID without truncation.
56+
string uniqueSlotName = environment?.GetAzureWebsiteUniqueSlotName();
57+
byte[] hash;
58+
using (MD5 md5 = MD5.Create())
59+
{
60+
hash = md5.ComputeHash(Encoding.UTF8.GetBytes(uniqueSlotName));
61+
}
62+
63+
result.HostId = new Guid(hash).ToString().Replace("-", string.Empty).ToLowerInvariant();
64+
65+
return result;
66+
}
67+
68+
// Note: HostIds must conform to the rules documented here: https://learn.microsoft.com/en-us/azure/azure-functions/functions-app-settings#azurefunctionswebhost__hostid
5469
string hostId = null;
5570
if (environment.IsAppService() || environment.IsAnyKubernetesEnvironment())
5671
{

‎test/WebJobs.Script.Tests.Integration/Management/FunctionsSyncManagerTests.cs‎

Lines changed: 90 additions & 26 deletions
Original file line numberDiff line numberDiff line change
@@ -139,6 +139,7 @@ public FunctionsSyncManagerTests()
139139
_mockEnvironment.Setup(p => p.GetEnvironmentVariable(EnvironmentSettingNames.AzureWebsiteName)).Returns((string)null);
140140
_mockEnvironment.Setup(p => p.GetEnvironmentVariable(EnvironmentSettingNames.AzureWebsiteSlotName)).Returns((string)null);
141141
_mockEnvironment.Setup(p => p.GetEnvironmentVariable(EnvironmentSettingNames.AzureWebJobsFeatureFlags)).Returns((string)null);
142+
_mockEnvironment.Setup(p => p.GetEnvironmentVariable(EnvironmentSettingNames.AzureWebsiteSku)).Returns((string)ScriptConstants.DynamicSku);
142143

143144
_hostNameProvider = new HostNameProvider(_mockEnvironment.Object);
144145

@@ -152,7 +153,7 @@ public FunctionsSyncManagerTests()
152153
_functionsSyncManager = new FunctionsSyncManager(configuration, hostIdProviderMock.Object, optionsMonitor, loggerFactory.CreateLogger<FunctionsSyncManager>(), httpClientFactory, _secretManagerProviderMock.Object, _mockWebHostEnvironment.Object, _mockEnvironment.Object, _hostNameProvider, functionMetadataManager, azureBlobStorageProvider);
153154
}
154155

155-
private string GetExpectedSyncTriggersPayload(string postedConnection = DefaultTestConnection, string postedTaskHub = DefaultTestTaskHub, string durableVersion = "V2")
156+
private string GetExpectedTriggersPayload(string postedConnection = DefaultTestConnection, string postedTaskHub = DefaultTestTaskHub, string durableVersion = "V2")
156157
{
157158
string taskHubSegment = postedTaskHub != null ? $",\"taskHubName\":\"{postedTaskHub}\"" : "";
158159
string storageProviderSegment = postedConnection != null && durableVersion == "V2" ? $",\"storageProvider\":{{\"connectionStringName\":\"DurableConnection\"}}" : "";
@@ -180,9 +181,13 @@ public async Task TrySyncTriggers_StandbyMode_ReturnsFalse()
180181
}
181182
}
182183

183-
[Fact]
184-
public async Task TrySyncTriggers_MaxSyncTriggersPayloadSize_Succeeds()
184+
[Theory]
185+
[InlineData(ScriptConstants.DynamicSku)]
186+
[InlineData(ScriptConstants.FlexConsumptionSku)]
187+
public async Task TrySyncTriggers_MaxSyncTriggersPayloadSize_Succeeds(string sku)
185188
{
189+
_mockEnvironment.Setup(p => p.GetEnvironmentVariable(EnvironmentSettingNames.AzureWebsiteSku)).Returns(sku);
190+
186191
// create a dummy file that pushes us over size
187192
string maxString = new string('x', ScriptConstants.MaxTriggersStringLength + 1);
188193
_function1 = $"{{ bindings: [], test: '{maxString}'}}";
@@ -196,8 +201,21 @@ public async Task TrySyncTriggers_MaxSyncTriggersPayloadSize_Succeeds()
196201

197202
string syncString = _contentBuilder.ToString();
198203
Assert.True(syncString.Length < ScriptConstants.MaxTriggersStringLength);
199-
var syncContent = JToken.Parse(syncString);
200-
Assert.Equal(JTokenType.Array, syncContent.Type);
204+
var syncContent = JObject.Parse(syncString);
205+
206+
if (_mockEnvironment.Object.IsFlexConsumptionSku())
207+
{
208+
Assert.Equal(2, syncContent.Count);
209+
Assert.Equal("testhostid123", syncContent["hostId"]);
210+
}
211+
else
212+
{
213+
Assert.Equal(1, syncContent.Count);
214+
Assert.Equal(null, syncContent["hostId"]);
215+
}
216+
217+
JArray triggers = (JArray)syncContent["triggers"];
218+
Assert.Equal(2, triggers.Count);
201219
}
202220
}
203221

@@ -256,13 +274,35 @@ public async Task TrySyncTriggers_PostsExpectedContent(bool cacheEnabled)
256274
}
257275
}
258276

277+
[Theory]
278+
[InlineData(ScriptConstants.DynamicSku)]
279+
[InlineData(ScriptConstants.FlexConsumptionSku)]
280+
public async Task TrySyncTriggers_PostsExpectedContent_BySku(string sku)
281+
{
282+
_mockEnvironment.Setup(p => p.GetEnvironmentVariable(EnvironmentSettingNames.AzureWebsiteSku)).Returns(sku);
283+
284+
using (var env = new TestScopedEnvironmentVariable(_vars))
285+
{
286+
// Act
287+
var syncResult = await _functionsSyncManager.TrySyncTriggersAsync();
288+
289+
// Assert
290+
Assert.True(syncResult.Success, "SyncTriggers should return success true");
291+
Assert.True(string.IsNullOrEmpty(syncResult.Error), "Error should be null or empty");
292+
293+
// verify expected headers
294+
Assert.Equal(ScriptConstants.FunctionsUserAgent, _mockHttpHandler.LastRequest.Headers.UserAgent.ToString());
295+
Assert.True(_mockHttpHandler.LastRequest.Headers.Contains(ScriptConstants.AntaresLogIdHeaderName));
296+
Assert.NotEmpty(_mockHttpHandler.LastRequest.Headers.GetValues(ScriptConstants.SiteRestrictedTokenHeaderName));
297+
Assert.NotEmpty(_mockHttpHandler.LastRequest.Headers.GetValues(ScriptConstants.SiteTokenHeaderName));
298+
299+
VerifyResultWithCacheOn(durableVersion: "V1");
300+
}
301+
}
302+
259303
private void VerifyResultWithCacheOn(string connection = DefaultTestConnection, string expectedTaskHub = "TestHubValue", string durableVersion = "V2")
260304
{
261-
string expectedSyncTriggersPayload = GetExpectedSyncTriggersPayload(postedConnection: connection, postedTaskHub: expectedTaskHub, durableVersion);
262-
// verify triggers
263-
var result = JObject.Parse(_contentBuilder.ToString());
264-
var triggers = result["triggers"];
265-
Assert.Equal(expectedSyncTriggersPayload, triggers.ToString(Formatting.None));
305+
var result = VerifyResultCommon(connection, expectedTaskHub, durableVersion);
266306

267307
// verify functions
268308
var functions = (JArray)result["functions"];
@@ -285,31 +325,55 @@ private void VerifyResultWithCacheOn(string connection = DefaultTestConnection,
285325
Assert.Equal("function1", function1Secrets["name"]);
286326
Assert.Equal("aaa", (string)function1Secrets["secrets"]["TestFunctionKey1"]);
287327
Assert.Equal("bbb", (string)function1Secrets["secrets"]["TestFunctionKey2"]);
328+
}
288329

289-
var logs = _loggerProvider.GetAllLogMessages().Where(m => m.Category.Equals(SyncManagerLogCategory)).ToList();
290-
291-
var log = logs[0];
292-
int startIdx = log.FormattedMessage.IndexOf("Content=") + 8;
293-
int endIdx = log.FormattedMessage.LastIndexOf(')');
294-
var triggersLog = log.FormattedMessage.Substring(startIdx, endIdx - startIdx).Trim();
295-
var logObject = JObject.Parse(triggersLog);
330+
private void VerifyResultWithCacheOff(string durableVersion)
331+
{
332+
var result = VerifyResultCommon(durableVersion: durableVersion);
296333

297-
Assert.Equal(expectedSyncTriggersPayload, logObject["triggers"].ToString(Formatting.None));
298-
Assert.False(triggersLog.Contains("secrets"));
334+
Assert.Null(result["functions"]);
299335
}
300336

301-
private void VerifyResultWithCacheOff(string durableVersion)
337+
public JObject VerifyResultCommon(string connection = DefaultTestConnection, string expectedTaskHub = "TestHubValue", string durableVersion = "V2")
302338
{
303-
string expectedSyncTriggersPayload = GetExpectedSyncTriggersPayload(durableVersion: durableVersion);
304-
var triggers = JArray.Parse(_contentBuilder.ToString());
305-
Assert.Equal(expectedSyncTriggersPayload, triggers.ToString(Formatting.None));
339+
string expectedTriggersPayload = GetExpectedTriggersPayload(postedConnection: connection, postedTaskHub: expectedTaskHub, durableVersion);
340+
341+
var result = JObject.Parse(_contentBuilder.ToString());
342+
343+
bool isFlexConsumptionSku = _mockEnvironment.Object.IsFlexConsumptionSku();
344+
if (isFlexConsumptionSku)
345+
{
346+
Assert.Equal("testhostid123", result["hostId"]);
347+
}
348+
else
349+
{
350+
Assert.Null(result["hostId"]);
351+
}
352+
353+
// verify triggers
354+
var triggers = result["triggers"];
355+
Assert.Equal(expectedTriggersPayload, triggers.ToString(Formatting.None));
306356

307357
var logs = _loggerProvider.GetAllLogMessages().Where(m => m.Category.Equals(SyncManagerLogCategory)).ToList();
308358
var log = logs[0];
309359
int startIdx = log.FormattedMessage.IndexOf("Content=") + 8;
310360
int endIdx = log.FormattedMessage.LastIndexOf(')');
311361
var triggersLog = log.FormattedMessage.Substring(startIdx, endIdx - startIdx).Trim();
312-
Assert.Equal(expectedSyncTriggersPayload, triggersLog);
362+
var logObject = JObject.Parse(triggersLog);
363+
364+
if (isFlexConsumptionSku)
365+
{
366+
Assert.Equal("testhostid123", logObject["hostId"]);
367+
}
368+
else
369+
{
370+
Assert.Null(logObject["hostId"]);
371+
}
372+
373+
Assert.Equal(expectedTriggersPayload, logObject["triggers"].ToString(Formatting.None));
374+
Assert.False(triggersLog.Contains("secrets"));
375+
376+
return result;
313377
}
314378

315379
[Fact]
@@ -375,7 +439,7 @@ public async Task TrySyncTriggers_BackgroundSync_PostsExpectedContent()
375439
Assert.Equal(1, _mockHttpHandler.RequestCount);
376440
var result = JObject.Parse(_contentBuilder.ToString());
377441
var triggers = result["triggers"];
378-
Assert.Equal(GetExpectedSyncTriggersPayload(durableVersion: "V1"), triggers.ToString(Formatting.None));
442+
Assert.Equal(GetExpectedTriggersPayload(durableVersion: "V1"), triggers.ToString(Formatting.None));
379443

380444
string hash = string.Empty;
381445
var downloadResponse = await hashBlob.DownloadAsync();
@@ -441,7 +505,7 @@ public async Task TrySyncTriggers_BackgroundSync_SetTriggersFailure_HashNotUpdat
441505
Assert.Equal(1, _mockHttpHandler.RequestCount);
442506
var result = JObject.Parse(_contentBuilder.ToString());
443507
var triggers = result["triggers"];
444-
Assert.Equal(GetExpectedSyncTriggersPayload(durableVersion: "V1"), triggers.ToString(Formatting.None));
508+
Assert.Equal(GetExpectedTriggersPayload(durableVersion: "V1"), triggers.ToString(Formatting.None));
445509
bool hashBlobExists = await hashBlob.ExistsAsync();
446510
Assert.False(hashBlobExists);
447511

‎test/WebJobs.Script.Tests/ScriptHostIdProviderTests.cs‎

Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -94,6 +94,29 @@ public void GetDefaultHostId_AzureHost_ReturnsExpectedResult(string siteName, st
9494
Assert.False(result.IsLocal);
9595
}
9696

97+
[Theory]
98+
[InlineData("myfunctionstestapp", "6606e225f163a70190e4f4357d3dad78")]
99+
[InlineData("TEST-FUNCTIONS-XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX", "9dd2d6d54f1135ee6db6116f084b29bd")]
100+
[InlineData("TEST-FUNCTIONS-XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXA", "c842e18afa2a84eac2818a956687a72e")]
101+
[InlineData("TEST-FUNCTIONS-XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXB", "b595edd5b794d34e79c81361c823487c")]
102+
public void GetDefaultHostId_FlexConsumption_ReturnsExpectedResult(string siteName, string expected)
103+
{
104+
var options = new ScriptApplicationHostOptions
105+
{
106+
ScriptPath = @"c:\testscripts"
107+
};
108+
109+
var environment = new TestEnvironment();
110+
environment.SetEnvironmentVariable(EnvironmentSettingNames.AzureWebsiteSku, ScriptConstants.FlexConsumptionSku);
111+
environment.SetEnvironmentVariable(EnvironmentSettingNames.AzureWebsiteName, siteName);
112+
113+
var result = ScriptHostIdProvider.GetDefaultHostId(environment, options);
114+
115+
Assert.Equal(expected, result.HostId);
116+
Assert.Equal(32, result.HostId.Length);
117+
Assert.False(result.IsTruncated);
118+
}
119+
97120
[Fact]
98121
public void GetDefaultHostId_SelfHost_ReturnsExpectedResult()
99122
{

0 commit comments

Comments
 (0)