Skip to content

Commit f5ad41b

Browse files
brettsambrettsam
authored
fixing some test races (hopefully) (#10802)
1 parent cded7b8 commit f5ad41b

File tree

8 files changed

+125
-84
lines changed

8 files changed

+125
-84
lines changed

‎eng/ci/templates/official/jobs/run-integration-tests.yml‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -46,6 +46,7 @@ jobs:
4646
inputs:
4747
azureSubscription: Azure-Functions-Host-CI-internal
4848
azurePowerShellVersion: 'LatestVersion'
49+
pwsh: true
4950
ScriptPath: eng/script/checkout-secrets.ps1
5051

5152
- task: AzureKeyVault@1
@@ -201,5 +202,6 @@ jobs:
201202
inputs:
202203
azureSubscription: Azure-Functions-Host-CI-internal
203204
azurePowerShellVersion: 'LatestVersion'
205+
pwsh: true
204206
ScriptPath: eng/script/checkin-secrets.ps1
205207
ScriptArguments: '-leaseBlob $(LeaseBlob) -leaseToken $(LeaseToken)'

‎eng/ci/templates/official/jobs/run-non-e2e-tests.yml‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -34,6 +34,7 @@ jobs:
3434
inputs:
3535
azureSubscription: Azure-Functions-Host-CI-internal
3636
azurePowerShellVersion: 'LatestVersion'
37+
pwsh: true
3738
ScriptPath: eng/script/checkout-secrets.ps1
3839

3940
- task: AzureKeyVault@1
@@ -91,5 +92,6 @@ jobs:
9192
inputs:
9293
azureSubscription: Azure-Functions-Host-CI-internal
9394
azurePowerShellVersion: 'LatestVersion'
95+
pwsh: true
9496
ScriptPath: eng/script/checkin-secrets.ps1
9597
ScriptArguments: '-leaseBlob $(LeaseBlob) -leaseToken $(LeaseToken)'

‎src/WebJobs.Script.WebHost/Diagnostics/MetricsEventManager.cs‎

Lines changed: 16 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -4,10 +4,8 @@
44
using System;
55
using System.Collections.Concurrent;
66
using System.Collections.Generic;
7-
using System.Diagnostics;
87
using System.Linq;
98
using System.Threading;
10-
using Microsoft.Azure.WebJobs.Logging;
119
using Microsoft.Azure.WebJobs.Script.Configuration;
1210
using Microsoft.Azure.WebJobs.Script.Description;
1311
using Microsoft.Azure.WebJobs.Script.Diagnostics;
@@ -29,6 +27,8 @@ public class MetricsEventManager : IDisposable
2927
private readonly int _functionActivityFlushIntervalSeconds;
3028
private readonly Timer _metricsFlushTimer;
3129
private readonly ILogger<MetricsEventManager> _logger;
30+
private readonly object _lock = new();
31+
3232
private bool _disposed;
3333
private AppServiceOptions _appServiceOptions;
3434

@@ -215,13 +215,24 @@ public void Flush()
215215

216216
private void FlushMetrics()
217217
{
218-
if (QueuedEvents.Count == 0)
218+
if (QueuedEvents.IsEmpty)
219219
{
220220
return;
221221
}
222222

223-
SystemMetricEvent[] eventsToFlush = QueuedEvents.Values.ToArray();
224-
QueuedEvents.Clear();
223+
SystemMetricEvent[] eventsToFlush;
224+
225+
// Prevent flushes from multiple threads (mostly from testing).
226+
lock (_lock)
227+
{
228+
if (QueuedEvents.IsEmpty)
229+
{
230+
return;
231+
}
232+
233+
eventsToFlush = QueuedEvents.Values.ToArray();
234+
QueuedEvents.Clear();
235+
}
225236

226237
// Use the same timestamp for all events. Since these are
227238
// aggregated events, individual timestamps for when the events were

‎test/WebJobs.Script.Tests.Integration/WebHostEndToEnd/DrainModeResumeEndToEndTests.cs‎

Lines changed: 6 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -1,13 +1,11 @@
1-
using Microsoft.Azure.WebJobs.Script.WebHost.Models;
2-
using Microsoft.Azure.WebJobs.Script.Workers.Rpc;
3-
using Microsoft.WebJobs.Script.Tests;
4-
using Newtonsoft.Json;
5-
using Newtonsoft.Json.Linq;
61
using System;
72
using System.IO;
83
using System.Net;
9-
using System.Threading;
104
using System.Threading.Tasks;
5+
using Microsoft.Azure.WebJobs.Script.WebHost.Models;
6+
using Microsoft.Azure.WebJobs.Script.Workers.Rpc;
7+
using Microsoft.WebJobs.Script.Tests;
8+
using Newtonsoft.Json;
119
using Xunit;
1210

1311
namespace Microsoft.Azure.WebJobs.Script.Tests.EndToEnd
@@ -27,7 +25,7 @@ public async Task DrainModeEnabled_RunningHost_StartsNewHost_ReturnsOk()
2725
Assert.Equal(drainStatus.State, DrainModeState.Disabled);
2826

2927
// Capture pre-drain instance ID
30-
var originalInstanceId = this.HostInstanceId;
28+
var originalInstanceId = await GetActiveHostInstanceIdAsync();
3129

3230
// Validate ability to call HttpTrigger without issues
3331
response = await SamplesTestHelpers.InvokeHttpTrigger(this, "HttpTrigger");
@@ -62,7 +60,7 @@ public async Task DrainModeEnabled_RunningHost_StartsNewHost_ReturnsOk()
6260
Assert.Equal(HttpStatusCode.OK, response.StatusCode);
6361

6462
// Validate the instance ID has changed
65-
Assert.NotEqual(originalInstanceId, this.HostInstanceId);
63+
Assert.NotEqual(originalInstanceId, await GetActiveHostInstanceIdAsync());
6664
}
6765

6866
[Fact]

‎test/WebJobs.Script.Tests.Integration/WebHostEndToEnd/EndToEndTestFixture.cs‎

Lines changed: 8 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -83,7 +83,14 @@ protected EndToEndTestFixture(
8383

8484
public TestEventGenerator EventGenerator { get; private set; } = new TestEventGenerator();
8585

86-
public string HostInstanceId => Host.JobHostServices.GetService<IOptions<ScriptJobHostOptions>>().Value.InstanceId;
86+
public async Task<string> GetActiveHostInstanceIdAsync()
87+
{
88+
// During restarts the ActiveHost can be null. Wait to see if it recovers.
89+
await TestHelpers.Await(() => Host.JobHostServices is not null,
90+
userMessageCallback: () => $"Timed out waiting for JobHostServices to be available. Logs:{Environment.NewLine}{Host.GetLog()}.");
91+
92+
return Host.JobHostServices.GetService<IOptions<ScriptJobHostOptions>>().Value.InstanceId;
93+
}
8794

8895
public string MasterKey { get; private set; }
8996

‎test/WebJobs.Script.Tests.Integration/WebHostEndToEnd/FunctionsControllerEndToEndTests.cs‎

Lines changed: 13 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -1,10 +1,10 @@
1-
using Microsoft.Azure.WebJobs.Script.WebHost.Models;
2-
using Microsoft.Azure.WebJobs.Script.Workers.Rpc;
3-
using Microsoft.WebJobs.Script.Tests;
41
using System;
52
using System.IO;
63
using System.Net;
74
using System.Threading.Tasks;
5+
using Microsoft.Azure.WebJobs.Script.WebHost.Models;
6+
using Microsoft.Azure.WebJobs.Script.Workers.Rpc;
7+
using Microsoft.WebJobs.Script.Tests;
88
using Xunit;
99

1010
namespace Microsoft.Azure.WebJobs.Script.Tests.EndToEnd
@@ -17,7 +17,7 @@ public class FunctionsControllerEndToEndTests : FunctionsControllerTestFixture
1717
public async Task FunctionsController_GetAllFunctions_ReturnsOk()
1818
{
1919
// Capture original instance ID
20-
var originalInstanceId = this.HostInstanceId;
20+
var originalInstanceId = await GetActiveHostInstanceIdAsync();
2121

2222
// Validate ability to call HttpTrigger without issues
2323
var response = await SamplesTestHelpers.InvokeHttpTrigger(this, "HttpTrigger");
@@ -32,14 +32,14 @@ public async Task FunctionsController_GetAllFunctions_ReturnsOk()
3232
Assert.Equal(HttpStatusCode.OK, response.StatusCode);
3333

3434
// Validate the instance ID is still the same
35-
Assert.Equal(originalInstanceId, this.HostInstanceId);
35+
Assert.Equal(originalInstanceId, await GetActiveHostInstanceIdAsync());
3636
}
3737

3838
[Fact]
3939
public async Task FunctionsController_GetSpecificFunction_ReturnsOk()
4040
{
4141
// Capture original instance ID
42-
var originalInstanceId = this.HostInstanceId;
42+
var originalInstanceId = await GetActiveHostInstanceIdAsync();
4343

4444
// Validate ability to call HttpTrigger without issues
4545
var response = await SamplesTestHelpers.InvokeHttpTrigger(this, "HttpTrigger");
@@ -55,14 +55,14 @@ public async Task FunctionsController_GetSpecificFunction_ReturnsOk()
5555
Assert.Equal(HttpStatusCode.OK, response.StatusCode);
5656

5757
// Validate the instance ID is still the same
58-
Assert.Equal(originalInstanceId, this.HostInstanceId);
58+
Assert.Equal(originalInstanceId, await GetActiveHostInstanceIdAsync());
5959
}
6060

6161
[Fact]
6262
public async Task FunctionsController_GetSpecificFunctionStatus_ReturnsOk()
6363
{
6464
// Capture original instance ID
65-
var originalInstanceId = this.HostInstanceId;
65+
var originalInstanceId = await GetActiveHostInstanceIdAsync();
6666

6767
// Validate ability to call HttpTrigger without issues
6868
var response = await SamplesTestHelpers.InvokeHttpTrigger(this, "HttpTrigger");
@@ -77,14 +77,14 @@ public async Task FunctionsController_GetSpecificFunctionStatus_ReturnsOk()
7777
Assert.Equal(HttpStatusCode.OK, response.StatusCode);
7878

7979
// Validate the instance ID is still the same
80-
Assert.Equal(originalInstanceId, this.HostInstanceId);
80+
Assert.Equal(originalInstanceId, await GetActiveHostInstanceIdAsync());
8181
}
8282

8383
[Fact]
8484
public async Task FunctionsController_CreateUpdate_NoFileChange_ReturnsCreated_NoRestart()
8585
{
8686
// Capture original instance ID
87-
var originalInstanceId = this.HostInstanceId;
87+
var originalInstanceId = await GetActiveHostInstanceIdAsync();
8888

8989
// Validate ability to call HttpTrigger without issues
9090
var response = await SamplesTestHelpers.InvokeHttpTrigger(this, "HttpTrigger");
@@ -99,14 +99,14 @@ public async Task FunctionsController_CreateUpdate_NoFileChange_ReturnsCreated_N
9999
Assert.Equal(HttpStatusCode.OK, response.StatusCode);
100100

101101
// Validate the instance ID is still the same
102-
Assert.Equal(originalInstanceId, this.HostInstanceId);
102+
Assert.Equal(originalInstanceId, await GetActiveHostInstanceIdAsync());
103103
}
104104

105105
[Fact]
106106
public async Task FunctionsController_CreateUpdate_FileChange_ReturnsCreated_RestartsJobHost()
107107
{
108108
// Capture pre-restart instance ID
109-
var originalInstanceId = this.HostInstanceId;
109+
var originalInstanceId = await GetActiveHostInstanceIdAsync();
110110

111111
// Validate ability to call HttpTrigger without issues
112112
var response = await SamplesTestHelpers.InvokeHttpTrigger(this, "HttpTrigger");
@@ -125,7 +125,7 @@ public async Task FunctionsController_CreateUpdate_FileChange_ReturnsCreated_Res
125125
Assert.Equal(HttpStatusCode.OK, response.StatusCode);
126126

127127
// Validate the instance ID has changed
128-
Assert.NotEqual(originalInstanceId, this.HostInstanceId);
128+
Assert.NotEqual(originalInstanceId, await GetActiveHostInstanceIdAsync());
129129

130130
// Reset config
131131
response = await SamplesTestHelpers.InvokeEndpointPut(this, "admin/functions/HttpTrigger", TestData());

0 commit comments

Comments
 (0)