Skip to content

Commit 3f4b27e

Browse files
author
Connor McMahon
authored
Improve CI reliability and performance (#1685)
Various improvements to the CI experience including: - The most common CI failure is related to when we try to listen to event source. I have attempted to fix this by both removing the lookup by event source name (instead defaulting to GUID), as well as updating the underlying dependency. - Reduced test length for some of our longer running tests. - Flaky tests are bad for both CI reliability and CI latency, as rerunning tests seems to add 1+ minutes per test to the CI. This PR tried to fix as many flaky tests as possible. - Reduced dotnet restore/build time by only building relevant projects for tests - Removed unnecessary publish step After this PR, the CI runs in 8-10 minutes assuming no flakey tests fail. resolves #1602
1 parent d70cc4c commit 3f4b27e

17 files changed

+136
-154
lines changed

‎azure-pipelines.yml‎

Lines changed: 25 additions & 54 deletions
Original file line numberDiff line numberDiff line change
@@ -7,23 +7,24 @@ jobs:
77
variables:
88
solution: 'WebJobs.Extensions.DurableTask.sln'
99
buildPlatform: 'Any CPU'
10-
buildConfiguration: 'Debug'
10+
buildConfiguration: 'Release'
1111

1212
steps:
1313
- task: NuGetToolInstaller@1
1414

1515
- task: DotNetCoreCLI@2
1616
inputs:
1717
command: 'restore'
18-
projects: '**/**/*.csproj'
18+
projects: 'test/FunctionsV1/*.csproj'
1919
feedsToUse: 'config'
2020
nugetConfigPath: '.nuget/nuget.config'
2121

22-
- task: VSBuild@1
22+
- task: DotNetCoreCLI@2
2323
inputs:
24-
solution: '$(solution)'
25-
platform: '$(buildPlatform)'
26-
configuration: '$(buildConfiguration)'
24+
command: 'build'
25+
projects: 'test/FunctionsV1/*.csproj'
26+
arguments: '--configuration $(buildConfiguration)'
27+
displayName: 'dotnet build $(buildConfiguration) '
2728

2829
- task: VSTest@2
2930
inputs:
@@ -49,34 +50,36 @@ jobs:
4950
variables:
5051
solution: 'WebJobs.Extensions.DurableTask.sln'
5152
buildPlatform: 'Any CPU'
52-
buildConfiguration: 'Debug'
53+
buildConfiguration: 'Release'
5354

5455
steps:
5556
- task: NuGetToolInstaller@1
5657

5758
- task: DotNetCoreCLI@2
5859
inputs:
5960
command: 'restore'
60-
projects: '**/**/*.csproj'
61+
projects: 'test/FunctionsV2/*.csproj'
6162
feedsToUse: 'config'
6263
nugetConfigPath: '.nuget/nuget.config'
6364

64-
- task: VSBuild@1
65+
- task: DotNetCoreCLI@2
6566
inputs:
66-
solution: '$(solution)'
67-
platform: '$(buildPlatform)'
68-
configuration: '$(buildConfiguration)'
67+
command: 'build'
68+
projects: 'test/FunctionsV2/*.csproj'
69+
arguments: '--configuration $(buildConfiguration)'
70+
displayName: 'dotnet build $(buildConfiguration)'
6971

7072
- task: VSTest@2
7173
inputs:
7274
testSelector: 'testAssemblies'
7375
testAssemblyVer2: |
74-
**/*tests.v2.dll
76+
**/bin/**/*tests.v2.dll
7577
distributionBatchType: basedOnExecutionTime
7678
diagnosticsEnabled: true
7779
rerunFailedTests: true
7880
rerunFailedThreshold: 10
7981
rerunMaxAttempts: 2
82+
runInParallel: true
8083
env:
8184
AzureWebJobsStorage: $(AzureWebJobsStorage)
8285
APPINSIGHTS_INSTRUMENTATIONKEY: $(APPINSIGHTS_INSTRUMENTATIONKEY)
@@ -88,67 +91,35 @@ jobs:
8891
variables:
8992
solution: '**/WebJobs.Extensions.DurableTask.Analyzers.sln'
9093
buildPlatform: 'Any CPU'
91-
buildConfiguration: 'Debug'
94+
buildConfiguration: 'Release'
9295

9396
steps:
9497
- task: NuGetToolInstaller@1
9598

9699
- task: DotNetCoreCLI@2
97100
inputs:
98101
command: 'restore'
99-
projects: '**/**/*.csproj'
102+
projects: 'test/WebJobs.Extensions.DurableTask.Analyzers.Test/*.csproj'
100103
feedsToUse: 'config'
101104
nugetConfigPath: '.nuget/nuget.config'
102105

103-
- task: VSBuild@1
106+
- task: DotNetCoreCLI@2
104107
inputs:
105-
solution: '$(solution)'
106-
platform: '$(buildPlatform)'
107-
configuration: '$(buildConfiguration)'
108+
command: 'build'
109+
projects: 'test/WebJobs.Extensions.DurableTask.Analyzers.Test/*.csproj'
110+
arguments: '--configuration $(buildConfiguration)'
111+
displayName: 'dotnet build $(buildConfiguration) '
108112

109113
- task: VSTest@2
110114
inputs:
111115
testSelector: 'testAssemblies'
112116
testAssemblyVer2: |
113-
**/*Analyzers.Test.dll
117+
**/bin/**/*Analyzers.Test.dll
114118
distributionBatchType: basedOnExecutionTime
115119
diagnosticsEnabled: true
116120
rerunFailedTests: true
117121
rerunFailedThreshold: 10
118122
rerunMaxAttempts: 2
119123
env:
120124
AzureWebJobsStorage: $(AzureWebJobsStorage)
121-
APPINSIGHTS_INSTRUMENTATIONKEY: $(APPINSIGHTS_INSTRUMENTATIONKEY)
122-
123-
- job: PublishPipelineArtifact
124-
dependsOn:
125-
- FunctionsV1Tests
126-
- FunctionsV2Tests
127-
- DurableAnalyzerTests
128-
129-
pool:
130-
vmImage: 'windows-latest'
131-
132-
variables:
133-
solution: 'WebJobs.Extensions.DurableTask.sln'
134-
buildPlatform: 'Any CPU'
135-
buildConfiguration: 'Debug'
136-
137-
steps:
138-
- task: NuGetToolInstaller@1
139-
140-
- task: DotNetCoreCLI@2
141-
inputs:
142-
command: 'restore'
143-
projects: '**/**/*.csproj'
144-
feedsToUse: 'config'
145-
nugetConfigPath: '.nuget/nuget.config'
146-
147-
- task: VSBuild@1
148-
inputs:
149-
solution: '$(solution)'
150-
platform: '$(buildPlatform)'
151-
configuration: '$(buildConfiguration)'
152-
153-
- publish: 'src/WebJobs.Extensions.DurableTask/bin'
154-
artifact: bin
125+
APPINSIGHTS_INSTRUMENTATIONKEY: $(APPINSIGHTS_INSTRUMENTATIONKEY)

‎test/Common/DurableHttpTests.cs‎

Lines changed: 3 additions & 35 deletions
Original file line numberDiff line numberDiff line change
@@ -31,47 +31,15 @@ public class DurableHttpTests : IDisposable
3131
private readonly ITestOutputHelper output;
3232

3333
private readonly TestLoggerProvider loggerProvider;
34-
private readonly bool useTestLogger = IsLogFriendlyPlatform();
35-
private readonly LogEventTraceListener eventSourceListener;
3634

3735
public DurableHttpTests(ITestOutputHelper output)
3836
{
3937
this.output = output;
4038
this.loggerProvider = new TestLoggerProvider(output);
41-
this.eventSourceListener = new LogEventTraceListener();
42-
this.StartLogCapture();
4339
}
4440

4541
public void Dispose()
4642
{
47-
this.eventSourceListener.Dispose();
48-
}
49-
50-
private void OnEventSourceListenerTraceLog(object sender, LogEventTraceListener.TraceLogEventArgs e)
51-
{
52-
this.output.WriteLine($" ETW: {e.ProviderName} [{e.Level}] : {e.Message}");
53-
}
54-
55-
private void StartLogCapture()
56-
{
57-
if (this.useTestLogger)
58-
{
59-
var traceConfig = new Dictionary<string, TraceEventLevel>
60-
{
61-
{ "DurableTask-AzureStorage", TraceEventLevel.Informational },
62-
{ "7DA4779A-152E-44A2-A6F2-F80D991A5BEE", TraceEventLevel.Warning }, // DurableTask.Core
63-
};
64-
65-
this.eventSourceListener.OnTraceLog += this.OnEventSourceListenerTraceLog;
66-
67-
string sessionName = "DTFxTrace" + Guid.NewGuid().ToString("N");
68-
this.eventSourceListener.CaptureLogs(sessionName, traceConfig);
69-
}
70-
}
71-
72-
private static bool IsLogFriendlyPlatform()
73-
{
74-
return !RuntimeInformation.IsOSPlatform(OSPlatform.Linux);
7543
}
7644

7745
[Fact]
@@ -138,7 +106,7 @@ public void SerializeManagedIdentityOptions()
138106
},
139107
""Content"": null,
140108
""TokenSource"": {
141-
""$type"": ""Microsoft.Azure.WebJobs.Extensions.DurableTask.Tests.DurableHttpTests+MockTokenSource, WebJobs.Extensions.DurableTask.Tests.V2"",
109+
""$type"": ""Microsoft.Azure.WebJobs.Extensions.DurableTask.Tests.DurableHttpTests+MockTokenSource, WebJobs.Extensions.DurableTask.Tests." + PlatformSpecificHelpers.VersionSuffix + @""",
142110
""testToken"": ""dummy token"",
143111
""options"": {
144112
""authorityhost"": ""https://dummy.login.microsoftonline.com/"",
@@ -809,8 +777,8 @@ public async Task DurableHttpAsync_AsynchronousAPI_ReturnsOK200(string storagePr
809777
asyncTestHeaders.Add("Location", "https://www.dummy-location-url.com");
810778

811779
HttpResponseMessage acceptedHttpResponseMessage = CreateTestHttpResponseMessage(
812-
statusCode: HttpStatusCode.Accepted,
813-
headers: asyncTestHeaders);
780+
statusCode: HttpStatusCode.Accepted,
781+
headers: asyncTestHeaders);
814782

815783
HttpMessageHandler httpMessageHandler = MockAsynchronousHttpMessageHandler(acceptedHttpResponseMessage);
816784

‎test/Common/DurableTaskEndToEndTests.cs‎

Lines changed: 20 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -63,9 +63,11 @@ private void StartLogCapture()
6363
{
6464
if (this.useTestLogger)
6565
{
66+
// Use GUID for eventsource, as TraceEventProviders.GetProviderGuidByName() is causing
67+
// the CI to abort runs.
6668
var traceConfig = new Dictionary<string, TraceEventLevel>
6769
{
68-
{ "DurableTask-AzureStorage", TraceEventLevel.Informational },
70+
{ "4c4ad4a2-f396-5e18-01b6-618c12a10433", TraceEventLevel.Informational }, // DurableTask.AzureStorage
6971
{ "7DA4779A-152E-44A2-A6F2-F80D991A5BEE", TraceEventLevel.Warning }, // DurableTask.Core
7072
};
7173

@@ -329,11 +331,12 @@ public async Task WritesToFile()
329331
var client = await host.StartOrchestratorAsync(orchestratorName, input: "World", this.output);
330332
var status = await client.WaitForCompletionAsync(this.output);
331333
await host.StopAsync();
332-
await Task.Delay(TimeSpan.FromSeconds(10)); // giving the logger enough time to flush
333334
}
334335

335-
// Ensure the logging file was at least generated
336-
Assert.True(File.Exists(LinuxAppServiceLogger.LoggingPath));
336+
await TestHelpers.WaitUntilTrue(
337+
predicate: () => File.Exists(LinuxAppServiceLogger.LoggingPath),
338+
conditionDescription: "Log file exists",
339+
timeout: TimeSpan.FromSeconds(20));
337340
}
338341

339342
/// <summary>
@@ -526,11 +529,12 @@ public async Task RemovesNewlinesFromExceptions()
526529
var client = await host.StartOrchestratorAsync(orchestratorName, input: null, this.output);
527530
var status = await client.WaitForCompletionAsync(this.output);
528531
await host.StopAsync();
529-
await Task.Delay(TimeSpan.FromSeconds(10)); // giving the logger enough time to flush
530532
}
531533

532-
// Ensure the logging file was at least generated
533-
Assert.True(File.Exists(LinuxAppServiceLogger.LoggingPath));
534+
await TestHelpers.WaitUntilTrue(
535+
predicate: () => File.Exists(LinuxAppServiceLogger.LoggingPath),
536+
conditionDescription: "Log file exists",
537+
timeout: TimeSpan.FromSeconds(20));
534538

535539
// string[] lines = File.ReadAllLines(LinuxAppServiceLogger.LoggingPath);
536540

@@ -1091,7 +1095,7 @@ public async Task ActorOrchestration(bool extendedSessions, string storageProvid
10911095
// Wait for the instance to go into the Running state. This is necessary to ensure log validation consistency.
10921096
await client.WaitForStartupAsync(this.output);
10931097

1094-
TimeSpan waitTimeout = TimeSpan.FromSeconds(Debugger.IsAttached ? 300 : 10);
1098+
TimeSpan waitTimeout = TimeSpan.FromSeconds(Debugger.IsAttached ? 300 : 5);
10951099

10961100
// Perform some operations
10971101
await client.RaiseEventAsync("operation", "incr", this.output);
@@ -1492,7 +1496,7 @@ public async Task TimerExpiration(bool extendedSessions, string storageProvider)
14921496
{
14931497
await host.StartAsync();
14941498

1495-
var timeout = TimeSpan.FromSeconds(10);
1499+
var timeout = TimeSpan.FromSeconds(2);
14961500
var client = await host.StartOrchestratorAsync(orchestratorFunctionNames[0], timeout, this.output);
14971501
await client.WaitForStartupAsync(this.output);
14981502

@@ -3289,7 +3293,7 @@ public async Task DurableEntity_EntityToAndFromBlob(bool extendedSessions)
32893293

32903294
while (true)
32913295
{
3292-
await Task.Delay(5000);
3296+
await Task.Delay(1000);
32933297

32943298
// while the orchestration is running, just for fun,
32953299
// send some deactivation signals which unload the entity from memory.
@@ -3696,9 +3700,7 @@ public async Task DurableEntity_AsyncIO()
36963700
await client.SignalEntity(this.output, "get", "https://www.microsoft.com");
36973701
await client.SignalEntity(this.output, "get", "https://bing.com");
36983702

3699-
await Task.Delay(TimeSpan.FromSeconds(10));
3700-
3701-
var state = await client.WaitForEntityState<IDictionary<string, string>>(this.output);
3703+
var state = await client.WaitForEntityState<IDictionary<string, string>>(this.output, TimeSpan.FromSeconds(10));
37023704
Assert.NotNull(state);
37033705

37043706
if (state.TryGetValue("error", out string error))
@@ -3935,7 +3937,7 @@ public async Task DurableEntity_EntityProxy_UsesBindings(bool extendedSessions)
39353937

39363938
using (var host = TestHelpers.GetJobHost(
39373939
this.loggerProvider,
3938-
nameof(this.DurableEntity_EntityProxy),
3940+
nameof(this.DurableEntity_EntityProxy_UsesBindings),
39393941
extendedSessions))
39403942
{
39413943
await host.StartAsync();
@@ -5265,8 +5267,11 @@ public async Task ReplaySafeLogger_LogsOnlyOnce()
52655267
var status = await client.WaitForCompletionAsync(this.output);
52665268

52675269
Assert.Equal(OrchestrationRuntimeStatus.Completed, status?.RuntimeStatus);
5268-
5270+
#if FUNCTIONS_V1
5271+
var logger = this.loggerProvider.CreatedLoggers.FirstOrDefault(l => l.Category.Equals("Function"));
5272+
#else
52695273
var logger = this.loggerProvider.CreatedLoggers.FirstOrDefault(l => l.Category.Equals("Function.ReplaySafeLogger_OneLogMessage.User"));
5274+
#endif
52705275
var logMessages = logger.LogMessages.Where(
52715276
msg => msg.FormattedMessage.Contains("ReplaySafeLogger Test: About to say Hello")).ToList();
52725277
Assert.Single(logMessages);

‎test/Common/HttpApiHandlerTests.cs‎

Lines changed: 8 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -217,8 +217,8 @@ public async Task WaitForCompletionOrCreateCheckStatusResponseAsync_Returns_Cust
217217
TaskHub = TestConstants.TaskHub,
218218
ConnectionName = TestConstants.ConnectionName,
219219
},
220-
TimeSpan.FromSeconds(100),
221220
TimeSpan.FromSeconds(10),
221+
TimeSpan.FromSeconds(3),
222222
true);
223223
stopWatch.Stop();
224224
Assert.Equal(HttpStatusCode.Accepted, httpResponseMessage.StatusCode);
@@ -237,7 +237,7 @@ public async Task WaitForCompletionOrCreateCheckStatusResponseAsync_Returns_Cust
237237
Assert.Equal(
238238
$"{TestConstants.NotificationUrlBase}/instances/9b59154ae666471993659902ed0ba749?taskHub=SampleHubVS&connection=Storage&code=mykey",
239239
(string)status["purgeHistoryDeleteUri"]);
240-
Assert.True(stopWatch.Elapsed > TimeSpan.FromSeconds(30));
240+
Assert.True(stopWatch.Elapsed > TimeSpan.FromSeconds(10));
241241
}
242242

243243
[Fact]
@@ -257,8 +257,8 @@ public async Task WaitForCompletionOrCreateCheckStatusResponseAsync_Returns_HTTP
257257
TaskHub = TestConstants.TaskHub,
258258
ConnectionName = TestConstants.ConnectionName,
259259
},
260-
TimeSpan.FromSeconds(100),
261-
TimeSpan.FromSeconds(10));
260+
TimeSpan.FromSeconds(10),
261+
TimeSpan.FromSeconds(3));
262262
stopWatch.Stop();
263263
Assert.Equal(HttpStatusCode.Accepted, httpResponseMessage.StatusCode);
264264
var content = await httpResponseMessage.Content.ReadAsStringAsync();
@@ -279,7 +279,7 @@ public async Task WaitForCompletionOrCreateCheckStatusResponseAsync_Returns_HTTP
279279
Assert.Equal(
280280
$"{TestConstants.NotificationUrlBase}/instances/9b59154ae666471993659902ed0ba749/restart?taskHub=SampleHubVS&connection=Storage&code=mykey",
281281
(string)status["restartPostUri"]);
282-
Assert.True(stopWatch.Elapsed > TimeSpan.FromSeconds(30));
282+
Assert.True(stopWatch.Elapsed > TimeSpan.FromSeconds(10));
283283
}
284284

285285
[Fact]
@@ -323,14 +323,14 @@ public async Task WaitForCompletionOrCreateCheckStatusResponseAsync_Returns_HTTP
323323
TaskHub = TestConstants.TaskHub,
324324
ConnectionName = TestConstants.ConnectionName,
325325
},
326-
TimeSpan.FromSeconds(30),
327-
TimeSpan.FromSeconds(8));
326+
TimeSpan.FromSeconds(10),
327+
TimeSpan.FromSeconds(3));
328328
stopwatch.Stop();
329329
Assert.Equal(HttpStatusCode.OK, httpResponseMessage.StatusCode);
330330
var content = await httpResponseMessage.Content.ReadAsStringAsync();
331331
var value = JsonConvert.DeserializeObject<string>(content);
332332
Assert.Equal("Hello Tokyo!", value);
333-
Assert.True(stopwatch.Elapsed < TimeSpan.FromSeconds(30));
333+
Assert.True(stopwatch.Elapsed < TimeSpan.FromSeconds(10));
334334
}
335335

336336
[Fact]

0 commit comments

Comments
 (0)