Skip to content

Commit d89209b

Browse files
sacOO7sacOO7
committed
Updated test script to detect failed tests on retry and throw error if found any
1 parent b478be5 commit d89209b

File tree

2 files changed

+167
-23
lines changed

2 files changed

+167
-23
lines changed

‎cake-build/helpers/test-retry.cake‎

Lines changed: 19 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -18,25 +18,41 @@ public class TestRetryHelper
1818
var failedTests = new List<string>();
1919

2020
if (!_context.FileExists(resultsPath))
21+
{
22+
_context.Warning($"Results file not found: {resultsPath}");
2123
return failedTests;
24+
}
2225

2326
try
2427
{
2528
var doc = System.Xml.Linq.XDocument.Load(resultsPath.FullPath);
26-
var nodes = doc.XPathSelectElements("//test-case[@success='False']");
2729

28-
foreach (var node in nodes)
30+
// XUnit v2 XML format uses @result='Fail' attribute
31+
// Try both formats for compatibility
32+
var failedNodes = doc.XPathSelectElements("//test[@result='Fail']")
33+
.Concat(doc.XPathSelectElements("//test-case[@result='Fail']"))
34+
.Concat(doc.XPathSelectElements("//test-case[@success='False']"));
35+
36+
foreach (var node in failedNodes)
2937
{
3038
var testName = node.Attribute("name")?.Value;
3139
if (!string.IsNullOrEmpty(testName))
3240
{
33-
failedTests.Add(TrimTestMethod(testName));
41+
var trimmedName = TrimTestMethod(testName);
42+
if (!failedTests.Contains(trimmedName))
43+
{
44+
_context.Information($"Found failed test: {trimmedName}");
45+
failedTests.Add(trimmedName);
46+
}
3447
}
3548
}
49+
50+
_context.Information($"Total failed tests found: {failedTests.Count}");
3651
}
3752
catch (Exception ex)
3853
{
3954
_context.Warning($"Error parsing XUnit results: {ex.Message}");
55+
_context.Warning($"Stack trace: {ex.StackTrace}");
4056
}
4157

4258
return failedTests;

‎cake-build/tasks/test.cake‎

Lines changed: 148 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -24,24 +24,56 @@ Task("_NetFramework_Unit_Tests_WithRetry")
2424
var testAssemblies = testExecutionHelper.FindTestAssemblies("IO.Ably.Tests.NETFramework");
2525
if (!testAssemblies.Any()) return;
2626

27-
var resultsPath = paths.TestResults.CombineWithFilePath("xunit-netframework-unit.xml");
27+
var baseReportName = "xunit-netframework-unit";
28+
var resultsPath = paths.TestResults.CombineWithFilePath($"{baseReportName}.xml");
2829

30+
var initialFailedTests = new List<string>();
2931
try
3032
{
31-
var settings = testExecutionHelper.CreateXUnitSettings("xunit-netframework-unit", isIntegration: false);
33+
var settings = testExecutionHelper.CreateXUnitSettings(baseReportName, isIntegration: false);
3234
testExecutionHelper.RunXUnitTests(testAssemblies, settings);
3335
}
3436
catch
3537
{
3638
Warning("Some tests failed. Retrying failed tests...");
39+
initialFailedTests = testRetryHelper.FindFailedXUnitTests(resultsPath);
3740
}
3841

39-
testExecutionHelper.RetryFailedXUnitTests(
40-
testAssemblies,
41-
resultsPath,
42-
testRetryHelper,
43-
(test) => testExecutionHelper.CreateXUnitSettings("retry", isIntegration: false, isRetry: true)
44-
);
42+
if (initialFailedTests.Any())
43+
{
44+
Information($"Retrying {initialFailedTests.Count} failed test(s)...");
45+
46+
testExecutionHelper.RetryFailedXUnitTests(
47+
testAssemblies,
48+
resultsPath,
49+
testRetryHelper,
50+
(test) => testExecutionHelper.CreateXUnitSettings(baseReportName, isIntegration: false, isRetry: true)
51+
);
52+
53+
// After retries complete, find all retry result files that were created
54+
// Pattern: xunit-netframework-unit-1.xml, xunit-netframework-unit-2.xml, etc.
55+
var retryPattern = $"{baseReportName}-*.xml";
56+
var retryResultFiles = GetFiles(paths.TestResults.Combine(retryPattern).FullPath);
57+
58+
Information($"Found {retryResultFiles.Count} retry result file(s)");
59+
60+
// Check each retry result file to see if tests still failed
61+
var stillFailedTests = new List<string>();
62+
foreach (var retryResultPath in retryResultFiles)
63+
{
64+
var retryFailed = testRetryHelper.FindFailedXUnitTests(retryResultPath);
65+
if (retryFailed.Any())
66+
{
67+
Information($"File {retryResultPath.GetFilename()} has {retryFailed.Count} failed test(s)");
68+
stillFailedTests.AddRange(retryFailed);
69+
}
70+
}
71+
72+
if (stillFailedTests.Any())
73+
{
74+
throw new Exception($"{stillFailedTests.Count} test(s) failed after retry");
75+
}
76+
}
4577
});
4678

4779
Task("_NetFramework_Integration_Tests")
@@ -66,24 +98,56 @@ Task("_NetFramework_Integration_Tests_WithRetry")
6698
var testAssemblies = testExecutionHelper.FindTestAssemblies("IO.Ably.Tests.NETFramework");
6799
if (!testAssemblies.Any()) return;
68100

69-
var resultsPath = paths.TestResults.CombineWithFilePath("xunit-netframework-integration.xml");
101+
var baseReportName = "xunit-netframework-integration";
102+
var resultsPath = paths.TestResults.CombineWithFilePath($"{baseReportName}.xml");
70103

104+
var initialFailedTests = new List<string>();
71105
try
72106
{
73-
var settings = testExecutionHelper.CreateXUnitSettings("xunit-netframework-integration", isIntegration: true);
107+
var settings = testExecutionHelper.CreateXUnitSettings(baseReportName, isIntegration: true);
74108
testExecutionHelper.RunXUnitTests(testAssemblies, settings);
75109
}
76110
catch
77111
{
78112
Warning("Some tests failed. Retrying failed tests...");
113+
initialFailedTests = testRetryHelper.FindFailedXUnitTests(resultsPath);
79114
}
80115

81-
testExecutionHelper.RetryFailedXUnitTests(
82-
testAssemblies,
83-
resultsPath,
84-
testRetryHelper,
85-
(test) => testExecutionHelper.CreateXUnitSettings("retry", isIntegration: true, isRetry: true)
86-
);
116+
if (initialFailedTests.Any())
117+
{
118+
Information($"Retrying {initialFailedTests.Count} failed test(s)...");
119+
120+
testExecutionHelper.RetryFailedXUnitTests(
121+
testAssemblies,
122+
resultsPath,
123+
testRetryHelper,
124+
(test) => testExecutionHelper.CreateXUnitSettings(baseReportName, isIntegration: true, isRetry: true)
125+
);
126+
127+
// After retries complete, find all retry result files that were created
128+
// Pattern: xunit-netframework-integration-1.xml, xunit-netframework-integration-2.xml, etc.
129+
var retryPattern = $"{baseReportName}-*.xml";
130+
var retryResultFiles = GetFiles(paths.TestResults.Combine(retryPattern).FullPath);
131+
132+
Information($"Found {retryResultFiles.Count} retry result file(s)");
133+
134+
// Check each retry result file to see if tests still failed
135+
var stillFailedTests = new List<string>();
136+
foreach (var retryResultPath in retryResultFiles)
137+
{
138+
var retryFailed = testRetryHelper.FindFailedXUnitTests(retryResultPath);
139+
if (retryFailed.Any())
140+
{
141+
Information($"File {retryResultPath.GetFilename()} has {retryFailed.Count} failed test(s)");
142+
stillFailedTests.AddRange(retryFailed);
143+
}
144+
}
145+
146+
if (stillFailedTests.Any())
147+
{
148+
throw new Exception($"{stillFailedTests.Count} test(s) failed after retry");
149+
}
150+
}
87151
});
88152

89153
///////////////////////////////////////////////////////////////////////////////
@@ -112,21 +176,53 @@ Task("_NetStandard_Unit_Tests_WithRetry")
112176
Information("Running .NET Standard unit tests with retry...");
113177

114178
var project = paths.Src.CombineWithFilePath("IO.Ably.Tests.DotNET/IO.Ably.Tests.DotNET.csproj");
115-
var resultsPath = paths.TestResults.CombineWithFilePath("tests-netstandard-unit.trx");
179+
var baseResultsName = "tests-netstandard-unit";
180+
var resultsPath = paths.TestResults.CombineWithFilePath($"{baseResultsName}.trx");
116181

117182
var filter = testExecutionHelper.CreateUnitTestFilter(IsRunningOnUnix());
118183
var settings = testExecutionHelper.CreateDotNetTestSettings(resultsPath, filter, framework, configuration);
119184

185+
var initialFailedTests = new List<string>();
120186
try
121187
{
122188
testExecutionHelper.RunDotNetTests(project, settings);
123189
}
124190
catch
125191
{
126192
Warning("Some tests failed. Retrying failed tests...");
193+
initialFailedTests = testRetryHelper.FindFailedDotNetTests(resultsPath);
127194
}
128195

129-
testExecutionHelper.RetryFailedDotNetTests(project, resultsPath, testRetryHelper, framework, configuration);
196+
if (initialFailedTests.Any())
197+
{
198+
Information($"Retrying {initialFailedTests.Count} failed test(s)...");
199+
200+
testExecutionHelper.RetryFailedDotNetTests(project, resultsPath, testRetryHelper, framework, configuration);
201+
202+
// After retries complete, find all retry result files that were created
203+
// Pattern: tests-netstandard-unit-1.trx, tests-netstandard-unit-2.trx, etc.
204+
var retryPattern = $"{baseResultsName}-*.trx";
205+
var retryResultFiles = GetFiles(paths.TestResults.Combine(retryPattern).FullPath);
206+
207+
Information($"Found {retryResultFiles.Count} retry result file(s)");
208+
209+
// Check each retry result file to see if tests still failed
210+
var stillFailedTests = new List<string>();
211+
foreach (var retryResultPath in retryResultFiles)
212+
{
213+
var retryFailed = testRetryHelper.FindFailedDotNetTests(retryResultPath);
214+
if (retryFailed.Any())
215+
{
216+
Information($"File {retryResultPath.GetFilename()} has {retryFailed.Count} failed test(s)");
217+
stillFailedTests.AddRange(retryFailed);
218+
}
219+
}
220+
221+
if (stillFailedTests.Any())
222+
{
223+
throw new Exception($"{stillFailedTests.Count} test(s) failed after retry");
224+
}
225+
}
130226
});
131227

132228
Task("_NetStandard_Integration_Tests")
@@ -151,21 +247,53 @@ Task("_NetStandard_Integration_Tests_WithRetry")
151247
Information("Running .NET Standard integration tests with retry...");
152248

153249
var project = paths.Src.CombineWithFilePath("IO.Ably.Tests.DotNET/IO.Ably.Tests.DotNET.csproj");
154-
var resultsPath = paths.TestResults.CombineWithFilePath("tests-netstandard-integration.trx");
250+
var baseResultsName = "tests-netstandard-integration";
251+
var resultsPath = paths.TestResults.CombineWithFilePath($"{baseResultsName}.trx");
155252

156253
var filter = testExecutionHelper.CreateIntegrationTestFilter();
157254
var settings = testExecutionHelper.CreateDotNetTestSettings(resultsPath, filter, framework, configuration);
158255

256+
var initialFailedTests = new List<string>();
159257
try
160258
{
161259
testExecutionHelper.RunDotNetTests(project, settings);
162260
}
163261
catch
164262
{
165263
Warning("Some tests failed. Retrying failed tests...");
264+
initialFailedTests = testRetryHelper.FindFailedDotNetTests(resultsPath);
166265
}
167266

168-
testExecutionHelper.RetryFailedDotNetTests(project, resultsPath, testRetryHelper, framework, configuration);
267+
if (initialFailedTests.Any())
268+
{
269+
Information($"Retrying {initialFailedTests.Count} failed test(s)...");
270+
271+
testExecutionHelper.RetryFailedDotNetTests(project, resultsPath, testRetryHelper, framework, configuration);
272+
273+
// After retries complete, find all retry result files that were created
274+
// Pattern: tests-netstandard-integration-1.trx, tests-netstandard-integration-2.trx, etc.
275+
var retryPattern = $"{baseResultsName}-*.trx";
276+
var retryResultFiles = GetFiles(paths.TestResults.Combine(retryPattern).FullPath);
277+
278+
Information($"Found {retryResultFiles.Count} retry result file(s)");
279+
280+
// Check each retry result file to see if tests still failed
281+
var stillFailedTests = new List<string>();
282+
foreach (var retryResultPath in retryResultFiles)
283+
{
284+
var retryFailed = testRetryHelper.FindFailedDotNetTests(retryResultPath);
285+
if (retryFailed.Any())
286+
{
287+
Information($"File {retryResultPath.GetFilename()} has {retryFailed.Count} failed test(s)");
288+
stillFailedTests.AddRange(retryFailed);
289+
}
290+
}
291+
292+
if (stillFailedTests.Any())
293+
{
294+
throw new Exception($"{stillFailedTests.Count} test(s) failed after retry");
295+
}
296+
}
169297
});
170298

171299
///////////////////////////////////////////////////////////////////////////////

0 commit comments

Comments
 (0)