Skip to content

Commit b2bce3a

Browse files
sacOO7sacOO7
committed
Updated test script to detect failed tests on retry and throw error if found any
1 parent b478be5 commit b2bce3a

File tree

3 files changed

+212
-23
lines changed

3 files changed

+212
-23
lines changed

‎.gitignore‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -175,3 +175,4 @@ cake-build/addins/
175175
cake-build/modules/
176176
build-output/
177177
unity-packager/
178+
test-results/

‎cake-build/helpers/test-retry.cake‎

Lines changed: 67 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -18,25 +18,41 @@ public class TestRetryHelper
1818
var failedTests = new List<string>();
1919

2020
if (!_context.FileExists(resultsPath))
21+
{
22+
_context.Warning($"Results file not found: {resultsPath}");
2123
return failedTests;
24+
}
2225

2326
try
2427
{
2528
var doc = System.Xml.Linq.XDocument.Load(resultsPath.FullPath);
26-
var nodes = doc.XPathSelectElements("//test-case[@success='False']");
2729

28-
foreach (var node in nodes)
30+
// XUnit v2 XML format uses @result='Fail' attribute
31+
// Try both formats for compatibility
32+
var failedNodes = doc.XPathSelectElements("//test[@result='Fail']")
33+
.Concat(doc.XPathSelectElements("//test-case[@result='Fail']"))
34+
.Concat(doc.XPathSelectElements("//test-case[@success='False']"));
35+
36+
foreach (var node in failedNodes)
2937
{
3038
var testName = node.Attribute("name")?.Value;
3139
if (!string.IsNullOrEmpty(testName))
3240
{
33-
failedTests.Add(TrimTestMethod(testName));
41+
var trimmedName = TrimTestMethod(testName);
42+
if (!failedTests.Contains(trimmedName))
43+
{
44+
_context.Information($"Found failed test: {trimmedName}");
45+
failedTests.Add(trimmedName);
46+
}
3447
}
3548
}
49+
50+
_context.Information($"Total failed tests found: {failedTests.Count}");
3651
}
3752
catch (Exception ex)
3853
{
3954
_context.Warning($"Error parsing XUnit results: {ex.Message}");
55+
_context.Warning($"Stack trace: {ex.StackTrace}");
4056
}
4157

4258
return failedTests;
@@ -97,6 +113,54 @@ public class TestRetryHelper
97113
}
98114
return basePath;
99115
}
116+
117+
/// <summary>
118+
/// Displays a formatted summary table of test retry results
119+
/// </summary>
120+
/// <param name="testType">The type of tests (e.g., ".NET Framework Unit", ".NET Standard Integration")</param>
121+
/// <param name="initialFailedTests">List of tests that failed initially</param>
122+
/// <param name="stillFailedTests">List of tests that still failed after retry</param>
123+
public void DisplayRetryResultsSummary(string testType, List<string> initialFailedTests, List<string> stillFailedTests)
124+
{
125+
var passedTests = initialFailedTests.Except(stillFailedTests).ToList();
126+
127+
// Display summary table
128+
_context.Information("");
129+
_context.Information("╔════════════════════════════════════════════════════════════════╗");
130+
_context.Information($"║ TEST RETRY SUMMARY - {testType,-25} ║");
131+
_context.Information("╠════════════════════════════════════════════════════════════════╣");
132+
_context.Information($"║ Total Tests Retried: {initialFailedTests.Count,3} ║");
133+
_context.Information($"║ Passed After Retry: {passedTests.Count,3} ║");
134+
_context.Information($"║ Still Failed After Retry: {stillFailedTests.Count,3} ║");
135+
_context.Information("╚════════════════════════════════════════════════════════════════╝");
136+
137+
if (passedTests.Any())
138+
{
139+
_context.Information("");
140+
_context.Information("✓ Tests that PASSED on retry:");
141+
foreach (var test in passedTests)
142+
{
143+
_context.Information($" • {test}");
144+
}
145+
}
146+
147+
if (stillFailedTests.Any())
148+
{
149+
_context.Information("");
150+
_context.Error("✗ Tests that FAILED after retry:");
151+
foreach (var test in stillFailedTests)
152+
{
153+
_context.Error($" • {test}");
154+
}
155+
}
156+
else
157+
{
158+
_context.Information("");
159+
_context.Information("✓ All retried tests passed!");
160+
}
161+
162+
_context.Information("");
163+
}
100164
}
101165

102166
var testRetryHelper = new TestRetryHelper(Context);

‎cake-build/tasks/test.cake‎

Lines changed: 144 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -24,24 +24,55 @@ Task("_NetFramework_Unit_Tests_WithRetry")
2424
var testAssemblies = testExecutionHelper.FindTestAssemblies("IO.Ably.Tests.NETFramework");
2525
if (!testAssemblies.Any()) return;
2626

27-
var resultsPath = paths.TestResults.CombineWithFilePath("xunit-netframework-unit.xml");
27+
var baseReportName = "xunit-netframework-unit";
28+
var resultsPath = paths.TestResults.CombineWithFilePath($"{baseReportName}.xml");
2829

30+
var initialFailedTests = new List<string>();
2931
try
3032
{
31-
var settings = testExecutionHelper.CreateXUnitSettings("xunit-netframework-unit", isIntegration: false);
33+
var settings = testExecutionHelper.CreateXUnitSettings(baseReportName, isIntegration: false);
3234
testExecutionHelper.RunXUnitTests(testAssemblies, settings);
3335
}
3436
catch
3537
{
3638
Warning("Some tests failed. Retrying failed tests...");
39+
initialFailedTests = testRetryHelper.FindFailedXUnitTests(resultsPath);
3740
}
3841

39-
testExecutionHelper.RetryFailedXUnitTests(
40-
testAssemblies,
41-
resultsPath,
42-
testRetryHelper,
43-
(test) => testExecutionHelper.CreateXUnitSettings("retry", isIntegration: false, isRetry: true)
44-
);
42+
if (initialFailedTests.Any())
43+
{
44+
Information($"Retrying {initialFailedTests.Count} failed test(s)...");
45+
46+
testExecutionHelper.RetryFailedXUnitTests(
47+
testAssemblies,
48+
resultsPath,
49+
testRetryHelper,
50+
(test) => testExecutionHelper.CreateXUnitSettings(baseReportName, isIntegration: false, isRetry: true)
51+
);
52+
53+
// After retries complete, find all retry result files that were created
54+
var retryPattern = $"{baseReportName}-*.xml";
55+
var retryResultFiles = GetFiles(paths.TestResults.Combine(retryPattern).FullPath);
56+
57+
// Analyze retry results
58+
var stillFailedTests = new List<string>();
59+
foreach (var retryResultPath in retryResultFiles)
60+
{
61+
var retryFailed = testRetryHelper.FindFailedXUnitTests(retryResultPath);
62+
if (retryFailed.Any())
63+
{
64+
stillFailedTests.AddRange(retryFailed);
65+
}
66+
}
67+
68+
// Display summary
69+
testRetryHelper.DisplayRetryResultsSummary(".NET Framework Unit", initialFailedTests, stillFailedTests);
70+
71+
if (stillFailedTests.Any())
72+
{
73+
throw new Exception($"{stillFailedTests.Count} test(s) failed after retry");
74+
}
75+
}
4576
});
4677

4778
Task("_NetFramework_Integration_Tests")
@@ -66,24 +97,55 @@ Task("_NetFramework_Integration_Tests_WithRetry")
6697
var testAssemblies = testExecutionHelper.FindTestAssemblies("IO.Ably.Tests.NETFramework");
6798
if (!testAssemblies.Any()) return;
6899

69-
var resultsPath = paths.TestResults.CombineWithFilePath("xunit-netframework-integration.xml");
100+
var baseReportName = "xunit-netframework-integration";
101+
var resultsPath = paths.TestResults.CombineWithFilePath($"{baseReportName}.xml");
70102

103+
var initialFailedTests = new List<string>();
71104
try
72105
{
73-
var settings = testExecutionHelper.CreateXUnitSettings("xunit-netframework-integration", isIntegration: true);
106+
var settings = testExecutionHelper.CreateXUnitSettings(baseReportName, isIntegration: true);
74107
testExecutionHelper.RunXUnitTests(testAssemblies, settings);
75108
}
76109
catch
77110
{
78111
Warning("Some tests failed. Retrying failed tests...");
112+
initialFailedTests = testRetryHelper.FindFailedXUnitTests(resultsPath);
79113
}
80114

81-
testExecutionHelper.RetryFailedXUnitTests(
82-
testAssemblies,
83-
resultsPath,
84-
testRetryHelper,
85-
(test) => testExecutionHelper.CreateXUnitSettings("retry", isIntegration: true, isRetry: true)
86-
);
115+
if (initialFailedTests.Any())
116+
{
117+
Information($"Retrying {initialFailedTests.Count} failed test(s)...");
118+
119+
testExecutionHelper.RetryFailedXUnitTests(
120+
testAssemblies,
121+
resultsPath,
122+
testRetryHelper,
123+
(test) => testExecutionHelper.CreateXUnitSettings(baseReportName, isIntegration: true, isRetry: true)
124+
);
125+
126+
// After retries complete, find all retry result files that were created
127+
var retryPattern = $"{baseReportName}-*.xml";
128+
var retryResultFiles = GetFiles(paths.TestResults.Combine(retryPattern).FullPath);
129+
130+
// Analyze retry results
131+
var stillFailedTests = new List<string>();
132+
foreach (var retryResultPath in retryResultFiles)
133+
{
134+
var retryFailed = testRetryHelper.FindFailedXUnitTests(retryResultPath);
135+
if (retryFailed.Any())
136+
{
137+
stillFailedTests.AddRange(retryFailed);
138+
}
139+
}
140+
141+
// Display summary
142+
testRetryHelper.DisplayRetryResultsSummary(".NET Framework Integration", initialFailedTests, stillFailedTests);
143+
144+
if (stillFailedTests.Any())
145+
{
146+
throw new Exception($"{stillFailedTests.Count} test(s) failed after retry");
147+
}
148+
}
87149
});
88150

89151
///////////////////////////////////////////////////////////////////////////////
@@ -112,21 +174,52 @@ Task("_NetStandard_Unit_Tests_WithRetry")
112174
Information("Running .NET Standard unit tests with retry...");
113175

114176
var project = paths.Src.CombineWithFilePath("IO.Ably.Tests.DotNET/IO.Ably.Tests.DotNET.csproj");
115-
var resultsPath = paths.TestResults.CombineWithFilePath("tests-netstandard-unit.trx");
177+
var baseResultsName = "tests-netstandard-unit";
178+
var resultsPath = paths.TestResults.CombineWithFilePath($"{baseResultsName}.trx");
116179

117180
var filter = testExecutionHelper.CreateUnitTestFilter(IsRunningOnUnix());
118181
var settings = testExecutionHelper.CreateDotNetTestSettings(resultsPath, filter, framework, configuration);
119182

183+
var initialFailedTests = new List<string>();
120184
try
121185
{
122186
testExecutionHelper.RunDotNetTests(project, settings);
123187
}
124188
catch
125189
{
126190
Warning("Some tests failed. Retrying failed tests...");
191+
initialFailedTests = testRetryHelper.FindFailedDotNetTests(resultsPath);
127192
}
128193

129-
testExecutionHelper.RetryFailedDotNetTests(project, resultsPath, testRetryHelper, framework, configuration);
194+
if (initialFailedTests.Any())
195+
{
196+
Information($"Retrying {initialFailedTests.Count} failed test(s)...");
197+
198+
testExecutionHelper.RetryFailedDotNetTests(project, resultsPath, testRetryHelper, framework, configuration);
199+
200+
// After retries complete, find all retry result files that were created
201+
var retryPattern = $"{baseResultsName}-*.trx";
202+
var retryResultFiles = GetFiles(paths.TestResults.Combine(retryPattern).FullPath);
203+
204+
// Analyze retry results
205+
var stillFailedTests = new List<string>();
206+
foreach (var retryResultPath in retryResultFiles)
207+
{
208+
var retryFailed = testRetryHelper.FindFailedDotNetTests(retryResultPath);
209+
if (retryFailed.Any())
210+
{
211+
stillFailedTests.AddRange(retryFailed);
212+
}
213+
}
214+
215+
// Display summary
216+
testRetryHelper.DisplayRetryResultsSummary(".NET Standard Unit", initialFailedTests, stillFailedTests);
217+
218+
if (stillFailedTests.Any())
219+
{
220+
throw new Exception($"{stillFailedTests.Count} test(s) failed after retry");
221+
}
222+
}
130223
});
131224

132225
Task("_NetStandard_Integration_Tests")
@@ -151,21 +244,52 @@ Task("_NetStandard_Integration_Tests_WithRetry")
151244
Information("Running .NET Standard integration tests with retry...");
152245

153246
var project = paths.Src.CombineWithFilePath("IO.Ably.Tests.DotNET/IO.Ably.Tests.DotNET.csproj");
154-
var resultsPath = paths.TestResults.CombineWithFilePath("tests-netstandard-integration.trx");
247+
var baseResultsName = "tests-netstandard-integration";
248+
var resultsPath = paths.TestResults.CombineWithFilePath($"{baseResultsName}.trx");
155249

156250
var filter = testExecutionHelper.CreateIntegrationTestFilter();
157251
var settings = testExecutionHelper.CreateDotNetTestSettings(resultsPath, filter, framework, configuration);
158252

253+
var initialFailedTests = new List<string>();
159254
try
160255
{
161256
testExecutionHelper.RunDotNetTests(project, settings);
162257
}
163258
catch
164259
{
165260
Warning("Some tests failed. Retrying failed tests...");
261+
initialFailedTests = testRetryHelper.FindFailedDotNetTests(resultsPath);
166262
}
167263

168-
testExecutionHelper.RetryFailedDotNetTests(project, resultsPath, testRetryHelper, framework, configuration);
264+
if (initialFailedTests.Any())
265+
{
266+
Information($"Retrying {initialFailedTests.Count} failed test(s)...");
267+
268+
testExecutionHelper.RetryFailedDotNetTests(project, resultsPath, testRetryHelper, framework, configuration);
269+
270+
// After retries complete, find all retry result files that were created
271+
var retryPattern = $"{baseResultsName}-*.trx";
272+
var retryResultFiles = GetFiles(paths.TestResults.Combine(retryPattern).FullPath);
273+
274+
// Analyze retry results
275+
var stillFailedTests = new List<string>();
276+
foreach (var retryResultPath in retryResultFiles)
277+
{
278+
var retryFailed = testRetryHelper.FindFailedDotNetTests(retryResultPath);
279+
if (retryFailed.Any())
280+
{
281+
stillFailedTests.AddRange(retryFailed);
282+
}
283+
}
284+
285+
// Display summary
286+
testRetryHelper.DisplayRetryResultsSummary(".NET Standard Integration", initialFailedTests, stillFailedTests);
287+
288+
if (stillFailedTests.Any())
289+
{
290+
throw new Exception($"{stillFailedTests.Count} test(s) failed after retry");
291+
}
292+
}
169293
});
170294

171295
///////////////////////////////////////////////////////////////////////////////

0 commit comments

Comments
 (0)