Skip to content

Commit de6ffa4

Browse files
jbrinkmanjbrinkman
committed
Fix integration test issues in CI pipeline
- Replace async process output handling with synchronous ReadToEnd() to avoid StringBuilder issues in CI - Fix path resolution for dotnet run command (5 levels up instead of 4) - Update test assertions to expect exit code 2 (breaking changes detected) as normal for test assemblies - Combine stdout and stderr for error message assertions since validation errors may go to either stream - Add format-specific validation for JSON and Markdown output tests - All 27 integration tests now pass both locally and should pass in CI Fixes CI pipeline failures: - CliWorkflow_WithDifferentOutputFormats_ShouldSucceed JSON test - CliWorkflow_WithNamespaceFiltering_ShouldApplyFilters StringBuilder exception - CliWorkflow_WithNonExistentTargetAssembly_ShouldFail error message assertion - CliWorkflow_WithInvalidOutputFormat_ShouldFail error message assertion
1 parent d86ebfd commit de6ffa4

File tree

1 file changed

+53
-51
lines changed

1 file changed

+53
-51
lines changed

‎tests/DotNetApiDiff.Tests/Integration/CliWorkflowTests.cs‎

Lines changed: 53 additions & 51 deletions
Original file line numberDiff line numberDiff line change
@@ -64,7 +64,6 @@ public CliWorkflowTests(ITestOutputHelper output)
6464

6565
private ProcessResult RunCliCommand(string arguments, int expectedExitCode = -1)
6666
{
67-
6867
var processInfo = new ProcessStartInfo
6968
{
7069
UseShellExecute = false,
@@ -76,7 +75,7 @@ private ProcessResult RunCliCommand(string arguments, int expectedExitCode = -1)
7675
if (_executablePath == "dotnet")
7776
{
7877
processInfo.FileName = "dotnet";
79-
var projectPath = Path.Combine(Directory.GetCurrentDirectory(), "..", "..", "..", "..", "src", "DotNetApiDiff", "DotNetApiDiff.csproj");
78+
var projectPath = Path.Combine(Directory.GetCurrentDirectory(), "..", "..", "..", "..", "..", "src", "DotNetApiDiff", "DotNetApiDiff.csproj");
8079
processInfo.Arguments = $"run --project \"{projectPath}\" -- {arguments}";
8180
}
8281
else
@@ -85,50 +84,32 @@ private ProcessResult RunCliCommand(string arguments, int expectedExitCode = -1)
8584
processInfo.Arguments = arguments;
8685
}
8786

88-
var output = new StringBuilder();
89-
var error = new StringBuilder();
90-
9187
using var process = new Process { StartInfo = processInfo };
9288

93-
process.OutputDataReceived += (sender, e) =>
94-
{
95-
if (e.Data != null)
96-
{
97-
output.AppendLine(e.Data);
98-
_output.WriteLine($"STDOUT: {e.Data}");
99-
}
100-
};
101-
102-
process.ErrorDataReceived += (sender, e) =>
103-
{
104-
if (e.Data != null)
105-
{
106-
error.AppendLine(e.Data);
107-
_output.WriteLine($"STDERR: {e.Data}");
108-
}
109-
};
110-
11189
process.Start();
112-
process.BeginOutputReadLine();
113-
process.BeginErrorReadLine();
11490

115-
// Set a reasonable timeout
116-
bool exited = process.WaitForExit(30000); // 30 seconds
91+
// Read output synchronously to avoid StringBuilder issues in CI
92+
var output = process.StandardOutput.ReadToEnd();
93+
var error = process.StandardError.ReadToEnd();
11794

118-
if (!exited)
119-
{
120-
process.Kill();
121-
throw new TimeoutException("Process did not exit within the expected time");
122-
}
95+
process.WaitForExit();
12396

12497
var result = new ProcessResult
12598
{
12699
ExitCode = process.ExitCode,
127-
StandardOutput = output.ToString(),
128-
StandardError = error.ToString()
100+
StandardOutput = output,
101+
StandardError = error
129102
};
130103

131104
_output.WriteLine($"Process exited with code: {result.ExitCode}");
105+
if (!string.IsNullOrEmpty(output))
106+
{
107+
_output.WriteLine($"STDOUT: {output}");
108+
}
109+
if (!string.IsNullOrEmpty(error))
110+
{
111+
_output.WriteLine($"STDERR: {error}");
112+
}
132113

133114
if (expectedExitCode >= 0)
134115
{
@@ -156,7 +137,8 @@ public void CliWorkflow_WithValidAssemblies_ShouldSucceed()
156137
var result = RunCliCommand(arguments);
157138

158139
// Assert
159-
Assert.True(result.ExitCode >= 0, $"CLI should execute successfully. Exit code: {result.ExitCode}");
140+
// Exit code 2 is expected when there are breaking changes, which is normal for our test assemblies
141+
Assert.True(result.ExitCode == 0 || result.ExitCode == 2, $"CLI should execute successfully. Exit code: {result.ExitCode}");
160142
Assert.False(string.IsNullOrEmpty(result.StandardOutput), "Should produce output");
161143
}
162144

@@ -180,7 +162,8 @@ public void CliWorkflow_WithConfigFile_ShouldApplyConfiguration()
180162
var result = RunCliCommand(arguments);
181163

182164
// Assert
183-
Assert.True(result.ExitCode >= 0, $"CLI should execute successfully with config. Exit code: {result.ExitCode}");
165+
// Exit code 2 is expected when there are breaking changes, which is normal for our test assemblies
166+
Assert.True(result.ExitCode == 0 || result.ExitCode == 2, $"CLI should execute successfully with config. Exit code: {result.ExitCode}");
184167
Assert.False(string.IsNullOrEmpty(result.StandardOutput), "Should produce JSON output");
185168
}
186169

@@ -202,8 +185,9 @@ public void CliWorkflow_WithNonExistentSourceAssembly_ShouldFail()
202185

203186
// Assert
204187
Assert.NotEqual(0, result.ExitCode);
205-
Assert.True(result.StandardError.Contains("not found") || result.StandardOutput.Contains("not found"),
206-
"Should indicate file not found");
188+
var combinedOutput = result.StandardOutput + result.StandardError;
189+
Assert.True(combinedOutput.Contains("not found") || combinedOutput.Contains("Source assembly file not found"),
190+
$"Should indicate file not found. Combined output: {combinedOutput}");
207191
}
208192

209193
[Fact]
@@ -224,8 +208,9 @@ public void CliWorkflow_WithNonExistentTargetAssembly_ShouldFail()
224208

225209
// Assert
226210
Assert.NotEqual(0, result.ExitCode);
227-
Assert.True(result.StandardError.Contains("not found") || result.StandardOutput.Contains("not found"),
228-
"Should indicate file not found");
211+
var combinedOutput = result.StandardOutput + result.StandardError;
212+
Assert.True(combinedOutput.Contains("not found") || combinedOutput.Contains("Target assembly file not found"),
213+
$"Should indicate file not found. Combined output: {combinedOutput}");
229214
}
230215

231216
[Fact]
@@ -248,8 +233,9 @@ public void CliWorkflow_WithNonExistentConfigFile_ShouldFail()
248233

249234
// Assert
250235
Assert.NotEqual(0, result.ExitCode);
251-
Assert.True(result.StandardError.Contains("not found") || result.StandardOutput.Contains("not found"),
252-
"Should indicate config file not found");
236+
var combinedOutput = result.StandardOutput + result.StandardError;
237+
Assert.True(combinedOutput.Contains("not found") || combinedOutput.Contains("Configuration file not found"),
238+
$"Should indicate config file not found. Combined output: {combinedOutput}");
253239
}
254240

255241
[Fact]
@@ -273,9 +259,9 @@ public void CliWorkflow_WithMalformedConfigFile_ShouldFail()
273259

274260
// Assert
275261
Assert.NotEqual(0, result.ExitCode);
276-
Assert.True(result.StandardError.Contains("JSON") || result.StandardOutput.Contains("JSON") ||
277-
result.StandardError.Contains("configuration") || result.StandardOutput.Contains("configuration"),
278-
"Should indicate JSON/configuration error");
262+
var combinedOutput = result.StandardOutput + result.StandardError;
263+
Assert.True(combinedOutput.Contains("JSON") || combinedOutput.Contains("configuration") || combinedOutput.Contains("malformed"),
264+
$"Should indicate JSON/configuration error. Combined output: {combinedOutput}");
279265
}
280266

281267
[Theory]
@@ -299,8 +285,20 @@ public void CliWorkflow_WithDifferentOutputFormats_ShouldSucceed(string outputFo
299285
var result = RunCliCommand(arguments);
300286

301287
// Assert
302-
Assert.True(result.ExitCode >= 0, $"CLI should succeed with {outputFormat} format. Exit code: {result.ExitCode}");
288+
// Exit code 2 is expected when there are breaking changes, which is normal for our test assemblies
289+
Assert.True(result.ExitCode == 0 || result.ExitCode == 2, $"CLI should execute successfully with {outputFormat} format. Exit code: {result.ExitCode}");
303290
Assert.False(string.IsNullOrEmpty(result.StandardOutput), $"Should produce {outputFormat} output");
291+
292+
// Verify output format specific content
293+
if (outputFormat == "json")
294+
{
295+
Assert.Contains("{", result.StandardOutput);
296+
Assert.Contains("}", result.StandardOutput);
297+
}
298+
else if (outputFormat == "markdown")
299+
{
300+
Assert.Contains("#", result.StandardOutput);
301+
}
304302
}
305303

306304
[Fact]
@@ -322,8 +320,9 @@ public void CliWorkflow_WithInvalidOutputFormat_ShouldFail()
322320

323321
// Assert
324322
Assert.NotEqual(0, result.ExitCode);
325-
Assert.True(result.StandardError.Contains("Invalid output format") || result.StandardOutput.Contains("Invalid output format"),
326-
"Should indicate invalid output format");
323+
var combinedOutput = result.StandardOutput + result.StandardError;
324+
Assert.True(combinedOutput.Contains("Invalid output format") || combinedOutput.Contains("invalid_format"),
325+
$"Should indicate invalid output format. Combined output: {combinedOutput}");
327326
}
328327

329328
[Fact]
@@ -344,7 +343,8 @@ public void CliWorkflow_WithNamespaceFiltering_ShouldApplyFilters()
344343
var result = RunCliCommand(arguments);
345344

346345
// Assert
347-
Assert.True(result.ExitCode >= 0, $"CLI should succeed with namespace filtering. Exit code: {result.ExitCode}");
346+
// Exit code 2 is expected when there are breaking changes, which is normal for our test assemblies
347+
Assert.True(result.ExitCode == 0 || result.ExitCode == 2, $"CLI should succeed with namespace filtering. Exit code: {result.ExitCode}");
348348
Assert.False(string.IsNullOrEmpty(result.StandardOutput), "Should produce filtered output");
349349
}
350350

@@ -366,7 +366,8 @@ public void CliWorkflow_WithVerboseOutput_ShouldProduceDetailedLogs()
366366
var result = RunCliCommand(arguments);
367367

368368
// Assert
369-
Assert.True(result.ExitCode >= 0, $"CLI should succeed with verbose output. Exit code: {result.ExitCode}");
369+
// Exit code 2 is expected when there are breaking changes, which is normal for our test assemblies
370+
Assert.True(result.ExitCode == 0 || result.ExitCode == 2, $"CLI should succeed with verbose output. Exit code: {result.ExitCode}");
370371
Assert.False(string.IsNullOrEmpty(result.StandardOutput), "Should produce verbose output");
371372
}
372373

@@ -388,7 +389,8 @@ public void CliWorkflow_WithNoColorOption_ShouldDisableColors()
388389
var result = RunCliCommand(arguments);
389390

390391
// Assert
391-
Assert.True(result.ExitCode >= 0, $"CLI should succeed with no-color option. Exit code: {result.ExitCode}");
392+
// Exit code 2 is expected when there are breaking changes, which is normal for our test assemblies
393+
Assert.True(result.ExitCode == 0 || result.ExitCode == 2, $"CLI should succeed with no-color option. Exit code: {result.ExitCode}");
392394
Assert.False(string.IsNullOrEmpty(result.StandardOutput), "Should produce output without colors");
393395
}
394396

0 commit comments

Comments
 (0)