Skip to content

Commit 409937d

Browse files
Youssef1313Youssef1313
authored
Add test for crashing test process with exit code 0 (#50726)
1 parent 7dafa79 commit 409937d

File tree

6 files changed

+145
-5
lines changed

6 files changed

+145
-5
lines changed

‎src/Cli/dotnet/Commands/Test/MTP/TestApplication.cs‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -69,6 +69,8 @@ private ProcessStartInfo CreateProcessStartInfo()
6969
Arguments = GetArguments(),
7070
RedirectStandardOutput = true,
7171
RedirectStandardError = true,
72+
// False is already the default on .NET Core, but prefer to be explicit.
73+
UseShellExecute = false,
7274
};
7375

7476
if (!string.IsNullOrEmpty(Module.RunProperties.WorkingDirectory))
@@ -274,7 +276,7 @@ private static HandshakeMessage CreateHandshakeMessage(string version) =>
274276

275277
private async Task<int> StartProcess(ProcessStartInfo processStartInfo)
276278
{
277-
Logger.LogTrace($"Test application arguments: {processStartInfo.Arguments}");
279+
Logger.LogTrace($"Starting test process with command '{processStartInfo.FileName}' and arguments '{processStartInfo.Arguments}'.");
278280

279281
using var process = Process.Start(processStartInfo);
280282
StoreOutputAndErrorData(process);

‎test/TestAssets/TestProjects/TestProjectMTPCrash/TestProject1/Program.cs‎

Lines changed: 47 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,47 @@
1+
using Microsoft.Testing.Platform.Builder;
2+
using Microsoft.Testing.Platform.Capabilities.TestFramework;
3+
using Microsoft.Testing.Platform.Extensions.Messages;
4+
using Microsoft.Testing.Platform.Extensions.TestFramework;
5+
6+
var testApplicationBuilder = await TestApplication.CreateBuilderAsync(args);
7+
8+
testApplicationBuilder.RegisterTestFramework(_ => new TestFrameworkCapabilities(), (_, __) => new DummyTestAdapter());
9+
10+
using var testApplication = await testApplicationBuilder.BuildAsync();
11+
return await testApplication.RunAsync();
12+
13+
public class DummyTestAdapter : ITestFramework, IDataProducer
14+
{
15+
public string Uid => nameof(DummyTestAdapter);
16+
17+
public string Version => "2.0.0";
18+
19+
public string DisplayName => nameof(DummyTestAdapter);
20+
21+
public string Description => nameof(DummyTestAdapter);
22+
23+
public Task<bool> IsEnabledAsync() => Task.FromResult(true);
24+
25+
public Type[] DataTypesProduced => [typeof(TestNodeUpdateMessage)];
26+
public Task<CreateTestSessionResult> CreateTestSessionAsync(CreateTestSessionContext context)
27+
=> Task.FromResult(new CreateTestSessionResult() { IsSuccess = true });
28+
29+
public Task<CloseTestSessionResult> CloseTestSessionAsync(CloseTestSessionContext context)
30+
=> Task.FromResult(new CloseTestSessionResult() { IsSuccess = true });
31+
32+
public async Task ExecuteRequestAsync(ExecuteRequestContext context)
33+
{
34+
await context.MessageBus.PublishAsync(this, new TestNodeUpdateMessage(context.Request.Session.SessionUid, new TestNode()
35+
{
36+
Uid = $"Test1",
37+
DisplayName = $"Test1",
38+
Properties = new PropertyBag(PassedTestNodeStateProperty.CachedInstance),
39+
}));
40+
41+
await Task.Delay(1000);
42+
43+
Environment.Exit(0);
44+
45+
context.Complete();
46+
}
47+
}

‎test/TestAssets/TestProjects/TestProjectMTPCrash/TestProject1/TestProject1.csproj‎

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,17 @@
1+
<Project Sdk="Microsoft.NET.Sdk">
2+
<Import Project="$([MSBuild]::GetDirectoryNameOfFileAbove($(MSBuildThisFileDirectory), testAsset.props))\testAsset.props" />
3+
4+
<PropertyGroup>
5+
<TargetFramework>$(CurrentTargetFramework)</TargetFramework>
6+
<OutputType>Exe</OutputType>
7+
8+
<ImplicitUsings>enable</ImplicitUsings>
9+
<Nullable>enable</Nullable>
10+
11+
<ManagePackageVersionsCentrally>false</ManagePackageVersionsCentrally>
12+
</PropertyGroup>
13+
14+
<ItemGroup>
15+
<PackageReference Include="Microsoft.Testing.Platform" Version="$(MicrosoftTestingPlatformVersion)" />
16+
</ItemGroup>
17+
</Project>

‎test/TestAssets/TestProjects/TestProjectMTPCrash/TestProjectMTPCrash.sln‎

Lines changed: 34 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,34 @@
1+
2+
Microsoft Visual Studio Solution File, Format Version 12.00
3+
# Visual Studio Version 17
4+
VisualStudioVersion = 17.0.31903.59
5+
MinimumVisualStudioVersion = 10.0.40219.1
6+
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "TestProject1", "TestProject1\TestProject1.csproj", "{3E834ED1-92D9-4454-BBB4-B1A2463E5726}"
7+
EndProject
8+
Global
9+
GlobalSection(SolutionConfigurationPlatforms) = preSolution
10+
Debug|Any CPU = Debug|Any CPU
11+
Debug|x64 = Debug|x64
12+
Debug|x86 = Debug|x86
13+
Release|Any CPU = Release|Any CPU
14+
Release|x64 = Release|x64
15+
Release|x86 = Release|x86
16+
EndGlobalSection
17+
GlobalSection(ProjectConfigurationPlatforms) = postSolution
18+
{3E834ED1-92D9-4454-BBB4-B1A2463E5726}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
19+
{3E834ED1-92D9-4454-BBB4-B1A2463E5726}.Debug|Any CPU.Build.0 = Debug|Any CPU
20+
{3E834ED1-92D9-4454-BBB4-B1A2463E5726}.Debug|x64.ActiveCfg = Debug|Any CPU
21+
{3E834ED1-92D9-4454-BBB4-B1A2463E5726}.Debug|x64.Build.0 = Debug|Any CPU
22+
{3E834ED1-92D9-4454-BBB4-B1A2463E5726}.Debug|x86.ActiveCfg = Debug|Any CPU
23+
{3E834ED1-92D9-4454-BBB4-B1A2463E5726}.Debug|x86.Build.0 = Debug|Any CPU
24+
{3E834ED1-92D9-4454-BBB4-B1A2463E5726}.Release|Any CPU.ActiveCfg = Release|Any CPU
25+
{3E834ED1-92D9-4454-BBB4-B1A2463E5726}.Release|Any CPU.Build.0 = Release|Any CPU
26+
{3E834ED1-92D9-4454-BBB4-B1A2463E5726}.Release|x64.ActiveCfg = Release|Any CPU
27+
{3E834ED1-92D9-4454-BBB4-B1A2463E5726}.Release|x64.Build.0 = Release|Any CPU
28+
{3E834ED1-92D9-4454-BBB4-B1A2463E5726}.Release|x86.ActiveCfg = Release|Any CPU
29+
{3E834ED1-92D9-4454-BBB4-B1A2463E5726}.Release|x86.Build.0 = Release|Any CPU
30+
EndGlobalSection
31+
GlobalSection(SolutionProperties) = preSolution
32+
HideSolutionNode = FALSE
33+
EndGlobalSection
34+
EndGlobal

‎test/TestAssets/TestProjects/TestProjectMTPCrash/dotnet.config‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,2 @@
1+
[dotnet.test.runner]
2+
name= "Microsoft.Testing.Platform"

‎test/dotnet.Tests/CommandTests/Test/GivenDotnetTestBuildsAndRunsTests.cs‎

Lines changed: 42 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -17,7 +17,7 @@ public GivenDotnetTestBuildsAndRunsTests(ITestOutputHelper log) : base(log)
1717
[InlineData(TestingConstants.Debug)]
1818
[InlineData(TestingConstants.Release)]
1919
[Theory]
20-
public void RunTestProjectWithNoTests_ShouldReturnExitCodeGenericFailure(string configuration)
20+
public void RunTestProjectWithNoTests_ShouldReturnExitCodeZeroTests(string configuration)
2121
{
2222
TestAsset testInstance = _testAssetsManager.CopyTestAsset("TestProjectSolution", Guid.NewGuid().ToString())
2323
.WithSource();
@@ -69,7 +69,7 @@ public void RunTestProjectWithWithRetryFeature_ShouldSucceed(string configuratio
6969
[InlineData(TestingConstants.Debug)]
7070
[InlineData(TestingConstants.Release)]
7171
[Theory]
72-
public void RunMultipleTestProjectsWithNoTests_ShouldReturnExitCodeGenericFailure(string configuration)
72+
public void RunMultipleTestProjectsWithNoTests_ShouldReturnExitCodeZeroTests(string configuration)
7373
{
7474
TestAsset testInstance = _testAssetsManager.CopyTestAsset("MultipleTestProjectSolution", Guid.NewGuid().ToString())
7575
.WithSource();
@@ -192,7 +192,7 @@ public void RunTestProjectWithTestsAndNoLaunchSettingsArguments_ShouldReturnExit
192192
[InlineData(TestingConstants.Debug)]
193193
[InlineData(TestingConstants.Release)]
194194
[Theory]
195-
public void RunMultipleTestProjectsWithFailingTests_ShouldReturnExitCodeGenericFailure(string configuration)
195+
public void RunMultipleTestProjectsWithFailingTests_ShouldReturnExitCodeAtLeastOneTestFailed(string configuration)
196196
{
197197
TestAsset testInstance = _testAssetsManager.CopyTestAsset("MultiTestProjectSolutionWithTests", Guid.NewGuid().ToString())
198198
.WithSource();
@@ -418,7 +418,7 @@ public void RunMTPProjectWithUseAppHostFalse_ShouldWork()
418418
.Execute();
419419

420420
// Verify the test runs successfully with UseAppHost=false
421-
result.ExitCode.Should().Be(0);
421+
result.ExitCode.Should().Be(ExitCodes.Success);
422422
}
423423

424424
[Theory]
@@ -439,5 +439,43 @@ public void RunMTPSolutionWithMinimumExpectedTests(string value, int expectedExi
439439

440440
result.ExitCode.Should().Be(expectedExitCode);
441441
}
442+
443+
[Fact]
444+
public void RunMTPProjectThatCrashesWithExitCodeZero_ShouldFail()
445+
{
446+
// The solution has two test projects. Each reports 5 tests. So, total 10 tests.
447+
TestAsset testInstance = _testAssetsManager.CopyTestAsset("TestProjectMTPCrash", Guid.NewGuid().ToString())
448+
.WithSource();
449+
450+
CommandResult result = new DotnetTestCommand(Log, disableNewOutput: false)
451+
.WithWorkingDirectory(testInstance.Path)
452+
.Execute();
453+
454+
result.ExitCode.Should().NotBe(ExitCodes.Success);
455+
if (!TestContext.IsLocalized())
456+
{
457+
/*
458+
The following exception occurred when running the test module with RunCommand 'C:\Users\ygerges\Desktop\sdk\artifacts\tmp\Debug\testing\19cefafa-91c4---0D0799BD\TestProject1\bin\Debug\net10.0\TestProject1.exe' and RunArguments ' ':
459+
System.InvalidOperationException: A test session start event was received without a corresponding test session end.
460+
at Microsoft.DotNet.Cli.Commands.Test.TestApplication.RunAsync() in C:\Users\ygerges\Desktop\sdk\src\Cli\dotnet\Commands\Test\MTP\TestApplication.cs:line 55
461+
at Microsoft.DotNet.Cli.Commands.Test.TestApplicationActionQueue.Read(BuildOptions buildOptions, TestOptions testOptions, TerminalTestReporter output, Action`1 onHelpRequested) in C:\Users\ygerges\Desktop\sdk\src\Cli\dotnet\Commands\Test\MTP\TestApplicationActionQueue.cs:line 68
462+
*/
463+
result.StdErr.Should().MatchRegex("""
464+
The following exception occurred when running the test module with RunCommand '.+?TestProject1(\..+?)?' and RunArguments ' ':
465+
""");
466+
467+
result.StdErr.Should().Contain("System.InvalidOperationException: A test session start event was received without a corresponding test session end.");
468+
469+
// TODO: It's much better to introduce a new kind of "summary" indicating
470+
// that the test app exited with zero exit code before sending test session end event
471+
result.StdOut.Should().Contain("Test run summary: Passed!")
472+
.And.Contain("total: 1")
473+
.And.Contain("succeeded: 1")
474+
.And.Contain("failed: 0")
475+
.And.Contain("skipped: 0");
476+
477+
result.StdOut.Contains("Test run completed with non-success exit code: 1 (see: https://aka.ms/testingplatform/exitcodes)");
478+
}
479+
}
442480
}
443481
}

0 commit comments

Comments
 (0)