Report captured output using OTR's new dedicated XML element (#4199)

If output capturing is enabled, captured output written to `System.out` 
and `System.err` is now included using OTR's new dedicated `output` 
element rather than as regular report entries.

Author: marcphilipp

documentation/documentation.gradle.kts

@@ -158,6 +158,8 @@ tasks {
 		args.addAll("execute")
 		args.addAll("--scan-classpath")
 		args.addAll("--config=junit.platform.reporting.open.xml.enabled=true")
+		args.addAll("--config=junit.platform.output.capture.stdout=true")
+		args.addAll("--config=junit.platform.output.capture.stderr=true")
 		outputs.dir(consoleLauncherTestReportsDir)
 		argumentProviders.add(CommandLineArgumentProvider {
 			listOf(

documentation/src/docs/asciidoc/release-notes/release-notes-5.12.0-M1.adoc

@@ -41,12 +41,17 @@ JUnit repository on GitHub.
   allow test engines to publish/attach files to containers and tests by calling
   `EngineExecutionListener.fileEntryPublished(...)`. Registered `TestExecutionListeners`
   can then access these files by overriding the `fileEntryPublished(...)` method.
-* The following improvements have been made to the open-test-reporting XML output:
+* The following improvements have been made to the
+  <<../user-guide/index.adoc#junit-platform-reporting-open-test-reporting, Open Test Reporting>>
+  XML output:
   - Information about the Git repository, the current branch, the commit hash, and the
     current worktree status are now included in the XML report, if applicable.
   - A section containing JUnit-specific metadata about each test/container to the HTML
     report is now written by open-test-reporting when added to the classpath/module path
   - Information about published files is now included as attachments.
+  - If <<../user-guide/index.adoc#running-tests-capturing-output, output capturing>> is
+    enabled, the captured output written to `System.out` and `System.err` is now included
+    in the XML report.
 * Introduced contracts for Kotlin-specific assertion methods.
 
 

documentation/src/docs/asciidoc/user-guide/advanced-topics/junit-platform-reporting.adoc

@@ -44,6 +44,9 @@ The listener is auto-registered and can be configured via the following
 If enabled, the listener creates an XML report file named `open-test-report.xml` in the
 configured <<junit-platform-reporting-output-directory, output directory>>.
 
+If <<running-tests-capturing-output, output capturing>> is enabled, the captured output
+written to `System.out` and `System.err` will be included in the report as well.
+
 TIP: The {OpenTestReportingCliTool} can be used to convert from the event-based format to
 the hierarchical format which is more human-readable.
 

gradle/plugins/common/src/main/kotlin/junitbuild.testing-conventions.gradle.kts

@@ -129,6 +129,8 @@ tasks.withType<Test>().configureEach {
 			"-Djunit.platform.reporting.output.dir=${reports.junitXml.outputLocation.get().asFile.absolutePath}/junit-{uniqueNumber}",
 		)
 	}
+	systemProperty("junit.platform.output.capture.stdout", "true")
+	systemProperty("junit.platform.output.capture.stderr", "true")
 
 	jvmArgumentProviders += objects.newInstance(JavaAgentArgumentProvider::class).apply {
 		classpath.from(javaAgentClasspath)

junit-platform-reporting/src/main/java/org/junit/platform/reporting/open/xml/OpenTestReportGeneratingListener.java

@@ -12,6 +12,8 @@
 
 import static org.apiguardian.api.API.Status.EXPERIMENTAL;
 import static org.junit.platform.commons.util.StringUtils.isNotBlank;
+import static org.junit.platform.launcher.LauncherConstants.STDERR_REPORT_ENTRY_KEY;
+import static org.junit.platform.launcher.LauncherConstants.STDOUT_REPORT_ENTRY_KEY;
 import static org.junit.platform.reporting.open.xml.JUnitFactory.legacyReportingName;
 import static org.junit.platform.reporting.open.xml.JUnitFactory.type;
 import static org.junit.platform.reporting.open.xml.JUnitFactory.uniqueId;
@@ -25,6 +27,7 @@
 import static org.opentest4j.reporting.events.core.CoreFactory.infrastructure;
 import static org.opentest4j.reporting.events.core.CoreFactory.metadata;
 import static org.opentest4j.reporting.events.core.CoreFactory.operatingSystem;
+import static org.opentest4j.reporting.events.core.CoreFactory.output;
 import static org.opentest4j.reporting.events.core.CoreFactory.reason;
 import static org.opentest4j.reporting.events.core.CoreFactory.result;
 import static org.opentest4j.reporting.events.core.CoreFactory.sources;
@@ -60,6 +63,7 @@
 import java.nio.file.Path;
 import java.nio.file.Paths;
 import java.time.Instant;
+import java.time.LocalDateTime;
 import java.util.Map;
 import java.util.Optional;
 import java.util.concurrent.ConcurrentHashMap;
@@ -90,6 +94,7 @@
 import org.junit.platform.launcher.TestPlan;
 import org.opentest4j.reporting.events.api.DocumentWriter;
 import org.opentest4j.reporting.events.api.NamespaceRegistry;
+import org.opentest4j.reporting.events.core.Attachments;
 import org.opentest4j.reporting.events.core.Infrastructure;
 import org.opentest4j.reporting.events.core.Result;
 import org.opentest4j.reporting.events.core.Sources;
@@ -345,8 +350,26 @@ public void reportingEntryPublished(TestIdentifier testIdentifier, ReportEntry e
 		String id = inProgressIds.get(testIdentifier.getUniqueIdObject());
 		eventsFileWriter.append(reported(id, Instant.now()), //
 			reported -> reported.append(attachments(), //
-				attachments -> attachments.append(data(entry.getTimestamp()), //
-					data -> entry.getKeyValuePairs().forEach(data::addEntry))));
+				attachments -> {
+					Map<String, String> keyValuePairs = entry.getKeyValuePairs();
+					if (keyValuePairs.containsKey(STDOUT_REPORT_ENTRY_KEY)
+							|| keyValuePairs.containsKey(STDERR_REPORT_ENTRY_KEY)) {
+						attachOutput(attachments, entry.getTimestamp(), keyValuePairs.get(STDOUT_REPORT_ENTRY_KEY),
+							"stdout");
+						attachOutput(attachments, entry.getTimestamp(), keyValuePairs.get(STDERR_REPORT_ENTRY_KEY),
+							"stderr");
+					}
+					else {
+						attachments.append(data(entry.getTimestamp()), //
+							data -> keyValuePairs.forEach(data::addEntry));
+					}
+				}));
+	}
+
+	private static void attachOutput(Attachments attachments, LocalDateTime timestamp, String content, String source) {
+		if (content != null) {
+			attachments.append(output(timestamp), output -> output.withSource(source).withContent(content));
+		}
 	}
 
 	@Override

platform-tests/src/test/java/org/junit/platform/reporting/open/xml/OpenTestReportGeneratingListenerTests.java

@@ -16,18 +16,23 @@
 import static org.junit.jupiter.api.Assertions.fail;
 import static org.junit.jupiter.api.Assumptions.assumeTrue;
 import static org.junit.platform.engine.discovery.DiscoverySelectors.selectUniqueId;
+import static org.junit.platform.launcher.LauncherConstants.CAPTURE_STDERR_PROPERTY_NAME;
+import static org.junit.platform.launcher.LauncherConstants.CAPTURE_STDOUT_PROPERTY_NAME;
 import static org.junit.platform.launcher.LauncherConstants.OUTPUT_DIR_PROPERTY_NAME;
 import static org.junit.platform.launcher.LauncherConstants.OUTPUT_DIR_UNIQUE_NUMBER_PLACEHOLDER;
 import static org.junit.platform.launcher.core.LauncherDiscoveryRequestBuilder.request;
 import static org.junit.platform.launcher.core.LauncherFactoryForTestingPurposesOnly.createLauncher;
 import static org.junit.platform.reporting.open.xml.OpenTestReportGeneratingListener.ENABLED_PROPERTY_NAME;
 import static org.junit.platform.reporting.testutil.FileUtils.findPath;
 
+import java.io.PrintStream;
 import java.net.URISyntaxException;
 import java.nio.file.Files;
 import java.nio.file.Path;
 import java.util.Map;
 
+import org.junit.jupiter.api.AfterEach;
+import org.junit.jupiter.api.BeforeEach;
 import org.junit.jupiter.api.Test;
 import org.junit.jupiter.api.io.TempDir;
 import org.junit.jupiter.params.ParameterizedTest;
@@ -51,12 +56,32 @@
  */
 public class OpenTestReportGeneratingListenerTests {
 
+	private PrintStream originalOut;
+	private PrintStream originalErr;
+
+	@BeforeEach
+	void wrapSystemPrintStreams() {
+		// Work around nesting check in org.junit.platform.launcher.core.StreamInterceptor
+		originalOut = System.out;
+		System.setOut(new PrintStream(System.out));
+		originalErr = System.err;
+		System.setErr(new PrintStream(System.err));
+	}
+
+	@AfterEach
+	void restoreSystemPrintStreams() {
+		System.setOut(originalOut);
+		System.setErr(originalErr);
+	}
+
 	@Test
 	void writesValidXmlReport(@TempDir Path tempDirectory) throws Exception {
 		var engine = new DemoHierarchicalTestEngine("dummy");
 		engine.addTest("failingTest", "display<-->Name 😎", (context, descriptor) -> {
 			var listener = context.request.getEngineExecutionListener();
 			listener.reportingEntryPublished(descriptor, ReportEntry.from("key", "value"));
+			System.out.println("Hello, stdout!");
+			System.err.println("Hello, stderr!");
 			fail("failure message");
 		});
 
@@ -105,6 +130,12 @@ void writesValidXmlReport(@TempDir Path tempDirectory) throws Exception {
 				            </data>
 				        </attachments>
 				    </e:reported>
+				    <e:reported id="2" time="${xmlunit.isDateTime}">
+				        <attachments>
+				            <output time="${xmlunit.isDateTime}" source="stdout"><![CDATA[Hello, stdout!]]></output>
+				            <output time="${xmlunit.isDateTime}" source="stderr"><![CDATA[Hello, stderr!]]></output>
+				        </attachments>
+				    </e:reported>
 				    <e:finished id="2" time="${xmlunit.isDateTime}">
 				        <result status="FAILED">
 				            <java:throwable assertionError="true" type="org.opentest4j.AssertionFailedError">
@@ -200,6 +231,8 @@ private void executeTests(Path tempDirectory, TestEngine engine, Path outputDir)
 		var build = request() //
 				.selectors(selectUniqueId(UniqueId.forEngine(engine.getId()))) //
 				.configurationParameter(ENABLED_PROPERTY_NAME, String.valueOf(true)) //
+				.configurationParameter(CAPTURE_STDOUT_PROPERTY_NAME, String.valueOf(true)) //
+				.configurationParameter(CAPTURE_STDERR_PROPERTY_NAME, String.valueOf(true)) //
 				.configurationParameter(OUTPUT_DIR_PROPERTY_NAME, outputDir.toString()) //
 				.build();
 		createLauncher(engine).execute(build, new OpenTestReportGeneratingListener(tempDirectory));

platform-tooling-support-tests/platform-tooling-support-tests.gradle.kts

@@ -165,6 +165,12 @@ tasks.test {
 		dir("${rootDir}/documentation/src/test").withPathSensitivity(RELATIVE)
 	}
 
+	// Disable capturing output since parallel execution is enabled and output of
+	// external processes happens on non-test threads which can't reliably be
+	// attributed to the test that started the process.
+	systemProperty("junit.platform.output.capture.stdout", "false")
+	systemProperty("junit.platform.output.capture.stderr", "false")
+
 	develocity {
 		testDistribution {
 			requirements.add("jdk=8")