Allow attaching files to test results (#4138)

Being able to attach files such as screenshots or extra log files to a 
test is useful to diagnose the outcome of tests. This PR adds an API for
Jupiter test authors to do so (`TestReporter.publishFile`) and includes
it when writing the Open Test Reporting XML output (in
`OpenTestReportGeneratingListener`) via a new method on 
`TestExecutionListener`. Moreover, it adds `OutputDirectoryProvider` to 
`EngineDiscoveryRequest` so other engines can also attach files and
write them to the same output directory and makes it available to 
`TestExecutionListener` implementations via `TestPlan`.

The default location of the XML output is changed from
`OUTPUT_DIR/junit-platform-events-*.xml` to 
`OUTPUT_DIR/open-test-report.xml`. The output directory can be made
unique by using the `{uniqueNumber}` placeholder in the
`junit.platform.reporting.output.dir` configuration parameter.

Author: marcphilipp

documentation/documentation.gradle.kts

@@ -147,7 +147,7 @@ tasks {
 
 	val consoleLauncherTestReportsDir = project.layout.buildDirectory.dir("console-launcher-test-results")
 	val consoleLauncherTestEventXmlFiles =
-		files(consoleLauncherTestReportsDir.map { it.asFileTree.matching { include("junit-platform-events-*.xml") } })
+		files(consoleLauncherTestReportsDir.map { it.asFileTree.matching { include("**/open-test-report.xml") } })
 
 	val consoleLauncherTest by registering(RunConsoleLauncher::class) {
 		args.addAll("execute")
@@ -157,7 +157,6 @@ tasks {
 		argumentProviders.add(CommandLineArgumentProvider {
 			listOf(
 				"--reports-dir=${consoleLauncherTestReportsDir.get()}",
-				"--config=junit.platform.reporting.output.dir=${consoleLauncherTestReportsDir.get()}",
 			)
 		})
 		args.addAll("--include-classname", ".*Tests")

documentation/src/docs/asciidoc/link-attributes.adoc

@@ -40,13 +40,15 @@ endif::[]
 :DiscoverySelectors_selectPackage:           {javadoc-root}/org.junit.platform.engine/org/junit/platform/engine/discovery/DiscoverySelectors.html#selectPackage(java.lang.String)[selectPackage]
 :DiscoverySelectors_selectUniqueId:          {javadoc-root}/org.junit.platform.engine/org/junit/platform/engine/discovery/DiscoverySelectors.html#selectUniqueId(java.lang.String)[selectUniqueId]
 :DiscoverySelectors_selectUri:               {javadoc-root}/org.junit.platform.engine/org/junit/platform/engine/discovery/DiscoverySelectors.html#selectUri(java.lang.String)[selectUri]
+:EngineDiscoveryRequest:                     {javadoc-root}/org.junit.platform.engine/org/junit/platform/engine/EngineDiscoveryRequest.html[EngineDiscoveryRequest]
 :FileSelector:                               {javadoc-root}/org.junit.platform.engine/org/junit/platform/engine/discovery/FileSelector.html[FileSelector]
 :HierarchicalTestEngine:                     {javadoc-root}/org.junit.platform.engine/org/junit/platform/engine/support/hierarchical/HierarchicalTestEngine.html[HierarchicalTestEngine]
 :IterationSelector:                          {javadoc-root}/org.junit.platform.engine/org/junit/platform/engine/discovery/IterationSelector.html[IterationSelector]
 :MethodSelector:                             {javadoc-root}/org.junit.platform.engine/org/junit/platform/engine/discovery/MethodSelector.html[MethodSelector]
 :ModuleSelector:                             {javadoc-root}/org.junit.platform.engine/org/junit/platform/engine/discovery/ModuleSelector.html[ModuleSelector]
 :NestedClassSelector:                        {javadoc-root}/org.junit.platform.engine/org/junit/platform/engine/discovery/NestedClassSelector.html[NestedClassSelector]
 :NestedMethodSelector:                       {javadoc-root}/org.junit.platform.engine/org/junit/platform/engine/discovery/NestedMethodSelector.html[NestedMethodSelector]
+:OutputDirectoryProvider:                    {javadoc-root}/org.junit.platform.engine/org/junit/platform/engine/reporting/OutputDirectoryProvider.html[OutputDirectoryProvider]
 :PackageSelector:                            {javadoc-root}/org.junit.platform.engine/org/junit/platform/engine/discovery/PackageSelector.html[PackageSelector]
 :ParallelExecutionConfigurationStrategy:     {javadoc-root}/org.junit.platform.engine/org/junit/platform/engine/support/hierarchical/ParallelExecutionConfigurationStrategy.html[ParallelExecutionConfigurationStrategy]
 :UniqueIdSelector:                           {javadoc-root}/org.junit.platform.engine/org/junit/platform/engine/discovery/UniqueIdSelector.html[UniqueIdSelector]

documentation/src/docs/asciidoc/release-notes/release-notes-5.12.0-M1.adoc

@@ -40,11 +40,16 @@ JUnit repository on GitHub.
   `--select-file` and `--select-resource`.
 * `ConsoleLauncher` now accepts multiple values for all `--select` options.
 * Add `--select-unique-id` support to ConsoleLauncher.
+* Add `getOutputDirectoryProvider()` method to `EngineDiscoveryRequest` and `TestPlan` to
+  allow test engines to publish/attach files to containers and tests by calling
+  `EngineExecutionListener.fileEntryPublished(...)`. Registered `TestExecutionListeners`
+  can then access these files by overriding the `fileEntryPublished(...)` method.
 * The following improvements have been made to the open-test-reporting XML output:
   - Information about the Git repository, the current branch, the commit hash, and the
     current worktree status are now included in the XML report, if applicable.
   - A section containing JUnit-specific metadata about each test/container to the HTML
     report is now written by open-test-reporting when added to the classpath/module path
+  - Information about published files is now included as attachments.
 
 
 [[release-notes-5.12.0-M1-junit-jupiter]]
@@ -112,6 +117,8 @@ JUnit repository on GitHub.
 * When enabled via the `junit.jupiter.execution.timeout.threaddump.enabled` configuration
   parameter, an implementation of `PreInterruptCallback` is registered that writes a
   thread dump to `System.out` prior to interrupting a test thread due to a timeout.
+* `TestReporter` now allows publishing files for a test method or test class which can be
+  used to include them in test reports, such as the Open Test Reporting format.
 
 
 [[release-notes-5.12.0-M1-junit-vintage]]

documentation/src/docs/asciidoc/user-guide/advanced-topics/junit-platform-reporting.adoc

@@ -3,41 +3,46 @@
 
 The `junit-platform-reporting` artifact contains `{TestExecutionListener}` implementations
 that generate XML test reports in two flavors:
-<<junit-platform-reporting-legacy-xml, legacy>> and
-<<junit-platform-reporting-open-test-reporting, Open Test Reporting>>.
+<<junit-platform-reporting-open-test-reporting, Open Test Reporting>> and
+<<junit-platform-reporting-legacy-xml, legacy>>.
 
 NOTE: The module also contains other `TestExecutionListener` implementations that can be
 used to build custom reporting. See <<running-tests-listeners>> for details.
 
-[[junit-platform-reporting-legacy-xml]]
-==== Legacy XML format
+[[junit-platform-reporting-output-directory]]
+==== Output Directory
 
-`{LegacyXmlReportGeneratingListener}` generates a separate XML report for each root in the
-`{TestPlan}`. Note that the generated XML format is compatible with the de facto standard
-for JUnit 4 based test reports that was made popular by the Ant build system.
+The JUnit Platform provides an `{OutputDirectoryProvider}` via
+`{EngineDiscoveryRequest}` and `{TestPlan}` to registered <<test-engines, test engines>>
+and <<running-tests-listeners, listeners>>, respectively. Its root directory can be
+configured via the following <<running-tests-config-params, configuration parameter>>:
 
-The `LegacyXmlReportGeneratingListener` is used by the <<running-tests-console-launcher>>
-as well.
+`junit.platform.reporting.output.dir=<path>`::
+  Configure the output directory for reporting. By default, `build` is used if a Gradle
+  build script is found, and `target` if a Maven POM is found; otherwise, the current
+  working directory is used.
+
+To create a unique output directory per test run, you can use the `\{uniqueNumber}`
+placeholder in the path. For example, `reports/junit-\{uniqueNumber}` will create
+directories like `reports/junit-8803697269315188212`. This can be useful when using
+Gradle's or Maven's parallel execution capabilities which create multiple JVM forks
+that run concurrently.
 
 [[junit-platform-reporting-open-test-reporting]]
-==== Open Test Reporting XML format
+==== Open Test Reporting
 
 `{OpenTestReportGeneratingListener}` writes an XML report for the entire execution in the
 event-based format specified by {OpenTestReporting} which supports all features of the
 JUnit Platform such as hierarchical test structures, display names, tags, etc.
 
 The listener is auto-registered and can be configured via the following
-<<running-tests-config-params>>:
+<<running-tests-config-params, configuration parameter>>:
 
 `junit.platform.reporting.open.xml.enabled=true|false`::
   Enable/disable writing the report.
-`junit.platform.reporting.output.dir=<path>`::
-  Configure the output directory for the reports. By default, `build` is used if a Gradle
-  build script is found, and `target` if a Maven POM is found; otherwise, the current
-  working directory is used.
 
-If enabled, the listener creates an XML report file named
-`junit-platform-events-<random-id>.xml` per test run in the configured output directory.
+If enabled, the listener creates an XML report file named `open-test-report.xml` in the
+configured <<junit-platform-reporting-output-directory, output directory>>.
 
 TIP: The {OpenTestReportingCliTool} can be used to convert from the event-based format to
 the hierarchical format which is more human-readable.
@@ -145,3 +150,13 @@ via the `--config-resource` option:
 $ java -jar junit-platform-console-standalone-{platform-version}.jar <OPTIONS> \
   --config-resource=configuration.properties
 ----
+
+[[junit-platform-reporting-legacy-xml]]
+==== Legacy XML format
+
+`{LegacyXmlReportGeneratingListener}` generates a separate XML report for each root in the
+`{TestPlan}`. Note that the generated XML format is compatible with the de facto standard
+for JUnit 4 based test reports that was made popular by the Ant build system.
+
+The `LegacyXmlReportGeneratingListener` is used by the <<running-tests-console-launcher>>
+as well.

documentation/src/docs/asciidoc/user-guide/writing-tests.adoc

@@ -1077,8 +1077,9 @@ include::{testDir}/example/TestInfoDemo.java[tags=user_guide]
 * `{TestReporterParameterResolver}`: if a constructor or method parameter is of type
   `{TestReporter}`, the `TestReporterParameterResolver` will supply an instance of
   `TestReporter`. The `TestReporter` can be used to publish additional data about the
-  current test run. The data can be consumed via the `reportingEntryPublished()` method in
-  a `{TestExecutionListener}`, allowing it to be viewed in IDEs or included in reports.
+  current test run or attach files to it. The data can be consumed in a
+  `{TestExecutionListener}` via the `reportingEntryPublished()` or `fileEntryPublished()`
+  method, respectively. This allows them to be viewed in IDEs or included in reports.
 +
 In JUnit Jupiter you should use `TestReporter` where you used to print information to
 `stdout` or `stderr` in JUnit 4. Using `@RunWith(JUnitPlatform.class)` will output all

documentation/src/test/java/example/TestReporterDemo.java

@@ -10,11 +10,16 @@
 
 package example;
 
+import static java.util.Collections.singletonList;
+
+import java.nio.file.Files;
+import java.nio.file.Path;
 import java.util.HashMap;
 import java.util.Map;
 
 import org.junit.jupiter.api.Test;
 import org.junit.jupiter.api.TestReporter;
+import org.junit.jupiter.api.io.TempDir;
 import org.junit.jupiter.api.parallel.Execution;
 import org.junit.jupiter.api.parallel.ExecutionMode;
 
@@ -41,5 +46,18 @@ void reportMultipleKeyValuePairs(TestReporter testReporter) {
 		testReporter.publishEntry(values);
 	}
 
+	@Test
+	void reportFiles(TestReporter testReporter, @TempDir Path tempDir) throws Exception {
+
+		testReporter.publishFile("test1.txt", file -> Files.write(file, singletonList("Test 1")));
+
+		Path existingFile = Files.write(tempDir.resolve("test2.txt"), singletonList("Test 2"));
+		testReporter.publishFile(existingFile);
+
+		testReporter.publishFile("test3", dir -> {
+			Path nestedFile = Files.createDirectory(dir).resolve("nested.txt");
+			Files.write(nestedFile, singletonList("Nested content"));
+		});
+	}
 }
 // end::user_guide[]

gradle/plugins/common/src/main/kotlin/junitbuild.testing-conventions.gradle.kts

@@ -5,7 +5,6 @@ import org.gradle.api.tasks.PathSensitivity.RELATIVE
 import org.gradle.api.tasks.testing.logging.TestExceptionFormat.FULL
 import org.gradle.api.tasks.testing.logging.TestLogEvent.FAILED
 import org.gradle.internal.os.OperatingSystem
-import java.nio.file.Files
 
 plugins {
 	`java-library`
@@ -31,7 +30,7 @@ val generateOpenTestHtmlReport by tasks.registering(JavaExec::class) {
 		eventXmlFiles.from(tasks.withType<Test>().map {
 			objects.fileTree()
 				.from(it.reports.junitXml.outputLocation)
-				.include("junit-platform-events-*.xml")
+				.include("junit-*/open-test-report.xml")
 		})
 		outputLocation = layout.buildDirectory.file("reports/open-test-report.html")
 	}
@@ -119,18 +118,20 @@ tasks.withType<Test>().configureEach {
 	jvmArgumentProviders += CommandLineArgumentProvider {
 		listOf(
 			"-Djunit.platform.reporting.open.xml.enabled=true",
-			"-Djunit.platform.reporting.output.dir=${reports.junitXml.outputLocation.get().asFile.absolutePath}"
+			"-Djunit.platform.reporting.output.dir=${reports.junitXml.outputLocation.get().asFile.absolutePath}/junit-{uniqueNumber}",
 		)
 	}
 
 	jvmArgumentProviders += objects.newInstance(JavaAgentArgumentProvider::class).apply {
 		classpath.from(javaAgentClasspath)
 	}
 
-	val reportFiles = objects.fileTree().from(reports.junitXml.outputLocation).matching { include("junit-platform-events-*.xml") }
+	val reportDirTree = objects.fileTree().from(reports.junitXml.outputLocation)
 	doFirst {
-		reportFiles.files.forEach {
-			Files.delete(it.toPath())
+		reportDirTree.visit {
+			if (name.startsWith("junit-")) {
+				file.deleteRecursively()
+			}
 		}
 	}
 

junit-jupiter-api/src/main/java/org/junit/jupiter/api/TestReporter.java

@@ -10,12 +10,17 @@
 
 package org.junit.jupiter.api;
 
+import static java.nio.file.StandardCopyOption.REPLACE_EXISTING;
+import static org.apiguardian.api.API.Status.EXPERIMENTAL;
 import static org.apiguardian.api.API.Status.STABLE;
 
+import java.nio.file.Files;
+import java.nio.file.Path;
 import java.util.Collections;
 import java.util.Map;
 
 import org.apiguardian.api.API;
+import org.junit.jupiter.api.function.ThrowingConsumer;
 
 /**
  * Parameters of type {@code TestReporter} can be injected into
@@ -77,4 +82,36 @@ default void publishEntry(String value) {
 		this.publishEntry("value", value);
 	}
 
+	/**
+	 * Publish the supplied file and attach it to the current test or container.
+	 * <p>
+	 * The file will be copied to the report output directory replacing any
+	 * potentially existing file with the same name.
+	 *
+	 * @param file the file to be attached; never {@code null} or blank
+	 * @since 5.12
+	 */
+	@API(status = EXPERIMENTAL, since = "5.12")
+	default void publishFile(Path file) {
+		publishFile(file.getFileName().toString(), path -> Files.copy(file, path, REPLACE_EXISTING));
+	}
+
+	/**
+	 * Publish a file with the supplied name written by the supplied action and
+	 * attach it to the current test or container.
+	 * <p>
+	 * The {@link Path} passed to the supplied action will be relative to the
+	 * report output directory, but it's up to the action to write the file or
+	 * directory.
+	 *
+	 * @param fileName the name of the file to be attached; never {@code null} or blank
+	 *                 and must not contain any path separators
+	 * @param action   the action to be executed to write the file; never {@code null}
+	 * @since 5.12
+	 */
+	@API(status = EXPERIMENTAL, since = "5.12")
+	default void publishFile(String fileName, ThrowingConsumer<Path> action) {
+		throw new UnsupportedOperationException();
+	}
+
 }

junit-jupiter-api/src/main/java/org/junit/jupiter/api/extension/ExtensionContext.java

@@ -10,10 +10,12 @@
 
 package org.junit.jupiter.api.extension;
 
+import static org.apiguardian.api.API.Status.EXPERIMENTAL;
 import static org.apiguardian.api.API.Status.STABLE;
 
 import java.lang.reflect.AnnotatedElement;
 import java.lang.reflect.Method;
+import java.nio.file.Path;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Collections;
@@ -25,6 +27,7 @@
 
 import org.apiguardian.api.API;
 import org.junit.jupiter.api.TestInstance.Lifecycle;
+import org.junit.jupiter.api.function.ThrowingConsumer;
 import org.junit.jupiter.api.parallel.ExecutionMode;
 import org.junit.platform.commons.PreconditionViolationException;
 import org.junit.platform.commons.support.ReflectionSupport;
@@ -364,6 +367,22 @@ default void publishReportEntry(String value) {
 		this.publishReportEntry("value", value);
 	}
 
+	/**
+	 * Publish a file with the supplied name written by the supplied action and
+	 * attach it to the current test or container.
+	 * <p>
+	 * The file will be resolved in the report output directory prior to
+	 * invoking the supplied action.
+	 *
+	 * @param fileName the name of the file to be attached; never {@code null} or blank
+	 *                 and must not contain any path separators
+	 * @param action   the action to be executed to write the file; never {@code null}
+	 * @since 5.12
+	 * @see org.junit.platform.engine.EngineExecutionListener#fileEntryPublished
+	 */
+	@API(status = EXPERIMENTAL, since = "5.12")
+	void publishFile(String fileName, ThrowingConsumer<Path> action);
+
 	/**
 	 * Get the {@link Store} for the supplied {@link Namespace}.
 	 *

junit-jupiter-engine/src/main/java/org/junit/jupiter/engine/JupiterTestEngine.java

@@ -63,8 +63,8 @@ public Optional<String> getArtifactId() {
 
 	@Override
 	public TestDescriptor discover(EngineDiscoveryRequest discoveryRequest, UniqueId uniqueId) {
-		JupiterConfiguration configuration = new CachingJupiterConfiguration(
-			new DefaultJupiterConfiguration(discoveryRequest.getConfigurationParameters()));
+		JupiterConfiguration configuration = new CachingJupiterConfiguration(new DefaultJupiterConfiguration(
+			discoveryRequest.getConfigurationParameters(), discoveryRequest.getOutputDirectoryProvider()));
 		JupiterEngineDescriptor engineDescriptor = new JupiterEngineDescriptor(uniqueId, configuration);
 		new DiscoverySelectorResolver().resolveSelectors(discoveryRequest, engineDescriptor);
 		return engineDescriptor;