Introduce discovery listeners and abort on failures by default

The `Launcher` now propagates errors during test discovery by default
instead of only logging and thereby potentially hiding them. In order
to restore the old, lenient behavior, the
`junit.platform.discovery.listener.default` configuration parameter can
be set to `logging`.

To support the above feature consistently, a new
`EngineDiscoveryListener` interface was introduced. `TestEngine`
implementations should now notify the listener that can be accessed via
the `EngineDiscoveryRequest.getDiscoveryListener()` method about each
processed `DiscoverySelector`. Test engines that use
`EngineDiscoveryRequestResolver` do not have to make any changes.

In addition, clients of the `Launcher` such as build tools and IDEs can
add `LauncherDiscoveryListeners` to `LauncherDiscoveryRequests` which
will be notified about the above events and the start and finish of each
engine's test discovery.

Resolves #2052.

Author: marcphilipp

documentation/src/docs/asciidoc/link-attributes.adoc

@@ -27,6 +27,8 @@ endif::[]
 // Platform Launcher API
 :junit-platform-launcher:                    {javadoc-root}/org/junit/platform/launcher/package-summary.html[junit-platform-launcher]
 :Launcher:                                   {javadoc-root}/org/junit/platform/launcher/Launcher.html[Launcher]
+:LauncherDiscoveryListener:                  {javadoc-root}/org/junit/platform/launcher/LauncherDiscoveryListener.html[LauncherDiscoveryListener]
+:LauncherDiscoveryRequestBuilder:            {javadoc-root}/org/junit/platform/launcher/core/LauncherDiscoveryRequestBuilder.html[LauncherDiscoveryRequestBuilder]
 :LoggingListener:                            {javadoc-root}/org/junit/platform/launcher/listeners/LoggingListener.html[LoggingListener]
 :SummaryGeneratingListener:                  {javadoc-root}/org/junit/platform/launcher/listeners/SummaryGeneratingListener.html[SummaryGeneratingListener]
 :TestExecutionListener:                      {javadoc-root}/org/junit/platform/launcher/TestExecutionListener.html[TestExecutionListener]

documentation/src/docs/asciidoc/release-notes/release-notes-5.6.0-M1.adoc

@@ -35,6 +35,15 @@ on GitHub.
 
 ==== Deprecations and Breaking Changes
 
+* The `Launcher` now propagates errors during test discovery by default instead of only
+  logging and thereby potentially hiding them. In order to restore the old, lenient
+  behavior, you can set the `junit.platform.discovery.listener.default` configuration
+  parameter to `logging`.
+* To support the above feature consistently, a new `EngineDiscoveryListener` interface was
+  introduced. `TestEngine` implementations should now notify the listener that can be
+  accessed via the `EngineDiscoveryRequest.getDiscoveryListener()` method about each
+  processed `DiscoverySelector`. Test engines that use `EngineDiscoveryRequestResolver` do
+  not have to make any changes.
 * In the `EngineTestKit` API, the `all()`, `containers()`, and `tests()` methods in
   `EngineExecutionResults` have been deprecated in favor of the new `allEvents()`,
   `containerEvents()`, and `testEvents()` methods, respectively. The deprecated methods

documentation/src/docs/asciidoc/user-guide/launcher-api.adoc

@@ -45,6 +45,13 @@ traverse the tree, retrieve details about a node, and get a link to the original
 (like class, method, or file position). Every node in the test plan has a _unique ID_
 that can be used to invoke a particular test or group of tests.
 
+Clients can register one or more `{LauncherDiscoveryListener}` implementations to get
+insights into events that occur during test discovery via the
+`{LauncherDiscoveryRequestBuilder}`. The builder registers a default listener that can be
+changed via the `junit.platform.discovery.listener.default` configuration parameter. If
+the parameter is not set, test discovery will be aborted after the first failure is
+encountered.
+
 [[launcher-api-execution]]
 ==== Executing Tests
 

junit-jupiter-engine/src/test/java/org/junit/jupiter/engine/discovery/DiscoverySelectorResolverTests.java

@@ -25,6 +25,9 @@
 import static org.junit.jupiter.engine.discovery.JupiterUniqueIdBuilder.uniqueIdForTestTemplateMethod;
 import static org.junit.jupiter.engine.discovery.JupiterUniqueIdBuilder.uniqueIdForTopLevelClass;
 import static org.junit.platform.commons.util.CollectionUtils.getOnlyElement;
+import static org.junit.platform.engine.SelectorResolutionResult.Status.FAILED;
+import static org.junit.platform.engine.SelectorResolutionResult.Status.RESOLVED;
+import static org.junit.platform.engine.SelectorResolutionResult.Status.UNRESOLVED;
 import static org.junit.platform.engine.discovery.ClassNameFilter.includeClassNamePatterns;
 import static org.junit.platform.engine.discovery.DiscoverySelectors.selectClass;
 import static org.junit.platform.engine.discovery.DiscoverySelectors.selectClasspathRoots;
@@ -33,8 +36,11 @@
 import static org.junit.platform.engine.discovery.DiscoverySelectors.selectUniqueId;
 import static org.junit.platform.engine.discovery.PackageNameFilter.excludePackageNames;
 import static org.junit.platform.engine.discovery.PackageNameFilter.includePackageNames;
-import static org.junit.platform.launcher.core.LauncherDiscoveryRequestBuilder.request;
+import static org.junit.platform.launcher.core.LauncherDiscoveryRequestBuilder.DEFAULT_DISCOVERY_LISTENER_CONFIGURATION_PROPERTY_NAME;
+import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.ArgumentMatchers.eq;
 import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.verify;
 import static org.mockito.Mockito.when;
 
 import java.lang.reflect.Method;
@@ -44,8 +50,7 @@
 import java.nio.file.Paths;
 import java.util.ArrayList;
 import java.util.List;
-import java.util.logging.Level;
-import java.util.logging.LogRecord;
+import java.util.function.Predicate;
 import java.util.stream.Stream;
 
 import org.junit.jupiter.api.BeforeEach;
@@ -55,7 +60,6 @@
 import org.junit.jupiter.api.Test;
 import org.junit.jupiter.api.TestFactory;
 import org.junit.jupiter.api.TestTemplate;
-import org.junit.jupiter.api.fixtures.TrackLogRecords;
 import org.junit.jupiter.api.parallel.ExecutionMode;
 import org.junit.jupiter.engine.config.JupiterConfiguration;
 import org.junit.jupiter.engine.descriptor.DynamicDescendantFilter;
@@ -68,24 +72,27 @@
 import org.junit.jupiter.engine.descriptor.subpackage.ClassWithStaticInnerTestCases;
 import org.junit.platform.commons.JUnitException;
 import org.junit.platform.commons.PreconditionViolationException;
-import org.junit.platform.commons.logging.LogRecordListener;
 import org.junit.platform.commons.util.ReflectionUtils;
+import org.junit.platform.engine.DiscoverySelector;
+import org.junit.platform.engine.SelectorResolutionResult;
 import org.junit.platform.engine.TestDescriptor;
 import org.junit.platform.engine.UniqueId;
 import org.junit.platform.engine.discovery.ClassSelector;
 import org.junit.platform.engine.discovery.ClasspathRootSelector;
 import org.junit.platform.engine.discovery.MethodSelector;
 import org.junit.platform.engine.discovery.PackageSelector;
 import org.junit.platform.engine.discovery.UniqueIdSelector;
-import org.junit.platform.engine.support.discovery.EngineDiscoveryRequestResolver;
+import org.junit.platform.launcher.LauncherDiscoveryListener;
 import org.junit.platform.launcher.core.LauncherDiscoveryRequestBuilder;
+import org.mockito.ArgumentCaptor;
 
 /**
  * @since 5.0
  */
 class DiscoverySelectorResolverTests {
 
 	private final JupiterConfiguration configuration = mock(JupiterConfiguration.class);
+	private final LauncherDiscoveryListener discoveryListener = mock(LauncherDiscoveryListener.class);
 	private final JupiterEngineDescriptor engineDescriptor = new JupiterEngineDescriptor(engineId(), configuration);
 
 	@BeforeEach
@@ -102,14 +109,11 @@ void nonTestClassResolution() {
 	}
 
 	@Test
-	@TrackLogRecords
-	void abstractClassResolution(LogRecordListener listener) {
+	void abstractClassResolution() {
 		resolve(request().selectors(selectClass(AbstractTestClass.class)));
 
 		assertTrue(engineDescriptor.getDescendants().isEmpty());
-		assertThat(firstDebugLogRecord(listener).getMessage())//
-				.isEqualTo(
-					"ClassSelector [className = '" + AbstractTestClass.class.getName() + "'] could not be resolved.");
+		assertUnresolved(selectClass(AbstractTestClass.class));
 	}
 
 	@Test
@@ -123,15 +127,15 @@ void singleClassResolution() {
 	}
 
 	@Test
-	@TrackLogRecords
-	void classResolutionForNonexistentClass(LogRecordListener listener) {
+	void classResolutionForNonexistentClass() {
 		ClassSelector selector = selectClass("org.example.DoesNotExist");
 
 		resolve(request().selectors(selector));
 
 		assertTrue(engineDescriptor.getDescendants().isEmpty());
-		assertThat(firstDebugLogRecord(listener).getMessage())//
-				.isEqualTo("ClassSelector [className = 'org.example.DoesNotExist'] could not be resolved.");
+		var result = verifySelectorProcessed(selector);
+		assertThat(result.getStatus()).isEqualTo(FAILED);
+		assertThat(result.getThrowable().get()).hasMessageContaining("Could not load class with name");
 	}
 
 	@Test
@@ -216,35 +220,31 @@ void resolvingSelectorOfNonTestMethodResolvesNothing() throws NoSuchMethodExcept
 	}
 
 	@Test
-	@TrackLogRecords
-	void methodResolutionForNonexistentClass(LogRecordListener listener) {
+	void methodResolutionForNonexistentClass() {
 		String className = "org.example.DoesNotExist";
 		String methodName = "bogus";
 		MethodSelector selector = selectMethod(className, methodName, "");
 
 		resolve(request().selectors(selector));
 
 		assertTrue(engineDescriptor.getDescendants().isEmpty());
-		LogRecord logRecord = firstDebugLogRecord(listener);
-		assertThat(logRecord.getMessage())//
-				.startsWith("MethodSelector").endsWith("could not be resolved.")//
-				.contains(className, methodName);
-		assertThat(logRecord.getThrown())//
+		var result = verifySelectorProcessed(selector);
+		assertThat(result.getStatus()).isEqualTo(FAILED);
+		assertThat(result.getThrowable().get())//
 				.isInstanceOf(PreconditionViolationException.class)//
 				.hasMessageStartingWith("Could not load class with name: " + className);
 	}
 
 	@Test
-	@TrackLogRecords
-	void methodResolutionForNonexistentMethod(LogRecordListener listener) {
+	void methodResolutionForNonexistentMethod() {
 		MethodSelector selector = selectMethod(MyTestClass.class, "bogus", "");
 
 		resolve(request().selectors(selector));
 
 		assertTrue(engineDescriptor.getDescendants().isEmpty());
-		assertThat(firstDebugLogRecord(listener).getMessage())//
-				.startsWith("MethodSelector").endsWith("could not be resolved.")//
-				.contains(MyTestClass.class.getName(), "bogus");
+		var result = verifySelectorProcessed(selector);
+		assertThat(result.getStatus()).isEqualTo(FAILED);
+		assertThat(result.getThrowable().get()).hasMessageContaining("Could not find method");
 	}
 
 	@Test
@@ -285,16 +285,14 @@ void methodOfInnerClassByUniqueId() {
 	}
 
 	@Test
-	@TrackLogRecords
-	void resolvingUniqueIdWithUnknownSegmentTypeResolvesNothing(LogRecordListener listener) {
+	void resolvingUniqueIdWithUnknownSegmentTypeResolvesNothing() {
 		UniqueId uniqueId = engineId().append("bogus", "enigma");
 		UniqueIdSelector selector = selectUniqueId(uniqueId);
 
 		resolve(request().selectors(selector));
 
 		assertTrue(engineDescriptor.getDescendants().isEmpty());
-		assertThat(firstWarningLogRecord(listener).getMessage()) //
-				.isEqualTo("UniqueIdSelector [uniqueId = " + uniqueId + "] could not be resolved.");
+		assertUnresolved(selector);
 	}
 
 	@Test
@@ -304,52 +302,47 @@ void resolvingUniqueIdOfNonTestMethodResolvesNothing() {
 		resolve(request().selectors(selector));
 
 		assertThat(engineDescriptor.getDescendants()).isEmpty();
+		assertUnresolved(selector);
 	}
 
 	@Test
-	@TrackLogRecords
-	void methodResolutionByUniqueIdWithMissingMethodName(LogRecordListener listener) {
+	void methodResolutionByUniqueIdWithMissingMethodName() {
 		UniqueId uniqueId = uniqueIdForMethod(getClass(), "()");
 
 		resolve(request().selectors(selectUniqueId(uniqueId)));
 
 		assertTrue(engineDescriptor.getDescendants().isEmpty());
-		LogRecord logRecord = firstWarningLogRecord(listener);
-		assertThat(logRecord.getMessage()).isEqualTo(
-			"UniqueIdSelector [uniqueId = " + uniqueId + "] could not be resolved.");
-		assertThat(logRecord.getThrown())//
+		var result = verifySelectorProcessed(selectUniqueId(uniqueId));
+		assertThat(result.getStatus()).isEqualTo(FAILED);
+		assertThat(result.getThrowable().get())//
 				.isInstanceOf(PreconditionViolationException.class)//
 				.hasMessageStartingWith("Method [()] does not match pattern");
 	}
 
 	@Test
-	@TrackLogRecords
-	void methodResolutionByUniqueIdWithMissingParameters(LogRecordListener listener) {
+	void methodResolutionByUniqueIdWithMissingParameters() {
 		UniqueId uniqueId = uniqueIdForMethod(getClass(), "methodName");
 
 		resolve(request().selectors(selectUniqueId(uniqueId)));
 
 		assertThat(engineDescriptor.getDescendants()).isEmpty();
-		LogRecord logRecord = firstWarningLogRecord(listener);
-		assertThat(logRecord.getMessage()).isEqualTo(
-			"UniqueIdSelector [uniqueId = " + uniqueId + "] could not be resolved.");
-		assertThat(logRecord.getThrown())//
+		var result = verifySelectorProcessed(selectUniqueId(uniqueId));
+		assertThat(result.getStatus()).isEqualTo(FAILED);
+		assertThat(result.getThrowable().get())//
 				.isInstanceOf(PreconditionViolationException.class)//
 				.hasMessageStartingWith("Method [methodName] does not match pattern");
 	}
 
 	@Test
-	@TrackLogRecords
-	void methodResolutionByUniqueIdWithBogusParameters(LogRecordListener listener) {
+	void methodResolutionByUniqueIdWithBogusParameters() {
 		UniqueId uniqueId = uniqueIdForMethod(getClass(), "methodName(java.lang.String, junit.foo.Enigma)");
 
 		resolve(request().selectors(selectUniqueId(uniqueId)));
 
 		assertTrue(engineDescriptor.getDescendants().isEmpty());
-		LogRecord logRecord = firstWarningLogRecord(listener);
-		assertThat(logRecord.getMessage()).isEqualTo(
-			"UniqueIdSelector [uniqueId = " + uniqueId + "] could not be resolved.");
-		assertThat(logRecord.getThrown())//
+		var result = verifySelectorProcessed(selectUniqueId(uniqueId));
+		assertThat(result.getStatus()).isEqualTo(FAILED);
+		assertThat(result.getThrowable().get())//
 				.isInstanceOf(JUnitException.class)//
 				.hasMessage("Failed to load parameter type [%s] for method [%s] in class [%s].", "junit.foo.Enigma",
 					"methodName", getClass().getName());
@@ -381,8 +374,7 @@ void methodResolutionByUniqueIdFromInheritedClass() {
 	}
 
 	@Test
-	@TrackLogRecords
-	void methodResolutionByUniqueIdWithParams(LogRecordListener listener) {
+	void methodResolutionByUniqueIdWithParams() {
 		UniqueIdSelector selector = selectUniqueId(
 			uniqueIdForMethod(HerTestClass.class, "test7(java.lang.String)").toString());
 
@@ -396,15 +388,13 @@ void methodResolutionByUniqueIdWithParams(LogRecordListener listener) {
 	}
 
 	@Test
-	@TrackLogRecords
-	void resolvingUniqueIdWithWrongParamsResolvesNothing(LogRecordListener listener) {
+	void resolvingUniqueIdWithWrongParamsResolvesNothing() {
 		UniqueId uniqueId = uniqueIdForMethod(HerTestClass.class, "test7(java.math.BigDecimal)");
 
 		resolve(request().selectors(selectUniqueId(uniqueId)));
 
 		assertTrue(engineDescriptor.getDescendants().isEmpty());
-		assertThat(firstWarningLogRecord(listener).getMessage())//
-				.isEqualTo("UniqueIdSelector [uniqueId = " + uniqueId + "] could not be resolved.");
+		assertUnresolved(selectUniqueId(uniqueId));
 	}
 
 	@Test
@@ -633,8 +623,7 @@ void testTemplateMethodResolutionByUniqueId() {
 	}
 
 	@Test
-	@TrackLogRecords
-	void resolvingDynamicTestByUniqueIdResolvesUpToParentTestFactory(LogRecordListener listener) {
+	void resolvingDynamicTestByUniqueIdResolvesUpToParentTestFactory() {
 		Class<?> clazz = MyTestClass.class;
 		UniqueId factoryUid = uniqueIdForTestFactoryMethod(clazz, "dynamicTest()");
 		UniqueId dynamicTestUid = factoryUid.append(DYNAMIC_TEST_SEGMENT_TYPE, "#1");
@@ -651,12 +640,11 @@ void resolvingDynamicTestByUniqueIdResolvesUpToParentTestFactory(LogRecordListen
 		assertThat(dynamicDescendantFilter.test(dynamicTestUid)).isTrue();
 		assertThat(dynamicDescendantFilter.test(differentDynamicTestUid)).isFalse();
 
-		assertZeroLogRecords(listener);
+		assertAllSelectorsResolved();
 	}
 
 	@Test
-	@TrackLogRecords
-	void resolvingDynamicContainerByUniqueIdResolvesUpToParentTestFactory(LogRecordListener listener) {
+	void resolvingDynamicContainerByUniqueIdResolvesUpToParentTestFactory() {
 		Class<?> clazz = MyTestClass.class;
 		UniqueId factoryUid = uniqueIdForTestFactoryMethod(clazz, "dynamicTest()");
 		UniqueId dynamicContainerUid = factoryUid.append(DYNAMIC_CONTAINER_SEGMENT_TYPE, "#1");
@@ -676,7 +664,7 @@ void resolvingDynamicContainerByUniqueIdResolvesUpToParentTestFactory(LogRecordL
 		assertThat(dynamicDescendantFilter.test(differentDynamicContainerUid)).isFalse();
 		assertThat(dynamicDescendantFilter.test(differentDynamicTestUid)).isFalse();
 
-		assertZeroLogRecords(listener);
+		assertAllSelectorsResolved();
 	}
 
 	@Test
@@ -765,18 +753,31 @@ private List<UniqueId> uniqueIds() {
 		return engineDescriptor.getDescendants().stream().map(TestDescriptor::getUniqueId).collect(toList());
 	}
 
-	private void assertZeroLogRecords(LogRecordListener listener) {
-		assertThat(listener.stream(EngineDiscoveryRequestResolver.class)).isEmpty();
+	private LauncherDiscoveryRequestBuilder request() {
+		return LauncherDiscoveryRequestBuilder.request() //
+				.configurationParameter(DEFAULT_DISCOVERY_LISTENER_CONFIGURATION_PROPERTY_NAME, "logging") //
+				.listeners(discoveryListener);
 	}
 
-	private LogRecord firstWarningLogRecord(LogRecordListener listener) throws AssertionError {
-		return listener.stream(EngineDiscoveryRequestResolver.class, Level.WARNING).findFirst().orElseThrow(
-			() -> new AssertionError("Failed to find warning log record"));
+	private void assertAllSelectorsResolved() {
+		ArgumentCaptor<SelectorResolutionResult> resultCaptor = ArgumentCaptor.forClass(SelectorResolutionResult.class);
+		verify(discoveryListener).selectorProcessed(eq(UniqueId.forEngine("junit-jupiter")), any(),
+			resultCaptor.capture());
+		assertThat(resultCaptor.getAllValues()) //
+				.flatExtracting(SelectorResolutionResult::getStatus) //
+				.allMatch(Predicate.isEqual(RESOLVED));
 	}
 
-	private LogRecord firstDebugLogRecord(LogRecordListener listener) throws AssertionError {
-		return listener.stream(EngineDiscoveryRequestResolver.class, Level.FINE).findFirst().orElseThrow(
-			() -> new AssertionError("Failed to find debug log record"));
+	private void assertUnresolved(DiscoverySelector selector) {
+		var result = verifySelectorProcessed(selector);
+		assertThat(result.getStatus()).isEqualTo(UNRESOLVED);
+	}
+
+	private SelectorResolutionResult verifySelectorProcessed(DiscoverySelector selector) {
+		ArgumentCaptor<SelectorResolutionResult> resultCaptor = ArgumentCaptor.forClass(SelectorResolutionResult.class);
+		verify(discoveryListener).selectorProcessed(eq(UniqueId.forEngine("junit-jupiter")), eq(selector),
+			resultCaptor.capture());
+		return resultCaptor.getValue();
 	}
 
 }

junit-jupiter-engine/src/test/resources/log4j2-test.xml

@@ -9,7 +9,7 @@
 		<Logger name="org.junit" level="warn" />
 		<Logger name="org.junit.jupiter.api" level="off" />
 		<Logger name="org.junit.jupiter.engine" level="off" />
-		<Logger name="org.junit.platform.engine.support.discovery" level="off" />
+		<Logger name="org.junit.platform.launcher.listeners.discovery.LoggingLauncherDiscoveryListener" level="off" />
 		<Root level="error">
 			<AppenderRef ref="Console" />
 		</Root>