Avoid reporting potentially misleading suppressed validation exception

BassemElMasry · marcphilipp · web-flow · commit c1e32b5d5ec4 · 2025-04-28T14:17:42.000+02:00
In case of earlier failures, the validation exception is no longer reported as suppressed. Resolves #3054. --------- Co-authored-by: Marc Philipp <mail@marcphilipp.de>
diff --git a/documentation/src/docs/asciidoc/release-notes/release-notes-5.13.0-M3.adoc b/documentation/src/docs/asciidoc/release-notes/release-notes-5.13.0-M3.adoc
@@ -105,11 +105,13 @@ to start reporting discovery issues.
   via a configuration parameter. Please also see the
   <<../user-guide/index.adoc#extensions-keeping-state-autocloseable-migration, migration note>>
   for third-party extensions wanting to support both JUnit 5.13 and earlier versions.
-
 * `java.util.Locale` arguments are now converted according to the IETF BCP 47 language tag
   format. See the
   <<../user-guide/index.adoc#writing-tests-parameterized-tests-argument-conversion-implicit, User Guide>>
   for details.
+* Avoid reporting potentially misleading validation exception for `@ParameterizedClass`
+  test classes and `@ParameterizedTest` methods as suppressed exception for earlier
+  failures.
 
 [[release-notes-5.13.0-M3-junit-vintage]]
 === JUnit Vintage
diff --git a/junit-jupiter-api/src/main/java/org/junit/jupiter/api/extension/ClassTemplateInvocationContextProvider.java b/junit-jupiter-api/src/main/java/org/junit/jupiter/api/extension/ClassTemplateInvocationContextProvider.java
@@ -90,6 +90,8 @@ public interface ClassTemplateInvocationContextProvider extends Extension {
 	 * invoked; never {@code null}
 	 * @return a {@code Stream} of {@code ClassTemplateInvocationContext}
 	 * instances for the invocation of the class template; never {@code null}
+	 * @throws TemplateInvocationValidationException if a validation fails when
+	 * while providing or closing the {@link java.util.stream.Stream}.
 	 * @see #supportsClassTemplate
 	 * @see ExtensionContext
 	 */
diff --git a/junit-jupiter-api/src/main/java/org/junit/jupiter/api/extension/TemplateInvocationValidationException.java b/junit-jupiter-api/src/main/java/org/junit/jupiter/api/extension/TemplateInvocationValidationException.java
@@ -0,0 +1,35 @@
+/*
+ * Copyright 2015-2025 the original author or authors.
+ *
+ * All rights reserved. This program and the accompanying materials are
+ * made available under the terms of the Eclipse Public License v2.0 which
+ * accompanies this distribution and is available at
+ *
+ * https://www.eclipse.org/legal/epl-v20.html
+ */
+
+package org.junit.jupiter.api.extension;
+
+import static org.apiguardian.api.API.Status.EXPERIMENTAL;
+
+import org.apiguardian.api.API;
+import org.junit.platform.commons.JUnitException;
+
+/**
+ * {@code TemplateInvocationValidationException} is an exception thrown by a
+ * {@link TestTemplateInvocationContextProvider} or
+ * {@link ClassTemplateInvocationContextProvider} if a validation fails when
+ * while providing or closing {@link java.util.stream.Stream} of invocation
+ * contexts.
+ *
+ * @since 5.13
+ */
+@API(status = EXPERIMENTAL, since = "5.13")
+public class TemplateInvocationValidationException extends JUnitException {
+
+	private static final long serialVersionUID = 1L;
+
+	public TemplateInvocationValidationException(String message) {
+		super(message);
+	}
+}
diff --git a/junit-jupiter-api/src/main/java/org/junit/jupiter/api/extension/TestTemplateInvocationContextProvider.java b/junit-jupiter-api/src/main/java/org/junit/jupiter/api/extension/TestTemplateInvocationContextProvider.java
@@ -86,6 +86,8 @@ public interface TestTemplateInvocationContextProvider extends Extension {
 	 * to be invoked; never {@code null}
 	 * @return a {@code Stream} of {@code TestTemplateInvocationContext}
 	 * instances for the invocation of the test template method; never {@code null}
+	 * @throws TemplateInvocationValidationException if a validation fails when
+	 * while providing or closing the {@link java.util.stream.Stream}.
 	 * @see #supportsTestTemplate
 	 * @see ExtensionContext
 	 */
diff --git a/junit-jupiter-engine/src/main/java/org/junit/jupiter/engine/descriptor/TemplateExecutor.java b/junit-jupiter-engine/src/main/java/org/junit/jupiter/engine/descriptor/TemplateExecutor.java
@@ -19,8 +19,10 @@
 
 import org.junit.jupiter.api.extension.Extension;
 import org.junit.jupiter.api.extension.ExtensionContext;
+import org.junit.jupiter.api.extension.TemplateInvocationValidationException;
 import org.junit.jupiter.engine.execution.JupiterEngineExecutionContext;
 import org.junit.jupiter.engine.extension.ExtensionRegistry;
+import org.junit.platform.commons.util.ExceptionUtils;
 import org.junit.platform.commons.util.Preconditions;
 import org.junit.platform.engine.TestDescriptor;
 import org.junit.platform.engine.UniqueId;
@@ -52,11 +54,24 @@ private void executeForProvider(P provider, AtomicInteger invocationIndex,
 
 		int initialValue = invocationIndex.get();
 
-		try (Stream<? extends C> stream = provideContexts(provider, extensionContext)) {
+		Stream<? extends C> stream = provideContexts(provider, extensionContext);
+		try {
 			stream.forEach(invocationContext -> createInvocationTestDescriptor(invocationContext,
 				invocationIndex.incrementAndGet()) //
 						.ifPresent(testDescriptor -> execute(dynamicTestExecutor, testDescriptor)));
 		}
+		catch (Throwable t) {
+			try {
+				stream.close();
+			}
+			catch (TemplateInvocationValidationException ignore) {
+				// ignore exceptions from close() to avoid masking the original failure
+			}
+			throw ExceptionUtils.throwAsUncheckedException(t);
+		}
+		finally {
+			stream.close();
+		}
 
 		Preconditions.condition(
 			invocationIndex.get() != initialValue || mayReturnZeroContexts(provider, extensionContext),
diff --git a/junit-jupiter-params/src/main/java/org/junit/jupiter/params/ParameterizedInvocationContextProvider.java b/junit-jupiter-params/src/main/java/org/junit/jupiter/params/ParameterizedInvocationContextProvider.java
@@ -17,6 +17,7 @@
 import java.util.stream.Stream;
 
 import org.junit.jupiter.api.extension.ExtensionContext;
+import org.junit.jupiter.api.extension.TemplateInvocationValidationException;
 import org.junit.jupiter.params.provider.Arguments;
 import org.junit.jupiter.params.provider.ArgumentsProvider;
 import org.junit.jupiter.params.provider.ArgumentsSource;
@@ -47,12 +48,20 @@ protected Stream<T> provideInvocationContexts(ExtensionContext extensionContext,
 					invocationCount.incrementAndGet();
 					return declarationContext.createInvocationContext(formatter, arguments, invocationCount.intValue());
 				})
-				.onClose(() ->
-						Preconditions.condition(invocationCount.get() > 0 || declarationContext.isAllowingZeroInvocations(),
-								() -> String.format("Configuration error: You must configure at least one set of arguments for this @%s", declarationContext.getAnnotationName())));
+				.onClose(() -> validateInvokedAtLeastOnce(invocationCount.get(),declarationContext ));
 		// @formatter:on
 	}
 
+	private static <T> void validateInvokedAtLeastOnce(long invocationCount,
+			ParameterizedDeclarationContext<T> declarationContext) {
+		if (invocationCount == 0 && !declarationContext.isAllowingZeroInvocations()) {
+			String message = String.format(
+				"Configuration error: You must configure at least one set of arguments for this @%s",
+				declarationContext.getAnnotationName());
+			throw new TemplateInvocationValidationException(message);
+		}
+	}
+
 	private static List<ArgumentsSource> collectArgumentSources(ParameterizedDeclarationContext<?> declarationContext) {
 		List<ArgumentsSource> argumentsSources = findRepeatableAnnotations(declarationContext.getAnnotatedElement(),
 			ArgumentsSource.class);
diff --git a/jupiter-tests/src/test/java/org/junit/jupiter/params/ParameterizedClassIntegrationTests.java b/jupiter-tests/src/test/java/org/junit/jupiter/params/ParameterizedClassIntegrationTests.java
@@ -40,6 +40,7 @@
 import static org.junit.platform.testkit.engine.EventConditions.started;
 import static org.junit.platform.testkit.engine.EventConditions.test;
 import static org.junit.platform.testkit.engine.EventConditions.uniqueId;
+import static org.junit.platform.testkit.engine.TestExecutionResultConditions.instanceOf;
 import static org.junit.platform.testkit.engine.TestExecutionResultConditions.message;
 import static org.junit.platform.testkit.engine.TestExecutionResultConditions.suppressed;
 
@@ -69,6 +70,7 @@
 import org.junit.jupiter.api.extension.AnnotatedElementContext;
 import org.junit.jupiter.api.extension.ExtensionContext;
 import org.junit.jupiter.api.extension.ParameterContext;
+import org.junit.jupiter.api.extension.TemplateInvocationValidationException;
 import org.junit.jupiter.engine.AbstractJupiterTestEngineTests;
 import org.junit.jupiter.engine.Constants;
 import org.junit.jupiter.engine.descriptor.ClassTemplateInvocationTestDescriptor;
@@ -368,8 +370,9 @@ void failsWhenInvocationIsRequiredButNoArgumentSetsAreProvided() {
 			var results = executeTestsForClass(ForbiddenZeroInvocationsTestCase.class);
 
 			results.containerEvents().assertThatEvents() //
-					.haveExactly(1, event(finishedWithFailure(message(
-						"Configuration error: You must configure at least one set of arguments for this @ParameterizedClass"))));
+					.haveExactly(1,
+						event(finishedWithFailure(instanceOf(TemplateInvocationValidationException.class), message(
+							"Configuration error: You must configure at least one set of arguments for this @ParameterizedClass"))));
 		}
 
 		@Test
diff --git a/jupiter-tests/src/test/java/org/junit/jupiter/params/ParameterizedTestExtensionTests.java b/jupiter-tests/src/test/java/org/junit/jupiter/params/ParameterizedTestExtensionTests.java
@@ -36,6 +36,7 @@
 import org.junit.jupiter.api.extension.ExecutableInvoker;
 import org.junit.jupiter.api.extension.ExtensionContext;
 import org.junit.jupiter.api.extension.MediaType;
+import org.junit.jupiter.api.extension.TemplateInvocationValidationException;
 import org.junit.jupiter.api.extension.TestInstances;
 import org.junit.jupiter.api.function.ThrowingConsumer;
 import org.junit.jupiter.api.parallel.ExecutionMode;
@@ -145,7 +146,7 @@ void throwsExceptionWhenParameterizedTestIsNotInvokedAtLeastOnce() {
 			extensionContextWithAnnotatedTestMethod);
 		// cause the stream to be evaluated
 		stream.toArray();
-		var exception = assertThrows(JUnitException.class, stream::close);
+		var exception = assertThrows(TemplateInvocationValidationException.class, stream::close);
 
 		assertThat(exception).hasMessage(
 			"Configuration error: You must configure at least one set of arguments for this @ParameterizedTest");
diff --git a/jupiter-tests/src/test/java/org/junit/jupiter/params/ParameterizedTestIntegrationTests.java b/jupiter-tests/src/test/java/org/junit/jupiter/params/ParameterizedTestIntegrationTests.java
@@ -24,7 +24,6 @@
 import static org.junit.jupiter.engine.discovery.JupiterUniqueIdBuilder.uniqueIdForTestTemplateMethod;
 import static org.junit.jupiter.params.converter.DefaultArgumentConverter.DEFAULT_LOCALE_CONVERSION_FORMAT_PROPERTY_NAME;
 import static org.junit.jupiter.params.provider.Arguments.arguments;
-import static org.junit.platform.engine.discovery.DiscoverySelectors.selectClass;
 import static org.junit.platform.engine.discovery.DiscoverySelectors.selectIteration;
 import static org.junit.platform.engine.discovery.DiscoverySelectors.selectMethod;
 import static org.junit.platform.engine.discovery.DiscoverySelectors.selectUniqueId;
@@ -88,6 +87,8 @@
 import org.junit.jupiter.api.extension.ParameterResolutionException;
 import org.junit.jupiter.api.extension.ParameterResolver;
 import org.junit.jupiter.api.extension.RegisterExtension;
+import org.junit.jupiter.api.extension.TemplateInvocationValidationException;
+import org.junit.jupiter.engine.AbstractJupiterTestEngineTests;
 import org.junit.jupiter.engine.JupiterTestEngine;
 import org.junit.jupiter.params.ParameterizedTestIntegrationTests.RepeatableSourcesTestCase.Action;
 import org.junit.jupiter.params.aggregator.AggregateWith;
@@ -112,8 +113,8 @@
 import org.junit.jupiter.params.support.ParameterDeclarations;
 import org.junit.platform.commons.PreconditionViolationException;
 import org.junit.platform.commons.util.ClassUtils;
-import org.junit.platform.engine.DiscoverySelector;
 import org.junit.platform.engine.TestDescriptor;
+import org.junit.platform.engine.TestExecutionResult;
 import org.junit.platform.testkit.engine.EngineExecutionResults;
 import org.junit.platform.testkit.engine.EngineTestKit;
 import org.junit.platform.testkit.engine.Event;
@@ -123,7 +124,7 @@
 /**
  * @since 5.0
  */
-class ParameterizedTestIntegrationTests {
+class ParameterizedTestIntegrationTests extends AbstractJupiterTestEngineTests {
 
 	private final Locale originalLocale = Locale.getDefault(Locale.Category.FORMAT);
 
@@ -395,7 +396,7 @@ void executesLifecycleMethods() {
 		LifecycleTestCase.lifecycleEvents.clear();
 		LifecycleTestCase.testMethods.clear();
 
-		var results = execute(selectClass(LifecycleTestCase.class));
+		var results = executeTestsForClass(LifecycleTestCase.class);
 		results.allEvents().assertThatEvents() //
 				.haveExactly(1,
 					event(test("test1"), displayName("[1] argument=foo"), finishedWithFailure(message("foo")))) //
@@ -456,8 +457,9 @@ void failsWhenInvocationIsRequiredButNoArgumentSetsAreProvided() {
 		var results = execute(ZeroInvocationsTestCase.class, "testThatRequiresInvocations", String.class);
 
 		results.containerEvents().assertThatEvents() //
-				.haveExactly(1, event(finishedWithFailure(message(
-					"Configuration error: You must configure at least one set of arguments for this @ParameterizedTest"))));
+				.haveExactly(1,
+					event(finishedWithFailure(instanceOf(TemplateInvocationValidationException.class), message(
+						"Configuration error: You must configure at least one set of arguments for this @ParameterizedTest"))));
 	}
 
 	@Test
@@ -500,12 +502,20 @@ void executesWithIso639LocaleConversionFormat() {
 		results.allEvents().assertStatistics(stats -> stats.started(4).succeeded(4));
 	}
 
-	private EngineExecutionResults execute(DiscoverySelector... selectors) {
-		return EngineTestKit.engine(new JupiterTestEngine()).selectors(selectors).execute();
-	}
+	@Test
+	void reportsExceptionInStaticInitializersWithoutInvocationCountValidation() {
+		var results = executeTestsForClass(ExceptionInStaticInitializerTestCase.class);
 
-	private EngineExecutionResults execute(Class<?> testClass, String methodName, Class<?>... methodParameterTypes) {
-		return execute(selectMethod(testClass, methodName, ClassUtils.nullSafeToString(methodParameterTypes)));
+		var failure = results.containerEvents().stream() //
+				.filter(finishedWithFailure()::matches) //
+				.findAny() //
+				.orElseThrow();
+
+		var throwable = failure.getRequiredPayload(TestExecutionResult.class).getThrowable().orElseThrow();
+
+		assertThat(throwable) //
+				.isInstanceOf(ExceptionInInitializerError.class) //
+				.hasNoSuppressedExceptions();
 	}
 
 	private EngineExecutionResults execute(Map<String, String> configurationParameters, Class<?> testClass,
@@ -520,6 +530,10 @@ private EngineExecutionResults execute(String methodName, Class<?>... methodPara
 		return execute(TestCase.class, methodName, methodParameterTypes);
 	}
 
+	private EngineExecutionResults execute(Class<?> testClass, String methodName, Class<?>... methodParameterTypes) {
+		return executeTests(selectMethod(testClass, methodName, ClassUtils.nullSafeToString(methodParameterTypes)));
+	}
+
 	/**
 	 * @since 5.4
 	 */
@@ -947,7 +961,7 @@ void duplicateMethodNames() {
 			// other words, we're not really testing the support for @RepeatedTest
 			// and @TestFactory, but their presence also contributes to the bug
 			// reported in #3001.
-			ParameterizedTestIntegrationTests.this.execute(selectClass(DuplicateMethodNamesMethodSourceTestCase.class))//
+			executeTestsForClass(DuplicateMethodNamesMethodSourceTestCase.class)//
 					.testEvents()//
 					.assertStatistics(stats -> stats.started(8).failed(0).finished(8));
 		}
@@ -1366,7 +1380,7 @@ void closeAutoCloseableArgumentsAfterTestDespiteEarlyFailure() {
 	@Test
 	void executesTwoIterationsBasedOnIterationAndUniqueIdSelector() {
 		var methodId = uniqueIdForTestTemplateMethod(TestCase.class, "testWithThreeIterations(int)");
-		var results = execute(selectUniqueId(appendTestTemplateInvocationSegment(methodId, 3)),
+		var results = executeTests(selectUniqueId(appendTestTemplateInvocationSegment(methodId, 3)),
 			selectIteration(selectMethod(TestCase.class, "testWithThreeIterations", "int"), 1));
 
 		results.allEvents().assertThatEvents() //
@@ -2648,4 +2662,24 @@ void test(AutoCloseableArgument autoCloseable) {
 		}
 	}
 
+	static class ExceptionInStaticInitializerTestCase {
+
+		static {
+			//noinspection ConstantValue
+			if (true)
+				throw new RuntimeException("boom");
+		}
+
+		private static Stream<String> getArguments() {
+			return Stream.of("foo", "bar");
+		}
+
+		@ParameterizedTest
+		@MethodSource("getArguments")
+		void test(String value) {
+			fail("should not be called: " + value);
+		}
+
+	}
+
 }
