Prune exceptions when JUnit.start is used (#5158)

Prune exceptions when JUnit.start is used

When using `JUnit.start` the `pruneStackTrace` algorithm immediately
sees the `TestClass.main` frame and assumes that this is the test
method because the test class name matches.

```
org.opentest4j.AssertionFailedError: expected: <11> but was: <12>
	at [email protected]/org.junit.jupiter.api.AssertionFailureBuilder.build(AssertionFailureBuilder.java:158)
	at [email protected]/org.junit.jupiter.api.AssertionFailureBuilder.buildAndThrow(AssertionFailureBuilder.java:139)
	at [email protected]/org.junit.jupiter.api.AssertEquals.failNotEqual(AssertEquals.java:201)
	at [email protected]/org.junit.jupiter.api.AssertEquals.assertEquals(AssertEquals.java:152)
	at [email protected]/org.junit.jupiter.api.AssertEquals.assertEquals(AssertEquals.java:147)
	at [email protected]/org.junit.jupiter.api.Assertions.assertEquals(Assertions.java:558)
	at com.examp.project/com.example.project.HelloTest.stringLength(HelloTest.java:14)
	at java.base/jdk.internal.reflect.DirectMethodHandleAccessor.invoke(DirectMethodHandleAccessor.java:104)
...
	at [email protected]/org.junit.platform.launcher.core.SessionPerRequestLauncher.execute(SessionPerRequestLauncher.java:81)
	at [email protected]/org.junit.start.JUnit.run(JUnit.java:63)
	at [email protected]/org.junit.start.JUnit.run(JUnit.java:37)
	at com.examp.project/com.example.project.HelloTest.main(HelloTest.java:9)
```

By checking if `org.junit.start` is involved further down the stack we
exclude this scenario. And can limit the stacktrace to a more readable.

```
org.opentest4j.AssertionFailedError: expected: <11> but was: <12>
	at [email protected]/org.junit.jupiter.api.Assertions.assertEquals(Assertions.java:558)
	at com.examp.project/com.example.project.HelloTest.stringLength(HelloTest.java:14)
	at [email protected]/org.junit.start.JUnit.run(JUnit.java:37)
	at com.examp.project/com.example.project.HelloTest.main(HelloTest.java:9)
```

Author: mpkorstanje

junit-platform-commons/src/main/java/org/junit/platform/commons/util/ExceptionUtils.java

@@ -101,16 +101,19 @@ public static String readStackTrace(Throwable throwable) {
 	/**
 	 * Prune the stack trace of the supplied {@link Throwable}.
 	 *
-	 * <p>Prune all {@linkplain StackTraceElement stack trace elements} up one
-	 * of the supplied {@code classNames} are pruned. All subsequent elements
-	 * in the stack trace will be retained.
+	 * <p>Prune all {@linkplain StackTraceElement stack trace elements} up to one
+	 * of the supplied {@code classNames}. All subsequent elements in the stack
+	 * trace will be retained.
 	 *
 	 * <p>If the {@code classNames} do not match any of the stacktrace elements
 	 * then the {@code org.junit}, {@code jdk.internal.reflect}, and
 	 * {@code sun.reflect} packages are pruned.
 	 *
-	 * <p>Additionally, all elements prior to and including the first JUnit Platform
-	 * Launcher call will be removed.
+	 * <p>Additionally:
+	 * <ul>
+	 *     <li>all elements prior to and including the first JUnit Platform Launcher call will be removed.
+	 *     <li>all elements prior to and including {@code org.junit.start} are kept.
+	 * </ul>
 	 *
 	 * @param throwable the {@code Throwable} whose stack trace should be pruned;
 	 * never {@code null}
@@ -126,14 +129,15 @@ public static void pruneStackTrace(Throwable throwable, List<String> classNames)
 
 		List<StackTraceElement> stackTrace = Arrays.asList(throwable.getStackTrace());
 		List<StackTraceElement> prunedStackTrace = new ArrayList<>();
+		List<StackTraceElement> junitStartStackTrace = new ArrayList<>(0);
 
 		Collections.reverse(stackTrace);
 
 		for (int i = 0; i < stackTrace.size(); i++) {
 			StackTraceElement element = stackTrace.get(i);
 			String className = element.getClassName();
 
-			if (classNames.contains(className)) {
+			if (classNames.contains(className) && !includesJunitStart(stackTrace, i + 1)) {
 				// We found the test
 				// everything before that is not informative.
 				prunedStackTrace.clear();
@@ -142,7 +146,9 @@ public static void pruneStackTrace(Throwable throwable, List<String> classNames)
 				break;
 			}
 			else if (className.startsWith(JUNIT_START_PACKAGE_PREFIX)) {
+				junitStartStackTrace.addAll(prunedStackTrace);
 				prunedStackTrace.clear();
+				junitStartStackTrace.add(element);
 			}
 			else if (className.startsWith(JUNIT_PLATFORM_LAUNCHER_PACKAGE_PREFIX)) {
 				prunedStackTrace.clear();
@@ -152,10 +158,22 @@ else if (STACK_TRACE_ELEMENT_FILTER.test(className)) {
 			}
 		}
 
+		if (!junitStartStackTrace.isEmpty()) {
+			junitStartStackTrace.addAll(prunedStackTrace);
+			prunedStackTrace = junitStartStackTrace;
+		}
+
 		Collections.reverse(prunedStackTrace);
 		throwable.setStackTrace(prunedStackTrace.toArray(new StackTraceElement[0]));
 	}
 
+	private static boolean includesJunitStart(List<StackTraceElement> stackTrace, int fromIndex) {
+		return stackTrace.stream() //
+				.skip(fromIndex) //
+				.map(StackTraceElement::getClassName) //
+				.anyMatch(className -> className.startsWith(JUNIT_START_PACKAGE_PREFIX));
+	}
+
 	/**
 	 * Find all causes and suppressed exceptions in the stack trace of the
 	 * supplied {@link Throwable}.

platform-tests/src/test/java/org/junit/platform/commons/util/ExceptionUtilsTests.java

@@ -19,6 +19,7 @@
 import static org.junit.platform.commons.util.ExceptionUtils.throwAsUncheckedException;
 
 import java.io.IOException;
+import java.util.Arrays;
 import java.util.List;
 
 import org.junit.jupiter.api.Test;
@@ -104,6 +105,30 @@ void pruneStackTraceOfEverythingPriorToFirstLauncherCall() {
 				.noneMatch(element -> element.toString().contains("org.example.Class.method(file:123)"));
 	}
 
+	@Test
+	void pruneStackTraceRetainsStackFramesFromJUnitStart() {
+		// Non-test class frames from org.junit are filtered.
+		var testClassName = "com.example.project.HelloTest";
+		var testFileName = "HelloTest.java";
+
+		var exception = new JUnitException("expected");
+		var stackTrace = exception.getStackTrace();
+		var extendedStacktrace = Arrays.copyOfRange(stackTrace, 0, stackTrace.length + 2);
+		extendedStacktrace[0] = new StackTraceElement(testClassName, "stringLength", testFileName, 10);
+		extendedStacktrace[stackTrace.length] = new StackTraceElement("org.junit.start.JUnit", "run", "JUnit.java", 3);
+		extendedStacktrace[stackTrace.length + 1] = new StackTraceElement(testClassName, "main", testFileName, 5);
+		exception.setStackTrace(extendedStacktrace);
+
+		pruneStackTrace(exception, List.of(testClassName));
+
+		assertThat(exception.getStackTrace()) //
+				.extracting(StackTraceElement::toString) //
+				.containsExactly( //
+					"com.example.project.HelloTest.stringLength(HelloTest.java:10)", //
+					"org.junit.start.JUnit.run(JUnit.java:3)", //
+					"com.example.project.HelloTest.main(HelloTest.java:5)");
+	}
+
 	@Test
 	void findSuppressedExceptionsAndCausesOfThrowable() {
 		Throwable t1 = new Throwable("#1");