tests: Make JUnit lifecycle tests more precise

We also track the instance on which a given method is invoked and verify
that the `ExtensionContext` contains consistent information. This
prepares for future changes to `Lifecycle.PER_EXECUTION`.

Author: fmeum

examples/junit/src/test/java/com/example/BUILD.bazel

@@ -21,6 +21,7 @@ java_binary(
         "//deploy:jazzer-junit",
         "//examples/junit/src/main/java/com/example:parser",
         "//examples/junit/src/test/resources:example_seed_corpora",
+        "@maven//:com_google_truth_extensions_truth_java8_extension",
         "@maven//:com_google_truth_truth",
         "@maven//:org_junit_jupiter_junit_jupiter_api",
         "@maven//:org_junit_jupiter_junit_jupiter_params",
@@ -84,6 +85,7 @@ java_fuzz_target_test(
         ":test_successful_exception",
         "//examples/junit/src/main/java/com/example:parser",
         "//src/main/java/com/code_intelligence/jazzer/junit:fuzz_test",
+        "@maven//:com_google_truth_extensions_truth_java8_extension",
         "@maven//:com_google_truth_truth",
         "@maven//:org_junit_jupiter_junit_jupiter_api",
     ],
@@ -105,6 +107,7 @@ java_fuzz_target_test(
         ":test_successful_exception",
         "//examples/junit/src/main/java/com/example:parser",
         "//src/main/java/com/code_intelligence/jazzer/junit:fuzz_test",
+        "@maven//:com_google_truth_extensions_truth_java8_extension",
         "@maven//:com_google_truth_truth",
         "@maven//:org_junit_jupiter_junit_jupiter_api",
     ],

examples/junit/src/test/java/com/example/PerExecutionLifecycleFuzzTest.java

@@ -17,16 +17,19 @@
 package com.example;
 
 import static com.google.common.truth.Truth.assertThat;
-import static java.util.Arrays.asList;
-import static java.util.Collections.unmodifiableList;
+import static com.google.common.truth.Truth8.assertThat;
+import static java.util.stream.Collectors.collectingAndThen;
+import static java.util.stream.Collectors.toList;
 
 import com.code_intelligence.jazzer.junit.FuzzTest;
 import com.code_intelligence.jazzer.junit.Lifecycle;
 import com.example.PerExecutionLifecycleFuzzTest.LifecycleCallbacks1;
 import com.example.PerExecutionLifecycleFuzzTest.LifecycleCallbacks2;
 import com.example.PerExecutionLifecycleFuzzTest.LifecycleCallbacks3;
 import java.util.ArrayList;
+import java.util.Collections;
 import java.util.List;
+import java.util.stream.Stream;
 import org.junit.jupiter.api.AfterAll;
 import org.junit.jupiter.api.AfterEach;
 import org.junit.jupiter.api.BeforeAll;
@@ -48,58 +51,83 @@
 class PerExecutionLifecycleFuzzTest {
   private static final ArrayList<String> events = new ArrayList<>();
   private static final long RUNS = 3;
+  private static int nextInstanceId = 1;
 
-  private boolean beforeEachCalledOnInstance = false;
-  private boolean testInstancePostProcessorCalledOnInstance = false;
+  private final int instanceId = nextInstanceId++;
 
   @BeforeAll
   static void beforeAll() {
     events.add("beforeAll");
   }
 
+  private void addEvent(String what) {
+    events.add(what + " on " + instanceId);
+  }
+
   @BeforeEach
   void beforeEach1() {
-    events.add("beforeEach1");
-    beforeEachCalledOnInstance = true;
+    addEvent("beforeEach1");
   }
 
   @BeforeEach
   void beforeEach2() {
-    events.add("beforeEach2");
+    addEvent("beforeEach2");
   }
 
   @BeforeEach
   void beforeEach3() {
-    events.add("beforeEach3");
+    addEvent("beforeEach3");
   }
 
   @Disabled
   @FuzzTest
   void disabledFuzz(byte[] data) {
-    events.add("disabledFuzz");
+    addEvent("disabledFuzz");
     throw new AssertionError("This test should not be executed");
   }
 
   @FuzzTest(maxExecutions = RUNS, lifecycle = Lifecycle.PER_EXECUTION)
   void lifecycleFuzz(byte[] data) {
-    events.add("lifecycleFuzz");
-    assertThat(beforeEachCalledOnInstance).isTrue();
-    assertThat(testInstancePostProcessorCalledOnInstance).isTrue();
+    addEvent("lifecycleFuzz");
   }
 
   @AfterEach
   void afterEach1() {
-    events.add("afterEach1");
+    addEvent("afterEach1");
   }
 
   @AfterEach
   void afterEach2() {
-    events.add("afterEach2");
+    addEvent("afterEach2");
   }
 
   @AfterEach
   void afterEach3() {
-    events.add("afterEach3");
+    addEvent("afterEach3");
+  }
+
+  static List<String> expectedBeforeEachEvents(int instanceId) {
+    return Stream.of(
+            "beforeEachCallback1",
+            "beforeEachCallback2",
+            "beforeEachCallback3",
+            "beforeEach1",
+            "beforeEach2",
+            "beforeEach3")
+        .map(s -> s + " on " + instanceId)
+        .collect(collectingAndThen(toList(), Collections::unmodifiableList));
+  }
+
+  static List<String> expectedAfterEachEvents(int instanceId) {
+    return Stream.of(
+            "afterEach1",
+            "afterEach2",
+            "afterEach3",
+            "afterEachCallback3",
+            "afterEachCallback2",
+            "afterEachCallback1")
+        .map(s -> s + " on " + instanceId)
+        .collect(collectingAndThen(toList(), Collections::unmodifiableList));
   }
 
   @AfterAll
@@ -110,39 +138,27 @@ static void afterAll() throws TestSuccessfulException {
     boolean isFuzzingFromCommandLine = System.getenv("JAZZER_FUZZ") == null;
     boolean isFuzzingFromJUnit = !isFuzzingFromCommandLine && !isRegressionTest;
 
-    final List<String> expectedBeforeEachEvents =
-        unmodifiableList(
-            asList(
-                "beforeEachCallback1",
-                "beforeEachCallback2",
-                "beforeEachCallback3",
-                "beforeEach1",
-                "beforeEach2",
-                "beforeEach3"));
-    final List<String> expectedAfterEachEvents =
-        unmodifiableList(
-            asList(
-                "afterEach1",
-                "afterEach2",
-                "afterEach3",
-                "afterEachCallback3",
-                "afterEachCallback2",
-                "afterEachCallback1"));
-
     ArrayList<String> expectedEvents = new ArrayList<>();
     expectedEvents.add("beforeAll");
 
+    int firstFuzzingInstanceId = 1;
     // When run from the command-line, the fuzz test is not separately executed on the empty seed.
     if (isRegressionTest || isFuzzingFromJUnit) {
-      expectedEvents.addAll(expectedBeforeEachEvents);
-      expectedEvents.add("lifecycleFuzz");
-      expectedEvents.addAll(expectedAfterEachEvents);
+      expectedEvents.add("postProcessTestInstance on 1");
+      expectedEvents.addAll(expectedBeforeEachEvents(1));
+      expectedEvents.add("lifecycleFuzz on 1");
+      expectedEvents.addAll(expectedAfterEachEvents(1));
+      firstFuzzingInstanceId++;
     }
     if (isFuzzingFromJUnit || isFuzzingFromCommandLine) {
-      for (int i = 0; i < RUNS; i++) {
-        expectedEvents.addAll(expectedBeforeEachEvents);
-        expectedEvents.add("lifecycleFuzz");
-        expectedEvents.addAll(expectedAfterEachEvents);
+      expectedEvents.add("postProcessTestInstance on " + firstFuzzingInstanceId);
+      for (int i = 1; i <= RUNS; i++) {
+        // TODO: Every execution should use its own instance, i.e., this should be
+        //  firstFuzzingInstanceId + i.
+        int expectedId = firstFuzzingInstanceId;
+        expectedEvents.addAll(expectedBeforeEachEvents(expectedId));
+        expectedEvents.add("lifecycleFuzz on " + expectedId);
+        expectedEvents.addAll(expectedAfterEachEvents(expectedId));
       }
     }
 
@@ -152,46 +168,69 @@ static void afterAll() throws TestSuccessfulException {
     throw new TestSuccessfulException("Lifecycle methods invoked as expected");
   }
 
+  static void assertConsistentTestInstances(ExtensionContext extensionContext) {
+    assertThat(extensionContext.getTestInstance().get())
+        .isSameInstanceAs(extensionContext.getRequiredTestInstance());
+    assertThat(extensionContext.getTestInstances().get())
+        .isSameInstanceAs(extensionContext.getRequiredTestInstances());
+    assertThat(extensionContext.getRequiredTestInstances().getAllInstances())
+        .containsExactly(extensionContext.getRequiredTestInstance());
+  }
+
   static class LifecycleInstancePostProcessor implements TestInstancePostProcessor {
     @Override
     public void postProcessTestInstance(Object o, ExtensionContext extensionContext) {
-      ((PerExecutionLifecycleFuzzTest) o).testInstancePostProcessorCalledOnInstance = true;
+      assertThat(extensionContext.getTestInstance()).isEmpty();
+      assertThat(extensionContext.getTestInstances()).isEmpty();
+      ((PerExecutionLifecycleFuzzTest) o).addEvent("postProcessTestInstance");
     }
   }
 
   static class LifecycleCallbacks1 implements BeforeEachCallback, AfterEachCallback {
     @Override
     public void beforeEach(ExtensionContext extensionContext) {
-      events.add("beforeEachCallback1");
+      assertConsistentTestInstances(extensionContext);
+      ((PerExecutionLifecycleFuzzTest) extensionContext.getRequiredTestInstance())
+          .addEvent("beforeEachCallback1");
     }
 
     @Override
     public void afterEach(ExtensionContext extensionContext) {
-      events.add("afterEachCallback1");
+      assertConsistentTestInstances(extensionContext);
+      ((PerExecutionLifecycleFuzzTest) extensionContext.getRequiredTestInstance())
+          .addEvent("afterEachCallback1");
     }
   }
 
   static class LifecycleCallbacks2 implements BeforeEachCallback, AfterEachCallback {
     @Override
     public void beforeEach(ExtensionContext extensionContext) {
-      events.add("beforeEachCallback2");
+      assertConsistentTestInstances(extensionContext);
+      ((PerExecutionLifecycleFuzzTest) extensionContext.getRequiredTestInstance())
+          .addEvent("beforeEachCallback2");
     }
 
     @Override
     public void afterEach(ExtensionContext extensionContext) {
-      events.add("afterEachCallback2");
+      assertConsistentTestInstances(extensionContext);
+      ((PerExecutionLifecycleFuzzTest) extensionContext.getRequiredTestInstance())
+          .addEvent("afterEachCallback2");
     }
   }
 
   static class LifecycleCallbacks3 implements BeforeEachCallback, AfterEachCallback {
     @Override
     public void beforeEach(ExtensionContext extensionContext) {
-      events.add("beforeEachCallback3");
+      assertConsistentTestInstances(extensionContext);
+      ((PerExecutionLifecycleFuzzTest) extensionContext.getRequiredTestInstance())
+          .addEvent("beforeEachCallback3");
     }
 
     @Override
     public void afterEach(ExtensionContext extensionContext) {
-      events.add("afterEachCallback3");
+      assertConsistentTestInstances(extensionContext);
+      ((PerExecutionLifecycleFuzzTest) extensionContext.getRequiredTestInstance())
+          .addEvent("afterEachCallback3");
     }
   }
 }