Skip to content

Commit 52e1a52

Browse files
fmeumfmeum
committed
junit: Make test instance handling in per-execution mode more realistic
Each execution uses its own dedicated test class instance and also runs preprocessors. See the comment in `JUnitLifecycleMethodsInvoker#beforeFirstExecution` for an explanation of how this still falls short of emulating default JUnit behavior.
1 parent 6d4fe06 commit 52e1a52

File tree

15 files changed

+724
-92
lines changed

15 files changed

+724
-92
lines changed

‎examples/junit/src/test/java/com/example/BUILD.bazel‎

Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -91,6 +91,29 @@ java_fuzz_target_test(
9191
],
9292
)
9393

94+
java_fuzz_target_test(
95+
name = "PerExecutionLifecycleWithFindingFuzzTest",
96+
srcs = ["PerExecutionLifecycleWithFindingFuzzTest.java"],
97+
# TODO: The TestSuccessfulException thrown in @AfterAll is swallowed when run from the CLI.
98+
allowed_findings = ["java.io.IOException"],
99+
fuzzer_args = [
100+
"-runs=3",
101+
],
102+
target_class = "com.example.PerExecutionLifecycleWithFindingFuzzTest",
103+
verify_crash_reproducer = False,
104+
runtime_deps = [
105+
":junit_runtime",
106+
],
107+
deps = [
108+
":test_successful_exception",
109+
"//examples/junit/src/main/java/com/example:parser",
110+
"//src/main/java/com/code_intelligence/jazzer/junit:fuzz_test",
111+
"@maven//:com_google_truth_extensions_truth_java8_extension",
112+
"@maven//:com_google_truth_truth",
113+
"@maven//:org_junit_jupiter_junit_jupiter_api",
114+
],
115+
)
116+
94117
java_fuzz_target_test(
95118
name = "PerTestLifecycleFuzzTest",
96119
srcs = ["PerTestLifecycleFuzzTest.java"],

‎examples/junit/src/test/java/com/example/PerExecutionLifecycleFuzzTest.java‎

Lines changed: 6 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -151,15 +151,18 @@ static void afterAll() throws TestSuccessfulException {
151151
firstFuzzingInstanceId++;
152152
}
153153
if (isFuzzingFromJUnit || isFuzzingFromCommandLine) {
154+
// See the comment in JUnitLifecycleMethodsInvoker#beforeFirstExecution() for an explanation
155+
// of why we see the lifecycle events on firstFuzzingInstanceId.
154156
expectedEvents.add("postProcessTestInstance on " + firstFuzzingInstanceId);
157+
expectedEvents.addAll(expectedBeforeEachEvents(firstFuzzingInstanceId));
155158
for (int i = 1; i <= RUNS; i++) {
156-
// TODO: Every execution should use its own instance, i.e., this should be
157-
// firstFuzzingInstanceId + i.
158-
int expectedId = firstFuzzingInstanceId;
159+
int expectedId = firstFuzzingInstanceId + i;
160+
expectedEvents.add("postProcessTestInstance on " + expectedId);
159161
expectedEvents.addAll(expectedBeforeEachEvents(expectedId));
160162
expectedEvents.add("lifecycleFuzz on " + expectedId);
161163
expectedEvents.addAll(expectedAfterEachEvents(expectedId));
162164
}
165+
expectedEvents.addAll(expectedAfterEachEvents(firstFuzzingInstanceId));
163166
}
164167

165168
expectedEvents.add("afterAll");

‎examples/junit/src/test/java/com/example/PerExecutionLifecycleWithFindingFuzzTest.java‎

Lines changed: 242 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,242 @@
1+
/*
2+
* Copyright 2022 Code Intelligence GmbH
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License");
5+
* you may not use this file except in compliance with the License.
6+
* You may obtain a copy of the License at
7+
*
8+
* http://www.apache.org/licenses/LICENSE-2.0
9+
*
10+
* Unless required by applicable law or agreed to in writing, software
11+
* distributed under the License is distributed on an "AS IS" BASIS,
12+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
* See the License for the specific language governing permissions and
14+
* limitations under the License.
15+
*/
16+
17+
package com.example;
18+
19+
import static com.google.common.truth.Truth.assertThat;
20+
import static com.google.common.truth.Truth8.assertThat;
21+
import static java.util.stream.Collectors.collectingAndThen;
22+
import static java.util.stream.Collectors.toList;
23+
24+
import com.code_intelligence.jazzer.junit.FuzzTest;
25+
import com.code_intelligence.jazzer.junit.Lifecycle;
26+
import com.example.PerExecutionLifecycleWithFindingFuzzTest.LifecycleCallbacks1;
27+
import com.example.PerExecutionLifecycleWithFindingFuzzTest.LifecycleCallbacks2;
28+
import com.example.PerExecutionLifecycleWithFindingFuzzTest.LifecycleCallbacks3;
29+
import java.io.IOException;
30+
import java.util.ArrayList;
31+
import java.util.Collections;
32+
import java.util.List;
33+
import java.util.stream.Stream;
34+
import org.junit.jupiter.api.AfterAll;
35+
import org.junit.jupiter.api.AfterEach;
36+
import org.junit.jupiter.api.BeforeAll;
37+
import org.junit.jupiter.api.BeforeEach;
38+
import org.junit.jupiter.api.Disabled;
39+
import org.junit.jupiter.api.MethodOrderer;
40+
import org.junit.jupiter.api.TestMethodOrder;
41+
import org.junit.jupiter.api.extension.AfterEachCallback;
42+
import org.junit.jupiter.api.extension.BeforeEachCallback;
43+
import org.junit.jupiter.api.extension.ExtendWith;
44+
import org.junit.jupiter.api.extension.ExtensionContext;
45+
import org.junit.jupiter.api.extension.TestInstancePostProcessor;
46+
47+
@TestMethodOrder(MethodOrderer.MethodName.class)
48+
@ExtendWith(PerExecutionLifecycleWithFindingFuzzTest.LifecycleInstancePostProcessor.class)
49+
@ExtendWith(LifecycleCallbacks1.class)
50+
@ExtendWith(LifecycleCallbacks2.class)
51+
@ExtendWith(LifecycleCallbacks3.class)
52+
class PerExecutionLifecycleWithFindingFuzzTest {
53+
private static final ArrayList<String> events = new ArrayList<>();
54+
private static final long RUNS = 3;
55+
private static int nextInstanceId = 1;
56+
private final int instanceId = nextInstanceId++;
57+
58+
@BeforeAll
59+
static void beforeAll() {
60+
events.add("beforeAll");
61+
}
62+
63+
private void addEvent(String what) {
64+
events.add(what + " on " + instanceId);
65+
}
66+
67+
@BeforeEach
68+
void beforeEach1() {
69+
addEvent("beforeEach1");
70+
}
71+
72+
@BeforeEach
73+
void beforeEach2() {
74+
addEvent("beforeEach2");
75+
}
76+
77+
@BeforeEach
78+
void beforeEach3() {
79+
addEvent("beforeEach3");
80+
}
81+
82+
@Disabled
83+
@FuzzTest
84+
void disabledFuzz(byte[] data) {
85+
addEvent("disabledFuzz");
86+
throw new AssertionError("This test should not be executed");
87+
}
88+
89+
@FuzzTest(maxExecutions = RUNS, lifecycle = Lifecycle.PER_EXECUTION)
90+
void lifecycleFuzz(byte[] data) throws IOException {
91+
addEvent("lifecycleFuzz");
92+
if (data.length != 0) {
93+
throw new IOException(
94+
"Planted finding on first non-trivial input (second execution during fuzzing)");
95+
}
96+
}
97+
98+
@AfterEach
99+
void afterEach1() {
100+
addEvent("afterEach1");
101+
}
102+
103+
@AfterEach
104+
void afterEach2() {
105+
addEvent("afterEach2");
106+
}
107+
108+
@AfterEach
109+
void afterEach3() {
110+
addEvent("afterEach3");
111+
}
112+
113+
static List<String> expectedBeforeEachEvents(int instanceId) {
114+
return Stream.of(
115+
"beforeEachCallback1",
116+
"beforeEachCallback2",
117+
"beforeEachCallback3",
118+
"beforeEach1",
119+
"beforeEach2",
120+
"beforeEach3")
121+
.map(s -> s + " on " + instanceId)
122+
.collect(collectingAndThen(toList(), Collections::unmodifiableList));
123+
}
124+
125+
static List<String> expectedAfterEachEvents(int instanceId) {
126+
return Stream.of(
127+
"afterEach1",
128+
"afterEach2",
129+
"afterEach3",
130+
"afterEachCallback3",
131+
"afterEachCallback2",
132+
"afterEachCallback1")
133+
.map(s -> s + " on " + instanceId)
134+
.collect(collectingAndThen(toList(), Collections::unmodifiableList));
135+
}
136+
137+
@AfterAll
138+
static void afterAll() throws TestSuccessfulException {
139+
events.add("afterAll");
140+
141+
boolean isRegressionTest = "".equals(System.getenv("JAZZER_FUZZ"));
142+
boolean isFuzzingFromCommandLine = System.getenv("JAZZER_FUZZ") == null;
143+
boolean isFuzzingFromJUnit = !isFuzzingFromCommandLine && !isRegressionTest;
144+
145+
ArrayList<String> expectedEvents = new ArrayList<>();
146+
expectedEvents.add("beforeAll");
147+
148+
int firstFuzzingInstanceId = 1;
149+
// When run from the command-line, the fuzz test is not separately executed on the empty seed.
150+
if (isRegressionTest || isFuzzingFromJUnit) {
151+
expectedEvents.add("postProcessTestInstance on 1");
152+
expectedEvents.addAll(expectedBeforeEachEvents(1));
153+
expectedEvents.add("lifecycleFuzz on 1");
154+
expectedEvents.addAll(expectedAfterEachEvents(1));
155+
firstFuzzingInstanceId++;
156+
}
157+
if (isFuzzingFromJUnit || isFuzzingFromCommandLine) {
158+
expectedEvents.add("postProcessTestInstance on " + firstFuzzingInstanceId);
159+
expectedEvents.addAll(expectedBeforeEachEvents(firstFuzzingInstanceId));
160+
// The fuzz test fails during the second run.
161+
for (int i = 1; i <= 2; i++) {
162+
int expectedId = firstFuzzingInstanceId + i;
163+
expectedEvents.add("postProcessTestInstance on " + expectedId);
164+
expectedEvents.addAll(expectedBeforeEachEvents(expectedId));
165+
expectedEvents.add("lifecycleFuzz on " + expectedId);
166+
expectedEvents.addAll(expectedAfterEachEvents(expectedId));
167+
}
168+
expectedEvents.addAll(expectedAfterEachEvents(firstFuzzingInstanceId));
169+
}
170+
171+
expectedEvents.add("afterAll");
172+
173+
assertThat(events).containsExactlyElementsIn(expectedEvents).inOrder();
174+
throw new TestSuccessfulException("Lifecycle methods invoked as expected");
175+
}
176+
177+
static void assertConsistentTestInstances(ExtensionContext extensionContext) {
178+
assertThat(extensionContext.getTestInstance().get())
179+
.isSameInstanceAs(extensionContext.getRequiredTestInstance());
180+
assertThat(extensionContext.getTestInstances().get())
181+
.isSameInstanceAs(extensionContext.getRequiredTestInstances());
182+
assertThat(extensionContext.getRequiredTestInstances().getAllInstances())
183+
.containsExactly(extensionContext.getRequiredTestInstance());
184+
}
185+
186+
static class LifecycleInstancePostProcessor implements TestInstancePostProcessor {
187+
@Override
188+
public void postProcessTestInstance(Object o, ExtensionContext extensionContext) {
189+
assertThat(extensionContext.getTestInstance()).isEmpty();
190+
assertThat(extensionContext.getTestInstances()).isEmpty();
191+
((PerExecutionLifecycleWithFindingFuzzTest) o).addEvent("postProcessTestInstance");
192+
}
193+
}
194+
195+
static class LifecycleCallbacks1 implements BeforeEachCallback, AfterEachCallback {
196+
@Override
197+
public void beforeEach(ExtensionContext extensionContext) {
198+
assertConsistentTestInstances(extensionContext);
199+
((PerExecutionLifecycleWithFindingFuzzTest) extensionContext.getRequiredTestInstance())
200+
.addEvent("beforeEachCallback1");
201+
}
202+
203+
@Override
204+
public void afterEach(ExtensionContext extensionContext) {
205+
assertConsistentTestInstances(extensionContext);
206+
((PerExecutionLifecycleWithFindingFuzzTest) extensionContext.getRequiredTestInstance())
207+
.addEvent("afterEachCallback1");
208+
}
209+
}
210+
211+
static class LifecycleCallbacks2 implements BeforeEachCallback, AfterEachCallback {
212+
@Override
213+
public void beforeEach(ExtensionContext extensionContext) {
214+
assertConsistentTestInstances(extensionContext);
215+
((PerExecutionLifecycleWithFindingFuzzTest) extensionContext.getRequiredTestInstance())
216+
.addEvent("beforeEachCallback2");
217+
}
218+
219+
@Override
220+
public void afterEach(ExtensionContext extensionContext) {
221+
assertConsistentTestInstances(extensionContext);
222+
((PerExecutionLifecycleWithFindingFuzzTest) extensionContext.getRequiredTestInstance())
223+
.addEvent("afterEachCallback2");
224+
}
225+
}
226+
227+
static class LifecycleCallbacks3 implements BeforeEachCallback, AfterEachCallback {
228+
@Override
229+
public void beforeEach(ExtensionContext extensionContext) {
230+
assertConsistentTestInstances(extensionContext);
231+
((PerExecutionLifecycleWithFindingFuzzTest) extensionContext.getRequiredTestInstance())
232+
.addEvent("beforeEachCallback3");
233+
}
234+
235+
@Override
236+
public void afterEach(ExtensionContext extensionContext) {
237+
assertConsistentTestInstances(extensionContext);
238+
((PerExecutionLifecycleWithFindingFuzzTest) extensionContext.getRequiredTestInstance())
239+
.addEvent("afterEachCallback3");
240+
}
241+
}
242+
}

‎src/main/java/com/code_intelligence/jazzer/driver/BUILD.bazel‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -78,6 +78,7 @@ java_library(
7878
"//src/test/java/com/code_intelligence/jazzer/driver:__pkg__",
7979
],
8080
deps = [
81+
":libfuzzer_lifecycle_methods_invoker",
8182
":lifecycle_methods_invoker",
8283
":opt",
8384
"//src/main/java/com/code_intelligence/jazzer/api",

‎src/main/java/com/code_intelligence/jazzer/driver/FuzzTargetFinder.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -105,6 +105,6 @@ private static FuzzTarget findFuzzTargetByMethodName(Class<?> clazz) {
105105
fuzzTargetMethod = dataFuzzTarget.orElseGet(bytesFuzzTarget::get);
106106
}
107107

108-
return new FuzzTarget(fuzzTargetMethod, () -> null, LibFuzzerLifecycleMethodsInvoker.of(clazz));
108+
return new FuzzTarget(fuzzTargetMethod, LibFuzzerLifecycleMethodsInvoker.of(clazz));
109109
}
110110
}

‎src/main/java/com/code_intelligence/jazzer/driver/FuzzTargetHolder.java‎

Lines changed: 5 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -18,43 +18,33 @@
1818

1919
import com.code_intelligence.jazzer.api.FuzzedDataProvider;
2020
import java.lang.reflect.Method;
21-
import java.util.concurrent.Callable;
2221

2322
public class FuzzTargetHolder {
24-
public static FuzzTarget autofuzzFuzzTarget(Callable<Object> newInstance) {
23+
public static FuzzTarget autofuzzFuzzTarget(LifecycleMethodsInvoker lifecycleMethodsInvoker) {
2524
try {
2625
Method fuzzerTestOneInput =
2726
com.code_intelligence.jazzer.autofuzz.FuzzTarget.class.getMethod(
2827
"fuzzerTestOneInput", FuzzedDataProvider.class);
29-
return new FuzzTargetHolder.FuzzTarget(
30-
fuzzerTestOneInput, newInstance, LifecycleMethodsInvoker.NOOP);
28+
return new FuzzTargetHolder.FuzzTarget(fuzzerTestOneInput, lifecycleMethodsInvoker);
3129
} catch (NoSuchMethodException e) {
3230
throw new IllegalStateException(e);
3331
}
3432
}
3533

3634
public static final FuzzTarget AUTOFUZZ_FUZZ_TARGET =
3735
autofuzzFuzzTarget(
38-
() -> {
39-
com.code_intelligence.jazzer.autofuzz.FuzzTarget.fuzzerInitialize(
40-
Opt.targetArgs.get().toArray(new String[0]));
41-
return null;
42-
});
36+
LibFuzzerLifecycleMethodsInvoker.of(
37+
com.code_intelligence.jazzer.autofuzz.FuzzTarget.class));
4338

4439
/** The fuzz target that {@link FuzzTargetRunner} should fuzz. */
4540
public static FuzzTarget fuzzTarget;
4641

4742
public static class FuzzTarget {
4843
public final Method method;
49-
public final Callable<Object> newInstance;
5044
public final LifecycleMethodsInvoker lifecycleMethodsInvoker;
5145

52-
public FuzzTarget(
53-
Method method,
54-
Callable<Object> newInstance,
55-
LifecycleMethodsInvoker lifecycleMethodsInvoker) {
46+
public FuzzTarget(Method method, LifecycleMethodsInvoker lifecycleMethodsInvoker) {
5647
this.method = method;
57-
this.newInstance = newInstance;
5848
this.lifecycleMethodsInvoker = lifecycleMethodsInvoker;
5949
}
6050

0 commit comments

Comments
 (0)