Skip to content

Commit abdd3e6

Browse files
fmeumfmeum
committed
junit: Make fuzz test lifecycle mode configurable
1 parent 993b70a commit abdd3e6

File tree

12 files changed

+510
-24
lines changed

12 files changed

+510
-24
lines changed

‎examples/junit/src/test/java/com/example/BUILD.bazel‎

Lines changed: 24 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -69,13 +69,34 @@ java_fuzz_target_test(
6969
)
7070

7171
java_fuzz_target_test(
72-
name = "LifecycleFuzzTest",
73-
srcs = ["LifecycleFuzzTest.java"],
72+
name = "PerExecutionLifecycleFuzzTest",
73+
srcs = ["PerExecutionLifecycleFuzzTest.java"],
7474
allowed_findings = ["com.example.TestSuccessfulException"],
7575
fuzzer_args = [
7676
"-runs=3",
7777
],
78-
target_class = "com.example.LifecycleFuzzTest",
78+
target_class = "com.example.PerExecutionLifecycleFuzzTest",
79+
verify_crash_reproducer = False,
80+
runtime_deps = [
81+
":junit_runtime",
82+
],
83+
deps = [
84+
":test_successful_exception",
85+
"//examples/junit/src/main/java/com/example:parser",
86+
"//src/main/java/com/code_intelligence/jazzer/junit:fuzz_test",
87+
"@maven//:com_google_truth_truth",
88+
"@maven//:org_junit_jupiter_junit_jupiter_api",
89+
],
90+
)
91+
92+
java_fuzz_target_test(
93+
name = "PerTestLifecycleFuzzTest",
94+
srcs = ["PerTestLifecycleFuzzTest.java"],
95+
allowed_findings = ["com.example.TestSuccessfulException"],
96+
fuzzer_args = [
97+
"-runs=3",
98+
],
99+
target_class = "com.example.PerTestLifecycleFuzzTest",
79100
verify_crash_reproducer = False,
80101
runtime_deps = [
81102
":junit_runtime",

examples/junit/src/test/java/com/example/LifecycleFuzzTest.java renamed to examples/junit/src/test/java/com/example/PerExecutionLifecycleFuzzTest.java

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -21,9 +21,9 @@
2121
import static java.util.Collections.unmodifiableList;
2222

2323
import com.code_intelligence.jazzer.junit.FuzzTest;
24-
import com.example.LifecycleFuzzTest.LifecycleCallbacks1;
25-
import com.example.LifecycleFuzzTest.LifecycleCallbacks2;
26-
import com.example.LifecycleFuzzTest.LifecycleCallbacks3;
24+
import com.example.PerExecutionLifecycleFuzzTest.LifecycleCallbacks1;
25+
import com.example.PerExecutionLifecycleFuzzTest.LifecycleCallbacks2;
26+
import com.example.PerExecutionLifecycleFuzzTest.LifecycleCallbacks3;
2727
import java.util.ArrayList;
2828
import java.util.List;
2929
import org.junit.jupiter.api.AfterAll;
@@ -40,11 +40,11 @@
4040
import org.junit.jupiter.api.extension.TestInstancePostProcessor;
4141

4242
@TestMethodOrder(MethodOrderer.MethodName.class)
43-
@ExtendWith(LifecycleFuzzTest.LifecycleInstancePostProcessor.class)
43+
@ExtendWith(PerExecutionLifecycleFuzzTest.LifecycleInstancePostProcessor.class)
4444
@ExtendWith(LifecycleCallbacks1.class)
4545
@ExtendWith(LifecycleCallbacks2.class)
4646
@ExtendWith(LifecycleCallbacks3.class)
47-
class LifecycleFuzzTest {
47+
class PerExecutionLifecycleFuzzTest {
4848
private static final ArrayList<String> events = new ArrayList<>();
4949
private static final long RUNS = 3;
5050

@@ -154,7 +154,7 @@ static void afterAll() throws TestSuccessfulException {
154154
static class LifecycleInstancePostProcessor implements TestInstancePostProcessor {
155155
@Override
156156
public void postProcessTestInstance(Object o, ExtensionContext extensionContext) {
157-
((LifecycleFuzzTest) o).testInstancePostProcessorCalledOnInstance = true;
157+
((PerExecutionLifecycleFuzzTest) o).testInstancePostProcessorCalledOnInstance = true;
158158
}
159159
}
160160

‎examples/junit/src/test/java/com/example/PerTestLifecycleFuzzTest.java‎

Lines changed: 197 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,197 @@
1+
/*
2+
* Copyright 2022 Code Intelligence GmbH
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License");
5+
* you may not use this file except in compliance with the License.
6+
* You may obtain a copy of the License at
7+
*
8+
* http://www.apache.org/licenses/LICENSE-2.0
9+
*
10+
* Unless required by applicable law or agreed to in writing, software
11+
* distributed under the License is distributed on an "AS IS" BASIS,
12+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
* See the License for the specific language governing permissions and
14+
* limitations under the License.
15+
*/
16+
17+
package com.example;
18+
19+
import static com.google.common.truth.Truth.assertThat;
20+
import static java.util.Arrays.asList;
21+
import static java.util.Collections.unmodifiableList;
22+
23+
import com.code_intelligence.jazzer.junit.FuzzTest;
24+
import com.code_intelligence.jazzer.junit.Lifecycle;
25+
import com.example.PerTestLifecycleFuzzTest.LifecycleCallbacks1;
26+
import com.example.PerTestLifecycleFuzzTest.LifecycleCallbacks2;
27+
import com.example.PerTestLifecycleFuzzTest.LifecycleCallbacks3;
28+
import java.util.ArrayList;
29+
import java.util.List;
30+
import org.junit.jupiter.api.AfterAll;
31+
import org.junit.jupiter.api.AfterEach;
32+
import org.junit.jupiter.api.BeforeAll;
33+
import org.junit.jupiter.api.BeforeEach;
34+
import org.junit.jupiter.api.Disabled;
35+
import org.junit.jupiter.api.MethodOrderer;
36+
import org.junit.jupiter.api.TestMethodOrder;
37+
import org.junit.jupiter.api.extension.AfterEachCallback;
38+
import org.junit.jupiter.api.extension.BeforeEachCallback;
39+
import org.junit.jupiter.api.extension.ExtendWith;
40+
import org.junit.jupiter.api.extension.ExtensionContext;
41+
import org.junit.jupiter.api.extension.TestInstancePostProcessor;
42+
43+
@TestMethodOrder(MethodOrderer.MethodName.class)
44+
@ExtendWith(PerTestLifecycleFuzzTest.LifecycleInstancePostProcessor.class)
45+
@ExtendWith(LifecycleCallbacks1.class)
46+
@ExtendWith(LifecycleCallbacks2.class)
47+
@ExtendWith(LifecycleCallbacks3.class)
48+
class PerTestLifecycleFuzzTest {
49+
private static final ArrayList<String> events = new ArrayList<>();
50+
private static final long RUNS = 3;
51+
52+
private boolean beforeEachCalledOnInstance = false;
53+
private boolean testInstancePostProcessorCalledOnInstance = false;
54+
55+
@BeforeAll
56+
static void beforeAll() {
57+
events.add("beforeAll");
58+
}
59+
60+
@BeforeEach
61+
void beforeEach1() {
62+
events.add("beforeEach1");
63+
beforeEachCalledOnInstance = true;
64+
}
65+
66+
@BeforeEach
67+
void beforeEach2() {
68+
events.add("beforeEach2");
69+
}
70+
71+
@BeforeEach
72+
void beforeEach3() {
73+
events.add("beforeEach3");
74+
}
75+
76+
@Disabled
77+
@FuzzTest
78+
void disabledFuzz(byte[] data) {
79+
events.add("disabledFuzz");
80+
throw new AssertionError("This test should not be executed");
81+
}
82+
83+
@FuzzTest(maxExecutions = RUNS, lifecycle = Lifecycle.PER_TEST)
84+
void lifecycleFuzz(byte[] data) {
85+
events.add("lifecycleFuzz");
86+
assertThat(beforeEachCalledOnInstance).isTrue();
87+
assertThat(testInstancePostProcessorCalledOnInstance).isTrue();
88+
}
89+
90+
@AfterEach
91+
void afterEach1() {
92+
events.add("afterEach1");
93+
}
94+
95+
@AfterEach
96+
void afterEach2() {
97+
events.add("afterEach2");
98+
}
99+
100+
@AfterEach
101+
void afterEach3() {
102+
events.add("afterEach3");
103+
}
104+
105+
@AfterAll
106+
static void afterAll() throws TestSuccessfulException {
107+
events.add("afterAll");
108+
109+
boolean isRegressionTest = "".equals(System.getenv("JAZZER_FUZZ"));
110+
boolean isFuzzingFromCommandLine = System.getenv("JAZZER_FUZZ") == null;
111+
boolean isFuzzingFromJUnit = !isFuzzingFromCommandLine && !isRegressionTest;
112+
113+
final List<String> expectedBeforeEachEvents =
114+
unmodifiableList(
115+
asList(
116+
"beforeEachCallback1",
117+
"beforeEachCallback2",
118+
"beforeEachCallback3",
119+
"beforeEach1",
120+
"beforeEach2",
121+
"beforeEach3"));
122+
final List<String> expectedAfterEachEvents =
123+
unmodifiableList(
124+
asList(
125+
"afterEach1",
126+
"afterEach2",
127+
"afterEach3",
128+
"afterEachCallback3",
129+
"afterEachCallback2",
130+
"afterEachCallback1"));
131+
132+
ArrayList<String> expectedEvents = new ArrayList<>();
133+
expectedEvents.add("beforeAll");
134+
135+
// When run from the command-line, the fuzz test is not separately executed on the empty seed.
136+
if (isRegressionTest || isFuzzingFromJUnit) {
137+
expectedEvents.addAll(expectedBeforeEachEvents);
138+
expectedEvents.add("lifecycleFuzz");
139+
expectedEvents.addAll(expectedAfterEachEvents);
140+
}
141+
if (isFuzzingFromJUnit || isFuzzingFromCommandLine) {
142+
expectedEvents.addAll(expectedBeforeEachEvents);
143+
for (int i = 0; i < RUNS; i++) {
144+
expectedEvents.add("lifecycleFuzz");
145+
}
146+
expectedEvents.addAll(expectedAfterEachEvents);
147+
}
148+
149+
expectedEvents.add("afterAll");
150+
151+
assertThat(events).containsExactlyElementsIn(expectedEvents).inOrder();
152+
throw new TestSuccessfulException("Lifecycle methods invoked as expected");
153+
}
154+
155+
static class LifecycleInstancePostProcessor implements TestInstancePostProcessor {
156+
@Override
157+
public void postProcessTestInstance(Object o, ExtensionContext extensionContext) {
158+
((PerTestLifecycleFuzzTest) o).testInstancePostProcessorCalledOnInstance = true;
159+
}
160+
}
161+
162+
static class LifecycleCallbacks1 implements BeforeEachCallback, AfterEachCallback {
163+
@Override
164+
public void beforeEach(ExtensionContext extensionContext) {
165+
events.add("beforeEachCallback1");
166+
}
167+
168+
@Override
169+
public void afterEach(ExtensionContext extensionContext) {
170+
events.add("afterEachCallback1");
171+
}
172+
}
173+
174+
static class LifecycleCallbacks2 implements BeforeEachCallback, AfterEachCallback {
175+
@Override
176+
public void beforeEach(ExtensionContext extensionContext) {
177+
events.add("beforeEachCallback2");
178+
}
179+
180+
@Override
181+
public void afterEach(ExtensionContext extensionContext) {
182+
events.add("afterEachCallback2");
183+
}
184+
}
185+
186+
static class LifecycleCallbacks3 implements BeforeEachCallback, AfterEachCallback {
187+
@Override
188+
public void beforeEach(ExtensionContext extensionContext) {
189+
events.add("beforeEachCallback3");
190+
}
191+
192+
@Override
193+
public void afterEach(ExtensionContext extensionContext) {
194+
events.add("afterEachCallback3");
195+
}
196+
}
197+
}

‎src/main/java/com/code_intelligence/jazzer/junit/BUILD.bazel‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -33,6 +33,9 @@ java_library(
3333
"//examples/junit/src/test/java/com/example:__pkg__",
3434
"//selffuzz/src/test/java/com/code_intelligence/selffuzz:__subpackages__",
3535
],
36+
exports = [
37+
":lifecycle",
38+
],
3639
runtime_deps = [
3740
# The JUnit launcher that is part of the Jazzer driver needs this on the classpath
3841
# to run an @FuzzTest with JUnit. This will also result in a transitive dependency
@@ -42,6 +45,7 @@ java_library(
4245
deps = [
4346
":fuzz_test_configuration_error",
4447
":fuzz_test_executor",
48+
":lifecycle",
4549
":seed_serializer",
4650
":utils",
4751
"@maven//:org_junit_jupiter_junit_jupiter_api",
@@ -67,6 +71,7 @@ java_jni_library(
6771
deps = [
6872
":agent_configurator",
6973
":junit_lifecycle_methods_invoker",
74+
":lifecycle",
7075
":seed_serializer",
7176
":utils",
7277
"//src/main/java/com/code_intelligence/jazzer/agent:agent_installer",
@@ -103,12 +108,18 @@ java_library(
103108
srcs = ["JUnitLifecycleMethodsInvoker.java"],
104109
deps = [
105110
":junit_internals_compile_only",
111+
":lifecycle",
106112
"//src/main/java/com/code_intelligence/jazzer/driver:lifecycle_methods_invoker",
107113
"//src/main/java/com/code_intelligence/jazzer/utils:unsafe_provider",
108114
"@maven//:org_junit_jupiter_junit_jupiter_api",
109115
],
110116
)
111117

118+
java_library(
119+
name = "lifecycle",
120+
srcs = ["Lifecycle.java"],
121+
)
122+
112123
java_library(
113124
name = "seed_serializer",
114125
srcs = ["SeedSerializer.java"],

‎src/main/java/com/code_intelligence/jazzer/junit/FuzzTest.java‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -124,6 +124,14 @@
124124
* <p>This option has no effect during regression testing.
125125
*/
126126
long maxExecutions() default 0;
127+
128+
/**
129+
* Controls the JUnit lifecycle of fuzz tests during fuzzing.
130+
*
131+
* <p>During regression testing, fuzz tests always go through the full JUnit lifecycle for every
132+
* execution regardless of the value of this option.
133+
*/
134+
Lifecycle lifecycle() default Lifecycle.PER_EXECUTION;
127135
}
128136

129137
// Internal use only.

‎src/main/java/com/code_intelligence/jazzer/junit/FuzzTestExecutor.java‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -312,7 +312,8 @@ public void addSeed(byte[] bytes) throws IOException {
312312
public Optional<Throwable> execute(
313313
ReflectiveInvocationContext<Method> invocationContext,
314314
ExtensionContext extensionContext,
315-
SeedSerializer seedSerializer) {
315+
SeedSerializer seedSerializer,
316+
Lifecycle lifecycle) {
316317
if (seedSerializer instanceof AutofuzzSeedSerializer) {
317318
FuzzTargetHolder.fuzzTarget =
318319
FuzzTargetHolder.autofuzzFuzzTarget(
@@ -336,7 +337,7 @@ public Optional<Throwable> execute(
336337
new FuzzTargetHolder.FuzzTarget(
337338
invocationContext.getExecutable(),
338339
() -> invocationContext.getTarget().get(),
339-
JUnitLifecycleMethodsInvoker.of(extensionContext));
340+
JUnitLifecycleMethodsInvoker.of(extensionContext, lifecycle));
340341
}
341342

342343
// Only register a finding handler in case the fuzz test is executed by JUnit.

‎src/main/java/com/code_intelligence/jazzer/junit/FuzzTestExtensions.java‎

Lines changed: 7 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -54,7 +54,7 @@ public void interceptTestTemplateMethod(
5454
// Skip the invocation of the test method with the special arguments provided by
5555
// FuzzTestArgumentsProvider and start fuzzing instead.
5656
if (Utils.isMarkedInvocation(invocationContext)) {
57-
startFuzzing(invocation, invocationContext, extensionContext);
57+
startFuzzing(invocation, invocationContext, extensionContext, fuzzTest.lifecycle());
5858
} else {
5959
// Blocked by https://github.com/junit-team/junit5/issues/3282:
6060
// TODO: The seeds from the input directory are duplicated here as there is no way to
@@ -113,13 +113,17 @@ private static void runWithHooks(Invocation<Void> invocation) throws Throwable {
113113
private static void startFuzzing(
114114
Invocation<Void> invocation,
115115
ReflectiveInvocationContext<Method> invocationContext,
116-
ExtensionContext extensionContext)
116+
ExtensionContext extensionContext,
117+
Lifecycle lifecycle)
117118
throws Throwable {
118119
invocation.skip();
119120
Optional<Throwable> throwable =
120121
FuzzTestExecutor.fromContext(extensionContext)
121122
.execute(
122-
invocationContext, extensionContext, getOrCreateSeedSerializer(extensionContext));
123+
invocationContext,
124+
extensionContext,
125+
getOrCreateSeedSerializer(extensionContext),
126+
lifecycle);
123127
if (throwable.isPresent()) {
124128
throw throwable.get();
125129
}

0 commit comments

Comments
 (0)