Add FuzzTest dictionary support (#862)

Add support for dictionaries in fuzz tests

This adds dictionary support to JUnit fuzz tests via 2 annotations: WithDictionary and WithDictionaryFile that allow dictionaries to be specified either as static arrays of tokens or by referring to a dictionary file. Multiple instances of both annotations are allowed and all values will be merged in the dictionary given to libfuzzer.
---------

Co-authored-by: Fabian Meumertzheim <[email protected]>

Author: br-lewis

examples/junit/src/test/java/com/example/BUILD.bazel

@@ -254,6 +254,33 @@ java_fuzz_target_test(
     ],
 )
 
+[
+    java_fuzz_target_test(
+        name = "DictionaryFuzzTests_" + method,
+        srcs = ["DictionaryFuzzTests.java"],
+        allowed_findings = ["com.example.TestSuccessfulException"],
+        env = {"JAZZER_FUZZ": "1"},
+        target_class = "com.example.DictionaryFuzzTests",
+        target_method = method,
+        verify_crash_reproducer = False,
+        runtime_deps = [
+            ":junit_runtime",
+        ],
+        deps = [
+            "//examples/junit/src/test/java/com/example:test_successful_exception",
+            "//examples/junit/src/test/resources:example_dictionaries",
+            "//src/main/java/com/code_intelligence/jazzer/junit:fuzz_test",
+            "@maven//:org_junit_jupiter_junit_jupiter_api",
+            "@maven//:org_junit_jupiter_junit_jupiter_params",
+        ],
+    )
+    for method in [
+        "inlineTest",
+        "fileTest",
+        "mixedTest",
+    ]
+]
+
 java_library(
     name = "junit_runtime",
     runtime_deps = [

examples/junit/src/test/java/com/example/DictionaryFuzzTests.java

@@ -0,0 +1,69 @@
+/*
+ * Copyright 2023 Code Intelligence GmbH
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.example;
+
+import com.code_intelligence.jazzer.api.FuzzedDataProvider;
+import com.code_intelligence.jazzer.junit.DictionaryEntries;
+import com.code_intelligence.jazzer.junit.DictionaryFile;
+import com.code_intelligence.jazzer.junit.FuzzTest;
+import java.nio.charset.StandardCharsets;
+import java.security.MessageDigest;
+import java.security.NoSuchAlgorithmException;
+import java.util.Base64;
+
+public class DictionaryFuzzTests {
+  // Generated via:
+  // printf 'a_53Cr3T_fl4G' | openssl dgst -binary -sha256 | openssl base64 -A
+  // Luckily the fuzzer can't read comments ;-)
+  private static final byte[] FLAG_SHA256 =
+      Base64.getDecoder().decode("IT7goSzYg6MXLugHl9H4oCswA+OEb4bGZmKrDzlZjO4=");
+
+  @DictionaryEntries(tokens = {"a_", "53Cr3T_", "fl4G"})
+  @FuzzTest
+  public void inlineTest(FuzzedDataProvider data)
+      throws NoSuchAlgorithmException, TestSuccessfulException {
+    String s = data.consumeRemainingAsString();
+    byte[] hash = MessageDigest.getInstance("SHA-256").digest(s.getBytes(StandardCharsets.UTF_8));
+    if (MessageDigest.isEqual(hash, FLAG_SHA256)) {
+      throw new TestSuccessfulException("error found");
+    }
+  }
+
+  @DictionaryFile(resourcePath = "com/example/test.dict")
+  @FuzzTest
+  public void fileTest(FuzzedDataProvider data)
+      throws NoSuchAlgorithmException, TestSuccessfulException {
+    String s = data.consumeRemainingAsString();
+    byte[] hash = MessageDigest.getInstance("SHA-256").digest(s.getBytes(StandardCharsets.UTF_8));
+    if (MessageDigest.isEqual(hash, FLAG_SHA256)) {
+      throw new TestSuccessfulException("error found");
+    }
+  }
+
+  @DictionaryEntries(tokens = {"a_"})
+  @DictionaryFile(resourcePath = "com/example/test2.dict")
+  @DictionaryFile(resourcePath = "com/example/test3.dict")
+  @FuzzTest
+  public void mixedTest(FuzzedDataProvider data)
+      throws NoSuchAlgorithmException, TestSuccessfulException {
+    String s = data.consumeRemainingAsString();
+    byte[] hash = MessageDigest.getInstance("SHA-256").digest(s.getBytes(StandardCharsets.UTF_8));
+    if (MessageDigest.isEqual(hash, FLAG_SHA256)) {
+      throw new TestSuccessfulException("error found");
+    }
+  }
+}

examples/junit/src/test/resources/BUILD.bazel

@@ -4,6 +4,12 @@ java_library(
     visibility = ["//examples/junit/src/test/java/com/example:__pkg__"],
 )
 
+java_library(
+    name = "example_dictionaries",
+    resources = glob(["**/*.dict"]),
+    visibility = ["//examples/junit/src/test/java/com/example:__pkg__"],
+)
+
 filegroup(
     name = "MutatorFuzzTestInputs",
     srcs = ["com/example/MutatorFuzzTestInputs"],

examples/junit/src/test/resources/com/example/test.dict

@@ -0,0 +1,4 @@
+# test dictionary
+"a_"
+"53Cr3T_"
+"fl4G"

examples/junit/src/test/resources/com/example/test2.dict

@@ -0,0 +1 @@
+"53Cr3T_"

examples/junit/src/test/resources/com/example/test3.dict

@@ -0,0 +1 @@
+"fl4G"

src/main/java/com/code_intelligence/jazzer/junit/AgentConfiguringArgumentsProvider.java

@@ -16,6 +16,8 @@
 
 package com.code_intelligence.jazzer.junit;
 
+import java.nio.file.Path;
+import java.util.Optional;
 import java.util.stream.Stream;
 import org.junit.jupiter.api.extension.ExtensionContext;
 import org.junit.jupiter.params.provider.Arguments;
@@ -36,11 +38,13 @@ public Stream<? extends Arguments> provideArguments(ExtensionContext extensionCo
     // FIXME(fmeum): Calling this here feels like a hack. There should be a lifecycle hook that runs
     //  before the argument discovery for a ParameterizedTest is kicked off, but I haven't found
     //  one.
+    Optional<Path> dictionaryPath =
+        FuzzerDictionary.createDictionaryFile(extensionContext.getRequiredTestMethod());
     // We need to call this method here in addition to the call in FuzzTestExtensions as our
     // ArgumentProviders need the bootstrap jar on the classpath and there may be no user-provided
     // ArgumentProviders to trigger the call in FuzzTestExtensions.
     FuzzTestExecutor.configureAndInstallAgent(
-        extensionContext, fuzzTest.maxDuration(), fuzzTest.maxExecutions());
+        extensionContext, fuzzTest.maxDuration(), fuzzTest.maxExecutions(), dictionaryPath);
     return Stream.empty();
   }
 }

src/main/java/com/code_intelligence/jazzer/junit/BUILD.bazel

@@ -24,8 +24,13 @@ java_library(
     name = "fuzz_test",
     srcs = [
         "AgentConfiguringArgumentsProvider.java",
+        "DictionaryEntries.java",
+        "DictionaryEntriesList.java",
+        "DictionaryFile.java",
+        "DictionaryFiles.java",
         "FuzzTest.java",
         "FuzzTestExtensions.java",
+        "FuzzerDictionary.java",
         "FuzzingArgumentsProvider.java",
         "SeedArgumentsProvider.java",
     ],
@@ -48,6 +53,7 @@ java_library(
         ":lifecycle",
         ":seed_serializer",
         ":utils",
+        "//src/main/java/com/code_intelligence/jazzer/utils:log",
         "@maven//:org_junit_jupiter_junit_jupiter_api",
         "@maven//:org_junit_jupiter_junit_jupiter_params",
         "@maven//:org_junit_platform_junit_platform_commons",

src/main/java/com/code_intelligence/jazzer/junit/DictionaryEntries.java

@@ -0,0 +1,34 @@
+/*
+ * Copyright 2023 Code Intelligence GmbH
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.code_intelligence.jazzer.junit;
+
+import java.lang.annotation.ElementType;
+import java.lang.annotation.Repeatable;
+import java.lang.annotation.Retention;
+import java.lang.annotation.RetentionPolicy;
+import java.lang.annotation.Target;
+
+/**
+ * Defines a reference to a dictionary within the resources directory. These should follow <a
+ * href="https://llvm.org/docs/LibFuzzer.html#dictionaries">libfuzzer's dictionary syntax</a>.
+ */
+@Target({ElementType.METHOD, ElementType.ANNOTATION_TYPE})
+@Retention(RetentionPolicy.RUNTIME)
+@Repeatable(DictionaryEntriesList.class)
+public @interface DictionaryEntries {
+  String[] tokens();
+}

src/main/java/com/code_intelligence/jazzer/junit/DictionaryEntriesList.java

@@ -0,0 +1,28 @@
+/*
+ * Copyright 2023 Code Intelligence GmbH
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.code_intelligence.jazzer.junit;
+
+import java.lang.annotation.ElementType;
+import java.lang.annotation.Retention;
+import java.lang.annotation.RetentionPolicy;
+import java.lang.annotation.Target;
+
+@Target({ElementType.METHOD, ElementType.ANNOTATION_TYPE})
+@Retention(RetentionPolicy.RUNTIME)
+public @interface DictionaryEntriesList {
+  DictionaryEntries[] value();
+}