tests: Verify emitted warnings and errors

Author: fmeum

bazel/fuzz_target.bzl

@@ -35,6 +35,8 @@ def java_fuzz_target_test(
         allowed_findings = [],
         # By default, expect a crash iff allowed_findings isn't empty.
         expect_crash = None,
+        # If empty, expect no warnings or errors, if not empty, expect one matching the given regex.
+        expected_warning_or_error = "",
         **kwargs):
     if target_class:
         fuzzer_args = fuzzer_args + ["--target_class=" + target_class]
@@ -115,6 +117,7 @@ def java_fuzz_target_test(
             str(verify_crash_reproducer),
             str(expect_crash),
             str(launcher_variant == "java"),
+            "'" + expected_warning_or_error + "'",
             "'" + ",".join(allowed_findings) + "'",
         ] + fuzzer_args,
         data = [

bazel/tools/java/com/code_intelligence/jazzer/tools/FuzzTargetTestWrapper.java

@@ -13,7 +13,10 @@
 // limitations under the License.
 package com.code_intelligence.jazzer.tools;
 
+import static java.util.Arrays.asList;
+import static java.util.Collections.unmodifiableSet;
 import static java.util.stream.Collectors.toList;
+import static java.util.stream.Collectors.toSet;
 
 import com.google.devtools.build.runfiles.AutoBazelRepository;
 import com.google.devtools.build.runfiles.Runfiles;
@@ -35,6 +38,7 @@
 import java.util.Collections;
 import java.util.Comparator;
 import java.util.List;
+import java.util.Optional;
 import java.util.Set;
 import java.util.regex.Pattern;
 import java.util.stream.Collectors;
@@ -47,13 +51,15 @@
 
 @AutoBazelRepository
 public class FuzzTargetTestWrapper {
+  private static final Set<String> IGNORED_WARNINGS =
+      // Triggered by BatikTranscoderFuzzer on macOS in Github Actions.
+      Collections.singleton("WARNING: GL pipe is running in software mode (Renderer ID=0x1020400)");
   private static final String EXCEPTION_PREFIX = "== Java Exception: ";
   private static final String FRAME_PREFIX = "\tat ";
   private static final Pattern SANITIZER_FINDING = Pattern.compile("^SUMMARY: \\w*Sanitizer");
   private static final String THREAD_DUMP_HEADER = "Stack traces of all JVM threads:";
   private static final Set<String> PUBLIC_JAZZER_PACKAGES =
-      Collections.unmodifiableSet(
-          Stream.of("api", "replay", "sanitizers").collect(Collectors.toSet()));
+      unmodifiableSet(Stream.of("api", "replay", "sanitizers").collect(toSet()));
 
   public static void main(String[] args) {
     Runfiles runfiles;
@@ -65,6 +71,7 @@ public static void main(String[] args) {
     boolean shouldVerifyCrashReproducer;
     boolean expectCrash;
     boolean usesJavaLauncher;
+    Optional<String> expectedWarningOrError;
     Set<String> allowedFindings;
     List<String> arguments;
     try {
@@ -78,12 +85,17 @@ public static void main(String[] args) {
       shouldVerifyCrashReproducer = Boolean.parseBoolean(args[5]);
       expectCrash = Boolean.parseBoolean(args[6]);
       usesJavaLauncher = Boolean.parseBoolean(args[7]);
+      if (args[8].isEmpty()) {
+        expectedWarningOrError = Optional.empty();
+      } else {
+        expectedWarningOrError = Optional.of(args[8]);
+      }
       allowedFindings =
-          Arrays.stream(args[8].split(",")).filter(s -> !s.isEmpty()).collect(Collectors.toSet());
+          Arrays.stream(args[9].split(",")).filter(s -> !s.isEmpty()).collect(toSet());
       // Map all files/dirs to real location
       arguments =
           Arrays.stream(args)
-              .skip(9)
+              .skip(10)
               .map(arg -> arg.startsWith("-") ? arg : runfiles.rlocation(arg))
               .collect(toList());
     } catch (IOException | ArrayIndexOutOfBoundsException e) {
@@ -150,9 +162,14 @@ public static void main(String[] args) {
 
     try {
       Process process = processBuilder.start();
+      boolean sawErrorWithStackTrace;
       try {
-        verifyFuzzerOutput(
-            process.getErrorStream(), allowedFindings, arguments.contains("--nohooks"));
+        sawErrorWithStackTrace =
+            verifyFuzzerOutput(
+                process.getErrorStream(),
+                allowedFindings,
+                arguments.contains("--nohooks"),
+                expectedWarningOrError);
       } finally {
         process.getErrorStream().close();
       }
@@ -166,10 +183,11 @@ public static void main(String[] args) {
         System.exit(0);
       }
       // Assert that we either found a crash in Java (exit code 77), a sanitizer crash (exit code
-      // 76), or a timeout (exit code 70).
+      // 76), a timeout (exit code 70) or an error with stack trace (exit code 1).
       if (exitCode != 76
           && exitCode != 77
-          && !(allowedFindings.contains("timeout") && exitCode == 70)) {
+          && !(allowedFindings.contains("timeout") && exitCode == 70)
+          && !(sawErrorWithStackTrace && exitCode == 1)) {
         System.err.printf("Did expect a crash, but Jazzer exited with exit code %d%n", exitCode);
         System.exit(1);
       }
@@ -207,25 +225,58 @@ public static void main(String[] args) {
     System.exit(0);
   }
 
-  private static void verifyFuzzerOutput(
-      InputStream fuzzerOutput, Set<String> expectedFindings, boolean noHooks) throws IOException {
-    List<String> stackTrace;
+  // Returns true if the fuzzer failed with an error and there was a stack trace.
+  private static boolean verifyFuzzerOutput(
+      InputStream fuzzerOutput,
+      Set<String> expectedFindings,
+      boolean noHooks,
+      Optional<String> expectedWarningOrError)
+      throws IOException {
+    List<String> lines;
     try (BufferedReader reader = new BufferedReader(new InputStreamReader(fuzzerOutput))) {
-      stackTrace =
-          reader
-              .lines()
-              .peek(System.err::println)
-              .filter(
-                  line ->
-                      line.startsWith(EXCEPTION_PREFIX)
-                          || line.startsWith(FRAME_PREFIX)
-                          || line.equals(THREAD_DUMP_HEADER)
-                          || SANITIZER_FINDING.matcher(line).find())
-              .collect(toList());
+      lines = reader.lines().collect(toList());
+    }
+
+    List<String> warningsAndErrors =
+        lines.stream()
+            .filter(line -> line.startsWith("WARN") || line.startsWith("ERROR"))
+            .filter(line -> !IGNORED_WARNINGS.contains(line))
+            .collect(toList());
+    boolean sawError = warningsAndErrors.stream().anyMatch(line -> line.startsWith("ERROR"));
+    if (!expectedWarningOrError.isPresent() && !warningsAndErrors.isEmpty()) {
+      throw new IllegalStateException(
+          "Did not expect warnings or errors, but got:\n" + String.join("\n", warningsAndErrors));
+    }
+    if (expectedWarningOrError.isPresent()) {
+      if (warningsAndErrors.isEmpty()) {
+        throw new IllegalStateException("Expected a warning or error, but did not get any");
+      }
+      String unexpectedWarningsAndErrors =
+          warningsAndErrors.stream()
+              .filter(line -> !expectedWarningOrError.get().equals(line))
+              .collect(Collectors.joining("\n"));
+      if (!unexpectedWarningsAndErrors.isEmpty()) {
+        throw new IllegalStateException(
+            "Got unexpected warnings or errors: " + unexpectedWarningsAndErrors);
+      }
     }
+
+    List<String> stackTrace =
+        lines.stream()
+            .peek(System.err::println)
+            .filter(
+                line ->
+                    line.startsWith(EXCEPTION_PREFIX)
+                        || line.startsWith(FRAME_PREFIX)
+                        || line.equals(THREAD_DUMP_HEADER)
+                        || SANITIZER_FINDING.matcher(line).find())
+            .collect(toList());
     if (expectedFindings.isEmpty()) {
       if (stackTrace.isEmpty()) {
-        return;
+        return false;
+      }
+      if (!warningsAndErrors.isEmpty()) {
+        return sawError;
       }
       throw new IllegalStateException(
           String.format(
@@ -244,7 +295,7 @@ private static void verifyFuzzerOutput(
       if (expectedFindings.size() != 1) {
         throw new IllegalStateException("Cannot expect both a native and other findings");
       }
-      return;
+      return false;
     }
     if (expectedFindings.contains("timeout")) {
       if (!stackTrace.contains(THREAD_DUMP_HEADER) || stackTrace.size() < 3) {
@@ -254,7 +305,7 @@ private static void verifyFuzzerOutput(
       if (expectedFindings.size() != 1) {
         throw new IllegalStateException("Cannot expect both a timeout and other findings");
       }
-      return;
+      return false;
     }
     List<String> findings =
         stackTrace.stream()
@@ -291,6 +342,7 @@ private static void verifyFuzzerOutput(
               "Unexpected strack trace frames:%n%n%s%n%nin:%n%s",
               String.join("\n", unexpectedFrames), String.join("\n", stackTrace)));
     }
+    return false;
   }
 
   private static void verifyCrashReproducer(
@@ -313,7 +365,7 @@ private static String compile(File source, Path api, Path targetJar) throws IOEx
     try (StandardJavaFileManager fileManager = compiler.getStandardFileManager(null, null, null)) {
       Iterable<? extends JavaFileObject> compilationUnits = fileManager.getJavaFileObjects(source);
       List<String> options =
-          Arrays.asList(
+          asList(
               "-classpath", String.join(File.pathSeparator, api.toString(), targetJar.toString()));
       System.out.printf(
           "Compile crash reproducer %s with options %s%n", source.getAbsolutePath(), options);

examples/BUILD.bazel

@@ -57,6 +57,7 @@ java_fuzz_target_test(
     allowed_findings = ["native"],
     env = {"EXAMPLE_NATIVE_LIB": "native_asan"},
     env_inherit = ["CC"],
+    expected_warning_or_error = "WARN: Jazzer is not compatible with LeakSanitizer. Leaks are not reported.",
     fuzzer_args = [
         "--asan",
     ],
@@ -351,6 +352,7 @@ java_fuzz_target_test(
         "src/main/java/com/example/FastJsonFuzzer.java",
     ],
     allowed_findings = ["java.lang.NumberFormatException"],
+    expected_warning_or_error = "WARN: Some hooks could not be applied to class files built for Java 7 or lower.",
     target_class = "com.example.FastJsonFuzzer",
     deps = [
         "@maven//:com_alibaba_fastjson",
@@ -376,6 +378,7 @@ java_fuzz_target_test(
         "java.lang.NumberFormatException",
         "java.lang.NullPointerException",
     ],
+    expected_warning_or_error = "WARN: Some hooks could not be applied to class files built for Java 7 or lower.",
     fuzzer_args = [
         "--keep_going=7",
     ],

examples/junit/src/test/java/com/example/BUILD.bazel

@@ -72,11 +72,13 @@ java_fuzz_target_test(
 java_fuzz_target_test(
     name = "PerExecutionLifecycleFuzzTest",
     srcs = ["PerExecutionLifecycleFuzzTest.java"],
-    allowed_findings = ["com.example.TestSuccessfulException"],
+    expect_crash = True,
+    expected_warning_or_error = "ERROR: com.example.TestSuccessfulException: Lifecycle methods invoked as expected",
     fuzzer_args = [
         "-runs=3",
     ],
     target_class = "com.example.PerExecutionLifecycleFuzzTest",
+    verify_crash_input = False,
     verify_crash_reproducer = False,
     runtime_deps = [
         ":junit_runtime",
@@ -94,8 +96,8 @@ java_fuzz_target_test(
 java_fuzz_target_test(
     name = "PerExecutionLifecycleWithFindingFuzzTest",
     srcs = ["PerExecutionLifecycleWithFindingFuzzTest.java"],
-    # TODO: The TestSuccessfulException thrown in @AfterAll is swallowed when run from the CLI.
     allowed_findings = ["java.io.IOException"],
+    expected_warning_or_error = "ERROR: com.example.TestSuccessfulException: Lifecycle methods invoked as expected",
     fuzzer_args = [
         "-runs=3",
     ],
@@ -117,11 +119,13 @@ java_fuzz_target_test(
 java_fuzz_target_test(
     name = "PerTestLifecycleFuzzTest",
     srcs = ["PerTestLifecycleFuzzTest.java"],
-    allowed_findings = ["com.example.TestSuccessfulException"],
+    expect_crash = True,
+    expected_warning_or_error = "ERROR: com.example.TestSuccessfulException: Lifecycle methods invoked as expected",
     fuzzer_args = [
         "-runs=3",
     ],
     target_class = "com.example.PerTestLifecycleFuzzTest",
+    verify_crash_input = False,
     verify_crash_reproducer = False,
     runtime_deps = [
         ":junit_runtime",

sanitizers/src/test/java/com/example/BUILD.bazel

@@ -48,6 +48,7 @@ java_fuzz_target_test(
         "ExpressionLanguageInjection.java",
     ],
     allowed_findings = ["com.code_intelligence.jazzer.api.FuzzerSecurityIssueHigh"],
+    expected_warning_or_error = "WARN: Some hooks could not be applied to class files built for Java 7 or lower.",
     target_class = "com.example.ExpressionLanguageInjection",
     # The reproducer can't find jaz.Zer and thus doesn't crash.
     verify_crash_reproducer = False,

src/main/java/com/code_intelligence/jazzer/api/BugDetectors.java

@@ -99,7 +99,7 @@ private static AtomicReference<BiPredicate<String, Integer>> getConnectionPermit
       return (AtomicReference<BiPredicate<String, Integer>>)
           ssrfSanitizer.getField("connectionPermitted").get(null);
     } catch (ClassNotFoundException | NoSuchFieldException | IllegalAccessException e) {
-      System.err.println("WARNING: ");
+      System.err.println("WARN: ");
       e.printStackTrace();
       return null;
     }

tests/BUILD.bazel

@@ -205,6 +205,7 @@ java_fuzz_target_test(
     name = "NoCoverageFuzzer",
     timeout = "short",
     srcs = ["src/test/java/com/example/NoCoverageFuzzer.java"],
+    expected_warning_or_error = "WARNING: no interesting inputs were found so far. Is the code instrumented for coverage?",
     fuzzer_args = [
         "-runs=10",
         "--instrumentation_excludes=**",
@@ -485,6 +486,11 @@ java_fuzz_target_test(
     timeout = "short",
     srcs = ["src/test/java/com/example/OfflineInstrumentedFuzzer.java"],
     allowed_findings = ["java.lang.IllegalStateException"],
+    fuzzer_args = [
+        # Do not instrument the JaCoCo agent.
+        "--instrumentation_includes=com.example.*",
+        "--custom_hook_includes=com.example.*",
+    ],
     target_class = "com.example.OfflineInstrumentedFuzzer",
     deps = [
         ":OfflineInstrumentedTargetInstrumented",