chore: exclude potentially dangerous tests by default

All tests that can perform potentially harmful side effects are tagged
and excluded by default. The CI still tests all.

Closes #971

Author: simonresch

.bazelrc

@@ -126,3 +126,9 @@ common:quiet --noshow_progress
 test:fail-fast --notest_keep_going
 # Instruct test runners to fail a test target on the first failing test.
 test:fail-fast --test_runner_fail_fast
+
+# Safety: by default on local runs, exclude tests tagged as dangerous
+# (e.g., those that could execute OS commands, touch the filesystem aggressively, or require network).
+test --test_tag_filters=-dangerous
+# In CI, include all tests.
+test:ci --test_tag_filters=

CONTRIBUTING.md

@@ -78,6 +78,21 @@ $ bazel test //...
 If you are bisecting a bug or otherwise want test execution to stop right after the first failure, use `--config=fail-fast`.
 This is especially useful with long-running or parameterized tests.
 
+#### Potentially harmful tests
+
+Some tests deliberately exercise vulnerable code with fuzzer input to e.g. assert specific findings reported by Jazzer.
+Due to the pseudo-random nature of fuzzing this can cause potentially harmful side effects on the host system which is
+why such tests are tagged with `"dangerous"` and not executed by default when running tests locally. You can run _all_
+tests by adding the flag
+
+```bash
+--test_tag_filters=
+```
+
+but this should be done with care. At worst the tests will execute arbitrary commands or open network connections to
+random addresses. Depending on the host OS the bazel test sandboxing can be a good first line of defence to prevent
+destructive side effects.
+
 #### Debugging
 
 ##### Internal debugging

examples/BUILD.bazel

@@ -343,7 +343,10 @@ java_fuzz_target_test(
         "com.code_intelligence.jazzer.api.FuzzerSecurityIssueHigh",
     ],
     fuzzer_args = ["-runs=100000"],
-    tags = ["no-jdk8"],
+    tags = [
+        "dangerous",
+        "no-jdk8",
+    ],
     target_class = "com.example.SpringCloudFunctionRoutingFuzzer",
     verify_crash_reproducer = False,
     deps = [

sanitizers/src/test/java/com/example/BUILD.bazel

@@ -10,6 +10,7 @@ java_fuzz_target_test(
         "com.code_intelligence.jazzer.api.FuzzerSecurityIssueHigh",
         "java.lang.ExceptionInInitializerError",
     ],
+    tags = ["dangerous"],
     target_class = "com.example.ObjectInputStreamDeserialization",
 )
 
@@ -22,6 +23,7 @@ java_fuzz_target_test(
         "com.code_intelligence.jazzer.api.FuzzerSecurityIssueHigh",
         "java.lang.ExceptionInInitializerError",
     ],
+    tags = ["dangerous"],
     target_class = "com.example.ReflectiveCall",
 )
 
@@ -33,6 +35,7 @@ java_fuzz_target_test(
     allowed_findings = [
         "com.code_intelligence.jazzer.api.FuzzerSecurityIssueHigh",
     ],
+    tags = ["dangerous"],
     target_class = "com.example.LibraryLoad",
     # loading of native libraries is very slow on macos,
     # especially using Java 17
@@ -49,6 +52,7 @@ java_fuzz_target_test(
     ],
     allowed_findings = ["com.code_intelligence.jazzer.api.FuzzerSecurityIssueHigh"],
     expected_warning_or_error = "WARN: Some hooks could not be applied to class files built for Java 7 or lower.",
+    tags = ["dangerous"],
     target_class = "com.example.ExpressionLanguageInjection",
     # The reproducer can't find jaz.Zer and thus doesn't crash.
     verify_crash_reproducer = False,
@@ -159,6 +163,7 @@ java_fuzz_target_test(
         "OsCommandInjectionProcessBuilder.java",
     ],
     allowed_findings = ["com.code_intelligence.jazzer.api.FuzzerSecurityIssueCritical"],
+    tags = ["dangerous"],
     target_class = "com.example.OsCommandInjectionProcessBuilder",
     verify_crash_reproducer = False,
 )
@@ -169,6 +174,7 @@ java_fuzz_target_test(
         "OsCommandInjectionRuntimeExec.java",
     ],
     allowed_findings = ["com.code_intelligence.jazzer.api.FuzzerSecurityIssueCritical"],
+    tags = ["dangerous"],
     target_class = "com.example.OsCommandInjectionRuntimeExec",
     verify_crash_reproducer = False,
 )
@@ -243,6 +249,7 @@ java_fuzz_target_test(
         # Reproducer does not find the honeypot library and doesn't have the hook.
         "java.lang.ExceptionInInitializerError",
     ],
+    tags = ["dangerous"],
     target_class = "com.example.ClassLoaderLoadClass",
 )
 
@@ -489,6 +496,7 @@ java_fuzz_target_test(
     allowed_findings = [
         "com.code_intelligence.jazzer.api.FuzzerSecurityIssueMedium",
     ],
+    tags = ["dangerous"],
     target_class = "com.example.SsrfSocketConnect",
     verify_crash_reproducer = False,
 )
@@ -501,6 +509,7 @@ java_fuzz_target_test(
     allowed_findings = [
         "com.code_intelligence.jazzer.api.FuzzerSecurityIssueMedium",
     ],
+    tags = ["dangerous"],
     target_class = "com.example.SsrfSocketConnectToHost",
     verify_crash_reproducer = False,
 )
@@ -513,6 +522,7 @@ java_fuzz_target_test(
     allowed_findings = [
         "com.code_intelligence.jazzer.api.FuzzerSecurityIssueMedium",
     ],
+    tags = ["dangerous"],
     target_class = "com.example.SsrfUrlConnection",
     verify_crash_reproducer = False,
 )
@@ -525,7 +535,10 @@ java_fuzz_target_test(
     allowed_findings = [
         "com.code_intelligence.jazzer.api.FuzzerSecurityIssueMedium",
     ],
-    tags = ["no-jdk8"],
+    tags = [
+        "dangerous",
+        "no-jdk8",
+    ],
     target_class = "com.example.SsrfHttpClient",
     verify_crash_reproducer = False,
 )