Use inline expectations for negative-example tests as well.

Author: Max Schaefer

java/ql/automodel/src/AutomodelApplicationModeCharacteristics.qll

@@ -397,6 +397,45 @@ predicate isCandidate(
   )
 }
 
+/**
+ * Holds if the given `endpoint` is a negative example for the `extensibleType`
+ * because of the `characteristic`.
+ *
+ * The other parameters record various other properties of interest.
+ */
+predicate isNegativeExample(
+  Endpoint endpoint, EndpointCharacteristic characteristic, float confidence, string package,
+  string type, string subtypes, string name, string signature, string input, string output,
+  string isVarargsArray, string extensibleType
+) {
+  characteristic.appliesToEndpoint(endpoint) and
+  // the node is known not to be an endpoint of any appropriate type
+  forall(AutomodelEndpointTypes::EndpointType tp |
+    tp = CharacteristicsImpl::getAPotentialType(endpoint)
+  |
+    characteristic.hasImplications(tp, false, _)
+  ) and
+  // the lowest confidence across all endpoint types should be at least highConfidence
+  confidence =
+    min(float c |
+      characteristic.hasImplications(CharacteristicsImpl::getAPotentialType(endpoint), false, c)
+    ) and
+  confidence >= SharedCharacteristics::highConfidence() and
+  any(ApplicationModeMetadataExtractor meta)
+      .hasMetadata(endpoint, package, type, subtypes, name, signature, input, output,
+        isVarargsArray, _, extensibleType) and
+  // It's valid for a node to be both a potential source/sanitizer and a sink. We don't want to include such nodes
+  // as negative examples in the prompt, because they're ambiguous and might confuse the model, so we explicitly exclude them here.
+  not exists(EndpointCharacteristic characteristic2, float confidence2 |
+    characteristic2 != characteristic
+  |
+    characteristic2.appliesToEndpoint(endpoint) and
+    confidence2 >= SharedCharacteristics::maximalConfidence() and
+    characteristic2
+        .hasImplications(CharacteristicsImpl::getAPotentialType(endpoint), true, confidence2)
+  )
+}
+
 /*
  * EndpointCharacteristic classes that are specific to Automodel for Java.
  */

java/ql/automodel/src/AutomodelApplicationModeExtractNegativeExamples.ql

@@ -40,45 +40,15 @@ Endpoint getSampleForCharacteristic(EndpointCharacteristic c, int limit) {
   )
 }
 
-predicate candidate(
-  Endpoint endpoint, EndpointCharacteristic characteristic, float confidence, string package,
-  string type, string subtypes, string name, string signature, string input, string output,
-  string isVarargsArray, string extensibleType
-) {
-  // the node is known not to be an endpoint of any appropriate type
-  forall(EndpointType tp | tp = CharacteristicsImpl::getAPotentialType(endpoint) |
-    characteristic.hasImplications(tp, false, _)
-  ) and
-  // the lowest confidence across all endpoint types should be at least highConfidence
-  confidence =
-    min(float c |
-      characteristic.hasImplications(CharacteristicsImpl::getAPotentialType(endpoint), false, c)
-    ) and
-  confidence >= SharedCharacteristics::highConfidence() and
-  any(ApplicationModeMetadataExtractor meta)
-      .hasMetadata(endpoint, package, type, subtypes, name, signature, input, output,
-        isVarargsArray, _, extensibleType) and
-  // It's valid for a node to be both a potential source/sanitizer and a sink. We don't want to include such nodes
-  // as negative examples in the prompt, because they're ambiguous and might confuse the model, so we explicitly exclude them here.
-  not exists(EndpointCharacteristic characteristic2, float confidence2 |
-    characteristic2 != characteristic
-  |
-    characteristic2.appliesToEndpoint(endpoint) and
-    confidence2 >= SharedCharacteristics::maximalConfidence() and
-    characteristic2
-        .hasImplications(CharacteristicsImpl::getAPotentialType(endpoint), true, confidence2)
-  )
-}
-
 from
   Endpoint endpoint, EndpointCharacteristic characteristic, float confidence, string message,
   DollarAtString package, DollarAtString type, DollarAtString subtypes, DollarAtString name,
   DollarAtString signature, DollarAtString input, DollarAtString output,
   DollarAtString isVarargsArray, DollarAtString extensibleType
 where
   endpoint = getSampleForCharacteristic(characteristic, 100) and
-  candidate(endpoint, characteristic, confidence, package, type, subtypes, name, signature, input,
-    output, isVarargsArray, extensibleType) and
+  isNegativeExample(endpoint, characteristic, confidence, package, type, subtypes, name, signature,
+    input, output, isVarargsArray, extensibleType) and
   message = characteristic
 select endpoint.asNode(),
   message + "\nrelated locations: $@, $@, $@." + "\nmetadata: $@, $@, $@, $@, $@, $@, $@, $@, $@.", //

java/ql/automodel/test/AutomodelApplicationModeExtraction/AutomodelApplicationModeExtractNegativeExamples.expected

@@ -1,3 +1,2 @@
-| Test.java:49:4:49:5 | f2 | known non-sink\nrelated locations: $@, $@, $@.\nmetadata: $@, $@, $@, $@, $@, $@, $@, $@, $@. | Test.java:48:10:50:3 | compareTo(...) | CallContext | Test.java:49:4:49:5 | f2 | MethodDoc | Test.java:49:4:49:5 | f2 | ClassDoc | file://java.io:1:1:1:1 | java.io | package | file://File:1:1:1:1 | File | type | file://true:1:1:1:1 | true | subtypes | file://compareTo:1:1:1:1 | compareTo | name | file://(File):1:1:1:1 | (File) | signature | file://Argument[0]:1:1:1:1 | Argument[0] | input | file://:1:1:1:1 |  | output | file://false:1:1:1:1 | false | isVarargsArray | file://sinkModel:1:1:1:1 | sinkModel | extensibleType |
-| Test.java:55:4:55:4 | p | taint step\nrelated locations: $@, $@, $@.\nmetadata: $@, $@, $@, $@, $@, $@, $@, $@, $@. | Test.java:54:3:59:3 | walk(...) | CallContext | Test.java:55:4:55:4 | p | MethodDoc | Test.java:55:4:55:4 | p | ClassDoc | file://java.nio.file:1:1:1:1 | java.nio.file | package | file://Files:1:1:1:1 | Files | type | file://false:1:1:1:1 | false | subtypes | file://walk:1:1:1:1 | walk | name | file://(Path,FileVisitOption[]):1:1:1:1 | (Path,FileVisitOption[]) | signature | file://Argument[0]:1:1:1:1 | Argument[0] | input | file://:1:1:1:1 |  | output | file://false:1:1:1:1 | false | isVarargsArray | file://sinkModel:1:1:1:1 | sinkModel | extensibleType |
-| Test.java:94:3:96:3 | deleteIfExists(...) | known sanitizer\nrelated locations: $@, $@, $@.\nmetadata: $@, $@, $@, $@, $@, $@, $@, $@, $@. | Test.java:94:3:96:3 | deleteIfExists(...) | CallContext | Test.java:94:3:96:3 | deleteIfExists(...) | MethodDoc | Test.java:94:3:96:3 | deleteIfExists(...) | ClassDoc | file://java.nio.file:1:1:1:1 | java.nio.file | package | file://Files:1:1:1:1 | Files | type | file://false:1:1:1:1 | false | subtypes | file://deleteIfExists:1:1:1:1 | deleteIfExists | name | file://(Path):1:1:1:1 | (Path) | signature | file://:1:1:1:1 |  | input | file://ReturnValue:1:1:1:1 | ReturnValue | output | file://false:1:1:1:1 | false | isVarargsArray | file://sourceModel:1:1:1:1 | sourceModel | extensibleType |
+testFailures
+failures

java/ql/automodel/test/AutomodelApplicationModeExtraction/AutomodelApplicationModeExtractNegativeExamples.ql

@@ -0,0 +1,27 @@
+import java
+import AutomodelApplicationModeCharacteristics
+import TestUtilities.InlineExpectationsTest
+
+module NegativeExampleTest implements TestSig {
+  string getARelevantTag() { result = "negativeExample" }
+
+  predicate hasActualResult(Location location, string element, string tag, string value) {
+    exists(
+      Endpoint endpoint, string name, string signature, string input,
+      string output, string extensibleType
+    |
+      isNegativeExample(endpoint, _, _, _, _, _, name, signature, input, output, _,
+        extensibleType)
+    |
+      endpoint.asTop().getLocation() = location and
+      endpoint.toString() = element and
+      tag = "negativeExample" and
+      // for source models only the output is relevant, and vice versa for sink models
+      if extensibleType = "sourceModel"
+      then value = name + signature + ":" + output
+      else value = name + signature + ":" + input
+    )
+  }
+}
+
+import MakeTest<NegativeExampleTest>

java/ql/automodel/test/AutomodelApplicationModeExtraction/AutomodelApplicationModeExtractNegativeExamples.qlref

@@ -1,7 +1,7 @@
-| Test.java:29:4:29:9 | source | path-injection\nrelated locations: $@, $@, $@.\nmetadata: $@, $@, $@, $@, $@, $@, $@, $@, $@. | Test.java:28:3:32:3 | copy(...) | CallContext | Test.java:29:4:29:9 | source | MethodDoc | Test.java:29:4:29:9 | source | ClassDoc | file://java.nio.file:1:1:1:1 | java.nio.file | package | file://Files:1:1:1:1 | Files | type | file://false:1:1:1:1 | false | subtypes | file://copy:1:1:1:1 | copy | name | file://(Path,Path,CopyOption[]):1:1:1:1 | (Path,Path,CopyOption[]) | signature | file://Argument[0]:1:1:1:1 | Argument[0] | input | file://:1:1:1:1 |  | output | file://false:1:1:1:1 | false | isVarargsArray | file://sinkModel:1:1:1:1 | sinkModel | extensibleType |
-| Test.java:30:4:30:9 | target | path-injection\nrelated locations: $@, $@, $@.\nmetadata: $@, $@, $@, $@, $@, $@, $@, $@, $@. | Test.java:28:3:32:3 | copy(...) | CallContext | Test.java:30:4:30:9 | target | MethodDoc | Test.java:30:4:30:9 | target | ClassDoc | file://java.nio.file:1:1:1:1 | java.nio.file | package | file://Files:1:1:1:1 | Files | type | file://false:1:1:1:1 | false | subtypes | file://copy:1:1:1:1 | copy | name | file://(Path,Path,CopyOption[]):1:1:1:1 | (Path,Path,CopyOption[]) | signature | file://Argument[1]:1:1:1:1 | Argument[1] | input | file://:1:1:1:1 |  | output | file://false:1:1:1:1 | false | isVarargsArray | file://sinkModel:1:1:1:1 | sinkModel | extensibleType |
-| Test.java:37:4:37:11 | openPath | path-injection\nrelated locations: $@, $@, $@.\nmetadata: $@, $@, $@, $@, $@, $@, $@, $@, $@. | Test.java:36:10:38:3 | newInputStream(...) | CallContext | Test.java:37:4:37:11 | openPath | MethodDoc | Test.java:37:4:37:11 | openPath | ClassDoc | file://java.nio.file:1:1:1:1 | java.nio.file | package | file://Files:1:1:1:1 | Files | type | file://false:1:1:1:1 | false | subtypes | file://newInputStream:1:1:1:1 | newInputStream | name | file://(Path,OpenOption[]):1:1:1:1 | (Path,OpenOption[]) | signature | file://Argument[0]:1:1:1:1 | Argument[0] | input | file://:1:1:1:1 |  | output | file://false:1:1:1:1 | false | isVarargsArray | file://sinkModel:1:1:1:1 | sinkModel | extensibleType |
-| Test.java:63:3:63:20 | getInputStream(...) | remote\nrelated locations: $@, $@, $@.\nmetadata: $@, $@, $@, $@, $@, $@, $@, $@, $@. | Test.java:63:3:63:20 | getInputStream(...) | CallContext | Test.java:63:3:63:20 | getInputStream(...) | MethodDoc | Test.java:63:3:63:20 | getInputStream(...) | ClassDoc | file://java.net:1:1:1:1 | java.net | package | file://URLConnection:1:1:1:1 | URLConnection | type | file://true:1:1:1:1 | true | subtypes | file://getInputStream:1:1:1:1 | getInputStream | name | file://():1:1:1:1 | () | signature | file://:1:1:1:1 |  | input | file://ReturnValue:1:1:1:1 | ReturnValue | output | file://false:1:1:1:1 | false | isVarargsArray | file://sourceModel:1:1:1:1 | sourceModel | extensibleType |
-| Test.java:87:28:87:28 | p | path-injection\nrelated locations: $@, $@, $@.\nmetadata: $@, $@, $@, $@, $@, $@, $@, $@, $@. | Test.java:87:4:87:29 | createDirectories(...) | CallContext | Test.java:87:28:87:28 | p | MethodDoc | Test.java:87:28:87:28 | p | ClassDoc | file://java.nio.file:1:1:1:1 | java.nio.file | package | file://Files:1:1:1:1 | Files | type | file://false:1:1:1:1 | false | subtypes | file://createDirectories:1:1:1:1 | createDirectories | name | file://(Path,FileAttribute[]):1:1:1:1 | (Path,FileAttribute[]) | signature | file://Argument[0]:1:1:1:1 | Argument[0] | input | file://:1:1:1:1 |  | output | file://false:1:1:1:1 | false | isVarargsArray | file://sinkModel:1:1:1:1 | sinkModel | extensibleType |
-| Test.java:91:4:91:4 | p | path-injection\nrelated locations: $@, $@, $@.\nmetadata: $@, $@, $@, $@, $@, $@, $@, $@, $@. | Test.java:90:3:92:3 | delete(...) | CallContext | Test.java:91:4:91:4 | p | MethodDoc | Test.java:91:4:91:4 | p | ClassDoc | file://java.nio.file:1:1:1:1 | java.nio.file | package | file://Files:1:1:1:1 | Files | type | file://false:1:1:1:1 | false | subtypes | file://delete:1:1:1:1 | delete | name | file://(Path):1:1:1:1 | (Path) | signature | file://Argument[0]:1:1:1:1 | Argument[0] | input | file://:1:1:1:1 |  | output | file://false:1:1:1:1 | false | isVarargsArray | file://sinkModel:1:1:1:1 | sinkModel | extensibleType |
-| Test.java:95:4:95:4 | p | path-injection\nrelated locations: $@, $@, $@.\nmetadata: $@, $@, $@, $@, $@, $@, $@, $@, $@. | Test.java:94:3:96:3 | deleteIfExists(...) | CallContext | Test.java:95:4:95:4 | p | MethodDoc | Test.java:95:4:95:4 | p | ClassDoc | file://java.nio.file:1:1:1:1 | java.nio.file | package | file://Files:1:1:1:1 | Files | type | file://false:1:1:1:1 | false | subtypes | file://deleteIfExists:1:1:1:1 | deleteIfExists | name | file://(Path):1:1:1:1 | (Path) | signature | file://Argument[0]:1:1:1:1 | Argument[0] | input | file://:1:1:1:1 |  | output | file://false:1:1:1:1 | false | isVarargsArray | file://sinkModel:1:1:1:1 | sinkModel | extensibleType |
+| Test.java:30:4:30:9 | source | path-injection\nrelated locations: $@, $@, $@.\nmetadata: $@, $@, $@, $@, $@, $@, $@, $@, $@. | Test.java:29:3:33:3 | copy(...) | CallContext | Test.java:30:4:30:9 | source | MethodDoc | Test.java:30:4:30:9 | source | ClassDoc | file://java.nio.file:1:1:1:1 | java.nio.file | package | file://Files:1:1:1:1 | Files | type | file://false:1:1:1:1 | false | subtypes | file://copy:1:1:1:1 | copy | name | file://(Path,Path,CopyOption[]):1:1:1:1 | (Path,Path,CopyOption[]) | signature | file://Argument[0]:1:1:1:1 | Argument[0] | input | file://:1:1:1:1 |  | output | file://false:1:1:1:1 | false | isVarargsArray | file://sinkModel:1:1:1:1 | sinkModel | extensibleType |
+| Test.java:31:4:31:9 | target | path-injection\nrelated locations: $@, $@, $@.\nmetadata: $@, $@, $@, $@, $@, $@, $@, $@, $@. | Test.java:29:3:33:3 | copy(...) | CallContext | Test.java:31:4:31:9 | target | MethodDoc | Test.java:31:4:31:9 | target | ClassDoc | file://java.nio.file:1:1:1:1 | java.nio.file | package | file://Files:1:1:1:1 | Files | type | file://false:1:1:1:1 | false | subtypes | file://copy:1:1:1:1 | copy | name | file://(Path,Path,CopyOption[]):1:1:1:1 | (Path,Path,CopyOption[]) | signature | file://Argument[1]:1:1:1:1 | Argument[1] | input | file://:1:1:1:1 |  | output | file://false:1:1:1:1 | false | isVarargsArray | file://sinkModel:1:1:1:1 | sinkModel | extensibleType |
+| Test.java:38:4:38:11 | openPath | path-injection\nrelated locations: $@, $@, $@.\nmetadata: $@, $@, $@, $@, $@, $@, $@, $@, $@. | Test.java:37:10:39:3 | newInputStream(...) | CallContext | Test.java:38:4:38:11 | openPath | MethodDoc | Test.java:38:4:38:11 | openPath | ClassDoc | file://java.nio.file:1:1:1:1 | java.nio.file | package | file://Files:1:1:1:1 | Files | type | file://false:1:1:1:1 | false | subtypes | file://newInputStream:1:1:1:1 | newInputStream | name | file://(Path,OpenOption[]):1:1:1:1 | (Path,OpenOption[]) | signature | file://Argument[0]:1:1:1:1 | Argument[0] | input | file://:1:1:1:1 |  | output | file://false:1:1:1:1 | false | isVarargsArray | file://sinkModel:1:1:1:1 | sinkModel | extensibleType |
+| Test.java:66:3:66:20 | getInputStream(...) | remote\nrelated locations: $@, $@, $@.\nmetadata: $@, $@, $@, $@, $@, $@, $@, $@, $@. | Test.java:66:3:66:20 | getInputStream(...) | CallContext | Test.java:66:3:66:20 | getInputStream(...) | MethodDoc | Test.java:66:3:66:20 | getInputStream(...) | ClassDoc | file://java.net:1:1:1:1 | java.net | package | file://URLConnection:1:1:1:1 | URLConnection | type | file://true:1:1:1:1 | true | subtypes | file://getInputStream:1:1:1:1 | getInputStream | name | file://():1:1:1:1 | () | signature | file://:1:1:1:1 |  | input | file://ReturnValue:1:1:1:1 | ReturnValue | output | file://false:1:1:1:1 | false | isVarargsArray | file://sourceModel:1:1:1:1 | sourceModel | extensibleType |
+| Test.java:90:28:90:28 | p | path-injection\nrelated locations: $@, $@, $@.\nmetadata: $@, $@, $@, $@, $@, $@, $@, $@, $@. | Test.java:90:4:90:29 | createDirectories(...) | CallContext | Test.java:90:28:90:28 | p | MethodDoc | Test.java:90:28:90:28 | p | ClassDoc | file://java.nio.file:1:1:1:1 | java.nio.file | package | file://Files:1:1:1:1 | Files | type | file://false:1:1:1:1 | false | subtypes | file://createDirectories:1:1:1:1 | createDirectories | name | file://(Path,FileAttribute[]):1:1:1:1 | (Path,FileAttribute[]) | signature | file://Argument[0]:1:1:1:1 | Argument[0] | input | file://:1:1:1:1 |  | output | file://false:1:1:1:1 | false | isVarargsArray | file://sinkModel:1:1:1:1 | sinkModel | extensibleType |
+| Test.java:94:4:94:4 | p | path-injection\nrelated locations: $@, $@, $@.\nmetadata: $@, $@, $@, $@, $@, $@, $@, $@, $@. | Test.java:93:3:95:3 | delete(...) | CallContext | Test.java:94:4:94:4 | p | MethodDoc | Test.java:94:4:94:4 | p | ClassDoc | file://java.nio.file:1:1:1:1 | java.nio.file | package | file://Files:1:1:1:1 | Files | type | file://false:1:1:1:1 | false | subtypes | file://delete:1:1:1:1 | delete | name | file://(Path):1:1:1:1 | (Path) | signature | file://Argument[0]:1:1:1:1 | Argument[0] | input | file://:1:1:1:1 |  | output | file://false:1:1:1:1 | false | isVarargsArray | file://sinkModel:1:1:1:1 | sinkModel | extensibleType |
+| Test.java:98:4:98:4 | p | path-injection\nrelated locations: $@, $@, $@.\nmetadata: $@, $@, $@, $@, $@, $@, $@, $@, $@. | Test.java:97:3:99:3 | deleteIfExists(...) | CallContext | Test.java:98:4:98:4 | p | MethodDoc | Test.java:98:4:98:4 | p | ClassDoc | file://java.nio.file:1:1:1:1 | java.nio.file | package | file://Files:1:1:1:1 | Files | type | file://false:1:1:1:1 | false | subtypes | file://deleteIfExists:1:1:1:1 | deleteIfExists | name | file://(Path):1:1:1:1 | (Path) | signature | file://Argument[0]:1:1:1:1 | Argument[0] | input | file://:1:1:1:1 |  | output | file://false:1:1:1:1 | false | isVarargsArray | file://sinkModel:1:1:1:1 | sinkModel | extensibleType |