Merge pull request github#13886 from github/kaeluka/java-automodel-variadic-args

Java: automodel application mode: use endpoint class like in framework mode

Author: Stephan Brandauer

java/ql/src/Telemetry/AutomodelApplicationModeCharacteristics.qll

@@ -21,21 +21,97 @@ import AutomodelEndpointTypes as AutomodelEndpointTypes
 
 newtype JavaRelatedLocationType = CallContext()
 
+newtype TApplicationModeEndpoint =
+  TExplicitArgument(Call call, DataFlow::Node arg) {
+    exists(Argument argExpr |
+      arg.asExpr() = argExpr and call = argExpr.getCall() and not argExpr.isVararg()
+    )
+  } or
+  TInstanceArgument(Call call, DataFlow::Node arg) { arg = DataFlow::getInstanceArgument(call) } or
+  TImplicitVarargsArray(Call call, DataFlow::Node arg, int idx) {
+    exists(Argument argExpr |
+      arg.asExpr() = argExpr and
+      call.getArgument(idx) = argExpr and
+      argExpr.isVararg() and
+      not exists(int i | i < idx and call.getArgument(i).(Argument).isVararg())
+    )
+  }
+
+/**
+ * An endpoint is a node that is a candidate for modeling.
+ */
+abstract private class ApplicationModeEndpoint extends TApplicationModeEndpoint {
+  abstract predicate isArgOf(Call c, int idx);
+
+  Call getCall() { this.isArgOf(result, _) }
+
+  int getArgIndex() { this.isArgOf(_, result) }
+
+  abstract Top asTop();
+
+  abstract DataFlow::Node asNode();
+
+  abstract string toString();
+}
+
 /**
  * A class representing nodes that are arguments to calls.
  */
-private class ArgumentNode extends DataFlow::Node {
-  Call c;
+class ExplicitArgument extends ApplicationModeEndpoint, TExplicitArgument {
+  Call call;
+  DataFlow::Node arg;
 
-  ArgumentNode() {
-    exists(Argument arg | this.asExpr() = arg and not arg.isVararg() and c = arg.getCall())
-    or
-    this.(DataFlow::ImplicitVarargsArray).getCall() = c
-    or
-    this = DataFlow::getInstanceArgument(c)
+  ExplicitArgument() { this = TExplicitArgument(call, arg) }
+
+  override predicate isArgOf(Call c, int idx) { c = call and this.asTop() = c.getArgument(idx) }
+
+  override Top asTop() { result = arg.asExpr() }
+
+  override DataFlow::Node asNode() { result = arg }
+
+  override string toString() { result = arg.toString() }
+}
+
+class InstanceArgument extends ApplicationModeEndpoint, TInstanceArgument {
+  Call call;
+  DataFlow::Node arg;
+
+  InstanceArgument() { this = TInstanceArgument(call, arg) }
+
+  override predicate isArgOf(Call c, int idx) {
+    c = call and this.asTop() = c.getQualifier() and idx = -1
   }
 
-  Call getCall() { result = c }
+  override Top asTop() { if exists(arg.asExpr()) then result = arg.asExpr() else result = call }
+
+  override DataFlow::Node asNode() { result = arg }
+
+  override string toString() { result = arg.toString() }
+}
+
+/**
+ * An endpoint that represents an implicit varargs array.
+ * We choose to represent the varargs array as a single endpoint, rather than as multiple endpoints.
+ *
+ * This avoids the problem of having to deal with redundant endpoints downstream.
+ *
+ * In order to be able to distinguish between varargs endpoints and regular endpoints, we export the `isVarargsArray`
+ * meta data field in the extraction queries.
+ */
+class ImplicitVarargsArray extends ApplicationModeEndpoint, TImplicitVarargsArray {
+  Call call;
+  DataFlow::Node vararg;
+  int idx;
+
+  ImplicitVarargsArray() { this = TImplicitVarargsArray(call, vararg, idx) }
+
+  override predicate isArgOf(Call c, int i) { c = call and i = idx }
+
+  override Top asTop() { result = this.getCall() }
+
+  override DataFlow::Node asNode() { result = vararg }
+
+  override string toString() { result = vararg.toString() }
 }
 
 /**
@@ -47,7 +123,7 @@ private class ArgumentNode extends DataFlow::Node {
  */
 module ApplicationCandidatesImpl implements SharedCharacteristics::CandidateSig {
   // for documentation of the implementations here, see the QLDoc in the CandidateSig signature module.
-  class Endpoint = ArgumentNode;
+  class Endpoint = ApplicationModeEndpoint;
 
   class EndpointType = AutomodelEndpointTypes::EndpointType;
 
@@ -61,18 +137,18 @@ module ApplicationCandidatesImpl implements SharedCharacteristics::CandidateSig
   predicate isSanitizer(Endpoint e, EndpointType t) {
     exists(t) and
     (
-      e.getType() instanceof BoxedType
+      e.asNode().getType() instanceof BoxedType
       or
-      e.getType() instanceof PrimitiveType
+      e.asNode().getType() instanceof PrimitiveType
       or
-      e.getType() instanceof NumberType
+      e.asNode().getType() instanceof NumberType
     )
     or
     t instanceof AutomodelEndpointTypes::PathInjectionSinkType and
-    e instanceof PathSanitizer::PathInjectionSanitizer
+    e.asNode() instanceof PathSanitizer::PathInjectionSanitizer
   }
 
-  RelatedLocation asLocation(Endpoint e) { result = e.asExpr() }
+  RelatedLocation asLocation(Endpoint e) { result = e.asTop() }
 
   predicate isKnownKind = AutomodelJavaUtil::isKnownKind/2;
 
@@ -98,16 +174,7 @@ module ApplicationCandidatesImpl implements SharedCharacteristics::CandidateSig
     ApplicationModeGetCallable::getCallable(e).hasQualifiedName(package, type, name) and
     signature = ExternalFlow::paramsString(ApplicationModeGetCallable::getCallable(e)) and
     ext = "" and
-    (
-      exists(Call c, int argIdx |
-        e.asExpr() = c.getArgument(argIdx) and
-        input = AutomodelJavaUtil::getArgumentForIndex(argIdx)
-      )
-      or
-      exists(Call c |
-        e.asExpr() = c.getQualifier() and input = AutomodelJavaUtil::getArgumentForIndex(-1)
-      )
-    )
+    input = AutomodelJavaUtil::getArgumentForIndex(e.getArgIndex())
   }
 
   /**
@@ -118,7 +185,7 @@ module ApplicationCandidatesImpl implements SharedCharacteristics::CandidateSig
    */
   RelatedLocation getRelatedLocation(Endpoint e, RelatedLocationType type) {
     type = CallContext() and
-    result = any(Call c | e.asExpr() = [c.getAnArgument(), c.getQualifier()])
+    result = e.getCall()
   }
 }
 
@@ -132,20 +199,15 @@ private module ApplicationModeGetCallable implements AutomodelSharedGetCallable:
   /**
    * Returns the API callable being modeled.
    */
-  Callable getCallable(Endpoint e) {
-    exists(Call c |
-      e.asExpr() = [c.getAnArgument(), c.getQualifier()] and
-      result = c.getCallee()
-    )
-  }
+  Callable getCallable(Endpoint e) { result = e.getCall().getCallee() }
 }
 
 /**
  * Contains endpoints that are defined in QL code rather than as a MaD model. Ideally this predicate
  * should be empty.
  */
 private predicate isCustomSink(Endpoint e, string kind) {
-  e instanceof QueryInjectionSink and kind = "sql"
+  e.asNode() instanceof QueryInjectionSink and kind = "sql"
 }
 
 module CharacteristicsImpl =
@@ -167,23 +229,21 @@ class ApplicationModeMetadataExtractor extends string {
 
   predicate hasMetadata(
     Endpoint e, string package, string type, string subtypes, string name, string signature,
-    string input
+    string input, string isVarargsArray
   ) {
-    exists(Call call, Callable callable, int argIdx |
-      call.getCallee() = callable and
-      (
-        e.asExpr() = call.getArgument(argIdx)
-        or
-        e.asExpr() = call.getQualifier() and argIdx = -1
-      ) and
-      input = AutomodelJavaUtil::getArgumentForIndex(argIdx) and
+    exists(Callable callable |
+      e.getCall().getCallee() = callable and
+      input = AutomodelJavaUtil::getArgumentForIndex(e.getArgIndex()) and
       package = callable.getDeclaringType().getPackage().getName() and
       // we're using the erased types because the MaD convention is to not specify type parameters.
       // Whether something is or isn't a sink doesn't usually depend on the type parameters.
       type = callable.getDeclaringType().getErasure().(RefType).nestedName() and
       subtypes = AutomodelJavaUtil::considerSubtypes(callable).toString() and
       name = callable.getName() and
-      signature = ExternalFlow::paramsString(callable)
+      signature = ExternalFlow::paramsString(callable) and
+      if e instanceof ImplicitVarargsArray
+      then isVarargsArray = "true"
+      else isVarargsArray = "false"
     )
   }
 }
@@ -253,28 +313,10 @@ private class IsMaDTaintStepCharacteristic extends CharacteristicsImpl::NotASink
   IsMaDTaintStepCharacteristic() { this = "taint step" }
 
   override predicate appliesToEndpoint(Endpoint e) {
-    FlowSummaryImpl::Private::Steps::summaryThroughStepValue(e, _, _) or
-    FlowSummaryImpl::Private::Steps::summaryThroughStepTaint(e, _, _) or
-    FlowSummaryImpl::Private::Steps::summaryGetterStep(e, _, _, _) or
-    FlowSummaryImpl::Private::Steps::summarySetterStep(e, _, _, _)
-  }
-}
-
-/**
- * A negative characteristic that filters out qualifiers that are classes (i.e. static calls). These
- * are unlikely to have any non-trivial flow going into them.
- *
- * Technically, an accessed type _could_ come from outside of the source code, but there's not
- * much likelihood of that being user-controlled.
- */
-private class ClassQualifierCharacteristic extends CharacteristicsImpl::NotASinkCharacteristic {
-  ClassQualifierCharacteristic() { this = "class qualifier" }
-
-  override predicate appliesToEndpoint(Endpoint e) {
-    exists(Call c |
-      e.asExpr() = c.getQualifier() and
-      c.getQualifier() instanceof TypeAccess
-    )
+    FlowSummaryImpl::Private::Steps::summaryThroughStepValue(e.asNode(), _, _) or
+    FlowSummaryImpl::Private::Steps::summaryThroughStepTaint(e.asNode(), _, _) or
+    FlowSummaryImpl::Private::Steps::summaryGetterStep(e.asNode(), _, _, _) or
+    FlowSummaryImpl::Private::Steps::summarySetterStep(e.asNode(), _, _, _)
   }
 }
 
@@ -351,7 +393,7 @@ private class OtherArgumentToModeledMethodCharacteristic extends Characteristics
 private class FunctionValueCharacteristic extends CharacteristicsImpl::LikelyNotASinkCharacteristic {
   FunctionValueCharacteristic() { this = "function value" }
 
-  override predicate appliesToEndpoint(Endpoint e) { e.asExpr() instanceof FunctionalExpr }
+  override predicate appliesToEndpoint(Endpoint e) { e.asNode().asExpr() instanceof FunctionalExpr }
 }
 
 /**
@@ -371,12 +413,12 @@ private class CannotBeTaintedCharacteristic extends CharacteristicsImpl::LikelyN
    * Holds if the node `n` is known as the predecessor in a modeled flow step.
    */
   private predicate isKnownOutNodeForStep(Endpoint e) {
-    e.asExpr() instanceof Call or // we just assume flow in that case
-    TaintTracking::localTaintStep(_, e) or
-    FlowSummaryImpl::Private::Steps::summaryThroughStepValue(_, e, _) or
-    FlowSummaryImpl::Private::Steps::summaryThroughStepTaint(_, e, _) or
-    FlowSummaryImpl::Private::Steps::summaryGetterStep(_, _, e, _) or
-    FlowSummaryImpl::Private::Steps::summarySetterStep(_, _, e, _)
+    e.asNode().asExpr() instanceof Call or // we just assume flow in that case
+    TaintTracking::localTaintStep(_, e.asNode()) or
+    FlowSummaryImpl::Private::Steps::summaryThroughStepValue(_, e.asNode(), _) or
+    FlowSummaryImpl::Private::Steps::summaryThroughStepTaint(_, e.asNode(), _) or
+    FlowSummaryImpl::Private::Steps::summaryGetterStep(_, _, e.asNode(), _) or
+    FlowSummaryImpl::Private::Steps::summarySetterStep(_, _, e.asNode(), _)
   }
 }
 

java/ql/src/Telemetry/AutomodelApplicationModeExtractCandidates.ql

@@ -25,16 +25,18 @@ private import AutomodelJavaUtil
 bindingset[limit]
 private Endpoint getSampleForSignature(
   int limit, string package, string type, string subtypes, string name, string signature,
-  string input
+  string input, string isVarargs
 ) {
   exists(int n, int num_endpoints, ApplicationModeMetadataExtractor meta |
     num_endpoints =
-      count(Endpoint e | meta.hasMetadata(e, package, type, subtypes, name, signature, input))
+      count(Endpoint e |
+        meta.hasMetadata(e, package, type, subtypes, name, signature, input, isVarargs)
+      )
   |
     result =
       rank[n](Endpoint e, Location loc |
-        loc = e.getLocation() and
-        meta.hasMetadata(e, package, type, subtypes, name, signature, input)
+        loc = e.asTop().getLocation() and
+        meta.hasMetadata(e, package, type, subtypes, name, signature, input, isVarargs)
       |
         e
         order by
@@ -53,19 +55,20 @@ private Endpoint getSampleForSignature(
 from
   Endpoint endpoint, string message, ApplicationModeMetadataExtractor meta, DollarAtString package,
   DollarAtString type, DollarAtString subtypes, DollarAtString name, DollarAtString signature,
-  DollarAtString input
+  DollarAtString input, DollarAtString isVarargsArray
 where
   not exists(CharacteristicsImpl::UninterestingToModelCharacteristic u |
     u.appliesToEndpoint(endpoint)
   ) and
-  endpoint = getSampleForSignature(9, package, type, subtypes, name, signature, input) and
+  endpoint =
+    getSampleForSignature(9, package, type, subtypes, name, signature, input, isVarargsArray) and
   // If a node is already a known sink for any of our existing ATM queries and is already modeled as a MaD sink, we
   // don't include it as a candidate. Otherwise, we might include it as a candidate for query A, but the model will
   // label it as a sink for one of the sink types of query B, for which it's already a known sink. This would result in
   // overlap between our detected sinks and the pre-existing modeling. We assume that, if a sink has already been
   // modeled in a MaD model, then it doesn't belong to any additional sink types, and we don't need to reexamine it.
   not CharacteristicsImpl::isSink(endpoint, _, _) and
-  meta.hasMetadata(endpoint, package, type, subtypes, name, signature, input) and
+  meta.hasMetadata(endpoint, package, type, subtypes, name, signature, input, isVarargsArray) and
   includeAutomodelCandidate(package, type, name, signature) and
   // The message is the concatenation of all sink types for which this endpoint is known neither to be a sink nor to be
   // a non-sink, and we surface only endpoints that have at least one such sink type.
@@ -76,11 +79,13 @@ where
     |
       sinkType, ", "
     )
-select endpoint, message + "\nrelated locations: $@." + "\nmetadata: $@, $@, $@, $@, $@, $@.", //
+select endpoint.asNode(),
+  message + "\nrelated locations: $@." + "\nmetadata: $@, $@, $@, $@, $@, $@, $@.", //
   CharacteristicsImpl::getRelatedLocationOrCandidate(endpoint, CallContext()), "CallContext", //
   package, "package", //
   type, "type", //
   subtypes, "subtypes", //
   name, "name", // method name
   signature, "signature", //
-  input, "input" //
+  input, "input", //
+  isVarargsArray, "isVarargsArray"

java/ql/src/Telemetry/AutomodelApplicationModeExtractNegativeExamples.ql

@@ -24,7 +24,7 @@ Endpoint getSampleForCharacteristic(EndpointCharacteristic c, int limit) {
   exists(int n, int num_endpoints | num_endpoints = count(Endpoint e | c.appliesToEndpoint(e)) |
     result =
       rank[n](Endpoint e, Location loc |
-        loc = e.getLocation() and c.appliesToEndpoint(e)
+        loc = e.asTop().getLocation() and c.appliesToEndpoint(e)
       |
         e
         order by
@@ -43,15 +43,16 @@ Endpoint getSampleForCharacteristic(EndpointCharacteristic c, int limit) {
 from
   Endpoint endpoint, EndpointCharacteristic characteristic, float confidence, string message,
   ApplicationModeMetadataExtractor meta, DollarAtString package, DollarAtString type,
-  DollarAtString subtypes, DollarAtString name, DollarAtString signature, DollarAtString input
+  DollarAtString subtypes, DollarAtString name, DollarAtString signature, DollarAtString input,
+  DollarAtString isVarargsArray
 where
   endpoint = getSampleForCharacteristic(characteristic, 100) and
   confidence >= SharedCharacteristics::highConfidence() and
   characteristic.hasImplications(any(NegativeSinkType negative), true, confidence) and
   // Exclude endpoints that have contradictory endpoint characteristics, because we only want examples we're highly
   // certain about in the prompt.
   not erroneousEndpoints(endpoint, _, _, _, _, false) and
-  meta.hasMetadata(endpoint, package, type, subtypes, name, signature, input) and
+  meta.hasMetadata(endpoint, package, type, subtypes, name, signature, input, isVarargsArray) and
   // It's valid for a node to satisfy the logic for both `isSink` and `isSanitizer`, but in that case it will be
   // treated by the actual query as a sanitizer, since the final logic is something like
   // `isSink(n) and not isSanitizer(n)`. We don't want to include such nodes as negative examples in the prompt, because
@@ -63,11 +64,13 @@ where
     characteristic2.hasImplications(positiveType, true, confidence2)
   ) and
   message = characteristic
-select endpoint, message + "\nrelated locations: $@." + "\nmetadata: $@, $@, $@, $@, $@, $@.", //
+select endpoint.asNode(),
+  message + "\nrelated locations: $@." + "\nmetadata: $@, $@, $@, $@, $@, $@, $@.", //
   CharacteristicsImpl::getRelatedLocationOrCandidate(endpoint, CallContext()), "CallContext", //
   package, "package", //
   type, "type", //
   subtypes, "subtypes", //
   name, "name", //
   signature, "signature", //
-  input, "input" //
+  input, "input", //
+  isVarargsArray, "isVarargsArray" //