Merge pull request github#10795 from aschackmull/java/synth-callable

Java: Add support for synthetic callables with flow summaries and model Stream.collect

Author: aschackmull

java/ql/lib/change-notes/2022-10-13-stream-collect.md

@@ -0,0 +1,4 @@
+---
+category: minorAnalysis
+---
+* Added support for common patterns involving `Stream.collect` and common collectors like `Collectors.toList()`.

java/ql/lib/semmle/code/java/dataflow/ExternalFlow.qll

@@ -75,7 +75,7 @@ import java
 private import semmle.code.java.dataflow.DataFlow::DataFlow
 private import internal.DataFlowPrivate
 private import internal.FlowSummaryImpl::Private::External
-private import internal.FlowSummaryImplSpecific
+private import internal.FlowSummaryImplSpecific as FlowSummaryImplSpecific
 private import internal.AccessPathSyntax
 private import FlowSummary
 
@@ -834,7 +834,7 @@ private module Cached {
    */
   cached
   predicate sourceNode(Node node, string kind) {
-    exists(InterpretNode n | isSourceNode(n, kind) and n.asNode() = node)
+    exists(FlowSummaryImplSpecific::InterpretNode n | isSourceNode(n, kind) and n.asNode() = node)
   }
 
   /**
@@ -843,7 +843,7 @@ private module Cached {
    */
   cached
   predicate sinkNode(Node node, string kind) {
-    exists(InterpretNode n | isSinkNode(n, kind) and n.asNode() = node)
+    exists(FlowSummaryImplSpecific::InterpretNode n | isSinkNode(n, kind) and n.asNode() = node)
   }
 }
 

java/ql/lib/semmle/code/java/dataflow/FlowSummary.qll

@@ -4,7 +4,6 @@
 
 import java
 private import internal.FlowSummaryImpl as Impl
-private import internal.DataFlowDispatch
 private import internal.DataFlowUtil
 
 // import all instances of SummarizedCallable below
@@ -24,6 +23,12 @@ module SummaryComponent {
   /** Gets a summary component for field `f`. */
   SummaryComponent field(Field f) { result = content(any(FieldContent c | c.getField() = f)) }
 
+  /** Gets a summary component for `Element`. */
+  SummaryComponent element() { result = content(any(CollectionContent c)) }
+
+  /** Gets a summary component for `MapValue`. */
+  SummaryComponent mapValue() { result = content(any(MapValueContent c)) }
+
   /** Gets a summary component that represents the return value of a call. */
   SummaryComponent return() { result = return(_) }
 }
@@ -42,10 +47,129 @@ module SummaryComponentStack {
     result = push(SummaryComponent::field(f), object)
   }
 
+  /** Gets a stack representing `Element` of `object`. */
+  SummaryComponentStack elementOf(SummaryComponentStack object) {
+    result = push(SummaryComponent::element(), object)
+  }
+
+  /** Gets a stack representing `MapValue` of `object`. */
+  SummaryComponentStack mapValueOf(SummaryComponentStack object) {
+    result = push(SummaryComponent::mapValue(), object)
+  }
+
   /** Gets a singleton stack representing a (normal) return. */
   SummaryComponentStack return() { result = singleton(SummaryComponent::return()) }
 }
 
+/** A synthetic callable with a set of concrete call sites and a flow summary. */
+abstract class SyntheticCallable extends string {
+  bindingset[this]
+  SyntheticCallable() { any() }
+
+  /** Gets a call that targets this callable. */
+  abstract Call getACall();
+
+  /**
+   * Holds if data may flow from `input` to `output` through this callable.
+   *
+   * See `SummarizedCallable::propagatesFlow` for details.
+   */
+  predicate propagatesFlow(
+    SummaryComponentStack input, SummaryComponentStack output, boolean preservesValue
+  ) {
+    none()
+  }
+
+  /**
+   * Gets the type of the parameter at the specified position with -1 indicating
+   * the instance parameter. If no types are provided then the types default to
+   * `Object`.
+   */
+  Type getParameterType(int pos) { none() }
+
+  /**
+   * Gets the return type of this callable. If no type is provided then the type
+   * defaults to `Object`.
+   */
+  Type getReturnType() { none() }
+}
+
+private newtype TSummarizedCallableBase =
+  TSimpleCallable(Callable c) { c.isSourceDeclaration() } or
+  TSyntheticCallable(SyntheticCallable c)
+
+/**
+ * A callable that may have a flow summary. This is either a regular `Callable`
+ * or a `SyntheticCallable`.
+ */
+class SummarizedCallableBase extends TSummarizedCallableBase {
+  /** Gets a textual representation of this callable. */
+  string toString() { result = this.asCallable().toString() or result = this.asSyntheticCallable() }
+
+  /** Gets the source location for this callable. */
+  Location getLocation() {
+    result = this.asCallable().getLocation()
+    or
+    result.hasLocationInfo("", 0, 0, 0, 0) and
+    this instanceof TSyntheticCallable
+  }
+
+  /** Gets this callable cast as a `Callable`. */
+  Callable asCallable() { this = TSimpleCallable(result) }
+
+  /** Gets this callable cast as a `SyntheticCallable`. */
+  SyntheticCallable asSyntheticCallable() { this = TSyntheticCallable(result) }
+
+  /** Gets a call that targets this callable. */
+  Call getACall() {
+    result.getCallee().getSourceDeclaration() = this.asCallable()
+    or
+    result = this.asSyntheticCallable().getACall()
+  }
+
+  /**
+   * Gets the type of the parameter at the specified position with -1 indicating
+   * the instance parameter.
+   */
+  Type getParameterType(int pos) {
+    result = this.asCallable().getParameterType(pos)
+    or
+    pos = -1 and result = this.asCallable().getDeclaringType()
+    or
+    result = this.asSyntheticCallable().getParameterType(pos)
+    or
+    exists(SyntheticCallable sc | sc = this.asSyntheticCallable() |
+      Impl::Private::summaryParameterNodeRange(this, pos) and
+      not exists(sc.getParameterType(pos)) and
+      result instanceof TypeObject
+    )
+  }
+
+  /** Gets the return type of this callable. */
+  Type getReturnType() {
+    result = this.asCallable().getReturnType()
+    or
+    exists(SyntheticCallable sc | sc = this.asSyntheticCallable() |
+      result = sc.getReturnType()
+      or
+      not exists(sc.getReturnType()) and
+      result instanceof TypeObject
+    )
+  }
+}
+
 class SummarizedCallable = Impl::Public::SummarizedCallable;
 
+/**
+ * An adapter class to add the flow summaries specified on `SyntheticCallable`
+ * to `SummarizedCallable`.
+ */
+private class SummarizedSyntheticCallableAdapter extends SummarizedCallable, TSyntheticCallable {
+  override predicate propagatesFlow(
+    SummaryComponentStack input, SummaryComponentStack output, boolean preservesValue
+  ) {
+    this.asSyntheticCallable().propagatesFlow(input, output, preservesValue)
+  }
+}
+
 class RequiredSummaryComponentStack = Impl::Public::RequiredSummaryComponentStack;

java/ql/lib/semmle/code/java/dataflow/internal/DataFlowDispatch.qll

@@ -9,9 +9,7 @@ private import semmle.code.java.dispatch.internal.Unification
 
 private module DispatchImpl {
   private predicate hasHighConfidenceTarget(Call c) {
-    exists(SummarizedCallable sc |
-      sc = c.getCallee().getSourceDeclaration() and not sc.isAutoGenerated()
-    )
+    exists(SummarizedCallable sc | sc.getACall() = c and not sc.isAutoGenerated())
     or
     exists(Callable srcTgt |
       srcTgt = VirtualDispatch::viableCallable(c) and
@@ -30,7 +28,7 @@ private module DispatchImpl {
   DataFlowCallable viableCallable(DataFlowCall c) {
     result.asCallable() = sourceDispatch(c.asCall())
     or
-    result.asSummarizedCallable() = c.asCall().getCallee().getSourceDeclaration()
+    result.asSummarizedCallable().getACall() = c.asCall()
   }
 
   /**
@@ -144,7 +142,7 @@ private module DispatchImpl {
         not Unification::failsUnification(t, t2)
       )
       or
-      result.asSummarizedCallable() = def
+      result.asSummarizedCallable().getACall() = ma
     )
   }
 

java/ql/lib/semmle/code/java/dataflow/internal/DataFlowNodes.qll

@@ -463,11 +463,7 @@ module Private {
       c.asSummarizedCallable() = sc and pos = pos_
     }
 
-    Type getTypeImpl() {
-      result = sc.getParameter(pos_).getType()
-      or
-      pos_ = -1 and result = sc.getDeclaringType()
-    }
+    Type getTypeImpl() { result = sc.getParameterType(pos_) }
   }
 }
 

java/ql/lib/semmle/code/java/dataflow/internal/DataFlowPrivate.qll

@@ -241,12 +241,6 @@ class DataFlowCallable extends TDataFlowCallable {
 
   Field asFieldScope() { this = TFieldScope(result) }
 
-  RefType getDeclaringType() {
-    result = this.asCallable().getDeclaringType() or
-    result = this.asSummarizedCallable().getDeclaringType() or
-    result = this.asFieldScope().getDeclaringType()
-  }
-
   string toString() {
     result = this.asCallable().toString() or
     result = "Synthetic: " + this.asSummarizedCallable().toString() or

java/ql/lib/semmle/code/java/dataflow/internal/FlowSummaryImplSpecific.qll

@@ -9,12 +9,9 @@ private import DataFlowUtil
 private import FlowSummaryImpl::Private
 private import FlowSummaryImpl::Public
 private import semmle.code.java.dataflow.ExternalFlow
+private import semmle.code.java.dataflow.FlowSummary as FlowSummary
 
-private module FlowSummaries {
-  private import semmle.code.java.dataflow.FlowSummary as F
-}
-
-class SummarizedCallableBase = Callable;
+class SummarizedCallableBase = FlowSummary::SummarizedCallableBase;
 
 DataFlowCallable inject(SummarizedCallable c) { result.asSummarizedCallable() = c }
 
@@ -67,26 +64,28 @@ private boolean isGenerated(string provenance) {
  * `input`, output specification `output`, kind `kind`, and a flag `generated`
  * stating whether the summary is autogenerated.
  */
-predicate summaryElement(Callable c, string input, string output, string kind, boolean generated) {
+predicate summaryElement(
+  SummarizedCallableBase c, string input, string output, string kind, boolean generated
+) {
   exists(
     string namespace, string type, boolean subtypes, string name, string signature, string ext,
     string provenance
   |
     summaryModel(namespace, type, subtypes, name, signature, ext, input, output, kind, provenance) and
     generated = isGenerated(provenance) and
-    c = interpretElement(namespace, type, subtypes, name, signature, ext)
+    c.asCallable() = interpretElement(namespace, type, subtypes, name, signature, ext)
   )
 }
 
 /**
  * Holds if a negative flow summary exists for `c`, which means that there is no
  * flow through `c`. The flag `generated` states whether the summary is autogenerated.
  */
-predicate negativeSummaryElement(Callable c, boolean generated) {
+predicate negativeSummaryElement(SummarizedCallableBase c, boolean generated) {
   exists(string namespace, string type, string name, string signature, string provenance |
     negativeSummaryModel(namespace, type, name, signature, provenance) and
     generated = isGenerated(provenance) and
-    c = interpretElement(namespace, type, false, name, signature, "")
+    c.asCallable() = interpretElement(namespace, type, false, name, signature, "")
   )
 }
 

java/ql/lib/semmle/code/java/frameworks/Stream.qll

@@ -1,6 +1,101 @@
 /** Definitions related to `java.util.stream`. */
 
 private import semmle.code.java.dataflow.ExternalFlow
+private import semmle.code.java.dataflow.FlowSummary
+
+private class CollectCall extends MethodAccess {
+  CollectCall() {
+    this.getMethod()
+        .getSourceDeclaration()
+        .hasQualifiedName("java.util.stream", "Stream", "collect")
+  }
+}
+
+private class Collector extends MethodAccess {
+  Collector() {
+    this.getMethod().getDeclaringType().hasQualifiedName("java.util.stream", "Collectors")
+  }
+
+  predicate hasName(string name) { this.getMethod().hasName(name) }
+}
+
+private class CollectToContainer extends SyntheticCallable {
+  CollectToContainer() { this = "java.util.stream.collect()+Collectors.[toList,...]" }
+
+  override Call getACall() {
+    result
+        .(CollectCall)
+        .getArgument(0)
+        .(Collector)
+        .hasName([
+            "maxBy", "minBy", "toCollection", "toList", "toSet", "toUnmodifiableList",
+            "toUnmodifiableSet"
+          ])
+  }
+
+  override predicate propagatesFlow(
+    SummaryComponentStack input, SummaryComponentStack output, boolean preservesValue
+  ) {
+    input = SummaryComponentStack::elementOf(SummaryComponentStack::qualifier()) and
+    output = SummaryComponentStack::elementOf(SummaryComponentStack::return()) and
+    preservesValue = true
+  }
+}
+
+private class CollectToJoining extends SyntheticCallable {
+  CollectToJoining() { this = "java.util.stream.collect()+Collectors.joining" }
+
+  override Call getACall() { result.(CollectCall).getArgument(0).(Collector).hasName("joining") }
+
+  override predicate propagatesFlow(
+    SummaryComponentStack input, SummaryComponentStack output, boolean preservesValue
+  ) {
+    input = SummaryComponentStack::elementOf(SummaryComponentStack::qualifier()) and
+    output = SummaryComponentStack::return() and
+    preservesValue = false
+  }
+
+  override Type getReturnType() { result instanceof TypeString }
+}
+
+private class CollectToGroupingBy extends SyntheticCallable {
+  CollectToGroupingBy() {
+    this = "java.util.stream.collect()+Collectors.[groupingBy(Function),...]"
+  }
+
+  override Call getACall() {
+    exists(Method m |
+      m = result.(CollectCall).getArgument(0).(Collector).getMethod() and
+      m.hasName(["groupingBy", "groupingByConcurrent", "partitioningBy"]) and
+      m.getNumberOfParameters() = 1
+    )
+  }
+
+  override predicate propagatesFlow(
+    SummaryComponentStack input, SummaryComponentStack output, boolean preservesValue
+  ) {
+    input = SummaryComponentStack::elementOf(SummaryComponentStack::qualifier()) and
+    output =
+      SummaryComponentStack::elementOf(SummaryComponentStack::mapValueOf(SummaryComponentStack::return())) and
+    preservesValue = true
+  }
+}
+
+private class RequiredComponentStackForCollect extends RequiredSummaryComponentStack {
+  override predicate required(SummaryComponent head, SummaryComponentStack tail) {
+    head = SummaryComponent::element() and
+    tail = SummaryComponentStack::qualifier()
+    or
+    head = SummaryComponent::element() and
+    tail = SummaryComponentStack::return()
+    or
+    head = SummaryComponent::element() and
+    tail = SummaryComponentStack::mapValueOf(SummaryComponentStack::return())
+    or
+    head = SummaryComponent::mapValue() and
+    tail = SummaryComponentStack::return()
+  }
+}
 
 private class StreamModel extends SummaryModelCsv {
   override predicate row(string s) {
@@ -19,7 +114,7 @@ private class StreamModel extends SummaryModelCsv {
         "java.util.stream;Stream;true;collect;(Supplier,BiConsumer,BiConsumer);;Argument[1].Parameter[0];Argument[2].Parameter[0..1];value;manual",
         "java.util.stream;Stream;true;collect;(Supplier,BiConsumer,BiConsumer);;Argument[2].Parameter[0..1];Argument[1].Parameter[0];value;manual",
         "java.util.stream;Stream;true;collect;(Supplier,BiConsumer,BiConsumer);;Argument[-1].Element;Argument[1].Parameter[1];value;manual",
-        // Missing: collect(Collector<T,A,R> collector)
+        // collect(Collector<T,A,R> collector) is handled separately on a case-by-case basis as it is too complex for MaD
         "java.util.stream;Stream;true;concat;(Stream,Stream);;Argument[0..1].Element;ReturnValue.Element;value;manual",
         "java.util.stream;Stream;true;distinct;();;Argument[-1].Element;ReturnValue.Element;value;manual",
         "java.util.stream;Stream;true;dropWhile;(Predicate);;Argument[-1].Element;Argument[0].Parameter[0];value;manual",