python: add summaries for

copy, pop, get, getitem, setdefault

Also add read steps to taint tracking.

Reading from a tainted collection can be done in two situations:
1. There is an acces path
    In this case a read step (possibly from a flow summary)
    gives rise to a taint step.
2. There is no access path
    In this case an explicit taint step (possibly via a flow
    summary) should exist.

Author: yoff

python/ql/lib/semmle/python/dataflow/new/internal/TaintTrackingPrivate.qll

@@ -202,6 +202,19 @@ predicate containerStep(DataFlow::Node nodeFrom, DataFlow::Node nodeTo) {
     obj = nodeTo.(DataFlow::PostUpdateNode).getPreUpdateNode() and
     call.getArg(0) = nodeFrom
   )
+  or
+  // Although flow through collections is modeled precisely using stores/reads, we still
+  // allow flow out of a _tainted_ collection. This is needed in order to support taint-
+  // tracking configurations where the source is a collection.
+  exists(DataFlow::Content c | DataFlowPrivate::readStep(nodeFrom, c, nodeTo) |
+    // c instanceof DataFlow::ListElementContent
+    // or
+    // c instanceof DataFlow::SetElementContent
+    // or
+    c instanceof DataFlow::DictionaryElementContent
+    // or
+    // c instanceof DataFlow::DictionaryElementAnyContent
+  )
 }
 
 /**

python/ql/lib/semmle/python/frameworks/Stdlib.qll

@@ -3939,6 +3939,176 @@ private module StdlibPrivate {
     }
   }
 
+  // ---------------------------------------------------------------------------
+  // Flow summaries for container methods
+  // ---------------------------------------------------------------------------
+  /** A flow summary for `copy`. */
+  class CopySummary extends SummarizedCallable {
+    CopySummary() { this = "collection.copy" }
+
+    override DataFlow::CallCfgNode getACall() {
+      result.(DataFlow::MethodCallNode).getMethodName() = "copy"
+    }
+
+    override DataFlow::ArgumentNode getACallback() { none() }
+
+    override predicate propagatesFlowExt(string input, string output, boolean preservesValue) {
+      exists(string content |
+        content = "ListElement"
+        or
+        content = "SetElement"
+        or
+        exists(DataFlow::TupleElementContent tc, int i | i = tc.getIndex() |
+          content = "TupleElement[" + i.toString() + "]"
+        )
+        or
+        exists(DataFlow::DictionaryElementContent dc, string key | key = dc.getKey() |
+          content = "DictionaryElement[" + key + "]"
+        )
+      |
+        input = "Argument[self]." + content and
+        output = "ReturnValue." + content and
+        preservesValue = true
+      )
+      or
+      input = "Argument[self]" and
+      output = "ReturnValue" and
+      preservesValue = false
+    }
+  }
+
+  /**
+   * A flow summary for `pop` either for list or set.
+   * This ignores the index if given, since content is
+   * imprecise anyway.
+   *
+   * I also handles the default value when `pop` is called
+   * on a dictionary, since that also does not depend on the key.
+   */
+  class PopSummary extends SummarizedCallable {
+    PopSummary() { this = "collection.pop" }
+
+    override DataFlow::CallCfgNode getACall() {
+      result.(DataFlow::MethodCallNode).getMethodName() = "pop"
+    }
+
+    override DataFlow::ArgumentNode getACallback() { none() }
+
+    override predicate propagatesFlowExt(string input, string output, boolean preservesValue) {
+      input = "Argument[self].ListElement" and
+      output = "ReturnValue" and
+      preservesValue = true
+      or
+      input = "Argument[self].SetElement" and
+      output = "ReturnValue" and
+      preservesValue = true
+      or
+      // default value for dictionary
+      input = "Argument[1]" and
+      output = "ReturnValue" and
+      preservesValue = true
+      or
+      // transfer taint on self to return value
+      input = "Argument[self]" and
+      output = "ReturnValue" and
+      preservesValue = false
+    }
+  }
+
+  /** A flow summary for `dict.pop` */
+  class DictPopSummary extends SummarizedCallable {
+    string key;
+
+    DictPopSummary() {
+      this = "dict.pop(" + key + ")" and
+      exists(DataFlow::DictionaryElementContent dc | key = dc.getKey())
+    }
+
+    override DataFlow::CallCfgNode getACall() {
+      result.(DataFlow::MethodCallNode).getMethodName() = "pop" and
+      result.getArg(0).getALocalSource().asExpr().(StrConst).getText() = key
+    }
+
+    override DataFlow::ArgumentNode getACallback() { none() }
+
+    override predicate propagatesFlowExt(string input, string output, boolean preservesValue) {
+      input = "Argument[self].DictionaryElement[" + key + "]" and
+      output = "ReturnValue" and
+      preservesValue = true
+    }
+  }
+
+  /** A flow summary for `dict.get` at specific content. */
+  class DictGetSummary extends SummarizedCallable {
+    string key;
+
+    DictGetSummary() {
+      this = "dict.get(" + key + ")" and
+      exists(DataFlow::DictionaryElementContent dc | key = dc.getKey())
+    }
+
+    override DataFlow::CallCfgNode getACall() {
+      result.(DataFlow::MethodCallNode).getMethodName() = "get" and
+      result.getArg(0).getALocalSource().asExpr().(StrConst).getText() = key
+    }
+
+    override DataFlow::ArgumentNode getACallback() { none() }
+
+    override predicate propagatesFlowExt(string input, string output, boolean preservesValue) {
+      input = "Argument[self].DictionaryElement[" + key + "]" and
+      output = "ReturnValue" and
+      preservesValue = true
+      or
+      // optional default value
+      input = "Argument[1]" and
+      output = "ReturnValue" and
+      preservesValue = true
+    }
+  }
+
+  /** A flow summary for `dict.get` disregarding content. */
+  class DictGetAnySummary extends SummarizedCallable {
+    DictGetAnySummary() { this = "dict.get" }
+
+    override DataFlow::CallCfgNode getACall() {
+      result.(DataFlow::MethodCallNode).getMethodName() = "get"
+    }
+
+    override DataFlow::ArgumentNode getACallback() { none() }
+
+    override predicate propagatesFlowExt(string input, string output, boolean preservesValue) {
+      // default value
+      input = "Argument[1]" and
+      output = "ReturnValue" and
+      preservesValue = true
+      or
+      // transfer taint from self to return value
+      input = "Argument[self]" and
+      output = "ReturnValue" and
+      preservesValue = false
+    }
+  }
+
+  /** A flow summary for `dict.popitem` */
+  class DictPopitemSummary extends SummarizedCallable {
+    DictPopitemSummary() { this = "dict.popitem" }
+
+    override DataFlow::CallCfgNode getACall() {
+      result.(DataFlow::MethodCallNode).getMethodName() = "popitem"
+    }
+
+    override DataFlow::ArgumentNode getACallback() { none() }
+
+    override predicate propagatesFlowExt(string input, string output, boolean preservesValue) {
+      exists(DataFlow::DictionaryElementContent dc, string key | key = dc.getKey() |
+        input = "Argument[self].DictionaryElement[" + key + "]" and
+        output = "ReturnValue.TupleElement[1]" and
+        preservesValue = true
+        // TODO: put `key` into "ReturnValue.TupleElement[0]"
+      )
+    }
+  }
+
   /**
    * A flow summary for `dict.setdefault`.
    *
@@ -3962,6 +4132,40 @@ private module StdlibPrivate {
       preservesValue = true
     }
   }
+
+  /**
+   * A flow summary for `dict.setdefault` at specifi key.
+   * See https://docs.python.org/3.10/library/stdtypes.html#dict.setdefault
+   * This summary handles read and store steps. See `DictSetdefaultSummary`
+   * for the dataflow steps.
+   */
+  class DictSetdefaultKeySummary extends SummarizedCallable {
+    string key;
+
+    DictSetdefaultKeySummary() {
+      this = "dict.setdefault(" + key + ")" and
+      exists(DataFlow::DictionaryElementContent dc | key = dc.getKey())
+    }
+
+    override DataFlow::CallCfgNode getACall() {
+      result.(DataFlow::MethodCallNode).getMethodName() = "setdefault" and
+      result.getArg(0).getALocalSource().asExpr().(StrConst).getText() = key
+    }
+
+    override DataFlow::ArgumentNode getACallback() { none() }
+
+    override predicate propagatesFlowExt(string input, string output, boolean preservesValue) {
+      // If key is in the dictionary, return its value.
+      input = "Argument[self].DictionaryElement[" + key + "]" and
+      output = "ReturnValue" and
+      preservesValue = true
+      or
+      // If not, insert key with a value of default.
+      input = "Argument[1]" and
+      output = "ReturnValue.DictionaryElement[" + key + "]" and
+      preservesValue = true
+    }
+  }
 }
 
 // ---------------------------------------------------------------------------

python/ql/test/experimental/dataflow/coverage/test.py

@@ -123,23 +123,23 @@ def test_nested_list_display():
 # 6.2.6. Set displays
 def test_set_display():
     x = {SOURCE}
-    SINK(x.pop()) #$ MISSING:flow="SOURCE, l:-1 -> x.pop()"
+    SINK(x.pop()) #$ flow="SOURCE, l:-1 -> x.pop()"
 
 
 def test_set_comprehension():
     x = {SOURCE for y in [NONSOURCE]}
-    SINK(x.pop()) #$ MISSING:flow="SOURCE, l:-1 -> x.pop()"
+    SINK(x.pop()) #$ flow="SOURCE, l:-1 -> x.pop()"
 
 
 def test_set_comprehension_flow():
     x = {y for y in [SOURCE]}
-    SINK(x.pop()) #$ MISSING:flow="SOURCE, l:-1 -> x.pop()"
+    SINK(x.pop()) #$ flow="SOURCE, l:-1 -> x.pop()"
 
 
 def test_set_comprehension_inflow():
     l = {SOURCE}
     x = {y for y in l}
-    SINK(x.pop()) #$ MISSING:flow="SOURCE, l:-2 -> x.pop()"
+    SINK(x.pop()) #$ flow="SOURCE, l:-2 -> x.pop()"
 
 
 def test_nested_set_display():
@@ -155,7 +155,7 @@ def test_dict_display():
 
 def test_dict_display_pop():
     x = {"s": SOURCE}
-    SINK(x.pop("s")) #$ MISSING:flow="SOURCE, l:-1 -> x.pop(..)"
+    SINK(x.pop("s")) #$ flow="SOURCE, l:-1 -> x.pop(..)"
 
 
 def test_dict_comprehension():

python/ql/test/experimental/dataflow/coverage/test_builtins.py

@@ -100,19 +100,19 @@ def test_set_from_list():
     l = [SOURCE]
     s = set(l)
     v = s.pop()
-    SINK(v) #$ MISSING:flow="SOURCE, l:-3 -> v"
+    SINK(v) #$ flow="SOURCE, l:-3 -> v"
 
 def test_set_from_tuple():
     t = (SOURCE,)
     s = set(t)
     v = s.pop()
-    SINK(v) #$ MISSING:flow="SOURCE, l:-3 -> v"
+    SINK(v) #$ flow="SOURCE, l:-3 -> v"
 
 def test_set_from_set():
     s0 = {SOURCE}
     s = set(s0)
     v = s.pop()
-    SINK(v) #$ MISSING:flow="SOURCE, l:-3 -> v"
+    SINK(v) #$ flow="SOURCE, l:-3 -> v"
 
 def test_set_from_dict():
     d = {SOURCE: "val"}
@@ -149,24 +149,24 @@ def test_dict_from_dict():
 def test_list_pop():
     l = [SOURCE]
     v = l.pop()
-    SINK(v) #$ MISSING:flow="SOURCE, l:-2 -> v"
+    SINK(v) #$ flow="SOURCE, l:-2 -> v"
 
 def test_list_pop_index():
     l = [SOURCE]
     v = l.pop(0)
-    SINK(v) #$ MISSING: flow="SOURCE, l:-2 -> v"
+    SINK(v) #$ flow="SOURCE, l:-2 -> v"
 
 def test_list_pop_index_imprecise():
     l = [SOURCE, NONSOURCE]
     v = l.pop(1)
-    SINK_F(v)
+    SINK_F(v) #$ SPURIOUS: flow="SOURCE, l:-2 -> v"
 
 @expects(2)
 def test_list_copy():
     l0 = [SOURCE, NONSOURCE]
     l = l0.copy()
-    SINK(l[0]) #$ MISSING: flow="SOURCE, l:-2 -> l[0]"
-    SINK_F(l[1])
+    SINK(l[0]) #$ flow="SOURCE, l:-2 -> l[0]"
+    SINK_F(l[1]) #$ SPURIOUS: flow="SOURCE, l:-3 -> l[1]"
 
 def test_list_append():
     l = [NONSOURCE]
@@ -178,12 +178,12 @@ def test_list_append():
 def test_set_pop():
     s = {SOURCE}
     v = s.pop()
-    SINK(v) #$ MISSING:flow="SOURCE, l:-2 -> v"
+    SINK(v) #$ flow="SOURCE, l:-2 -> v"
 
 def test_set_copy():
     s0 = {SOURCE}
     s = s0.copy()
-    SINK(s.pop()) #$ MISSING: flow="SOURCE, l:-2 -> s.pop()"
+    SINK(s.pop()) #$ flow="SOURCE, l:-2 -> s.pop()"
 
 def test_set_add():
     s = set([])
@@ -218,32 +218,32 @@ def test_dict_items():
 def test_dict_pop():
     d = {'k': SOURCE}
     v = d.pop("k")
-    SINK(v) #$ MISSING:flow="SOURCE, l:-2 -> v"
+    SINK(v) #$ flow="SOURCE, l:-2 -> v"
     v1 = d.pop("k", NONSOURCE)
-    SINK_F(v1)
+    SINK_F(v1) #$ SPURIOUS: flow="SOURCE, l:-4 -> v1"
     v2 = d.pop("non-existing", SOURCE)
-    SINK(v2) #$ MISSING: flow="SOURCE, l:-1 -> v2"
+    SINK(v2) #$ flow="SOURCE, l:-1 -> v2"
 
 @expects(2)
 def test_dict_get():
     d = {'k': SOURCE}
     v = d.get("k")
-    SINK(v) #$ MISSING:flow="SOURCE, l:-2 -> v"
+    SINK(v) #$ flow="SOURCE, l:-2 -> v"
     v1 = d.get("non-existing", SOURCE)
-    SINK(v1) #$ MISSING: flow="SOURCE, l:-1 -> v1"
+    SINK(v1) #$ flow="SOURCE, l:-1 -> v1"
 
 @expects(2)
 def test_dict_popitem():
     d = {'k': SOURCE}
     t = d.popitem() # could be any pair (before 3.7), but we only have one
     SINK_F(t[0])
-    SINK(t[1]) #$ MISSING: flow="SOURCE, l:-3 -> t[1]"
+    SINK(t[1]) #$ flow="SOURCE, l:-3 -> t[1]"
 
 @expects(2)
 def test_dict_copy():
     d = {'k': SOURCE, 'k1': NONSOURCE}
     d1 = d.copy()
-    SINK(d1["k"]) #$ MISSING: flow="SOURCE, l:-2 -> d[k]"
+    SINK(d1["k"]) #$ flow="SOURCE, l:-2 -> d1['k']"
     SINK_F(d1["k1"])