Merge pull request github#12537 from yoff/python/captured-variables-for-typetracking

Python: Captured variables for type tracking and the API graph

Author: yoff

python/ql/lib/change-notes/2023-03-16-typetracking-read-captured-variables.md

@@ -0,0 +1,4 @@
+---
+category: minorAnalysis
+---
+* Type tracking is now aware of reads of captured variables (variables defined in an outer scope). This leads to a richer API graph, and may lead to more results in some queries.

python/ql/lib/semmle/python/ApiGraphs.qll

@@ -987,7 +987,7 @@ module API {
     DataFlow::LocalSourceNode trackUseNode(DataFlow::LocalSourceNode src) {
       Stages::TypeTracking::ref() and
       result = trackUseNode(src, DataFlow::TypeTracker::end()) and
-      result instanceof DataFlow::ExprNode
+      result instanceof DataFlow::LocalSourceNodeNotModuleVariableNode
     }
 
     /**

python/ql/lib/semmle/python/dataflow/new/internal/LocalSources.qll

@@ -51,6 +51,10 @@ class LocalSourceNode extends Node {
     // We explicitly include any read of a global variable, as some of these may have local flow going
     // into them.
     this = any(ModuleVariableNode mvn).getARead()
+    or
+    // We include all scope entry definitions, as these act as the local source within the scope they
+    // enter.
+    this.asVar() instanceof ScopeEntryDefinition
   }
 
   /** Holds if this `LocalSourceNode` can flow to `nodeTo` in one or more local flow steps. */
@@ -133,6 +137,21 @@ class LocalSourceNode extends Node {
   LocalSourceNode backtrack(TypeBackTracker t2, TypeBackTracker t) { t2 = t.step(result, this) }
 }
 
+/**
+ * A LocalSourceNode that is not a ModuleVariableNode
+ * This class provides a positive formulation of that in its charpred.
+ *
+ * Aka FutureLocalSourceNode (see FutureWork below), but until the future is here...
+ */
+class LocalSourceNodeNotModuleVariableNode extends LocalSourceNode {
+  cached
+  LocalSourceNodeNotModuleVariableNode() {
+    this instanceof ExprNode
+    or
+    this.asVar() instanceof ScopeEntryDefinition
+  }
+}
+
 /**
  * A node that can be used for type tracking or type back-tracking.
  *

python/ql/lib/semmle/python/dataflow/new/internal/TypeTrackerSpecific.qll

@@ -43,7 +43,19 @@ predicate compatibleContents(TypeTrackerContent storeContent, TypeTrackerContent
 
 predicate simpleLocalFlowStep = DataFlowPrivate::simpleLocalFlowStepForTypetracking/2;
 
-predicate jumpStep = DataFlowPrivate::jumpStepSharedWithTypeTracker/2;
+predicate jumpStep(Node nodeFrom, Node nodeTo) {
+  DataFlowPrivate::jumpStepSharedWithTypeTracker(nodeFrom, nodeTo)
+  or
+  capturedJumpStep(nodeFrom, nodeTo)
+}
+
+predicate capturedJumpStep(Node nodeFrom, Node nodeTo) {
+  exists(SsaSourceVariable var, DefinitionNode def | var.hasDefiningNode(def) |
+    nodeTo.asVar().(ScopeEntryDefinition).getSourceVariable() = var and
+    nodeFrom.asCfgNode() = def.getValue() and
+    var.getScope().getScope*() = nodeFrom.getScope()
+  )
+}
 
 /** Holds if there is a level step from `nodeFrom` to `nodeTo`, which may depend on the call graph. */
 predicate levelStepCall(Node nodeFrom, Node nodeTo) { none() }

python/ql/test/experimental/dataflow/coverage/test.py

@@ -726,15 +726,15 @@ def f6(arg):
         return f5(arg)
 
     x = f6(SOURCE)
-    SINK(x) #$ MISSING:flow="SOURCE, l:-1 -> x"
+    SINK(x) #$ flow="SOURCE, l:-1 -> x"
     x = f5(SOURCE)
-    SINK(x) #$ MISSING:flow="SOURCE, l:-1 -> x"
+    SINK(x) #$ flow="SOURCE, l:-1 -> x"
     x = f4(SOURCE)
-    SINK(x) #$ MISSING:flow="SOURCE, l:-1 -> x"
+    SINK(x) #$ flow="SOURCE, l:-1 -> x"
     x = f3(SOURCE)
-    SINK(x) #$ MISSING:flow="SOURCE, l:-1 -> x"
+    SINK(x) #$ flow="SOURCE, l:-1 -> x"
     x = f2(SOURCE)
-    SINK(x) #$ MISSING:flow="SOURCE, l:-1 -> x"
+    SINK(x) #$ flow="SOURCE, l:-1 -> x"
     x = f1(SOURCE)
     SINK(x) #$ flow="SOURCE, l:-1 -> x"
 

python/ql/test/experimental/dataflow/typetracking/moduleattr.expected

@@ -6,5 +6,6 @@ module_attr_tracker
 | import_as_attr.py:1:28:1:35 | GSSA Variable attr_ref |
 | import_as_attr.py:3:1:3:1 | GSSA Variable x |
 | import_as_attr.py:3:5:3:12 | ControlFlowNode for attr_ref |
+| import_as_attr.py:5:1:5:10 | GSSA Variable attr_ref |
 | import_as_attr.py:6:5:6:5 | SSA variable y |
 | import_as_attr.py:6:9:6:16 | ControlFlowNode for attr_ref |

python/ql/test/experimental/dataflow/typetracking/test.py

@@ -60,10 +60,10 @@ def test_import():
 def to_inner_scope():
     x = tracked # $tracked
     def foo():
-        y = x # $ MISSING: tracked
-        return y # $ MISSING: tracked
-    also_x = foo() # $ MISSING: tracked
-    print(also_x) # $ MISSING: tracked
+        y = x # $ tracked
+        return y # $ tracked
+    also_x = foo() # $ tracked
+    print(also_x) # $ tracked
 
 # ------------------------------------------------------------------------------
 # Function decorator

python/ql/test/experimental/dataflow/typetracking/tracked.ql

@@ -24,6 +24,11 @@ class TrackedTest extends InlineExpectationsTest {
       tracked(t).flowsTo(e) and
       // Module variables have no sensible location, and hence can't be annotated.
       not e instanceof DataFlow::ModuleVariableNode and
+      // Global variables on line 0 also cannot be annotated
+      not e.getLocation().getStartLine() = 0 and
+      // We do not wish to annotate scope entry definitions,
+      // as they do not appear in the source code.
+      not e.asVar() instanceof ScopeEntryDefinition and
       tag = "tracked" and
       location = e.getLocation() and
       value = t.getAttr() and

python/ql/test/experimental/dataflow/validTest.py

@@ -68,8 +68,10 @@ def check_tests_valid_after_version(testFile, version):
     check_tests_valid("coverage-py3.classes")
     check_tests_valid("variable-capture.in")
     check_tests_valid("variable-capture.nonlocal")
+    check_tests_valid("variable-capture.global")
     check_tests_valid("variable-capture.dict")
-    check_tests_valid("variable-capture.collections")
+    check_tests_valid("variable-capture.test_collections")
+    check_tests_valid("variable-capture.by_value")
     check_tests_valid("module-initialization.multiphase")
     check_tests_valid("fieldflow.test")
     check_tests_valid("fieldflow.test_dict")

python/ql/test/experimental/dataflow/variable-capture/by_value.py

@@ -0,0 +1,52 @@
+# Here we test capturing the _value_ of a variable (by using it as the default value for a parameter)
+# All functions starting with "test_" should run and execute `print("OK")` exactly once.
+# This can be checked by running validTest.py.
+
+import sys
+import os
+
+sys.path.append(os.path.dirname(os.path.dirname((__file__))))
+from testlib import expects
+
+# These are defined so that we can evaluate the test code.
+NONSOURCE = "not a source"
+SOURCE = "source"
+
+def is_source(x):
+    return x == "source" or x == b"source" or x == 42 or x == 42.0 or x == 42j
+
+
+def SINK(x):
+    if is_source(x):
+        print("OK")
+    else:
+        print("Unexpected flow", x)
+
+
+def SINK_F(x):
+    if is_source(x):
+        print("Unexpected flow", x)
+    else:
+        print("OK")
+
+
+def by_value1():
+    a = SOURCE
+    def inner(a_val=a):
+        SINK(a_val) #$ captured
+        SINK_F(a)
+    a = NONSOURCE
+    inner()
+
+def by_value2():
+    a = NONSOURCE
+    def inner(a_val=a):
+        SINK(a) #$ MISSING:captured
+        SINK_F(a_val)
+    a = SOURCE
+    inner()
+
+@expects(4)
+def test_by_value():
+    by_value1()
+    by_value2()