Python: rewrite simpleLocalFlowStep

to take into account the split between
import time and runtime.

Author: yoff

python/ql/lib/semmle/python/dataflow/new/internal/DataFlowPrivate.qll

@@ -152,6 +152,7 @@ class DataFlowExpr = Expr;
  * Flow comes from definitions, uses and refinements.
  */
 // TODO: Consider constraining `nodeFrom` and `nodeTo` to be in the same scope.
+// If they have different enclosing callables, we get consistency errors.
 module EssaFlow {
   predicate essaFlowStep(Node nodeFrom, Node nodeTo) {
     // Definition
@@ -225,41 +226,64 @@ module EssaFlow {
 //--------
 /**
  * This is the local flow predicate that is used as a building block in global
- * data flow. It is a strict subset of the `localFlowStep` predicate, as it
- * excludes SSA flow through instance fields.
+ * data flow.
+ *
+ * Local flow can happen either at import time, when the module is initialised
+ * or at runtime when callables in the module are called.
  */
 predicate simpleLocalFlowStep(Node nodeFrom, Node nodeTo) {
-  // If there is ESSA-flow out of a node `node`, we want flow
+  // If there is local flow out of a node `node`, we want flow
   // both out of `node` and any post-update node of `node`.
   exists(Node node |
-    EssaFlow::essaFlowStep(node, nodeTo) and
     nodeFrom = update(node) and
     (
-      not node instanceof EssaNode or
-      not nodeTo instanceof EssaNode or
-      localEssaStep(node, nodeTo)
+      importTimeLocalFlowStep(node, nodeTo) or
+      runtimeLocalFlowStep(node, nodeTo)
     )
   )
 }
 
 /**
- * Holds if there is an Essa flow step from `nodeFrom` to `nodeTo` that does not switch between
- * local and global SSA variables.
- *
- * This predicate is currently empty, since `EssaFlow::essaFlowStep` never goes between `EssaNode`s.
- * (It only starts in an `EssaNode` in a single case, namely `defToFirstUse` which ends in a `CfgNode`.)
+ * Holds if `node` is found at the top level of a module.
  */
-private predicate localEssaStep(EssaNode nodeFrom, EssaNode nodeTo) {
-  EssaFlow::essaFlowStep(nodeFrom, nodeTo) and
+pragma[inline]
+predicate isTopLevel(Node node) { node.getScope() instanceof Module }
+
+/** Holds if there is local flow from `nodeFrom` to `nodeTo` at import time. */
+predicate importTimeLocalFlowStep(Node nodeFrom, Node nodeTo) {
+  // As a proxy for whether statements can be executed at import time,
+  // we check if they appear at the top level.
+  // This will miss statements inside functions called from the top level.
+  isTopLevel(nodeFrom) and
+  isTopLevel(nodeTo) and
   (
-    nodeFrom.getVar() instanceof GlobalSsaVariable and
-    nodeTo.getVar() instanceof GlobalSsaVariable
+    EssaFlow::essaFlowStep(nodeFrom, nodeTo)
     or
-    not nodeFrom.getVar() instanceof GlobalSsaVariable and
-    not nodeTo.getVar() instanceof GlobalSsaVariable
+    exists(SsaVariable def |
+      def = any(SsaVariable var).getAnUltimateDefinition() and
+      def.getDefinition() = nodeFrom.asCfgNode() and
+      def.getVariable() = nodeTo.(ModuleVariableNode).getVariable()
+    )
   )
 }
 
+/** Holds if there is local flow from `nodeFrom` to `nodeTo` at runtime. */
+predicate runtimeLocalFlowStep(Node nodeFrom, Node nodeTo) {
+  // Anything not at the top level can be executed at runtime.
+  not isTopLevel(nodeFrom) and
+  not isTopLevel(nodeTo) and
+  EssaFlow::essaFlowStep(nodeFrom, nodeTo)
+}
+
+/** `ModuleVariable`s are accessed via jump steps at runtime. */
+predicate runtimeJumpStep(Node nodeFrom, Node nodeTo) {
+  // Module variable read
+  nodeFrom.(ModuleVariableNode).getARead() = nodeTo
+  or
+  // Module variable write
+  nodeFrom = nodeTo.(ModuleVariableNode).getAWrite()
+}
+
 /**
  * Holds if `result` is either `node`, or the post-update node for `node`.
  */
@@ -860,11 +884,7 @@ string ppReprType(DataFlowType t) { none() }
  * taken into account.
  */
 predicate jumpStep(Node nodeFrom, Node nodeTo) {
-  // Module variable read
-  nodeFrom.(ModuleVariableNode).getARead() = nodeTo
-  or
-  // Module variable write
-  nodeFrom = nodeTo.(ModuleVariableNode).getAWrite()
+  runtimeJumpStep(nodeFrom, nodeTo)
   or
   // Read of module attribute:
   exists(AttrRead r, ModuleValue mv |

python/ql/test/experimental/dataflow/fieldflow/allLocalFlow.expected

@@ -2,6 +2,7 @@
 | examples.py:0:0:0:0 | GSSA Variable SOURCE | examples.py:27:15:27:20 | ControlFlowNode for SOURCE |
 | examples.py:0:0:0:0 | GSSA Variable object | examples.py:6:13:6:18 | ControlFlowNode for object |
 | examples.py:6:1:6:20 | ControlFlowNode for ClassExpr | examples.py:6:7:6:11 | GSSA Variable MyObj |
+| examples.py:6:7:6:11 | ControlFlowNode for MyObj | examples.py:0:0:0:0 | ModuleVariableNode for Global Variable MyObj in Module examples |
 | examples.py:6:7:6:11 | GSSA Variable MyObj | examples.py:25:9:25:13 | ControlFlowNode for MyObj |
 | examples.py:6:13:6:18 | ControlFlowNode for object | examples.py:11:17:11:22 | ControlFlowNode for object |
 | examples.py:7:5:7:28 | ControlFlowNode for FunctionExpr | examples.py:7:9:7:16 | SSA variable __init__ |
@@ -67,8 +68,10 @@
 | examples.py:55:5:55:5 | SSA variable a | examples.py:56:12:56:12 | ControlFlowNode for a |
 | examples.py:55:9:55:15 | ControlFlowNode for Attribute | examples.py:55:5:55:5 | SSA variable a |
 | test.py:2:13:2:26 | ControlFlowNode for Str | test.py:2:1:2:9 | GSSA Variable NONSOURCE |
+| test.py:3:1:3:6 | ControlFlowNode for SOURCE | test.py:0:0:0:0 | ModuleVariableNode for Global Variable SOURCE in Module test |
 | test.py:3:10:3:17 | ControlFlowNode for Str | test.py:3:1:3:6 | GSSA Variable SOURCE |
 | test.py:6:1:6:17 | ControlFlowNode for FunctionExpr | test.py:6:5:6:13 | GSSA Variable is_source |
+| test.py:6:5:6:13 | ControlFlowNode for is_source | test.py:0:0:0:0 | ModuleVariableNode for Global Variable is_source in Module test |
 | test.py:6:15:6:15 | ControlFlowNode for x | test.py:6:15:6:15 | SSA variable x |
 | test.py:6:15:6:15 | SSA variable x | test.py:7:12:7:12 | ControlFlowNode for x |
 | test.py:7:12:7:12 | ControlFlowNode for x | test.py:7:29:7:29 | ControlFlowNode for x |
@@ -77,17 +80,20 @@
 | test.py:7:58:7:58 | ControlFlowNode for x | test.py:7:71:7:71 | ControlFlowNode for x |
 | test.py:10:1:10:12 | ControlFlowNode for FunctionExpr | test.py:10:5:10:8 | GSSA Variable SINK |
 | test.py:10:1:10:12 | GSSA Variable is_source | test.py:11:8:11:16 | ControlFlowNode for is_source |
+| test.py:10:5:10:8 | ControlFlowNode for SINK | test.py:0:0:0:0 | ModuleVariableNode for Global Variable SINK in Module test |
 | test.py:10:10:10:10 | ControlFlowNode for x | test.py:10:10:10:10 | SSA variable x |
 | test.py:10:10:10:10 | SSA variable x | test.py:11:18:11:18 | ControlFlowNode for x |
 | test.py:11:18:11:18 | ControlFlowNode for x | test.py:14:34:14:34 | ControlFlowNode for x |
 | test.py:11:18:11:18 | [post arg] ControlFlowNode for x | test.py:14:34:14:34 | ControlFlowNode for x |
 | test.py:17:1:17:14 | ControlFlowNode for FunctionExpr | test.py:17:5:17:10 | GSSA Variable SINK_F |
 | test.py:17:1:17:14 | GSSA Variable is_source | test.py:18:8:18:16 | ControlFlowNode for is_source |
+| test.py:17:5:17:10 | ControlFlowNode for SINK_F | test.py:0:0:0:0 | ModuleVariableNode for Global Variable SINK_F in Module test |
 | test.py:17:12:17:12 | ControlFlowNode for x | test.py:17:12:17:12 | SSA variable x |
 | test.py:17:12:17:12 | SSA variable x | test.py:18:18:18:18 | ControlFlowNode for x |
 | test.py:18:18:18:18 | ControlFlowNode for x | test.py:19:34:19:34 | ControlFlowNode for x |
 | test.py:18:18:18:18 | [post arg] ControlFlowNode for x | test.py:19:34:19:34 | ControlFlowNode for x |
 | test.py:25:1:25:20 | ControlFlowNode for ClassExpr | test.py:25:7:25:11 | GSSA Variable MyObj |
+| test.py:25:7:25:11 | ControlFlowNode for MyObj | test.py:0:0:0:0 | ModuleVariableNode for Global Variable MyObj in Module test |
 | test.py:25:13:25:18 | ControlFlowNode for object | test.py:33:17:33:22 | ControlFlowNode for object |
 | test.py:26:5:26:28 | ControlFlowNode for FunctionExpr | test.py:26:9:26:16 | SSA variable __init__ |
 | test.py:26:18:26:21 | ControlFlowNode for self | test.py:26:18:26:21 | SSA variable self |
@@ -100,6 +106,7 @@
 | test.py:29:22:29:24 | ControlFlowNode for foo | test.py:29:22:29:24 | SSA variable foo |
 | test.py:29:22:29:24 | SSA variable foo | test.py:30:20:30:22 | ControlFlowNode for foo |
 | test.py:33:1:33:24 | ControlFlowNode for ClassExpr | test.py:33:7:33:15 | GSSA Variable NestedObj |
+| test.py:33:7:33:15 | ControlFlowNode for NestedObj | test.py:0:0:0:0 | ModuleVariableNode for Global Variable NestedObj in Module test |
 | test.py:34:5:34:23 | ControlFlowNode for FunctionExpr | test.py:34:9:34:16 | SSA variable __init__ |
 | test.py:34:5:34:23 | GSSA Variable MyObj | test.py:35:20:35:24 | ControlFlowNode for MyObj |
 | test.py:34:18:34:21 | ControlFlowNode for self | test.py:34:18:34:21 | SSA variable self |
@@ -109,6 +116,7 @@
 | test.py:37:16:37:19 | SSA variable self | test.py:38:16:38:19 | ControlFlowNode for self |
 | test.py:41:1:41:19 | ControlFlowNode for FunctionExpr | test.py:41:5:41:10 | GSSA Variable setFoo |
 | test.py:41:1:41:19 | GSSA Variable SINK_F | test.py:42:5:42:10 | ControlFlowNode for SINK_F |
+| test.py:41:5:41:10 | ControlFlowNode for setFoo | test.py:0:0:0:0 | ModuleVariableNode for Global Variable setFoo in Module test |
 | test.py:41:12:41:14 | ControlFlowNode for obj | test.py:41:12:41:14 | SSA variable obj |
 | test.py:41:12:41:14 | SSA variable obj | test.py:42:12:42:14 | ControlFlowNode for obj |
 | test.py:41:17:41:17 | ControlFlowNode for x | test.py:41:17:41:17 | SSA variable x |
@@ -166,6 +174,7 @@
 | test.py:86:11:86:27 | ControlFlowNode for MyObj() | test.py:86:5:86:7 | SSA variable obj |
 | test.py:90:1:90:30 | ControlFlowNode for FunctionExpr | test.py:90:5:90:26 | GSSA Variable fields_with_local_flow |
 | test.py:90:1:90:30 | GSSA Variable MyObj | test.py:91:11:91:15 | ControlFlowNode for MyObj |
+| test.py:90:5:90:26 | ControlFlowNode for fields_with_local_flow | test.py:0:0:0:0 | ModuleVariableNode for Global Variable fields_with_local_flow in Module test |
 | test.py:90:28:90:28 | ControlFlowNode for x | test.py:90:28:90:28 | SSA variable x |
 | test.py:90:28:90:28 | SSA variable x | test.py:91:17:91:17 | ControlFlowNode for x |
 | test.py:91:5:91:7 | SSA variable obj | test.py:92:9:92:11 | ControlFlowNode for obj |

python/ql/test/experimental/dataflow/fieldflow/globalStep.expected

@@ -42,6 +42,8 @@
 | examples.py:6:1:6:20 | ControlFlowNode for ClassExpr | examples.py:25:9:25:13 | ControlFlowNode for MyObj |
 | examples.py:6:1:6:20 | ControlFlowNode for ClassExpr | examples.py:49:7:49:11 | ControlFlowNode for MyObj |
 | examples.py:6:1:6:20 | ControlFlowNode for ClassExpr | examples.py:49:7:49:11 | ControlFlowNode for MyObj |
+| examples.py:6:7:6:11 | ControlFlowNode for MyObj | examples.py:0:0:0:0 | ModuleVariableNode for Global Variable MyObj in Module examples |
+| examples.py:6:7:6:11 | ControlFlowNode for MyObj | examples.py:0:0:0:0 | ModuleVariableNode for Global Variable MyObj in Module examples |
 | examples.py:6:7:6:11 | GSSA Variable MyObj | examples.py:0:0:0:0 | ModuleVariableNode for Global Variable MyObj in Module examples |
 | examples.py:6:7:6:11 | GSSA Variable MyObj | examples.py:0:0:0:0 | ModuleVariableNode for Global Variable MyObj in Module examples |
 | examples.py:6:7:6:11 | GSSA Variable MyObj | examples.py:25:9:25:13 | ControlFlowNode for MyObj |
@@ -534,12 +536,16 @@
 | test.py:0:0:0:0 | ModuleVariableNode for Global Variable setFoo in Module test | test.py:49:5:49:10 | ControlFlowNode for setFoo |
 | test.py:2:13:2:26 | ControlFlowNode for Str | test.py:2:1:2:9 | GSSA Variable NONSOURCE |
 | test.py:2:13:2:26 | ControlFlowNode for Str | test.py:2:1:2:9 | GSSA Variable NONSOURCE |
+| test.py:3:1:3:6 | ControlFlowNode for SOURCE | test.py:0:0:0:0 | ModuleVariableNode for Global Variable SOURCE in Module test |
+| test.py:3:1:3:6 | ControlFlowNode for SOURCE | test.py:0:0:0:0 | ModuleVariableNode for Global Variable SOURCE in Module test |
 | test.py:3:1:3:6 | GSSA Variable SOURCE | test.py:0:0:0:0 | ModuleVariableNode for Global Variable SOURCE in Module test |
 | test.py:3:1:3:6 | GSSA Variable SOURCE | test.py:0:0:0:0 | ModuleVariableNode for Global Variable SOURCE in Module test |
 | test.py:3:10:3:17 | ControlFlowNode for Str | test.py:3:1:3:6 | GSSA Variable SOURCE |
 | test.py:3:10:3:17 | ControlFlowNode for Str | test.py:3:1:3:6 | GSSA Variable SOURCE |
 | test.py:6:1:6:17 | ControlFlowNode for FunctionExpr | test.py:6:5:6:13 | GSSA Variable is_source |
 | test.py:6:1:6:17 | ControlFlowNode for FunctionExpr | test.py:6:5:6:13 | GSSA Variable is_source |
+| test.py:6:5:6:13 | ControlFlowNode for is_source | test.py:0:0:0:0 | ModuleVariableNode for Global Variable is_source in Module test |
+| test.py:6:5:6:13 | ControlFlowNode for is_source | test.py:0:0:0:0 | ModuleVariableNode for Global Variable is_source in Module test |
 | test.py:6:5:6:13 | GSSA Variable is_source | test.py:0:0:0:0 | ModuleVariableNode for Global Variable is_source in Module test |
 | test.py:6:5:6:13 | GSSA Variable is_source | test.py:0:0:0:0 | ModuleVariableNode for Global Variable is_source in Module test |
 | test.py:6:15:6:15 | ControlFlowNode for x | test.py:6:15:6:15 | SSA variable x |
@@ -634,6 +640,8 @@
 | test.py:10:1:10:12 | ControlFlowNode for FunctionExpr | test.py:10:5:10:8 | GSSA Variable SINK |
 | test.py:10:1:10:12 | GSSA Variable is_source | test.py:11:8:11:16 | ControlFlowNode for is_source |
 | test.py:10:1:10:12 | GSSA Variable is_source | test.py:11:8:11:16 | ControlFlowNode for is_source |
+| test.py:10:5:10:8 | ControlFlowNode for SINK | test.py:0:0:0:0 | ModuleVariableNode for Global Variable SINK in Module test |
+| test.py:10:5:10:8 | ControlFlowNode for SINK | test.py:0:0:0:0 | ModuleVariableNode for Global Variable SINK in Module test |
 | test.py:10:5:10:8 | GSSA Variable SINK | test.py:0:0:0:0 | ModuleVariableNode for Global Variable SINK in Module test |
 | test.py:10:5:10:8 | GSSA Variable SINK | test.py:0:0:0:0 | ModuleVariableNode for Global Variable SINK in Module test |
 | test.py:10:10:10:10 | ControlFlowNode for x | test.py:10:10:10:10 | SSA variable x |
@@ -684,6 +692,8 @@
 | test.py:17:1:17:14 | ControlFlowNode for FunctionExpr | test.py:17:5:17:10 | GSSA Variable SINK_F |
 | test.py:17:1:17:14 | GSSA Variable is_source | test.py:18:8:18:16 | ControlFlowNode for is_source |
 | test.py:17:1:17:14 | GSSA Variable is_source | test.py:18:8:18:16 | ControlFlowNode for is_source |
+| test.py:17:5:17:10 | ControlFlowNode for SINK_F | test.py:0:0:0:0 | ModuleVariableNode for Global Variable SINK_F in Module test |
+| test.py:17:5:17:10 | ControlFlowNode for SINK_F | test.py:0:0:0:0 | ModuleVariableNode for Global Variable SINK_F in Module test |
 | test.py:17:5:17:10 | GSSA Variable SINK_F | test.py:0:0:0:0 | ModuleVariableNode for Global Variable SINK_F in Module test |
 | test.py:17:5:17:10 | GSSA Variable SINK_F | test.py:0:0:0:0 | ModuleVariableNode for Global Variable SINK_F in Module test |
 | test.py:17:12:17:12 | ControlFlowNode for x | test.py:17:12:17:12 | SSA variable x |
@@ -722,6 +732,8 @@
 | test.py:19:34:19:34 | [post arg] ControlFlowNode for x | test.py:42:12:42:18 | [post arg] ControlFlowNode for Attribute |
 | test.py:25:1:25:20 | ControlFlowNode for ClassExpr | test.py:25:7:25:11 | GSSA Variable MyObj |
 | test.py:25:1:25:20 | ControlFlowNode for ClassExpr | test.py:25:7:25:11 | GSSA Variable MyObj |
+| test.py:25:7:25:11 | ControlFlowNode for MyObj | test.py:0:0:0:0 | ModuleVariableNode for Global Variable MyObj in Module test |
+| test.py:25:7:25:11 | ControlFlowNode for MyObj | test.py:0:0:0:0 | ModuleVariableNode for Global Variable MyObj in Module test |
 | test.py:25:7:25:11 | GSSA Variable MyObj | test.py:0:0:0:0 | ModuleVariableNode for Global Variable MyObj in Module test |
 | test.py:25:7:25:11 | GSSA Variable MyObj | test.py:0:0:0:0 | ModuleVariableNode for Global Variable MyObj in Module test |
 | test.py:25:13:25:18 | ControlFlowNode for object | test.py:33:17:33:22 | ControlFlowNode for object |
@@ -805,6 +817,8 @@
 | test.py:30:20:30:22 | ControlFlowNode for foo | test.py:30:9:30:12 | [post store] ControlFlowNode for self [Attribute foo] |
 | test.py:33:1:33:24 | ControlFlowNode for ClassExpr | test.py:33:7:33:15 | GSSA Variable NestedObj |
 | test.py:33:1:33:24 | ControlFlowNode for ClassExpr | test.py:33:7:33:15 | GSSA Variable NestedObj |
+| test.py:33:7:33:15 | ControlFlowNode for NestedObj | test.py:0:0:0:0 | ModuleVariableNode for Global Variable NestedObj in Module test |
+| test.py:33:7:33:15 | ControlFlowNode for NestedObj | test.py:0:0:0:0 | ModuleVariableNode for Global Variable NestedObj in Module test |
 | test.py:33:7:33:15 | GSSA Variable NestedObj | test.py:0:0:0:0 | ModuleVariableNode for Global Variable NestedObj in Module test |
 | test.py:33:7:33:15 | GSSA Variable NestedObj | test.py:0:0:0:0 | ModuleVariableNode for Global Variable NestedObj in Module test |
 | test.py:34:5:34:23 | ControlFlowNode for FunctionExpr | test.py:34:9:34:16 | SSA variable __init__ |
@@ -865,6 +879,8 @@
 | test.py:41:1:41:19 | ControlFlowNode for FunctionExpr | test.py:41:5:41:10 | GSSA Variable setFoo |
 | test.py:41:1:41:19 | GSSA Variable SINK_F | test.py:42:5:42:10 | ControlFlowNode for SINK_F |
 | test.py:41:1:41:19 | GSSA Variable SINK_F | test.py:42:5:42:10 | ControlFlowNode for SINK_F |
+| test.py:41:5:41:10 | ControlFlowNode for setFoo | test.py:0:0:0:0 | ModuleVariableNode for Global Variable setFoo in Module test |
+| test.py:41:5:41:10 | ControlFlowNode for setFoo | test.py:0:0:0:0 | ModuleVariableNode for Global Variable setFoo in Module test |
 | test.py:41:5:41:10 | GSSA Variable setFoo | test.py:0:0:0:0 | ModuleVariableNode for Global Variable setFoo in Module test |
 | test.py:41:5:41:10 | GSSA Variable setFoo | test.py:0:0:0:0 | ModuleVariableNode for Global Variable setFoo in Module test |
 | test.py:41:12:41:14 | ControlFlowNode for obj | test.py:41:12:41:14 | SSA variable obj |
@@ -1191,6 +1207,8 @@
 | test.py:90:1:90:30 | ControlFlowNode for FunctionExpr | test.py:90:5:90:26 | GSSA Variable fields_with_local_flow |
 | test.py:90:1:90:30 | GSSA Variable MyObj | test.py:91:11:91:15 | ControlFlowNode for MyObj |
 | test.py:90:1:90:30 | GSSA Variable MyObj | test.py:91:11:91:15 | ControlFlowNode for MyObj |
+| test.py:90:5:90:26 | ControlFlowNode for fields_with_local_flow | test.py:0:0:0:0 | ModuleVariableNode for Global Variable fields_with_local_flow in Module test |
+| test.py:90:5:90:26 | ControlFlowNode for fields_with_local_flow | test.py:0:0:0:0 | ModuleVariableNode for Global Variable fields_with_local_flow in Module test |
 | test.py:90:5:90:26 | GSSA Variable fields_with_local_flow | test.py:0:0:0:0 | ModuleVariableNode for Global Variable fields_with_local_flow in Module test |
 | test.py:90:5:90:26 | GSSA Variable fields_with_local_flow | test.py:0:0:0:0 | ModuleVariableNode for Global Variable fields_with_local_flow in Module test |
 | test.py:90:28:90:28 | ControlFlowNode for x | test.py:90:28:90:28 | SSA variable x |