Merge pull request #7246 from tausbn/python/import-star-flow

Python: Support flow through `import *`

Author: yoff

python/ql/lib/semmle/python/ApiGraphs.qll

@@ -358,134 +358,26 @@ module API {
       )
     }
 
-    /** Gets the name of a known built-in. */
-    private string getBuiltInName() {
-      // These lists were created by inspecting the `builtins` and `__builtin__` modules in
-      // Python 3 and 2 respectively, using the `dir` built-in.
-      // Built-in functions and exceptions shared between Python 2 and 3
-      result in [
-          "abs", "all", "any", "bin", "bool", "bytearray", "callable", "chr", "classmethod",
-          "compile", "complex", "delattr", "dict", "dir", "divmod", "enumerate", "eval", "filter",
-          "float", "format", "frozenset", "getattr", "globals", "hasattr", "hash", "help", "hex",
-          "id", "input", "int", "isinstance", "issubclass", "iter", "len", "list", "locals", "map",
-          "max", "memoryview", "min", "next", "object", "oct", "open", "ord", "pow", "print",
-          "property", "range", "repr", "reversed", "round", "set", "setattr", "slice", "sorted",
-          "staticmethod", "str", "sum", "super", "tuple", "type", "vars", "zip", "__import__",
-          // Exceptions
-          "ArithmeticError", "AssertionError", "AttributeError", "BaseException", "BufferError",
-          "BytesWarning", "DeprecationWarning", "EOFError", "EnvironmentError", "Exception",
-          "FloatingPointError", "FutureWarning", "GeneratorExit", "IOError", "ImportError",
-          "ImportWarning", "IndentationError", "IndexError", "KeyError", "KeyboardInterrupt",
-          "LookupError", "MemoryError", "NameError", "NotImplemented", "NotImplementedError",
-          "OSError", "OverflowError", "PendingDeprecationWarning", "ReferenceError", "RuntimeError",
-          "RuntimeWarning", "StandardError", "StopIteration", "SyntaxError", "SyntaxWarning",
-          "SystemError", "SystemExit", "TabError", "TypeError", "UnboundLocalError",
-          "UnicodeDecodeError", "UnicodeEncodeError", "UnicodeError", "UnicodeTranslateError",
-          "UnicodeWarning", "UserWarning", "ValueError", "Warning", "ZeroDivisionError",
-          // Added for compatibility
-          "exec"
-        ]
-      or
-      // Built-in constants shared between Python 2 and 3
-      result in ["False", "True", "None", "NotImplemented", "Ellipsis", "__debug__"]
-      or
-      // Python 3 only
-      result in [
-          "ascii", "breakpoint", "bytes", "exec", "aiter", "anext",
-          // Exceptions
-          "BlockingIOError", "BrokenPipeError", "ChildProcessError", "ConnectionAbortedError",
-          "ConnectionError", "ConnectionRefusedError", "ConnectionResetError", "FileExistsError",
-          "FileNotFoundError", "InterruptedError", "IsADirectoryError", "ModuleNotFoundError",
-          "NotADirectoryError", "PermissionError", "ProcessLookupError", "RecursionError",
-          "ResourceWarning", "StopAsyncIteration", "TimeoutError"
-        ]
-      or
-      // Python 2 only
-      result in [
-          "basestring", "cmp", "execfile", "file", "long", "raw_input", "reduce", "reload",
-          "unichr", "unicode", "xrange"
-        ]
-    }
-
-    /**
-     * Gets a data flow node that is likely to refer to a built-in with the name `name`.
-     *
-     * Currently this is an over-approximation, and may not account for things like overwriting a
-     * built-in with a different value.
-     */
-    private DataFlow::Node likely_builtin(string name) {
-      exists(Module m |
-        result.asCfgNode() =
-          any(NameNode n |
-            possible_builtin_accessed_in_module(n, name, m) and
-            not possible_builtin_defined_in_module(name, m)
-          )
-      )
-    }
-
-    /**
-     * Holds if a global variable called `name` (which is also the name of a built-in) is assigned
-     * a value in the module `m`.
-     */
-    private predicate possible_builtin_defined_in_module(string name, Module m) {
-      global_name_defined_in_module(name, m) and
-      name = getBuiltInName()
-    }
-
-    /**
-     * Holds if `n` is an access of a global variable called `name` (which is also the name of a
-     * built-in) inside the module `m`.
-     */
-    private predicate possible_builtin_accessed_in_module(NameNode n, string name, Module m) {
-      n.isGlobal() and
-      n.isLoad() and
-      name = n.getId() and
-      name = getBuiltInName() and
-      m = n.getEnclosingModule()
-    }
-
-    /**
-     * Holds if `n` is an access of a variable called `name` (which is _not_ the name of a
-     * built-in, and which is _not_ a global defined in the enclosing module) inside the scope `s`.
-     */
-    private predicate name_possibly_defined_in_import_star(NameNode n, string name, Scope s) {
-      n.isLoad() and
-      name = n.getId() and
-      // Not already defined in an enclosing scope.
-      not exists(LocalVariable v |
-        v.getId() = name and v.getScope() = n.getScope().getEnclosingScope*()
-      ) and
-      not name = getBuiltInName() and
-      s = n.getScope().getEnclosingScope*() and
-      exists(potential_import_star_base(s)) and
-      not global_name_defined_in_module(name, n.getEnclosingModule())
-    }
-
-    /** Holds if a global variable called `name` is assigned a value in the module `m`. */
-    private predicate global_name_defined_in_module(string name, Module m) {
-      exists(NameNode n |
-        not exists(LocalVariable v | n.defines(v)) and
-        n.isStore() and
-        name = n.getId() and
-        m = n.getEnclosingModule()
-      )
-    }
+    private import semmle.python.dataflow.new.internal.Builtins
+    private import semmle.python.dataflow.new.internal.ImportStar
 
     /**
      * Gets the API graph node for all modules imported with `from ... import *` inside the scope `s`.
      *
      * For example, given
      *
-     * `from foo.bar import *`
+     * ```python
+     * from foo.bar import *
+     * ```
      *
      * this would be the API graph node with the path
      *
      * `moduleImport("foo").getMember("bar")`
      */
     private TApiNode potential_import_star_base(Scope s) {
-      exists(DataFlow::Node ref |
-        ref.asCfgNode() = any(ImportStarNode n | n.getScope() = s).getModule() and
-        use(result, ref)
+      exists(DataFlow::Node n |
+        n.asCfgNode() = ImportStar::potentialImportStarBase(s) and
+        use(result, n)
       )
     }
 
@@ -529,14 +421,14 @@ module API {
       or
       // Built-ins, treated as members of the module `builtins`
       base = MkModuleImport("builtins") and
-      lbl = Label::member(any(string name | ref = likely_builtin(name)))
+      lbl = Label::member(any(string name | ref = Builtins::likelyBuiltin(name)))
       or
       // Unknown variables that may belong to a module imported with `import *`
       exists(Scope s |
         base = potential_import_star_base(s) and
         lbl =
           Label::member(any(string name |
-              name_possibly_defined_in_import_star(ref.asCfgNode(), name, s)
+              ImportStar::namePossiblyDefinedInImportStar(ref.asCfgNode(), name, s)
             ))
       )
     }

python/ql/lib/semmle/python/dataflow/new/internal/Builtins.qll

@@ -0,0 +1,93 @@
+/** Provides predicates for reasoning about built-ins in Python. */
+
+private import python
+private import semmle.python.dataflow.new.DataFlow
+private import semmle.python.dataflow.new.internal.ImportStar
+
+module Builtins {
+  /** Gets the name of a known built-in. */
+  string getBuiltinName() {
+    // These lists were created by inspecting the `builtins` and `__builtin__` modules in
+    // Python 3 and 2 respectively, using the `dir` built-in.
+    // Built-in functions and exceptions shared between Python 2 and 3
+    result in [
+        "abs", "all", "any", "bin", "bool", "bytearray", "callable", "chr", "classmethod",
+        "compile", "complex", "delattr", "dict", "dir", "divmod", "enumerate", "eval", "filter",
+        "float", "format", "frozenset", "getattr", "globals", "hasattr", "hash", "help", "hex",
+        "id", "input", "int", "isinstance", "issubclass", "iter", "len", "list", "locals", "map",
+        "max", "memoryview", "min", "next", "object", "oct", "open", "ord", "pow", "print",
+        "property", "range", "repr", "reversed", "round", "set", "setattr", "slice", "sorted",
+        "staticmethod", "str", "sum", "super", "tuple", "type", "vars", "zip", "__import__",
+        // Exceptions
+        "ArithmeticError", "AssertionError", "AttributeError", "BaseException", "BufferError",
+        "BytesWarning", "DeprecationWarning", "EOFError", "EnvironmentError", "Exception",
+        "FloatingPointError", "FutureWarning", "GeneratorExit", "IOError", "ImportError",
+        "ImportWarning", "IndentationError", "IndexError", "KeyError", "KeyboardInterrupt",
+        "LookupError", "MemoryError", "NameError", "NotImplemented", "NotImplementedError",
+        "OSError", "OverflowError", "PendingDeprecationWarning", "ReferenceError", "RuntimeError",
+        "RuntimeWarning", "StandardError", "StopIteration", "SyntaxError", "SyntaxWarning",
+        "SystemError", "SystemExit", "TabError", "TypeError", "UnboundLocalError",
+        "UnicodeDecodeError", "UnicodeEncodeError", "UnicodeError", "UnicodeTranslateError",
+        "UnicodeWarning", "UserWarning", "ValueError", "Warning", "ZeroDivisionError",
+        // Added for compatibility
+        "exec"
+      ]
+    or
+    // Built-in constants shared between Python 2 and 3
+    result in ["False", "True", "None", "NotImplemented", "Ellipsis", "__debug__"]
+    or
+    // Python 3 only
+    result in [
+        "ascii", "breakpoint", "bytes", "exec",
+        // Exceptions
+        "BlockingIOError", "BrokenPipeError", "ChildProcessError", "ConnectionAbortedError",
+        "ConnectionError", "ConnectionRefusedError", "ConnectionResetError", "FileExistsError",
+        "FileNotFoundError", "InterruptedError", "IsADirectoryError", "ModuleNotFoundError",
+        "NotADirectoryError", "PermissionError", "ProcessLookupError", "RecursionError",
+        "ResourceWarning", "StopAsyncIteration", "TimeoutError"
+      ]
+    or
+    // Python 2 only
+    result in [
+        "basestring", "cmp", "execfile", "file", "long", "raw_input", "reduce", "reload", "unichr",
+        "unicode", "xrange"
+      ]
+  }
+
+  /**
+   * Gets a data flow node that is likely to refer to a built-in with the name `name`.
+   *
+   * Currently this is an over-approximation, and may not account for things like overwriting a
+   * built-in with a different value.
+   */
+  DataFlow::Node likelyBuiltin(string name) {
+    exists(Module m |
+      result.asCfgNode() =
+        any(NameNode n |
+          possible_builtin_accessed_in_module(n, name, m) and
+          not possible_builtin_defined_in_module(name, m)
+        )
+    )
+  }
+
+  /**
+   * Holds if a global variable called `name` (which is also the name of a built-in) is assigned
+   * a value in the module `m`.
+   */
+  private predicate possible_builtin_defined_in_module(string name, Module m) {
+    ImportStar::globalNameDefinedInModule(name, m) and
+    name = getBuiltinName()
+  }
+
+  /**
+   * Holds if `n` is an access of a global variable called `name` (which is also the name of a
+   * built-in) inside the module `m`.
+   */
+  private predicate possible_builtin_accessed_in_module(NameNode n, string name, Module m) {
+    n.isGlobal() and
+    n.isLoad() and
+    name = n.getId() and
+    name = getBuiltinName() and
+    m = n.getEnclosingModule()
+  }
+}

python/ql/lib/semmle/python/dataflow/new/internal/DataFlowPrivate.qll

@@ -2,6 +2,7 @@ private import python
 private import DataFlowPublic
 import semmle.python.SpecialMethods
 private import semmle.python.essa.SsaCompute
+private import semmle.python.dataflow.new.internal.ImportStar
 
 /** Gets the callable in which this node occurs. */
 DataFlowCallable nodeGetEnclosingCallable(Node n) { result = n.getEnclosingCallable() }
@@ -948,7 +949,7 @@ predicate jumpStep(Node nodeFrom, Node nodeTo) {
 private predicate module_export(Module m, string name, CfgNode defn) {
   exists(EssaVariable v |
     v.getName() = name and
-    v.getAUse() = m.getANormalExit()
+    v.getAUse() = ImportStar::getStarImported*(m).getANormalExit()
   |
     defn.getNode() = v.getDefinition().(AssignmentDefinition).getValue()
     or

python/ql/lib/semmle/python/dataflow/new/internal/DataFlowPublic.qll

@@ -8,6 +8,7 @@ import semmle.python.dataflow.new.TypeTracker
 import Attributes
 import LocalSources
 private import semmle.python.essa.SsaCompute
+private import semmle.python.dataflow.new.internal.ImportStar
 
 /**
  * IPA type for data flow nodes.
@@ -30,7 +31,15 @@ newtype TNode =
   /** A synthetic node representing the value of an object after a state change. */
   TSyntheticPostUpdateNode(NeedsSyntheticPostUpdateNode pre) or
   /** A node representing a global (module-level) variable in a specific module. */
-  TModuleVariableNode(Module m, GlobalVariable v) { v.getScope() = m and v.escapes() } or
+  TModuleVariableNode(Module m, GlobalVariable v) {
+    v.getScope() = m and
+    (
+      v.escapes()
+      or
+      isAccessedThroughImportStar(m) and
+      ImportStar::globalNameDefinedInModule(v.getId(), m)
+    )
+  } or
   /**
    * A node representing the overflow positional arguments to a call.
    * That is, `call` contains more positional arguments than there are
@@ -346,6 +355,8 @@ class ModuleVariableNode extends Node, TModuleVariableNode {
     result.asCfgNode() = var.getALoad().getAFlowNode() and
     // Ignore reads that happen when the module is imported. These are only executed once.
     not result.getScope() = mod
+    or
+    this = import_star_read(result)
   }
 
   /** Gets an `EssaNode` that corresponds to an assignment of this global variable. */
@@ -358,6 +369,13 @@ class ModuleVariableNode extends Node, TModuleVariableNode {
   override Location getLocation() { result = mod.getLocation() }
 }
 
+private predicate isAccessedThroughImportStar(Module m) { m = ImportStar::getStarImported(_) }
+
+private ModuleVariableNode import_star_read(Node n) {
+  ImportStar::importStarResolvesTo(n.asCfgNode(), result.getModule()) and
+  n.asCfgNode().(NameNode).getId() = result.getVariable().getId()
+}
+
 /**
  * The node holding the extra positional arguments to a call. This node is passed as a tuple
  * to the starred parameter of the callable.