Merge pull request github#8883 from erik-krogh/pyMaD

Python: add MaD implementation

Author: RasmusWL

config/identical-files.json

@@ -525,7 +525,8 @@
     "csharp/ql/lib/semmle/code/csharp/dataflow/internal/AccessPathSyntax.qll",
     "java/ql/lib/semmle/code/java/dataflow/internal/AccessPathSyntax.qll",
     "javascript/ql/lib/semmle/javascript/frameworks/data/internal/AccessPathSyntax.qll",
-    "ruby/ql/lib/codeql/ruby/dataflow/internal/AccessPathSyntax.qll"
+    "ruby/ql/lib/codeql/ruby/dataflow/internal/AccessPathSyntax.qll",
+    "python/ql/lib/semmle/python/frameworks/data/internal/AccessPathSyntax.qll"
   ],
   "IncompleteUrlSubstringSanitization": [
     "javascript/ql/src/Security/CWE-020/IncompleteUrlSubstringSanitization.qll",
@@ -543,7 +544,8 @@
   ],
   "ApiGraphModels": [
     "javascript/ql/lib/semmle/javascript/frameworks/data/internal/ApiGraphModels.qll",
-    "ruby/ql/lib/codeql/ruby/frameworks/data/internal/ApiGraphModels.qll"
+    "ruby/ql/lib/codeql/ruby/frameworks/data/internal/ApiGraphModels.qll",
+    "python/ql/lib/semmle/python/frameworks/data/internal/ApiGraphModels.qll"
   ],
   "TaintedFormatStringQuery Ruby/JS": [
     "javascript/ql/lib/semmle/javascript/security/dataflow/TaintedFormatStringQuery.qll",

javascript/ql/lib/semmle/javascript/frameworks/data/internal/ApiGraphModels.qll

@@ -299,7 +299,7 @@ private class AccessPathRange extends AccessPath::Range {
 bindingset[token]
 API::Node getSuccessorFromNode(API::Node node, AccessPathToken token) {
   // API graphs use the same label for arguments and parameters. An edge originating from a
-  // use-node represents be an argument, and an edge originating from a def-node represents a parameter.
+  // use-node represents an argument, and an edge originating from a def-node represents a parameter.
   // We just map both to the same thing.
   token.getName() = ["Argument", "Parameter"] and
   result = node.getParameter(AccessPath::parseIntUnbounded(token.getAnArgument()))

python/ql/lib/semmle/python/ApiGraphs.qll

@@ -136,6 +136,9 @@ module API {
       result = this.getASuccessor(Label::keywordParameter(name))
     }
 
+    /** Gets the node representing the self parameter */
+    Node getSelfParameter() { result = this.getASuccessor(Label::selfParameter()) }
+
     /**
      * Gets the number of parameters of the function represented by this node.
      */
@@ -321,6 +324,12 @@ module API {
     /** Gets the API node for a parameter of this invocation. */
     Node getAParameter() { result = this.getParameter(_) }
 
+    /** Gets the object that this method-call is being called on, if this is a method-call */
+    Node getSelfParameter() {
+      result.getARhs() = this.(DataFlow::MethodCallNode).getObject() and
+      result = callee.getSelfParameter()
+    }
+
     /** Gets the API node for the keyword parameter `name` of this invocation. */
     Node getKeywordParameter(string name) {
       result = callee.getKeywordParameter(name) and
@@ -345,6 +354,14 @@ module API {
       result = callee.getReturn() and
       result.getAnImmediateUse() = this
     }
+
+    /**
+     * Gets the number of positional arguments of this call.
+     *
+     * Note: This is used for `WithArity[<n>]` in modeling-as-data, where we thought
+     * including keyword arguments didn't make much sense.
+     */
+    int getNumArgument() { result = count(this.getArg(_)) }
   }
 
   /**
@@ -589,15 +606,24 @@ module API {
       exists(DataFlow::Node def, PY::CallableExpr fn |
         rhs(base, def) and fn = trackDefNode(def).asExpr()
       |
-        exists(int i |
-          lbl = Label::parameter(i) and
+        exists(int i, int offset |
+          if exists(PY::Parameter p | p = fn.getInnerScope().getAnArg() and p.isSelf())
+          then offset = 1
+          else offset = 0
+        |
+          lbl = Label::parameter(i - offset) and
           ref.asExpr() = fn.getInnerScope().getArg(i)
         )
         or
-        exists(string name |
+        exists(string name, PY::Parameter param |
           lbl = Label::keywordParameter(name) and
-          ref.asExpr() = fn.getInnerScope().getArgByName(name)
+          param = fn.getInnerScope().getArgByName(name) and
+          not param.isSelf() and
+          ref.asExpr() = param
         )
+        or
+        lbl = Label::selfParameter() and
+        ref.asExpr() = any(PY::Parameter p | p = fn.getInnerScope().getAnArg() and p.isSelf())
       )
       or
       // Built-ins, treated as members of the module `builtins`
@@ -664,6 +690,9 @@ module API {
         exists(string name | lbl = Label::keywordParameter(name) |
           arg = pred.getACall().getArgByName(name)
         )
+        or
+        lbl = Label::selfParameter() and
+        arg = pred.getACall().(DataFlow::MethodCallNode).getObject()
       )
     }
 
@@ -780,6 +809,7 @@ module API {
           or
           exists(any(PY::Function f).getArgByName(name))
         } or
+        MkLabelSelfParameter() or
         MkLabelReturn() or
         MkLabelSubclass() or
         MkLabelAwait()
@@ -837,6 +867,11 @@ module API {
         string getName() { result = name }
       }
 
+      /** A label for the self parameter. */
+      class LabelSelfParameter extends ApiLabel, MkLabelSelfParameter {
+        override string toString() { result = "getSelfParameter()" }
+      }
+
       /** A label that gets the return value of a function. */
       class LabelReturn extends ApiLabel, MkLabelReturn {
         override string toString() { result = "getReturn()" }
@@ -876,6 +911,9 @@ module API {
     /** Gets the `parameter` edge label for the keyword parameter `name`. */
     LabelKeywordParameter keywordParameter(string name) { result.getName() = name }
 
+    /** Gets the edge label for the self parameter. */
+    LabelSelfParameter selfParameter() { any() }
+
     /** Gets the `return` edge label. */
     LabelReturn return() { any() }
 

python/ql/lib/semmle/python/Frameworks.qll

@@ -12,6 +12,7 @@ private import semmle.python.frameworks.Asyncpg
 private import semmle.python.frameworks.ClickhouseDriver
 private import semmle.python.frameworks.Cryptodome
 private import semmle.python.frameworks.Cryptography
+private import semmle.python.frameworks.data.ModelsAsData
 private import semmle.python.frameworks.Dill
 private import semmle.python.frameworks.Django
 private import semmle.python.frameworks.Fabric

python/ql/lib/semmle/python/frameworks/Asyncpg.qll

@@ -7,91 +7,42 @@ private import python
 private import semmle.python.dataflow.new.DataFlow
 private import semmle.python.Concepts
 private import semmle.python.ApiGraphs
+private import semmle.python.frameworks.data.ModelsAsData
 
 /** Provides models for the `asyncpg` PyPI package. */
 private module Asyncpg {
-  private import semmle.python.internal.Awaited
-
-  /** Gets a `ConnectionPool` that is created when the result of `asyncpg.create_pool()` is awaited. */
-  API::Node connectionPool() {
-    result = API::moduleImport("asyncpg").getMember("create_pool").getReturn().getAwaited()
-  }
-
-  /**
-   * Gets a `Connection` that is created when
-   * - the result of `asyncpg.connect()` is awaited.
-   * - the result of calling `acquire` on a `ConnectionPool` is awaited.
-   */
-  API::Node connection() {
-    result = API::moduleImport("asyncpg").getMember("connect").getReturn().getAwaited()
-    or
-    result = connectionPool().getMember("acquire").getReturn().getAwaited()
-  }
-
-  /** `Connection`s and `ConnectionPool`s provide some methods that execute SQL. */
-  class SqlExecutionOnConnection extends SqlExecution::Range, DataFlow::MethodCallNode {
-    string methodName;
-
-    SqlExecutionOnConnection() {
-      this = [connectionPool(), connection()].getMember(methodName).getACall() and
-      methodName in ["copy_from_query", "execute", "fetch", "fetchrow", "fetchval", "executemany"]
-    }
-
-    override DataFlow::Node getSql() {
-      methodName in ["copy_from_query", "execute", "fetch", "fetchrow", "fetchval"] and
-      result in [this.getArg(0), this.getArgByName("query")]
-      or
-      methodName = "executemany" and
-      result in [this.getArg(0), this.getArgByName("command")]
-    }
-  }
-
-  /** A model of `Connection` and `ConnectionPool`, which provide some methods that access the file system. */
-  class FileAccessOnConnection extends FileSystemAccess::Range, DataFlow::MethodCallNode {
-    string methodName;
-
-    FileAccessOnConnection() {
-      this = [connectionPool(), connection()].getMember(methodName).getACall() and
-      methodName in ["copy_from_query", "copy_from_table", "copy_to_table"]
-    }
-
-    // The path argument is keyword only.
-    override DataFlow::Node getAPathArgument() {
-      methodName in ["copy_from_query", "copy_from_table"] and
-      result = this.getArgByName("output")
-      or
-      methodName = "copy_to_table" and
-      result = this.getArgByName("source")
+  class AsyncpgModel extends ModelInput::TypeModelCsv {
+    override predicate row(string row) {
+      // package1;type1;package2;type2;path
+      row =
+        [
+          // a `ConnectionPool` that is created when the result of `asyncpg.create_pool()` is awaited.
+          "asyncpg;ConnectionPool;asyncpg;;Member[create_pool].ReturnValue.Awaited",
+          // a `Connection` that is created when
+          // * - the result of `asyncpg.connect()` is awaited.
+          // * - the result of calling `acquire` on a `ConnectionPool` is awaited.
+          "asyncpg;Connection;asyncpg;;Member[connect].ReturnValue.Awaited",
+          "asyncpg;Connection;asyncpg;ConnectionPool;Member[acquire].ReturnValue.Awaited",
+          // Creating an internal `~Connection` type that contains both `Connection` and `ConnectionPool`.
+          "asyncpg;~Connection;asyncpg;Connection;", "asyncpg;~Connection;asyncpg;ConnectionPool;"
+        ]
     }
   }
 
-  /**
-   * Provides models of the `PreparedStatement` class in `asyncpg`.
-   * `PreparedStatement`s are created when the result of calling `prepare(query)` on a connection is awaited.
-   * The result of calling `prepare(query)` is a `PreparedStatementFactory` and the argument, `query` needs to
-   * be tracked to the place where a `PreparedStatement` is created and then further to any executing methods.
-   * Hence the two type trackers.
-   */
-  module PreparedStatement {
-    class PreparedStatementConstruction extends SqlConstruction::Range, API::CallNode {
-      PreparedStatementConstruction() { this = connection().getMember("prepare").getACall() }
-
-      override DataFlow::Node getSql() { result = this.getParameter(0, "query").getARhs() }
-    }
-
-    class PreparedStatementExecution extends SqlExecution::Range, API::CallNode {
-      PreparedStatementConstruction prepareCall;
-
-      PreparedStatementExecution() {
-        this =
-          prepareCall
-              .getReturn()
-              .getAwaited()
-              .getMember(["executemany", "fetch", "fetchrow", "fetchval"])
-              .getACall()
-      }
-
-      override DataFlow::Node getSql() { result = prepareCall.getSql() }
+  class AsyncpgSink extends ModelInput::SinkModelCsv {
+    // package;type;path;kind
+    override predicate row(string row) {
+      row =
+        [
+          // `Connection`s and `ConnectionPool`s provide some methods that execute SQL.
+          "asyncpg;~Connection;Member[copy_from_query,execute,fetch,fetchrow,fetchval].Argument[0,query:];sql-injection",
+          "asyncpg;~Connection;Member[executemany].Argument[0,command:];sql-injection",
+          // A model of `Connection` and `ConnectionPool`, which provide some methods that access the file system.
+          "asyncpg;~Connection;Member[copy_from_query,copy_from_table].Argument[output:];path-injection",
+          "asyncpg;~Connection;Member[copy_to_table].Argument[source:];path-injection",
+          // the `PreparedStatement` class in `asyncpg`.
+          "asyncpg;Connection;Member[prepare].Argument[0,query:];sql-injection",
+        ]
     }
   }
 
@@ -106,7 +57,9 @@ private module Asyncpg {
    */
   module Cursor {
     class CursorConstruction extends SqlConstruction::Range, API::CallNode {
-      CursorConstruction() { this = connection().getMember("cursor").getACall() }
+      CursorConstruction() {
+        this = ModelOutput::getATypeNode("asyncpg", "Connection").getMember("cursor").getACall()
+      }
 
       override DataFlow::Node getSql() { result = this.getParameter(0, "query").getARhs() }
     }
@@ -121,8 +74,11 @@ private module Asyncpg {
           this = c.getReturn().getAwaited().getAnImmediateUse()
         )
         or
-        exists(PreparedStatement::PreparedStatementConstruction prepareCall |
-          sql = prepareCall.getSql() and
+        exists(API::CallNode prepareCall |
+          prepareCall =
+            ModelOutput::getATypeNode("asyncpg", "Connection").getMember("prepare").getACall()
+        |
+          sql = prepareCall.getParameter(0, "query").getARhs() and
           this =
             prepareCall
                 .getReturn()

python/ql/lib/semmle/python/frameworks/data/ModelsAsData.qll

@@ -0,0 +1,47 @@
+/**
+ * Provides classes for contributing a model, or using the interpreted results
+ * of a model represented as data.
+ *
+ * - Use the `ModelInput` module to contribute new models.
+ * - Use the `ModelOutput` module to access the model results in terms of API nodes.
+ *
+ * The package name refers to the top-level module the import comes from, and not a PyPI package.
+ * So for `from foo.bar import baz`, the package will be `foo`.
+ */
+
+private import python
+private import internal.ApiGraphModels as Shared
+private import internal.ApiGraphModelsSpecific as Specific
+import Shared::ModelInput as ModelInput
+import Shared::ModelOutput as ModelOutput
+private import semmle.python.dataflow.new.RemoteFlowSources
+private import semmle.python.dataflow.new.DataFlow
+private import semmle.python.ApiGraphs
+private import semmle.python.dataflow.new.TaintTracking
+
+/**
+ * A remote flow source originating from a CSV source row.
+ */
+private class RemoteFlowSourceFromCsv extends RemoteFlowSource {
+  RemoteFlowSourceFromCsv() { this = ModelOutput::getASourceNode("remote").getAnImmediateUse() }
+
+  override string getSourceType() { result = "Remote flow (from model)" }
+}
+
+/**
+ * Like `ModelOutput::summaryStep` but with API nodes mapped to data-flow nodes.
+ */
+private predicate summaryStepNodes(DataFlow::Node pred, DataFlow::Node succ, string kind) {
+  exists(API::Node predNode, API::Node succNode |
+    Specific::summaryStep(predNode, succNode, kind) and
+    pred = predNode.getARhs() and
+    succ = succNode.getAnImmediateUse()
+  )
+}
+
+/** Taint steps induced by summary models of kind `taint`. */
+private class TaintStepFromSummary extends TaintTracking::AdditionalTaintStep {
+  override predicate step(DataFlow::Node pred, DataFlow::Node succ) {
+    summaryStepNodes(pred, succ, "taint")
+  }
+}