Merge pull request github#16789 from yoff/python/document-models-as-data

python: Document MaD format

Author: yoff

docs/codeql/codeql-language-guides/codeql-for-python.rst

@@ -15,6 +15,7 @@ Experiment and learn how to write effective and efficient queries for CodeQL dat
    functions-in-python
    expressions-and-statements-in-python
    analyzing-control-flow-in-python
+   customizing-library-models-for-python
 
 -  :doc:`Basic query for Python code <basic-query-for-python-code>`: Learn to write and run a simple CodeQL query.
 
@@ -29,3 +30,5 @@ Experiment and learn how to write effective and efficient queries for CodeQL dat
 -  :doc:`Expressions and statements in Python <expressions-and-statements-in-python>`: You can use syntactic classes from the CodeQL library to explore how Python expressions and statements are used in a codebase.
 
 -  :doc:`Analyzing control flow in Python <analyzing-control-flow-in-python>`: You can write CodeQL queries to explore the control-flow graph of a Python program, for example, to discover unreachable code or mutually exclusive blocks of code.
+
+-  :doc:`Customizing library models for Python <customizing-library-models-for-python>`: You can model frameworks and libraries that your codebase depends on using data extensions and publish them as CodeQL model packs.

docs/codeql/codeql-language-guides/customizing-library-models-for-python.rst

@@ -0,0 +1,4 @@
+---
+category: minorAnalysis
+---
+* A number of Python queries now support sinks defined vi data extensions. The format of data extensions for Python has been documented.

python/ql/lib/change-notes/2024-06-24-Data-extensions-supported-by-queries-and-documented.md

@@ -29,8 +29,8 @@ private module FabricV1 {
     // -------------------------------------------------------------------------
     // fabric.api
     // -------------------------------------------------------------------------
-    /** Gets a reference to the `fabric.api` module. */
-    API::Node api() { result = fabric().getMember("api") }
+    /** Gets a reference to the `fabric.api` module. Also known as `fabric.operations` */
+    API::Node api() { result = fabric().getMember(["api", "operations"]) }
 
     /** Provides models for the `fabric.api` module */
     module Api {

python/ql/lib/semmle/python/frameworks/Fabric.qll

@@ -9,6 +9,7 @@ private import semmle.python.dataflow.new.DataFlow
 private import semmle.python.Concepts
 private import semmle.python.dataflow.new.RemoteFlowSources
 private import semmle.python.dataflow.new.BarrierGuards
+private import semmle.python.frameworks.data.ModelsAsData
 
 /**
  * Provides default sources, sinks and sanitizers for detecting
@@ -43,6 +44,10 @@ module CodeInjection {
     CodeExecutionAsSink() { this = any(CodeExecution e).getCode() }
   }
 
+  private class SinkFromModel extends Sink {
+    SinkFromModel() { this = ModelOutput::getASinkNode("code-injection").asSink() }
+  }
+
   /**
    * A comparison with a constant string, considered as a sanitizer-guard.
    */

python/ql/lib/semmle/python/security/dataflow/CodeInjectionCustomizations.qll

@@ -9,6 +9,7 @@ private import semmle.python.dataflow.new.DataFlow
 private import semmle.python.Concepts
 private import semmle.python.dataflow.new.RemoteFlowSources
 private import semmle.python.dataflow.new.BarrierGuards
+private import semmle.python.frameworks.data.ModelsAsData
 
 /**
  * Provides default sources, sinks and sanitizers for detecting
@@ -78,6 +79,10 @@ module CommandInjection {
     }
   }
 
+  private class SinkFromModel extends Sink {
+    SinkFromModel() { this = ModelOutput::getASinkNode("command-injection").asSink() }
+  }
+
   /**
    * A comparison with a constant string, considered as a sanitizer-guard.
    */

python/ql/lib/semmle/python/security/dataflow/CommandInjectionCustomizations.qll

@@ -9,6 +9,7 @@ private import semmle.python.dataflow.new.DataFlow
 private import semmle.python.Concepts
 private import semmle.python.dataflow.new.RemoteFlowSources
 private import semmle.python.dataflow.new.BarrierGuards
+private import semmle.python.frameworks.data.ModelsAsData
 
 /**
  * Provides default sources, sinks and sanitizers for detecting
@@ -71,6 +72,10 @@ module LogInjection {
     }
   }
 
+  private class SinkFromModel extends Sink {
+    SinkFromModel() { this = ModelOutput::getASinkNode("log-injection").asSink() }
+  }
+
   /**
    * A comparison with a constant string, considered as a sanitizer-guard.
    */

python/ql/lib/semmle/python/security/dataflow/LogInjectionCustomizations.qll

@@ -9,6 +9,7 @@ private import semmle.python.dataflow.new.DataFlow
 private import semmle.python.Concepts
 private import semmle.python.dataflow.new.RemoteFlowSources
 private import semmle.python.dataflow.new.BarrierGuards
+private import semmle.python.frameworks.data.ModelsAsData
 
 /**
  * Provides default sources, sinks and sanitizers for detecting
@@ -48,6 +49,10 @@ module UnsafeDeserialization {
     }
   }
 
+  private class SinkFromModel extends Sink {
+    SinkFromModel() { this = ModelOutput::getASinkNode("unsafe-deserialization").asSink() }
+  }
+
   /**
    * A comparison with a constant string, considered as a sanitizer-guard.
    */

python/ql/lib/semmle/python/security/dataflow/UnsafeDeserializationCustomizations.qll

@@ -9,6 +9,7 @@ private import semmle.python.dataflow.new.DataFlow
 private import semmle.python.Concepts
 private import semmle.python.dataflow.new.RemoteFlowSources
 private import semmle.python.dataflow.new.BarrierGuards
+private import semmle.python.frameworks.data.ModelsAsData
 
 /**
  * Provides default sources, sinks and sanitizers for detecting
@@ -89,6 +90,10 @@ module UrlRedirect {
     }
   }
 
+  private class SinkFromModel extends Sink {
+    SinkFromModel() { this = ModelOutput::getASinkNode("url-redirection").asSink() }
+  }
+
   /**
    * The right side of a string-concat, considered as a sanitizer.
    */

python/ql/lib/semmle/python/security/dataflow/UrlRedirectCustomizations.qll

@@ -25,3 +25,15 @@ extensions:
       - ["foo.MS_Class", "Member[instance_method]", "Argument[0]", "ReturnValue.TupleElement[1]", "value"]
       - ["foo.MS_Class", "Member[explicit_self]", "Argument[self:]", "ReturnValue", "value"]
       - ["json", "Member[MS_loads]", "Argument[0]", "ReturnValue", "taint"]
+
+  - addsTo:
+      pack: codeql/python-all
+      extensible: typeModel
+    data:
+      - ["foo.MS_Class", "foo", "Member[get_instance].ReturnValue"]
+      - ["foo.MS_Class!", "foo", "Member[get_class].ReturnValue"]
+      # Ideally this would be a consequence of the above line
+      - ["foo.MS_Class", "foo", "Member[get_class].ReturnValue.Instance"]
+      - ["foo.MS_Class", "foo.MS_Factory!", "Member[get_instance].ReturnValue"]
+      - ["foo.MS_Class", "foo.MS_Factory", "Member[make].ReturnValue"]
+      - ["foo.MS_Class", "foo.Impl.MS_Class_Impl", ""]