Merge pull request github#13729 from yoff/python/model-aws-lambdas

Python/JavaScript: Shared module for serverless functions

Author: yoff

javascript/ql/lib/semmle/javascript/frameworks/ServerLess.qll

@@ -1,120 +1,52 @@
 /**
  * Provides classes and predicates for working with serverless handlers.
  * E.g. [AWS](https://docs.aws.amazon.com/lambda/latest/dg/nodejs-handler.html) or [serverless](https://npmjs.com/package/serverless)
- */
-
-import javascript
-
-/**
- * Provides classes and predicates for working with serverless handlers.
  * In particular a `RemoteFlowSource` is added for AWS, Alibaba, and serverless.
  */
-private module ServerLess {
-  /**
-   * Holds if the `.yml` file `ymlFile` contains a serverless configuration with `handler` and `codeURI` properties.
-   * `codeURI` defaults to the empty string if no explicit value is set in the configuration.
-   */
-  private predicate hasServerlessHandler(File ymlFile, string handler, string codeUri) {
-    exists(YamlMapping resource | ymlFile = resource.getFile() |
-      // There exists at least "AWS::Serverless::Function" and "Aliyun::Serverless::Function"
-      resource.lookup("Type").(YamlScalar).getValue().regexpMatch(".*::Serverless::Function") and
-      exists(YamlMapping properties | properties = resource.lookup("Properties") |
-        handler = properties.lookup("Handler").(YamlScalar).getValue() and
-        if exists(properties.lookup("CodeUri"))
-        then codeUri = properties.lookup("CodeUri").(YamlScalar).getValue()
-        else codeUri = ""
-      )
-      or
-      // The `serverless` library, which specifies a top-level `functions` property
-      exists(YamlMapping functions |
-        functions = resource.lookup("functions") and
-        not exists(resource.getParentNode()) and
-        handler = functions.getValue(_).(YamlMapping).lookup("handler").(YamlScalar).getValue() and
-        codeUri = ""
-      )
-    )
-  }
-
-  /**
-   * Gets a string where an ending "/." is simplified to "/" (if it exists).
-   */
-  bindingset[base]
-  private string removeTrailingDot(string base) {
-    if base.regexpMatch(".*/\\.")
-    then result = base.substring(0, base.length() - 1)
-    else result = base
-  }
 
-  /**
-   * Gets a string where a leading "./" is simplified to "" (if it exists).
-   */
-  bindingset[base]
-  private string removeLeadingDotSlash(string base) {
-    if base.regexpMatch("\\./.*") then result = base.substring(2, base.length()) else result = base
-  }
+import javascript
+import codeql.serverless.ServerLess
 
-  /**
-   * Gets a path to a file from a `codeURI` property and a file name from a serverless configuration.
-   *
-   * For example if `codeURI` is "function/." and `file` is "index", then the result becomes "function/index.js".
-   */
-  bindingset[codeUri, file]
-  private string getPathFromHandlerProperties(string codeUri, string file) {
-    exists(string folder | folder = removeLeadingDotSlash(removeTrailingDot(codeUri)) |
-      result = folder + file + ".js"
-    )
-  }
+private module YamlImpl implements Input {
+  import semmle.javascript.Files
+  import semmle.javascript.YAML
+}
 
-  /**
-   * Holds if `file` has a serverless handler function with name `func`.
-   */
-  private predicate hasServerlessHandler(File file, string func) {
-    exists(File ymlFile, string handler, string codeUri, string fileName |
-      hasServerlessHandler(ymlFile, handler, codeUri) and
-      // Splits a `handler` into two components. The `fileName` to the left of the dot, and the `func` to the right.
-      // E.g. if `handler` is "index.foo", then `fileName` is "index" and `func` is "foo".
-      exists(string pattern | pattern = "(.*)\\.(.*)" |
-        fileName = handler.regexpCapture(pattern, 1) and
-        func = handler.regexpCapture(pattern, 2)
-      )
-    |
-      file.getAbsolutePath() =
-        ymlFile.getParentContainer().getAbsolutePath() + "/" +
-          getPathFromHandlerProperties(codeUri, fileName)
-    )
-  }
+module SL = ServerLess<YamlImpl>;
 
-  /**
-   * Gets a function that is a serverless request handler.
-   *
-   * For example: if an AWS serverless resource contains the following properties (in the "template.yml" file):
-   * ```yaml
-   * Handler: mylibrary.handler
-   * Runtime: nodejs12.x
-   * CodeUri: backend/src/
-   * ```
-   *
-   * And a file "mylibrary.js" exists in the folder "backend/src" (relative to the "template.yml" file).
-   * Then the result of this predicate is a function exported as "handler" from "mylibrary.js".
-   * The "mylibrary.js" file could for example look like:
-   *
-   * ```JavaScript
-   * module.exports.handler = function (event) { ... }
-   * ```
-   */
-  private DataFlow::FunctionNode getAServerlessHandler() {
-    exists(File file, string handler, Module mod | hasServerlessHandler(file, handler) |
-      mod.getFile() = file and
-      result = mod.getAnExportedValue(handler).getAFunctionValue()
-    )
-  }
+/**
+ * Gets a function that is a serverless request handler.
+ *
+ * For example: if an AWS serverless resource contains the following properties (in the "template.yml" file):
+ * ```yaml
+ * Handler: mylibrary.handler
+ * Runtime: nodejs12.x
+ * CodeUri: backend/src/
+ * ```
+ *
+ * And a file "mylibrary.js" exists in the folder "backend/src" (relative to the "template.yml" file).
+ * Then the result of this predicate is a function exported as "handler" from "mylibrary.js".
+ * The "mylibrary.js" file could for example look like:
+ *
+ * ```JavaScript
+ * module.exports.handler = function (event) { ... }
+ * ```
+ */
+private DataFlow::FunctionNode getAServerlessHandler() {
+  exists(File file, string stem, string handler, Module mod |
+    SL::hasServerlessHandler(stem, handler, _, _) and
+    file.getAbsolutePath() = stem + ".js"
+  |
+    mod.getFile() = file and
+    result = mod.getAnExportedValue(handler).getAFunctionValue()
+  )
+}
 
-  /**
-   * A serverless request handler event, seen as a RemoteFlowSource.
-   */
-  private class ServerlessHandlerEventAsRemoteFlow extends RemoteFlowSource {
-    ServerlessHandlerEventAsRemoteFlow() { this = getAServerlessHandler().getParameter(0) }
+/**
+ * A serverless request handler event, seen as a RemoteFlowSource.
+ */
+private class ServerlessHandlerEventAsRemoteFlow extends RemoteFlowSource {
+  ServerlessHandlerEventAsRemoteFlow() { this = getAServerlessHandler().getParameter(0) }
 
-    override string getSourceType() { result = "Serverless event" }
-  }
+  override string getSourceType() { result = "Serverless event" }
 }

python/ql/lib/change-notes/2023-08-07-serverless-sources.md

@@ -0,0 +1,4 @@
+---
+category: minorAnalysis
+---
+* Added modeling of AWS Lambda handlers that can be identified with `AWS::Serverless::Function` in YAML files, where the event parameter is modeled as a remote-flow-source.

python/ql/lib/semmle/python/Frameworks.qll

@@ -49,6 +49,7 @@ private import semmle.python.frameworks.Requests
 private import semmle.python.frameworks.RestFramework
 private import semmle.python.frameworks.Rsa
 private import semmle.python.frameworks.RuamelYaml
+private import semmle.python.frameworks.ServerLess
 private import semmle.python.frameworks.Simplejson
 private import semmle.python.frameworks.SqlAlchemy
 private import semmle.python.frameworks.Starlette

python/ql/lib/semmle/python/frameworks/ServerLess.qll

@@ -0,0 +1,67 @@
+/**
+ * Provides classes and predicates for working with those serverless handlers,
+ * handled by the shared library.
+ *
+ * E.g. [AWS](https://docs.aws.amazon.com/lambda/latest/dg/python-handler.html).
+ *
+ * In particular a `RemoteFlowSource` is added for each.
+ */
+
+import python
+import codeql.serverless.ServerLess
+import semmle.python.dataflow.new.DataFlow
+import semmle.python.dataflow.new.RemoteFlowSources
+
+private module YamlImpl implements Input {
+  import semmle.python.Files
+  import semmle.python.Yaml
+}
+
+module SL = ServerLess<YamlImpl>;
+
+/**
+ * Gets a function that is a serverless request handler.
+ *
+ * For example: if an AWS serverless resource contains the following properties (in the "template.yml" file):
+ * ```yaml
+ * Handler: mylibrary.handler
+ * Runtime: pythonXXX
+ * CodeUri: backend/src/
+ * ```
+ *
+ * And a file "mylibrary.py" exists in the folder "backend/src" (relative to the "template.yml" file).
+ * Then the result of this predicate is a function exported as "handler" from "mylibrary.py".
+ * The "mylibrary.py" file could for example look like:
+ *
+ * ```python
+ * def handler(event):
+ *  ...
+ * ```
+ */
+private Function getAServerlessHandler() {
+  exists(File file, string stem, string handler, string runtime, Module mod |
+    SL::hasServerlessHandler(stem, handler, _, runtime) and
+    file.getAbsolutePath() = stem + ".py" and
+    // if runtime is specified, it should be python
+    (runtime = "" or runtime.matches("python%"))
+  |
+    mod.getFile() = file and
+    result.getScope() = mod and
+    result.getName() = handler
+  )
+}
+
+private DataFlow::ParameterNode getAHandlerEventParameter() {
+  exists(Function func | func = getAServerlessHandler() |
+    result.getParameter() in [func.getArg(0), func.getArgByName("event")]
+  )
+}
+
+/**
+ * A serverless request handler event, seen as a RemoteFlowSource.
+ */
+private class ServerlessHandlerEventAsRemoteFlow extends RemoteFlowSource::Range {
+  ServerlessHandlerEventAsRemoteFlow() { this = getAHandlerEventParameter() }
+
+  override string getSourceType() { result = "Serverless event" }
+}

python/ql/test/library-tests/frameworks/serverless/InlineTaintTest.expected

@@ -0,0 +1,4 @@
+failures
+argumentToEnsureNotTaintedNotMarkedAsSpurious
+untaintedArgumentToEnsureTaintedNotMarkedAsMissing
+testFailures

python/ql/test/library-tests/frameworks/serverless/InlineTaintTest.ql

@@ -0,0 +1,2 @@
+import experimental.meta.InlineTaintTest
+import MakeInlineTaintTest<TestTaintTrackingConfig>

python/ql/test/library-tests/frameworks/serverless/aws_lambda/function/extra_lambdas.py

@@ -0,0 +1,13 @@
+def handler1(event, context):
+    ensure_tainted(event) # $ tainted
+    return "Hello World!"
+
+def handler2(event, context):
+    ensure_tainted(event) # $ tainted
+    return "Hello World!"
+
+# This function is not mentioned in template.yml
+# and so it is not receiving user input.
+def non_handler(event, context):
+    ensure_not_tainted(event)
+    return "Hello World!"

python/ql/test/library-tests/frameworks/serverless/aws_lambda/function/lambda_function.py

@@ -0,0 +1,11 @@
+def lambda_handler(event, context):
+    ensure_tainted(
+        event, # $ tainted
+        # event is usually a dict, see https://docs.aws.amazon.com/lambda/latest/dg/python-handler.html
+        event["key"], # $ tainted
+        event["key"]["key2"], # $ tainted
+        event["key"][0], # $ tainted
+        # but can also be a list
+        event[0], # $ tainted
+    )
+    return "OK"

python/ql/test/library-tests/frameworks/serverless/aws_lambda/template.yml

@@ -0,0 +1,62 @@
+# inspired by https://github.com/awsdocs/aws-lambda-developer-guide/blob/main/sample-apps/blank-python/template.yml
+# but we have added extra handlers
+AWSTemplateFormatVersion: '2010-09-09'
+Transform: 'AWS::Serverless-2016-10-31'
+Description: An AWS Lambda application that calls the Lambda API.
+Resources:
+  function:
+    Type: AWS::Serverless::Function
+    Properties:
+      Handler: lambda_function.lambda_handler
+      Runtime: python3.8
+      CodeUri: function/.
+      Description: Call the AWS Lambda API
+      Timeout: 10
+      # Function's execution role
+      Policies:
+        - AWSLambdaBasicExecutionRole
+        - AWSLambda_ReadOnlyAccess
+        - AWSXrayWriteOnlyAccess
+      Tracing: Active
+      Layers:
+        - !Ref libs
+  function:
+    Type: AWS::Serverless::Function
+    Properties:
+      Handler: extra_lambdas.handler1
+      Runtime: python3.8
+      CodeUri: function/.
+      Description: Call the AWS Lambda API
+      Timeout: 10
+      # Function's execution role
+      Policies:
+        - AWSLambdaBasicExecutionRole
+        - AWSLambda_ReadOnlyAccess
+        - AWSXrayWriteOnlyAccess
+      Tracing: Active
+      Layers:
+        - !Ref libs
+  function:
+    Type: AWS::Serverless::Function
+    Properties:
+      Handler: extra_lambdas.handler2
+      Runtime: python3.8
+      CodeUri: function/.
+      Description: Call the AWS Lambda API
+      Timeout: 10
+      # Function's execution role
+      Policies:
+        - AWSLambdaBasicExecutionRole
+        - AWSLambda_ReadOnlyAccess
+        - AWSXrayWriteOnlyAccess
+      Tracing: Active
+      Layers:
+        - !Ref libs
+  libs:
+    Type: AWS::Serverless::LayerVersion
+    Properties:
+      LayerName: blank-python-lib
+      Description: Dependencies for the blank-python sample app.
+      ContentUri: package/.
+      CompatibleRuntimes:
+        - python3.8

python/ql/test/library-tests/frameworks/serverless/options

@@ -0,0 +1 @@
+semmle-extractor-options: -R .