Merge branch 'main' into swift/js-injection

Author: d10c

.github/ISSUE_TEMPLATE/lgtm-com---false-positive.md

@@ -0,0 +1,36 @@
+---
+name: CodeQL False positive
+about: Report CodeQL alerts that you think should not have been detected (not applicable, not exploitable, etc.)
+title: False positive
+labels: false-positive
+assignees: ''
+
+---
+
+**Description of the false positive**
+
+<!-- Please explain briefly why you think it shouldn't be included. -->
+
+**Code samples or links to source code**
+
+<!--
+For open source code: file links with line numbers on GitHub, for example:
+https://github.com/github/codeql/blob/dc440aaee6695deb0d9676b87e06ea984e1b4ae5/javascript/ql/test/query-tests/Security/CWE-078/CommandInjection/exec-sh2.js#L10
+
+For closed source code: (redacted) code samples that illustrate the problem, for example:
+
+```
+function execSh(command, options) {
+    return cp.spawn(getShell(), ["-c", command], options) // <- command line injection
+};
+```
+-->
+
+**URL to the alert on GitHub code scanning (optional)**
+
+<!--
+1. Open the project on GitHub.com.
+2. Switch to the `Security` tab.
+3. Browse to the alert that you would like to report.
+4. Copy and paste the page URL here.
+-->

.github/ISSUE_TEMPLATE/ql--false-positive.md

@@ -26,7 +26,7 @@ jobs:
         if: ${{ github.event_name == 'pull_request' }}
         uses: actions/cache@v3
         with:
-          path: '*/ql/src/.cache'
+          path: '**/.cache'
           key: codeql-compile-pr-${{ github.sha }} # deliberately not using the `compile-compile-main` keys here.
           restore-keys: |
             codeql-compile-${{ github.base_ref }}-${{ env.merge-base }}
@@ -36,7 +36,7 @@ jobs:
         if: ${{ github.event_name != 'pull_request' }}
         uses: actions/cache@v3
         with:
-          path: '*/ql/src/.cache'
+          path: '**/.cache'
           key: codeql-compile-${{ github.ref_name }}-${{ github.sha }} # just fill on main
           restore-keys: | # restore from another random commit, to speed up compilation.
             codeql-compile-${{ github.ref_name }}- 
@@ -46,14 +46,26 @@ jobs:
         with:
           channel: 'release'
       - name: check formatting
-        run: codeql query format */ql/**/*.{qll,ql} --check-only
+        run: find */ql -type f \( -name "*.qll" -o -name "*.ql" \) -print0 | xargs -0 codeql query format --check-only
       - name: compile queries - check-only
         # run with --check-only if running in a PR (github.sha != main)
         if : ${{ github.event_name == 'pull_request' }}
         shell: bash
-        run: codeql query compile -j0 */ql/src --keep-going --warnings=error --check-only
+        run: codeql query compile -j0 */ql/{src,examples} --keep-going --warnings=error --check-only
       - name: compile queries - full
         # do full compile if running on main - this populates the cache
         if : ${{ github.event_name != 'pull_request' }}
         shell: bash
-        run: codeql query compile -j0 */ql/src --keep-going --warnings=error
+        run: |
+          # Move all the existing cache into another folder, so we only preserve the cache for the current queries.
+          mkdir -p ${COMBINED_CACHE_DIR}
+          rm -f */ql/{src,examples}/.cache/{lock,size} # -f to avoid errors if the cache is empty.
+          # copy the contents of the .cache folders into the combined cache folder.
+          cp -r */ql/{src,examples}/.cache/* ${COMBINED_CACHE_DIR}/ || : # ignore missing files
+          # clean up the .cache folders
+          rm -rf */ql/{src,examples}/.cache/*
+
+          # compile the queries
+          codeql query compile -j0 */ql/{src,examples} --keep-going --warnings=error --compilation-cache ${COMBINED_CACHE_DIR}
+        env: 
+          COMBINED_CACHE_DIR: ${{ github.workspace }}/compilation-dir

.github/workflows/compile-queries.yml

@@ -27,7 +27,7 @@ jobs:
         with:
           filters: |
             codegen:
-              - 'github/workflows/swift.yml'
+              - '.github/workflows/swift.yml'
               - "misc/bazel/**"
               - "*.bazel*"
               - 'swift/actions/setup-env/**'
@@ -111,4 +111,10 @@ jobs:
       - uses: actions/upload-artifact@v3
         with:
           name: swift-generated-cpp-files
-          path: swift/generated-cpp-files/**
+          path: generated-cpp-files/**
+  database-upgrade-scripts:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+      - uses: ./.github/actions/fetch-codeql
+      - uses: ./swift/actions/database-upgrade-scripts

.github/workflows/swift.yml

@@ -19,7 +19,7 @@ repos:
     rev: v1.6.0
     hooks:
       - id: autopep8
-        files: ^swift/codegen/.*\.py
+        files: ^swift/.*\.py
 
   - repo: local
     hooks:
@@ -44,7 +44,7 @@ repos:
 
       - id: swift-codegen
         name: Run Swift checked in code generation
-        files: ^swift/(codegen/|.*/generated/|ql/lib/(swift\.dbscheme$|codeql/swift/elements))
+        files: ^swift/(schema.py$|codegen/|.*/generated/|ql/lib/(swift\.dbscheme$|codeql/swift/elements))
         language: system
         entry: bazel run //swift/codegen -- --quiet
         pass_filenames: false

.pre-commit-config.yaml

@@ -45,4 +45,4 @@ WORKSPACE.bazel @github/codeql-ci-reviewers
 /.github/workflows/js-ml-tests.yml @github/codeql-ml-powered-queries-reviewers
 /.github/workflows/ql-for-ql-* @github/codeql-ql-for-ql-reviewers
 /.github/workflows/ruby-* @github/codeql-ruby
-/.github/workflows/swift-* @github/codeql-c
+/.github/workflows/swift.yml @github/codeql-c

CODEOWNERS

@@ -486,40 +486,6 @@
     "python/ql/lib/semmle/python/security/internal/SensitiveDataHeuristics.qll",
     "ruby/ql/lib/codeql/ruby/security/internal/SensitiveDataHeuristics.qll"
   ],
-  "ReDoS Util Python/JS/Ruby/Java": [
-    "javascript/ql/lib/semmle/javascript/security/regexp/NfaUtils.qll",
-    "python/ql/lib/semmle/python/security/regexp/NfaUtils.qll",
-    "ruby/ql/lib/codeql/ruby/security/regexp/NfaUtils.qll",
-    "java/ql/lib/semmle/code/java/security/regexp/NfaUtils.qll"
-  ],
-  "ReDoS Exponential Python/JS/Ruby/Java": [
-    "javascript/ql/lib/semmle/javascript/security/regexp/ExponentialBackTracking.qll",
-    "python/ql/lib/semmle/python/security/regexp/ExponentialBackTracking.qll",
-    "ruby/ql/lib/codeql/ruby/security/regexp/ExponentialBackTracking.qll",
-    "java/ql/lib/semmle/code/java/security/regexp/ExponentialBackTracking.qll"
-  ],
-  "ReDoS Polynomial Python/JS/Ruby/Java": [
-    "javascript/ql/lib/semmle/javascript/security/regexp/SuperlinearBackTracking.qll",
-    "python/ql/lib/semmle/python/security/regexp/SuperlinearBackTracking.qll",
-    "ruby/ql/lib/codeql/ruby/security/regexp/SuperlinearBackTracking.qll",
-    "java/ql/lib/semmle/code/java/security/regexp/SuperlinearBackTracking.qll"
-  ],
-  "RegexpMatching Python/JS/Ruby": [
-    "javascript/ql/lib/semmle/javascript/security/regexp/RegexpMatching.qll",
-    "python/ql/lib/semmle/python/security/regexp/RegexpMatching.qll",
-    "ruby/ql/lib/codeql/ruby/security/regexp/RegexpMatching.qll"
-  ],
-  "BadTagFilterQuery Python/JS/Ruby": [
-    "javascript/ql/lib/semmle/javascript/security/BadTagFilterQuery.qll",
-    "python/ql/lib/semmle/python/security/BadTagFilterQuery.qll",
-    "ruby/ql/lib/codeql/ruby/security/BadTagFilterQuery.qll"
-  ],
-  "OverlyLargeRange Python/JS/Ruby/Java": [
-    "javascript/ql/lib/semmle/javascript/security/OverlyLargeRangeQuery.qll",
-    "python/ql/lib/semmle/python/security/OverlyLargeRangeQuery.qll",
-    "ruby/ql/lib/codeql/ruby/security/OverlyLargeRangeQuery.qll",
-    "java/ql/lib/semmle/code/java/security/OverlyLargeRangeQuery.qll"
-  ],
   "CFG": [
     "csharp/ql/lib/semmle/code/csharp/controlflow/internal/ControlFlowGraphImplShared.qll",
     "ruby/ql/lib/codeql/ruby/controlflow/internal/ControlFlowGraphImplShared.qll",

config/identical-files.json

@@ -11,11 +11,12 @@
   <ItemGroup>
     <PackageReference Include="System.IO.FileSystem" Version="4.3.0" />
     <PackageReference Include="System.IO.FileSystem.Primitives" Version="4.3.0" />
-    <PackageReference Include="xunit" Version="2.4.1" />
-    <PackageReference Include="xunit.runner.visualstudio" Version="2.4.1">
+    <PackageReference Include="xunit" Version="2.4.2" />
+    <PackageReference Include="xunit.runner.visualstudio" Version="2.4.5">
       <PrivateAssets>all</PrivateAssets>
       <IncludeAssets>runtime; build; native; contentfiles; analyzers</IncludeAssets>
     </PackageReference>
+    <PackageReference Include="Microsoft.NET.Test.Sdk" Version="17.4.0" />
   </ItemGroup>
 
   <ItemGroup>

cpp/autobuilder/Semmle.Autobuild.Cpp.Tests/Semmle.Autobuild.Cpp.Tests.csproj

@@ -17,7 +17,7 @@
   </ItemGroup>
 
   <ItemGroup>
-    <PackageReference Include="Microsoft.Build" Version="16.11.0" />
+    <PackageReference Include="Microsoft.Build" Version="17.3.2" />
   </ItemGroup>
 
   <ItemGroup>

cpp/autobuilder/Semmle.Autobuild.Cpp/Semmle.Autobuild.Cpp.csproj

@@ -147,6 +147,12 @@ abstract class Configuration extends string {
    */
   FlowFeature getAFeature() { none() }
 
+  /** Holds if sources should be grouped in the result of `hasFlowPath`. */
+  predicate sourceGrouping(Node source, string sourceGroup) { none() }
+
+  /** Holds if sinks should be grouped in the result of `hasFlowPath`. */
+  predicate sinkGrouping(Node sink, string sinkGroup) { none() }
+
   /**
    * Holds if data may flow from `source` to `sink` for this configuration.
    */
@@ -158,7 +164,7 @@ abstract class Configuration extends string {
    * The corresponding paths are generated from the end-points and the graph
    * included in the module `PathGraph`.
    */
-  predicate hasFlowPath(PathNode source, PathNode sink) { flowsTo(source, sink, _, _, this) }
+  predicate hasFlowPath(PathNode source, PathNode sink) { hasFlowPath(source, sink, this) }
 
   /**
    * Holds if data may flow from some source to `sink` for this configuration.
@@ -2712,6 +2718,18 @@ private newtype TPathNode =
       state = sink.getState() and
       config = sink.getConfiguration()
     )
+  } or
+  TPathNodeSourceGroup(string sourceGroup, Configuration config) {
+    exists(PathNodeImpl source |
+      sourceGroup = source.getSourceGroup() and
+      config = source.getConfiguration()
+    )
+  } or
+  TPathNodeSinkGroup(string sinkGroup, Configuration config) {
+    exists(PathNodeSink sink |
+      sinkGroup = sink.getSinkGroup() and
+      config = sink.getConfiguration()
+    )
   }
 
 /**
@@ -2920,6 +2938,22 @@ abstract private class PathNodeImpl extends TPathNode {
     )
   }
 
+  string getSourceGroup() {
+    this.isSource() and
+    this.getConfiguration().sourceGrouping(this.getNodeEx().asNode(), result)
+  }
+
+  predicate isFlowSource() {
+    this.isSource() and not exists(this.getSourceGroup())
+    or
+    this instanceof PathNodeSourceGroup
+  }
+
+  predicate isFlowSink() {
+    this = any(PathNodeSink sink | not exists(sink.getSinkGroup())) or
+    this instanceof PathNodeSinkGroup
+  }
+
   private string ppAp() {
     this instanceof PathNodeSink and result = ""
     or
@@ -2959,7 +2993,9 @@ abstract private class PathNodeImpl extends TPathNode {
 
 /** Holds if `n` can reach a sink. */
 private predicate directReach(PathNodeImpl n) {
-  n instanceof PathNodeSink or directReach(n.getANonHiddenSuccessor())
+  n instanceof PathNodeSink or
+  n instanceof PathNodeSinkGroup or
+  directReach(n.getANonHiddenSuccessor())
 }
 
 /** Holds if `n` can reach a sink or is used in a subpath that can reach a sink. */
@@ -3015,6 +3051,12 @@ class PathNode instanceof PathNodeImpl {
 
   /** Holds if this node is a source. */
   final predicate isSource() { super.isSource() }
+
+  /** Holds if this node is a grouping of source nodes. */
+  final predicate isSourceGroup(string group) { this = TPathNodeSourceGroup(group, _) }
+
+  /** Holds if this node is a grouping of sink nodes. */
+  final predicate isSinkGroup(string group) { this = TPathNodeSinkGroup(group, _) }
 }
 
 /**
@@ -3136,9 +3178,66 @@ private class PathNodeSink extends PathNodeImpl, TPathNodeSink {
 
   override Configuration getConfiguration() { result = config }
 
-  override PathNodeImpl getASuccessorImpl() { none() }
+  override PathNodeImpl getASuccessorImpl() {
+    result = TPathNodeSinkGroup(this.getSinkGroup(), config)
+  }
 
   override predicate isSource() { sourceNode(node, state, config) }
+
+  string getSinkGroup() { config.sinkGrouping(node.asNode(), result) }
+}
+
+private class PathNodeSourceGroup extends PathNodeImpl, TPathNodeSourceGroup {
+  string sourceGroup;
+  Configuration config;
+
+  PathNodeSourceGroup() { this = TPathNodeSourceGroup(sourceGroup, config) }
+
+  override NodeEx getNodeEx() { none() }
+
+  override FlowState getState() { none() }
+
+  override Configuration getConfiguration() { result = config }
+
+  override PathNodeImpl getASuccessorImpl() {
+    result.getSourceGroup() = sourceGroup and
+    result.getConfiguration() = config
+  }
+
+  override predicate isSource() { none() }
+
+  override string toString() { result = sourceGroup }
+
+  override predicate hasLocationInfo(
+    string filepath, int startline, int startcolumn, int endline, int endcolumn
+  ) {
+    filepath = "" and startline = 0 and startcolumn = 0 and endline = 0 and endcolumn = 0
+  }
+}
+
+private class PathNodeSinkGroup extends PathNodeImpl, TPathNodeSinkGroup {
+  string sinkGroup;
+  Configuration config;
+
+  PathNodeSinkGroup() { this = TPathNodeSinkGroup(sinkGroup, config) }
+
+  override NodeEx getNodeEx() { none() }
+
+  override FlowState getState() { none() }
+
+  override Configuration getConfiguration() { result = config }
+
+  override PathNodeImpl getASuccessorImpl() { none() }
+
+  override predicate isSource() { none() }
+
+  override string toString() { result = sinkGroup }
+
+  override predicate hasLocationInfo(
+    string filepath, int startline, int startcolumn, int endline, int endcolumn
+  ) {
+    filepath = "" and startline = 0 and startcolumn = 0 and endline = 0 and endcolumn = 0
+  }
 }
 
 private predicate pathNode(
@@ -3495,6 +3594,15 @@ private module Subpaths {
  * Will only have results if `configuration` has non-empty sources and
  * sinks.
  */
+private predicate hasFlowPath(
+  PathNodeImpl flowsource, PathNodeImpl flowsink, Configuration configuration
+) {
+  flowsource.isFlowSource() and
+  flowsource.getConfiguration() = configuration and
+  (flowsource = flowsink or pathSuccPlus(flowsource, flowsink)) and
+  flowsink.isFlowSink()
+}
+
 private predicate flowsTo(
   PathNodeImpl flowsource, PathNodeSink flowsink, Node source, Node sink,
   Configuration configuration