Merge branch 'main' into exit-code

Author: erik-krogh

.github/actions/cache-query-compilation/action.yml

@@ -14,8 +14,7 @@ outputs:
 runs:
   using: composite
   steps:
-    # Cache the query compilation caches.
-    # calculate the merge-base with main, in a way that works both on PRs and pushes to main. 
+    # calculate the merge-base with main, in a way that works both on PRs and pushes to main.
     - name: Calculate merge-base
       shell: bash
       if: ${{ github.event_name == 'pull_request' }}
@@ -24,38 +23,33 @@ runs:
       run: |
         MERGE_BASE=$(git cat-file commit $GITHUB_SHA | grep '^parent ' | head -1 | cut -f 2 -d " ")
         echo "merge_base=$MERGE_BASE" >> $GITHUB_ENV
-    - name: Read CodeQL query compilation - PR
+    - name: Restore read-only cache (PR)
       if: ${{ github.event_name == 'pull_request' }}
       uses: erik-krogh/actions-cache@a88d0603fe5fb5606db9f002dfcadeb32b5f84c6
       with:
         path: '**/.cache'
         read-only: true
-        key: codeql-compile-${{ inputs.key }}-pr-${{ github.sha }} # deliberately not using the `compile-compile-main` keys here.
+        key: codeql-compile-${{ inputs.key }}-pr-${{ github.sha }}
         restore-keys: |
           codeql-compile-${{ inputs.key }}-${{ github.base_ref }}-${{ env.merge_base }}
           codeql-compile-${{ inputs.key }}-${{ github.base_ref }}-
           codeql-compile-${{ inputs.key }}-main-
-    - name: Fill CodeQL query compilation cache - main
+    - name: Fill cache (push)
       if: ${{ github.event_name != 'pull_request' }}
       uses: erik-krogh/actions-cache@a88d0603fe5fb5606db9f002dfcadeb32b5f84c6
       with:
         path: '**/.cache'
         key: codeql-compile-${{ inputs.key }}-${{ github.ref_name }}-${{ github.sha }} # just fill on main
-        restore-keys: | # restore from another random commit, to speed up compilation.
-          codeql-compile-${{ inputs.key }}-${{ github.ref_name }}- 
+        restore-keys: | # restore the latest cache if the exact cache is unavailable, to speed up compilation.
+          codeql-compile-${{ inputs.key }}-${{ github.ref_name }}-
           codeql-compile-${{ inputs.key }}-main-
     - name: Fill compilation cache directory
       id: fill-compilation-dir
-      shell: bash 
+      shell: bash
       run: |
         # Move all the existing cache into another folder, so we only preserve the cache for the current queries.
-        mkdir -p ${COMBINED_CACHE_DIR}
-        rm -f **/.cache/{lock,size} # -f to avoid errors if the cache is empty.
-        # copy the contents of the .cache folders into the combined cache folder.
-        cp -r **/.cache/* ${COMBINED_CACHE_DIR}/ || : # ignore missing files
-        # clean up the .cache folders
-        rm -rf **/.cache/*
+        node $GITHUB_WORKSPACE/.github/actions/cache-query-compilation/move-caches.js ${COMBINED_CACHE_DIR}
 
         echo "compdir=${COMBINED_CACHE_DIR}" >> $GITHUB_OUTPUT
-      env: 
-        COMBINED_CACHE_DIR: ${{ github.workspace }}/compilation-dir
+      env:
+        COMBINED_CACHE_DIR: ${{ runner.temp }}/compilation-dir

.github/actions/cache-query-compilation/move-caches.js

@@ -0,0 +1,75 @@
+// # Move all the existing cache into another folder, so we only preserve the cache for the current queries.
+// mkdir -p ${COMBINED_CACHE_DIR}
+// rm -f **/.cache/{lock,size} # -f to avoid errors if the cache is empty.
+// # copy the contents of the .cache folders into the combined cache folder.
+// cp -r **/.cache/* ${COMBINED_CACHE_DIR}/ || : # ignore missing files
+// # clean up the .cache folders
+// rm -rf **/.cache/*
+
+const fs = require("fs");
+const path = require("path");
+
+// the first argv is the cache folder to create.
+const COMBINED_CACHE_DIR = process.argv[2];
+
+function* walkCaches(dir) {
+  const files = fs.readdirSync(dir, { withFileTypes: true });
+  for (const file of files) {
+    if (file.isDirectory()) {
+      const filePath = path.join(dir, file.name);
+      yield* walkCaches(filePath);
+      if (file.name === ".cache") {
+        yield filePath;
+      }
+    }
+  }
+}
+
+async function copyDir(src, dest) {
+  for await (const file of await fs.promises.readdir(src, { withFileTypes: true })) {
+    const srcPath = path.join(src, file.name);
+    const destPath = path.join(dest, file.name);
+    if (file.isDirectory()) {
+      if (!fs.existsSync(destPath)) {
+        fs.mkdirSync(destPath);
+      }
+      await copyDir(srcPath, destPath);
+    } else {
+      await fs.promises.copyFile(srcPath, destPath);
+    }
+  }
+}
+
+async function main() {
+  const cacheDirs = [...walkCaches(".")];
+
+  for (const dir of cacheDirs) {
+    console.log(`Found .cache dir at ${dir}`);
+  }
+
+  // mkdir -p ${COMBINED_CACHE_DIR}
+  fs.mkdirSync(COMBINED_CACHE_DIR, { recursive: true });
+
+  // rm -f **/.cache/{lock,size} # -f to avoid errors if the cache is empty.
+  await Promise.all(
+    cacheDirs.map((cacheDir) =>
+      (async function () {
+        await fs.promises.rm(path.join(cacheDir, "lock"), { force: true });
+        await fs.promises.rm(path.join(cacheDir, "size"), { force: true });
+      })()
+    )
+  );
+
+  // # copy the contents of the .cache folders into the combined cache folder.
+  // cp -r **/.cache/* ${COMBINED_CACHE_DIR}/ || : # ignore missing files
+  await Promise.all(
+    cacheDirs.map((cacheDir) => copyDir(cacheDir, COMBINED_CACHE_DIR))
+  );
+
+  // # clean up the .cache folders
+  // rm -rf **/.cache/*
+  await Promise.all(
+    cacheDirs.map((cacheDir) => fs.promises.rm(cacheDir, { recursive: true }))
+  );
+}
+main();

.github/workflows/compile-queries.yml

@@ -35,5 +35,3 @@ jobs:
         if : ${{ github.event_name != 'pull_request' }}
         shell: bash
         run: codeql query compile -j0 */ql/{src,examples} --keep-going --warnings=error --compilation-cache "${{ steps.query-cache.outputs.cache-dir }}"
-        env: 
-          COMBINED_CACHE_DIR: ${{ github.workspace }}/compilation-dir

.github/workflows/go-tests-other-os.yml

@@ -5,8 +5,7 @@ on:
       - "go/**"
       - "!go/ql/**" # don't run other-os if only ql/ files changed
       - .github/workflows/go-tests-other-os.yml
-      - .github/actions/fetch-codeql/action.yml
-      - .github/actions/cache-query-compilation/action.yml
+      - .github/actions/**
       - codeql-workspace.yml
 jobs:
   test-mac:

.github/workflows/go-tests.yml

@@ -4,8 +4,7 @@ on:
     paths:
       - "go/**"
       - .github/workflows/go-tests.yml
-      - .github/actions/fetch-codeql/action.yml
-      - .github/actions/cache-query-compilation/action.yml
+      - .github/actions/**
       - codeql-workspace.yml
     branches:
       - main
@@ -14,8 +13,7 @@ on:
     paths:
       - "go/**"
       - .github/workflows/go-tests.yml
-      - .github/actions/fetch-codeql/action.yml
-      - .github/actions/cache-query-compilation/action.yml
+      - .github/actions/**
       - codeql-workspace.yml
 jobs:
   test-linux:
@@ -64,7 +62,7 @@ jobs:
         uses: ./.github/actions/cache-query-compilation
         with:
           key: go-qltest
-    
+
       - name: Test
         run: |
           cd go

.github/workflows/swift.yml

@@ -7,8 +7,7 @@ on:
       - "misc/bazel/**"
       - "*.bazel*"
       - .github/workflows/swift.yml
-      - .github/actions/fetch-codeql/action.yml
-      - .github/actions/cache-query-compilation/action.yml
+      - .github/actions/**
       - codeql-workspace.yml
       - .pre-commit-config.yaml
       - "!**/*.md"
@@ -22,8 +21,7 @@ on:
       - "misc/bazel/**"
       - "*.bazel*"
       - .github/workflows/swift.yml
-      - .github/actions/fetch-codeql/action.yml
-      - .github/actions/cache-query-compilation/action.yml
+      - .github/actions/**
       - codeql-workspace.yml
       - "!**/*.md"
       - "!**/*.qhelp"
@@ -35,20 +33,15 @@ jobs:
   # not using a matrix as you cannot depend on a specific job in a matrix, and we want to start linux checks
   # without waiting for the macOS build
   build-and-test-macos:
-    if: ${{ github.event_name == 'pull_request' }}
     runs-on: macos-12-xl
     steps:
       - uses: actions/checkout@v3
-      - uses: ./swift/actions/create-extractor-pack
-      - uses: ./swift/actions/run-quick-tests
-      - uses: ./swift/actions/print-unextracted
+      - uses: ./swift/actions/build-and-test
   build-and-test-linux:
     runs-on: ubuntu-latest-xl
     steps:
       - uses: actions/checkout@v3
-      - uses: ./swift/actions/create-extractor-pack
-      - uses: ./swift/actions/run-quick-tests
-      - uses: ./swift/actions/print-unextracted
+      - uses: ./swift/actions/build-and-test
   qltests-linux:
     needs: build-and-test-linux
     runs-on: ubuntu-latest-xl
@@ -80,7 +73,10 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v3
-      - uses: ./swift/actions/setup-env
+      - uses: bazelbuild/setup-bazelisk@v2
+      - uses: actions/setup-python@v4
+        with:
+          python-version-file: 'swift/.python-version'
       - uses: pre-commit/[email protected]
         name: Check that python code is properly formatted
         with:

CODEOWNERS

@@ -12,13 +12,6 @@
 # ML-powered queries
 /javascript/ql/experimental/adaptivethreatmodeling/ @github/codeql-ml-powered-queries-reviewers
 
-# Notify members of codeql-go about PRs to the shared data-flow library files
-/java/ql/src/semmle/code/java/dataflow/internal/DataFlowImpl.qll @github/codeql-java @github/codeql-go
-/java/ql/src/semmle/code/java/dataflow/internal/DataFlowImpl2.qll @github/codeql-java @github/codeql-go
-/java/ql/src/semmle/code/java/dataflow/internal/DataFlowImplCommon.qll @github/codeql-java @github/codeql-go
-/java/ql/src/semmle/code/java/dataflow/internal/tainttracking1/TaintTrackingImpl.qll @github/codeql-java @github/codeql-go
-/java/ql/src/semmle/code/java/dataflow/internal/tainttracking2/TaintTrackingImpl.qll @github/codeql-java @github/codeql-go
-
 # CodeQL tools and associated docs
 /docs/codeql/codeql-cli/ @github/codeql-cli-reviewers
 /docs/codeql/codeql-for-visual-studio-code/ @github/codeql-vscode-reviewers

config/identical-files.json

@@ -1,5 +1,5 @@
 {
-  "DataFlow Java/C++/C#/Python": [
+  "DataFlow Java/C++/C#/Go/Python/Ruby/Swift": [
     "java/ql/lib/semmle/code/java/dataflow/internal/DataFlowImpl.qll",
     "java/ql/lib/semmle/code/java/dataflow/internal/DataFlowImpl2.qll",
     "java/ql/lib/semmle/code/java/dataflow/internal/DataFlowImpl3.qll",
@@ -27,6 +27,8 @@
     "csharp/ql/lib/semmle/code/csharp/dataflow/internal/DataFlowImpl4.qll",
     "csharp/ql/lib/semmle/code/csharp/dataflow/internal/DataFlowImpl5.qll",
     "csharp/ql/lib/semmle/code/csharp/dataflow/internal/DataFlowImplForContentDataFlow.qll",
+    "go/ql/lib/semmle/go/dataflow/internal/DataFlowImpl.qll",
+    "go/ql/lib/semmle/go/dataflow/internal/DataFlowImpl2.qll",
     "python/ql/lib/semmle/python/dataflow/new/internal/DataFlowImpl.qll",
     "python/ql/lib/semmle/python/dataflow/new/internal/DataFlowImpl2.qll",
     "python/ql/lib/semmle/python/dataflow/new/internal/DataFlowImpl3.qll",
@@ -38,17 +40,18 @@
     "ruby/ql/lib/codeql/ruby/dataflow/internal/DataFlowImplForPathname.qll",
     "swift/ql/lib/codeql/swift/dataflow/internal/DataFlowImpl.qll"
   ],
-  "DataFlow Java/C++/C#/Python Common": [
+  "DataFlow Java/C++/C#/Go/Python/Ruby/Swift Common": [
     "java/ql/lib/semmle/code/java/dataflow/internal/DataFlowImplCommon.qll",
     "cpp/ql/lib/semmle/code/cpp/dataflow/internal/DataFlowImplCommon.qll",
     "cpp/ql/lib/semmle/code/cpp/ir/dataflow/internal/DataFlowImplCommon.qll",
     "cpp/ql/lib/experimental/semmle/code/cpp/ir/dataflow/internal/DataFlowImplCommon.qll",
     "csharp/ql/lib/semmle/code/csharp/dataflow/internal/DataFlowImplCommon.qll",
+    "go/ql/lib/semmle/go/dataflow/internal/DataFlowImplCommon.qll",
     "python/ql/lib/semmle/python/dataflow/new/internal/DataFlowImplCommon.qll",
     "ruby/ql/lib/codeql/ruby/dataflow/internal/DataFlowImplCommon.qll",
     "swift/ql/lib/codeql/swift/dataflow/internal/DataFlowImplCommon.qll"
   ],
-  "TaintTracking::Configuration Java/C++/C#/Python": [
+  "TaintTracking::Configuration Java/C++/C#/Go/Python/Ruby/Swift": [
     "cpp/ql/lib/semmle/code/cpp/dataflow/internal/tainttracking1/TaintTrackingImpl.qll",
     "cpp/ql/lib/semmle/code/cpp/dataflow/internal/tainttracking2/TaintTrackingImpl.qll",
     "cpp/ql/lib/semmle/code/cpp/ir/dataflow/internal/tainttracking1/TaintTrackingImpl.qll",
@@ -62,6 +65,8 @@
     "csharp/ql/lib/semmle/code/csharp/dataflow/internal/tainttracking3/TaintTrackingImpl.qll",
     "csharp/ql/lib/semmle/code/csharp/dataflow/internal/tainttracking4/TaintTrackingImpl.qll",
     "csharp/ql/lib/semmle/code/csharp/dataflow/internal/tainttracking5/TaintTrackingImpl.qll",
+    "go/ql/lib/semmle/go/dataflow/internal/tainttracking1/TaintTrackingImpl.qll",
+    "go/ql/lib/semmle/go/dataflow/internal/tainttracking2/TaintTrackingImpl.qll",
     "java/ql/lib/semmle/code/java/dataflow/internal/tainttracking1/TaintTrackingImpl.qll",
     "java/ql/lib/semmle/code/java/dataflow/internal/tainttracking2/TaintTrackingImpl.qll",
     "java/ql/lib/semmle/code/java/dataflow/internal/tainttracking3/TaintTrackingImpl.qll",
@@ -72,7 +77,7 @@
     "ruby/ql/lib/codeql/ruby/dataflow/internal/tainttracking1/TaintTrackingImpl.qll",
     "swift/ql/lib/codeql/swift/dataflow/internal/tainttracking1/TaintTrackingImpl.qll"
   ],
-  "DataFlow Java/C++/C#/Python Consistency checks": [
+  "DataFlow Java/C++/C#/Python/Ruby/Swift Consistency checks": [
     "java/ql/lib/semmle/code/java/dataflow/internal/DataFlowImplConsistency.qll",
     "cpp/ql/lib/semmle/code/cpp/dataflow/internal/DataFlowImplConsistency.qll",
     "cpp/ql/lib/semmle/code/cpp/ir/dataflow/internal/DataFlowImplConsistency.qll",
@@ -82,9 +87,10 @@
     "ruby/ql/lib/codeql/ruby/dataflow/internal/DataFlowImplConsistency.qll",
     "swift/ql/lib/codeql/swift/dataflow/internal/DataFlowImplConsistency.qll"
   ],
-  "DataFlow Java/C#/Ruby/Python/Swift Flow Summaries": [
+  "DataFlow Java/C#/Go/Ruby/Python/Swift Flow Summaries": [
     "java/ql/lib/semmle/code/java/dataflow/internal/FlowSummaryImpl.qll",
     "csharp/ql/lib/semmle/code/csharp/dataflow/internal/FlowSummaryImpl.qll",
+    "go/ql/lib/semmle/go/dataflow/internal/FlowSummaryImpl.qll",
     "ruby/ql/lib/codeql/ruby/dataflow/internal/FlowSummaryImpl.qll",
     "python/ql/lib/semmle/python/dataflow/new/internal/FlowSummaryImpl.qll",
     "swift/ql/lib/codeql/swift/dataflow/internal/FlowSummaryImpl.qll"
@@ -505,6 +511,7 @@
   ],
   "AccessPathSyntax": [
     "csharp/ql/lib/semmle/code/csharp/dataflow/internal/AccessPathSyntax.qll",
+    "go/ql/lib/semmle/go/dataflow/internal/AccessPathSyntax.qll",
     "java/ql/lib/semmle/code/java/dataflow/internal/AccessPathSyntax.qll",
     "javascript/ql/lib/semmle/javascript/frameworks/data/internal/AccessPathSyntax.qll",
     "ruby/ql/lib/codeql/ruby/dataflow/internal/AccessPathSyntax.qll",

cpp/ql/lib/experimental/semmle/code/cpp/ir/dataflow/internal/DataFlowImpl.qll

@@ -70,7 +70,7 @@ abstract class Configuration extends string {
   /**
    * Holds if `sink` is a relevant data flow sink accepting `state`.
    */
-  predicate isSink(Node source, FlowState state) { none() }
+  predicate isSink(Node sink, FlowState state) { none() }
 
   /**
    * Holds if data flow through `node` is prohibited. This completely removes

cpp/ql/lib/experimental/semmle/code/cpp/ir/dataflow/internal/DataFlowImpl2.qll

@@ -70,7 +70,7 @@ abstract class Configuration extends string {
   /**
    * Holds if `sink` is a relevant data flow sink accepting `state`.
    */
-  predicate isSink(Node source, FlowState state) { none() }
+  predicate isSink(Node sink, FlowState state) { none() }
 
   /**
    * Holds if data flow through `node` is prohibited. This completely removes