Merge branch 'main' into rbPoly

Author: erik-krogh

.github/ISSUE_TEMPLATE/ql---general.md

@@ -10,5 +10,5 @@ assignees: ''
 **Description of the issue**
 
 <!-- Please explain briefly what is the problem.
-If it is about an LGTM project, please include its URL.-->
+If it is about a GitHub project, please include its URL. -->
 

.github/ISSUE_TEMPLATE/ql--false-positive.md

@@ -1,5 +1,5 @@
 ---
-name: CodeQL False positive
+name: CodeQL false positive
 about: Report CodeQL alerts that you think should not have been detected (not applicable, not exploitable, etc.)
 title: False positive
 labels: false-positive

.github/actions/cache-query-compilation/action.yml

@@ -14,8 +14,7 @@ outputs:
 runs:
   using: composite
   steps:
-    # Cache the query compilation caches.
-    # calculate the merge-base with main, in a way that works both on PRs and pushes to main. 
+    # calculate the merge-base with main, in a way that works both on PRs and pushes to main.
     - name: Calculate merge-base
       shell: bash
       if: ${{ github.event_name == 'pull_request' }}
@@ -24,38 +23,32 @@ runs:
       run: |
         MERGE_BASE=$(git cat-file commit $GITHUB_SHA | grep '^parent ' | head -1 | cut -f 2 -d " ")
         echo "merge_base=$MERGE_BASE" >> $GITHUB_ENV
-    - name: Read CodeQL query compilation - PR
+    - name: Restore cache (PR)
       if: ${{ github.event_name == 'pull_request' }}
-      uses: erik-krogh/actions-cache@a88d0603fe5fb5606db9f002dfcadeb32b5f84c6
+      uses: actions/cache/restore@v3
       with:
         path: '**/.cache'
-        read-only: true
-        key: codeql-compile-${{ inputs.key }}-pr-${{ github.sha }} # deliberately not using the `compile-compile-main` keys here.
+        key: codeql-compile-${{ inputs.key }}-pr-${{ github.sha }}
         restore-keys: |
           codeql-compile-${{ inputs.key }}-${{ github.base_ref }}-${{ env.merge_base }}
           codeql-compile-${{ inputs.key }}-${{ github.base_ref }}-
           codeql-compile-${{ inputs.key }}-main-
-    - name: Fill CodeQL query compilation cache - main
+    - name: Fill cache (only branch push)
       if: ${{ github.event_name != 'pull_request' }}
-      uses: erik-krogh/actions-cache@a88d0603fe5fb5606db9f002dfcadeb32b5f84c6
+      uses: actions/cache@v3
       with:
         path: '**/.cache'
         key: codeql-compile-${{ inputs.key }}-${{ github.ref_name }}-${{ github.sha }} # just fill on main
-        restore-keys: | # restore from another random commit, to speed up compilation.
-          codeql-compile-${{ inputs.key }}-${{ github.ref_name }}- 
+        restore-keys: | # restore the latest cache if the exact cache is unavailable, to speed up compilation.
+          codeql-compile-${{ inputs.key }}-${{ github.ref_name }}-
           codeql-compile-${{ inputs.key }}-main-
     - name: Fill compilation cache directory
       id: fill-compilation-dir
-      shell: bash 
+      shell: bash
       run: |
         # Move all the existing cache into another folder, so we only preserve the cache for the current queries.
-        mkdir -p ${COMBINED_CACHE_DIR}
-        rm -f **/.cache/{lock,size} # -f to avoid errors if the cache is empty.
-        # copy the contents of the .cache folders into the combined cache folder.
-        cp -r **/.cache/* ${COMBINED_CACHE_DIR}/ || : # ignore missing files
-        # clean up the .cache folders
-        rm -rf **/.cache/*
+        node $GITHUB_WORKSPACE/.github/actions/cache-query-compilation/move-caches.js ${COMBINED_CACHE_DIR}
 
         echo "compdir=${COMBINED_CACHE_DIR}" >> $GITHUB_OUTPUT
-      env: 
-        COMBINED_CACHE_DIR: ${{ github.workspace }}/compilation-dir
+      env:
+        COMBINED_CACHE_DIR: ${{ runner.temp }}/compilation-dir

.github/actions/cache-query-compilation/move-caches.js

@@ -0,0 +1,75 @@
+// # Move all the existing cache into another folder, so we only preserve the cache for the current queries.
+// mkdir -p ${COMBINED_CACHE_DIR}
+// rm -f **/.cache/{lock,size} # -f to avoid errors if the cache is empty.
+// # copy the contents of the .cache folders into the combined cache folder.
+// cp -r **/.cache/* ${COMBINED_CACHE_DIR}/ || : # ignore missing files
+// # clean up the .cache folders
+// rm -rf **/.cache/*
+
+const fs = require("fs");
+const path = require("path");
+
+// the first argv is the cache folder to create.
+const COMBINED_CACHE_DIR = process.argv[2];
+
+function* walkCaches(dir) {
+  const files = fs.readdirSync(dir, { withFileTypes: true });
+  for (const file of files) {
+    if (file.isDirectory()) {
+      const filePath = path.join(dir, file.name);
+      yield* walkCaches(filePath);
+      if (file.name === ".cache") {
+        yield filePath;
+      }
+    }
+  }
+}
+
+async function copyDir(src, dest) {
+  for await (const file of await fs.promises.readdir(src, { withFileTypes: true })) {
+    const srcPath = path.join(src, file.name);
+    const destPath = path.join(dest, file.name);
+    if (file.isDirectory()) {
+      if (!fs.existsSync(destPath)) {
+        fs.mkdirSync(destPath);
+      }
+      await copyDir(srcPath, destPath);
+    } else {
+      await fs.promises.copyFile(srcPath, destPath);
+    }
+  }
+}
+
+async function main() {
+  const cacheDirs = [...walkCaches(".")];
+
+  for (const dir of cacheDirs) {
+    console.log(`Found .cache dir at ${dir}`);
+  }
+
+  // mkdir -p ${COMBINED_CACHE_DIR}
+  fs.mkdirSync(COMBINED_CACHE_DIR, { recursive: true });
+
+  // rm -f **/.cache/{lock,size} # -f to avoid errors if the cache is empty.
+  await Promise.all(
+    cacheDirs.map((cacheDir) =>
+      (async function () {
+        await fs.promises.rm(path.join(cacheDir, "lock"), { force: true });
+        await fs.promises.rm(path.join(cacheDir, "size"), { force: true });
+      })()
+    )
+  );
+
+  // # copy the contents of the .cache folders into the combined cache folder.
+  // cp -r **/.cache/* ${COMBINED_CACHE_DIR}/ || : # ignore missing files
+  await Promise.all(
+    cacheDirs.map((cacheDir) => copyDir(cacheDir, COMBINED_CACHE_DIR))
+  );
+
+  // # clean up the .cache folders
+  // rm -rf **/.cache/*
+  await Promise.all(
+    cacheDirs.map((cacheDir) => fs.promises.rm(cacheDir, { recursive: true }))
+  );
+}
+main();

.github/actions/fetch-codeql/action.yml

@@ -19,4 +19,6 @@ runs:
         gh extension install github/gh-codeql
         gh codeql set-channel "$CHANNEL"
         gh codeql version
+        printf "CODEQL_FETCHED_CODEQL_PATH=" >> "${GITHUB_ENV}"
+        gh codeql version --format=json | jq -r .unpackedLocation >> "${GITHUB_ENV}"
         gh codeql version --format=json | jq -r .unpackedLocation >> "${GITHUB_PATH}"

.github/actions/find-latest-bundle/action.yml

@@ -0,0 +1,26 @@
+name: Find Latest CodeQL Bundle
+description: Finds the URL of the latest released version of the CodeQL bundle.
+outputs:
+  url:
+    description: The download URL of the latest CodeQL bundle release
+    value: ${{ steps.find-latest.outputs.url }}
+runs:
+  using: composite
+  steps:
+    - name: Find Latest Release
+      id: find-latest
+      shell: pwsh
+      run: |
+        $Latest = gh release list --repo github/codeql-action --exclude-drafts --limit 1000 |
+          ForEach-Object { $C = $_ -split "`t"; return @{ type = $C[1]; tag = $C[2]; } } |
+          Where-Object { $_.type -eq 'Latest' }
+
+        $Tag = $Latest.tag
+        if ($Tag -eq '') {
+          throw 'Failed to find latest bundle release.'
+        }
+
+        Write-Output "Latest bundle tag is '${Tag}'."
+        "url=https://github.com/github/codeql-action/releases/download/${Tag}/codeql-bundle-linux64.tar.gz" >> $env:GITHUB_OUTPUT
+      env:
+        GITHUB_TOKEN: ${{ github.token }}

.github/actions/os-version/action.yml

@@ -0,0 +1,32 @@
+name: OS Version
+description: Get OS version.
+
+outputs:
+  version:
+    description: "OS version"
+    value: ${{ steps.version.outputs.version }}
+
+runs:
+  using: composite
+  steps:
+    - if: runner.os == 'Linux'
+      shell: bash
+      run: |
+        . /etc/os-release
+        echo "VERSION=${NAME} ${VERSION}" >> $GITHUB_ENV
+    - if: runner.os == 'Windows'
+      shell: powershell
+      run: |
+        $objects = systeminfo.exe /FO CSV | ConvertFrom-Csv
+        "VERSION=$($objects.'OS Name') $($objects.'OS Version')" >> $env:GITHUB_ENV
+    - if: runner.os == 'macOS'
+      shell: bash
+      run: |
+        echo "VERSION=$(sw_vers -productName) $(sw_vers -productVersion)" >> $GITHUB_ENV
+    - name: Emit OS version
+      id: version
+      shell: bash
+      run: |
+        echo "$VERSION"
+        echo "version=${VERSION}" >> $GITHUB_OUTPUT
+

.github/dependabot.yml

@@ -1,19 +1,7 @@
 version: 2
 updates:
   - package-ecosystem: "cargo"
-    directory: "ruby/node-types"
-    schedule:
-      interval: "daily"
-  - package-ecosystem: "cargo"
-    directory: "ruby/generator"
-    schedule:
-      interval: "daily"
-  - package-ecosystem: "cargo"
-    directory: "ruby/extractor"
-    schedule:
-      interval: "daily"
-  - package-ecosystem: "cargo"
-    directory: "ruby/autobuilder"
+    directory: "ruby"
     schedule:
       interval: "daily"
 

.github/workflows/atm-check-query-suite.yml

@@ -13,7 +13,7 @@ on:
 
 jobs:
   atm-check-query-suite:
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-latest-xl
 
     steps:
       - uses: actions/checkout@v3
@@ -23,6 +23,12 @@ jobs:
         with:
           channel: release
 
+      - name: Cache compilation cache
+        id: query-cache
+        uses: ./.github/actions/cache-query-compilation
+        with: 
+          key: atm-suite
+
       - name: Install ATM model
         run: |
           set -exu
@@ -50,10 +56,13 @@ jobs:
           echo "SARIF_PATH=${SARIF_PATH}" >> "${GITHUB_ENV}"
 
           codeql database analyze \
+            --threads=0 \
+            --ram 50000 \
             --format sarif-latest \
             --output "${SARIF_PATH}" \
             --sarif-group-rules-by-pack \
             -vv \
+            --compilation-cache "${{ steps.query-cache.outputs.cache-dir }}" \
             -- \
             "${DB_PATH}" \
             "${QUERY_PACK}/${QUERY_SUITE}"

.github/workflows/check-change-note.yml

@@ -26,3 +26,9 @@ jobs:
         run: |
           gh api 'repos/${{github.repository}}/pulls/${{github.event.number}}/files' --paginate --jq 'any(.[].filename ; test("/change-notes/.*[.]md$"))' |
           grep true -c
+      - name: Fail if the change note filename doesn't match the expected format. The file name must be of the form 'YYYY-MM-DD.md' or 'YYYY-MM-DD-{title}.md', where '{title}' is arbitrary text.
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        run: |
+          gh api 'repos/${{github.repository}}/pulls/${{github.event.number}}/files' --paginate --jq '[.[].filename | select(test("/change-notes/.*[.]md$"))] | all(test("/change-notes/[0-9]{4}-[0-9]{2}-[0-9]{2}.*[.]md$"))' |
+          grep true -c