Merge branch 'main' into codeql-ci/atm/release-0.4.7

Author: henrymercer

.github/ISSUE_TEMPLATE/lgtm-com---false-positive.md

@@ -10,5 +10,5 @@ assignees: ''
 **Description of the issue**
 
 <!-- Please explain briefly what is the problem.
-If it is about an LGTM project, please include its URL.-->
+If it is about a GitHub project, please include its URL. -->
 

.github/ISSUE_TEMPLATE/ql---general.md

@@ -0,0 +1,36 @@
+---
+name: CodeQL false positive
+about: Report CodeQL alerts that you think should not have been detected (not applicable, not exploitable, etc.)
+title: False positive
+labels: false-positive
+assignees: ''
+
+---
+
+**Description of the false positive**
+
+<!-- Please explain briefly why you think it shouldn't be included. -->
+
+**Code samples or links to source code**
+
+<!--
+For open source code: file links with line numbers on GitHub, for example:
+https://github.com/github/codeql/blob/dc440aaee6695deb0d9676b87e06ea984e1b4ae5/javascript/ql/test/query-tests/Security/CWE-078/CommandInjection/exec-sh2.js#L10
+
+For closed source code: (redacted) code samples that illustrate the problem, for example:
+
+```
+function execSh(command, options) {
+    return cp.spawn(getShell(), ["-c", command], options) // <- command line injection
+};
+```
+-->
+
+**URL to the alert on GitHub code scanning (optional)**
+
+<!--
+1. Open the project on GitHub.com.
+2. Switch to the `Security` tab.
+3. Browse to the alert that you would like to report.
+4. Copy and paste the page URL here.
+-->

.github/ISSUE_TEMPLATE/ql--false-positive.md

@@ -0,0 +1,54 @@
+name: Cache query compilation
+description: Caches CodeQL compilation caches - should be run both on PRs and pushes to main.
+
+inputs:
+  key:
+    description: 'The cache key to use - should be unique to the workflow'
+    required: true
+
+outputs:
+  cache-dir:
+    description: "The directory where the cache was stored"
+    value: ${{ steps.fill-compilation-dir.outputs.compdir }}
+
+runs:
+  using: composite
+  steps:
+    # calculate the merge-base with main, in a way that works both on PRs and pushes to main.
+    - name: Calculate merge-base
+      shell: bash
+      if: ${{ github.event_name == 'pull_request' }}
+      env:
+        BASE_BRANCH: ${{ github.base_ref }}
+      run: |
+        MERGE_BASE=$(git cat-file commit $GITHUB_SHA | grep '^parent ' | head -1 | cut -f 2 -d " ")
+        echo "merge_base=$MERGE_BASE" >> $GITHUB_ENV
+    - name: Restore cache (PR)
+      if: ${{ github.event_name == 'pull_request' }}
+      uses: actions/cache/restore@v3
+      with:
+        path: '**/.cache'
+        key: codeql-compile-${{ inputs.key }}-pr-${{ github.sha }}
+        restore-keys: |
+          codeql-compile-${{ inputs.key }}-${{ github.base_ref }}-${{ env.merge_base }}
+          codeql-compile-${{ inputs.key }}-${{ github.base_ref }}-
+          codeql-compile-${{ inputs.key }}-main-
+    - name: Fill cache (only branch push)
+      if: ${{ github.event_name != 'pull_request' }}
+      uses: actions/cache@v3
+      with:
+        path: '**/.cache'
+        key: codeql-compile-${{ inputs.key }}-${{ github.ref_name }}-${{ github.sha }} # just fill on main
+        restore-keys: | # restore the latest cache if the exact cache is unavailable, to speed up compilation.
+          codeql-compile-${{ inputs.key }}-${{ github.ref_name }}-
+          codeql-compile-${{ inputs.key }}-main-
+    - name: Fill compilation cache directory
+      id: fill-compilation-dir
+      shell: bash
+      run: |
+        # Move all the existing cache into another folder, so we only preserve the cache for the current queries.
+        node $GITHUB_WORKSPACE/.github/actions/cache-query-compilation/move-caches.js ${COMBINED_CACHE_DIR}
+
+        echo "compdir=${COMBINED_CACHE_DIR}" >> $GITHUB_OUTPUT
+      env:
+        COMBINED_CACHE_DIR: ${{ runner.temp }}/compilation-dir

.github/actions/cache-query-compilation/action.yml

@@ -0,0 +1,75 @@
+// # Move all the existing cache into another folder, so we only preserve the cache for the current queries.
+// mkdir -p ${COMBINED_CACHE_DIR}
+// rm -f **/.cache/{lock,size} # -f to avoid errors if the cache is empty.
+// # copy the contents of the .cache folders into the combined cache folder.
+// cp -r **/.cache/* ${COMBINED_CACHE_DIR}/ || : # ignore missing files
+// # clean up the .cache folders
+// rm -rf **/.cache/*
+
+const fs = require("fs");
+const path = require("path");
+
+// the first argv is the cache folder to create.
+const COMBINED_CACHE_DIR = process.argv[2];
+
+function* walkCaches(dir) {
+  const files = fs.readdirSync(dir, { withFileTypes: true });
+  for (const file of files) {
+    if (file.isDirectory()) {
+      const filePath = path.join(dir, file.name);
+      yield* walkCaches(filePath);
+      if (file.name === ".cache") {
+        yield filePath;
+      }
+    }
+  }
+}
+
+async function copyDir(src, dest) {
+  for await (const file of await fs.promises.readdir(src, { withFileTypes: true })) {
+    const srcPath = path.join(src, file.name);
+    const destPath = path.join(dest, file.name);
+    if (file.isDirectory()) {
+      if (!fs.existsSync(destPath)) {
+        fs.mkdirSync(destPath);
+      }
+      await copyDir(srcPath, destPath);
+    } else {
+      await fs.promises.copyFile(srcPath, destPath);
+    }
+  }
+}
+
+async function main() {
+  const cacheDirs = [...walkCaches(".")];
+
+  for (const dir of cacheDirs) {
+    console.log(`Found .cache dir at ${dir}`);
+  }
+
+  // mkdir -p ${COMBINED_CACHE_DIR}
+  fs.mkdirSync(COMBINED_CACHE_DIR, { recursive: true });
+
+  // rm -f **/.cache/{lock,size} # -f to avoid errors if the cache is empty.
+  await Promise.all(
+    cacheDirs.map((cacheDir) =>
+      (async function () {
+        await fs.promises.rm(path.join(cacheDir, "lock"), { force: true });
+        await fs.promises.rm(path.join(cacheDir, "size"), { force: true });
+      })()
+    )
+  );
+
+  // # copy the contents of the .cache folders into the combined cache folder.
+  // cp -r **/.cache/* ${COMBINED_CACHE_DIR}/ || : # ignore missing files
+  await Promise.all(
+    cacheDirs.map((cacheDir) => copyDir(cacheDir, COMBINED_CACHE_DIR))
+  );
+
+  // # clean up the .cache folders
+  // rm -rf **/.cache/*
+  await Promise.all(
+    cacheDirs.map((cacheDir) => fs.promises.rm(cacheDir, { recursive: true }))
+  );
+}
+main();

.github/actions/cache-query-compilation/move-caches.js

@@ -19,4 +19,6 @@ runs:
         gh extension install github/gh-codeql
         gh codeql set-channel "$CHANNEL"
         gh codeql version
+        printf "CODEQL_FETCHED_CODEQL_PATH=" >> "${GITHUB_ENV}"
+        gh codeql version --format=json | jq -r .unpackedLocation >> "${GITHUB_ENV}"
         gh codeql version --format=json | jq -r .unpackedLocation >> "${GITHUB_PATH}"

.github/actions/fetch-codeql/action.yml

@@ -0,0 +1,32 @@
+name: OS Version
+description: Get OS version.
+
+outputs:
+  version:
+    description: "OS version"
+    value: ${{ steps.version.outputs.version }}
+
+runs:
+  using: composite
+  steps:
+    - if: runner.os == 'Linux'
+      shell: bash
+      run: |
+        . /etc/os-release
+        echo "VERSION=${NAME} ${VERSION}" >> $GITHUB_ENV
+    - if: runner.os == 'Windows'
+      shell: powershell
+      run: |
+        $objects = systeminfo.exe /FO CSV | ConvertFrom-Csv
+        "VERSION=$($objects.'OS Name') $($objects.'OS Version')" >> $env:GITHUB_ENV
+    - if: runner.os == 'macOS'
+      shell: bash
+      run: |
+        echo "VERSION=$(sw_vers -productName) $(sw_vers -productVersion)" >> $GITHUB_ENV
+    - name: Emit OS version
+      id: version
+      shell: bash
+      run: |
+        echo "$VERSION"
+        echo "version=${VERSION}" >> $GITHUB_OUTPUT
+

.github/actions/os-version/action.yml

@@ -1,19 +1,12 @@
 version: 2
 updates:
   - package-ecosystem: "cargo"
-    directory: "ruby/node-types"
-    schedule:
-      interval: "daily"
-  - package-ecosystem: "cargo"
-    directory: "ruby/generator"
-    schedule:
-      interval: "daily"
-  - package-ecosystem: "cargo"
-    directory: "ruby/extractor"
+    directory: "ruby"
     schedule:
       interval: "daily"
+
   - package-ecosystem: "cargo"
-    directory: "ruby/autobuilder"
+    directory: "ql"
     schedule:
       interval: "daily"
 

.github/dependabot.yml

@@ -13,7 +13,7 @@ on:
 
 jobs:
   atm-check-query-suite:
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-latest-xl
 
     steps:
       - uses: actions/checkout@v3
@@ -23,6 +23,12 @@ jobs:
         with:
           channel: release
 
+      - name: Cache compilation cache
+        id: query-cache
+        uses: ./.github/actions/cache-query-compilation
+        with: 
+          key: atm-suite
+
       - name: Install ATM model
         run: |
           set -exu
@@ -50,10 +56,13 @@ jobs:
           echo "SARIF_PATH=${SARIF_PATH}" >> "${GITHUB_ENV}"
 
           codeql database analyze \
+            --threads=0 \
+            --ram 50000 \
             --format sarif-latest \
             --output "${SARIF_PATH}" \
             --sarif-group-rules-by-pack \
             -vv \
+            --compilation-cache "${{ steps.query-cache.outputs.cache-dir }}" \
             -- \
             "${DB_PATH}" \
             "${QUERY_PACK}/${QUERY_SUITE}"

.github/workflows/atm-check-query-suite.yml

@@ -26,3 +26,9 @@ jobs:
         run: |
           gh api 'repos/${{github.repository}}/pulls/${{github.event.number}}/files' --paginate --jq 'any(.[].filename ; test("/change-notes/.*[.]md$"))' |
           grep true -c
+      - name: Fail if the change note filename doesn't match the expected format. The file name must be of the form 'YYYY-MM-DD.md' or 'YYYY-MM-DD-{title}.md', where '{title}' is arbitrary text.
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        run: |
+          gh api 'repos/${{github.repository}}/pulls/${{github.event.number}}/files' --paginate --jq '[.[].filename | select(test("/change-notes/.*[.]md$"))] | all(test("/change-notes/[0-9]{4}-[0-9]{2}-[0-9]{2}.*[.]md$"))' |
+          grep true -c