Merge branch 'main' into py-shell

Author: erik-krogh

.git-blame-ignore-revs

@@ -0,0 +1,21 @@
+# .git-blame-ignore-revs
+# Auto-formatted Java
+730eae952139209fe9fdf598541d608f4c0c0c84
+# Auto-formatted C#
+5ad7ed49dd3de03ec6dcfcb6848758a6a987e11c
+# Auto-formatted C/C++
+ef97e539ec1971494d4bba5cafe82e00bc8217ac
+# Auto-formatted Python
+21d5fa836b3a7d020ba45e8b8168b145a9772131
+# Auto-formatted JavaScript
+8d97fe9ed327a9546ff2eaf515cf0f5214deddd9
+# Auto-formatted Ruby
+a5d229903d2f12d45f2c2c38822f1d0e7504ae7f
+# Auto-formatted Go
+08c658e66bf867090033ea096e244a93d46c0aa7
+# Auto-formatted Swift
+711d7057f79fb7d72fc3b35e010bd018f9009169
+# Auto-formatted shared ql packs
+3640b6d3a8ce9edf8e1d3ed106fe8526cf255bc0
+# Auto-formatted taint tracking files
+159d8e978c51959b380838c080d891b66e763b19

.github/actions/cache-query-compilation/action.yml

@@ -9,7 +9,7 @@ inputs:
 outputs:
   cache-dir:
     description: "The directory where the cache was stored"
-    value: ${{ steps.fill-compilation-dir.outputs.compdir }}
+    value: ${{ steps.output-compilation-dir.outputs.compdir }}
 
 runs:
   using: composite
@@ -27,7 +27,9 @@ runs:
       if: ${{ github.event_name == 'pull_request' }}
       uses: actions/cache/restore@v3
       with:
-        path: '**/.cache'
+        path: |
+          **/.cache
+          ~/.codeql/compile-cache
         key: codeql-compile-${{ inputs.key }}-pr-${{ github.sha }}
         restore-keys: |
           codeql-compile-${{ inputs.key }}-${{ github.base_ref }}-${{ env.merge_base }}
@@ -37,18 +39,111 @@ runs:
       if: ${{ github.event_name != 'pull_request' }}
       uses: actions/cache@v3
       with:
-        path: '**/.cache'
+        path: |
+          **/.cache
+          ~/.codeql/compile-cache
         key: codeql-compile-${{ inputs.key }}-${{ github.ref_name }}-${{ github.sha }} # just fill on main
         restore-keys: | # restore the latest cache if the exact cache is unavailable, to speed up compilation.
           codeql-compile-${{ inputs.key }}-${{ github.ref_name }}-
           codeql-compile-${{ inputs.key }}-main-
-    - name: Fill compilation cache directory
-      id: fill-compilation-dir
+    - name: Output-compilationdir
+      id: output-compilation-dir
       shell: bash
       run: |
-        # Move all the existing cache into another folder, so we only preserve the cache for the current queries.
-        node $GITHUB_WORKSPACE/.github/actions/cache-query-compilation/move-caches.js ${COMBINED_CACHE_DIR}
-
         echo "compdir=${COMBINED_CACHE_DIR}" >> $GITHUB_OUTPUT
       env:
         COMBINED_CACHE_DIR: ${{ runner.temp }}/compilation-dir
+    - name: Fill compilation cache directory
+      id: fill-compilation-dir
+      uses: actions/github-script@v6
+      env: 
+        COMBINED_CACHE_DIR: ${{ runner.temp }}/compilation-dir
+      with:
+        script: |
+          // # Move all the existing cache into another folder, so we only preserve the cache for the current queries.
+          // mkdir -p ${COMBINED_CACHE_DIR}
+          // rm -f **/.cache/{lock,size} # -f to avoid errors if the cache is empty.
+          // # copy the contents of the .cache folders into the combined cache folder.
+          // cp -r **/.cache/* ${COMBINED_CACHE_DIR}/ || : # ignore missing files
+          // # clean up the .cache folders
+          // rm -rf **/.cache/*
+
+          const fs = require("fs");
+          const path = require("path");
+          const os = require("os");
+
+          // the first argv is the cache folder to create.
+          const COMBINED_CACHE_DIR = process.env.COMBINED_CACHE_DIR;
+
+          function* walkCaches(dir) {
+            const files = fs.readdirSync(dir, { withFileTypes: true });
+            for (const file of files) {
+              if (file.isDirectory()) {
+                const filePath = path.join(dir, file.name);
+                yield* walkCaches(filePath);
+                if (file.name === ".cache") {
+                  yield filePath;
+                }
+              }
+            }
+          }
+
+          async function copyDir(src, dest) {
+            for await (const file of await fs.promises.readdir(src, { withFileTypes: true })) {
+              const srcPath = path.join(src, file.name);
+              const destPath = path.join(dest, file.name);
+              if (file.isDirectory()) {
+                if (!fs.existsSync(destPath)) {
+                  fs.mkdirSync(destPath);
+                }
+                await copyDir(srcPath, destPath);
+              } else {
+                await fs.promises.copyFile(srcPath, destPath);
+              }
+            }
+          }
+
+          async function main() {
+            const cacheDirs = [...walkCaches(".")];
+
+            for (const dir of cacheDirs) {
+              console.log(`Found .cache dir at ${dir}`);
+            }
+
+            const globalCacheDir = path.join(os.homedir(), ".codeql", "compile-cache");
+            if (fs.existsSync(globalCacheDir)) {
+              console.log("Found global home dir: " + globalCacheDir);
+              cacheDirs.push(globalCacheDir);
+            }
+
+            if (cacheDirs.length === 0) {
+              console.log("No cache dirs found");
+              return;
+            }
+
+            // mkdir -p ${COMBINED_CACHE_DIR}
+            fs.mkdirSync(COMBINED_CACHE_DIR, { recursive: true });
+
+            // rm -f **/.cache/{lock,size} # -f to avoid errors if the cache is empty.
+            await Promise.all(
+              cacheDirs.map((cacheDir) =>
+                (async function () {
+                  await fs.promises.rm(path.join(cacheDir, "lock"), { force: true });
+                  await fs.promises.rm(path.join(cacheDir, "size"), { force: true });
+                })()
+              )
+            );
+
+            // # copy the contents of the .cache folders into the combined cache folder.
+            // cp -r **/.cache/* ${COMBINED_CACHE_DIR}/ || : # ignore missing files
+            await Promise.all(
+              cacheDirs.map((cacheDir) => copyDir(cacheDir, COMBINED_CACHE_DIR))
+            );
+
+            // # clean up the .cache folders
+            // rm -rf **/.cache/*
+            await Promise.all(
+              cacheDirs.map((cacheDir) => fs.promises.rm(cacheDir, { recursive: true }))
+            );
+          }
+          main();

.github/actions/cache-query-compilation/move-caches.js

@@ -26,9 +26,8 @@ jobs:
         shell: bash
         run: |
           EXIT_CODE=0
-          # TODO: remove the swift exception from the regex when we fix generated QLdoc
           # TODO: remove the shared exception from the regex when coverage of qlpacks without dbschemes is supported
-          changed_lib_packs="$(git diff --name-only --diff-filter=ACMRT HEAD^ HEAD | { grep -Po '^(?!(swift|shared))[a-z]*/ql/lib' || true; } | sort -u)"
+          changed_lib_packs="$(git diff --name-only --diff-filter=ACMRT HEAD^ HEAD | { grep -Po '^(?!(shared))[a-z]*/ql/lib' || true; } | sort -u)"
           for pack_dir in ${changed_lib_packs}; do
             lang="${pack_dir%/ql/lib}"
             codeql generate library-doc-coverage --output="${RUNNER_TEMP}/${lang}-current.txt" --dir="${pack_dir}"

.github/actions/find-latest-bundle/action.yml

@@ -24,14 +24,14 @@ jobs:
         with: 
           key: all-queries
       - name: check formatting
-        run: find */ql -type f \( -name "*.qll" -o -name "*.ql" \) -print0 | xargs -0 codeql query format --check-only
+        run: find */ql -type f \( -name "*.qll" -o -name "*.ql" \) -print0 | xargs -0 -n 3000 -P 10 codeql query format -q --check-only
       - name: compile queries - check-only
         # run with --check-only if running in a PR (github.sha != main)
         if : ${{ github.event_name == 'pull_request' }}
         shell: bash
-        run: codeql query compile -j0 */ql/{src,examples} --keep-going --warnings=error --check-only --compilation-cache "${{ steps.query-cache.outputs.cache-dir }}"
+        run: codeql query compile -q -j0 */ql/{src,examples} --keep-going --warnings=error --check-only --compilation-cache "${{ steps.query-cache.outputs.cache-dir }}"
       - name: compile queries - full
         # do full compile if running on main - this populates the cache
         if : ${{ github.event_name != 'pull_request' }}
         shell: bash
-        run: codeql query compile -j0 */ql/{src,examples} --keep-going --warnings=error --compilation-cache "${{ steps.query-cache.outputs.cache-dir }}"
+        run: codeql query compile -q -j0 */ql/{src,examples} --keep-going --warnings=error --compilation-cache "${{ steps.query-cache.outputs.cache-dir }}"

.github/workflows/check-qldoc.yml

@@ -12,10 +12,10 @@ jobs:
     name: Test MacOS
     runs-on: macos-latest
     steps:
-      - name: Set up Go 1.19
+      - name: Set up Go 1.20
         uses: actions/setup-go@v3
         with:
-          go-version: 1.19
+          go-version: 1.20.0
         id: go
 
       - name: Check out code
@@ -47,10 +47,10 @@ jobs:
     name: Test Windows
     runs-on: windows-latest-xl
     steps:
-      - name: Set up Go 1.19
+      - name: Set up Go 1.20
         uses: actions/setup-go@v3
         with:
-          go-version: 1.19
+          go-version: 1.20.0
         id: go
 
       - name: Check out code

.github/workflows/compile-queries.yml

@@ -20,10 +20,10 @@ jobs:
     name: Test Linux (Ubuntu)
     runs-on: ubuntu-latest-xl
     steps:
-      - name: Set up Go 1.19
+      - name: Set up Go 1.20
         uses: actions/setup-go@v3
         with:
-          go-version: 1.19
+          go-version: 1.20.0
         id: go
 
       - name: Check out code