Merge branch 'main' into import-refined

Author: RasmusWL

.github/actions/cache-query-compilation/action.yml

@@ -9,7 +9,7 @@ inputs:
 outputs:
   cache-dir:
     description: "The directory where the cache was stored"
-    value: ${{ steps.fill-compilation-dir.outputs.compdir }}
+    value: ${{ steps.output-compilation-dir.outputs.compdir }}
 
 runs:
   using: composite
@@ -27,7 +27,9 @@ runs:
       if: ${{ github.event_name == 'pull_request' }}
       uses: actions/cache/restore@v3
       with:
-        path: '**/.cache'
+        path: |
+          **/.cache
+          ~/.codeql/compile-cache
         key: codeql-compile-${{ inputs.key }}-pr-${{ github.sha }}
         restore-keys: |
           codeql-compile-${{ inputs.key }}-${{ github.base_ref }}-${{ env.merge_base }}
@@ -37,12 +39,22 @@ runs:
       if: ${{ github.event_name != 'pull_request' }}
       uses: actions/cache@v3
       with:
-        path: '**/.cache'
+        path: |
+          **/.cache
+          ~/.codeql/compile-cache
         key: codeql-compile-${{ inputs.key }}-${{ github.ref_name }}-${{ github.sha }} # just fill on main
         restore-keys: | # restore the latest cache if the exact cache is unavailable, to speed up compilation.
           codeql-compile-${{ inputs.key }}-${{ github.ref_name }}-
           codeql-compile-${{ inputs.key }}-main-
+    - name: Output-compilationdir
+      id: output-compilation-dir
+      shell: bash
+      run: |
+        echo "compdir=${COMBINED_CACHE_DIR}" >> $GITHUB_OUTPUT
+      env:
+        COMBINED_CACHE_DIR: ${{ runner.temp }}/compilation-dir
     - name: Fill compilation cache directory
+      id: fill-compilation-dir
       uses: actions/github-script@v6
       env: 
         COMBINED_CACHE_DIR: ${{ runner.temp }}/compilation-dir
@@ -58,6 +70,7 @@ runs:
 
           const fs = require("fs");
           const path = require("path");
+          const os = require("os");
 
           // the first argv is the cache folder to create.
           const COMBINED_CACHE_DIR = process.env.COMBINED_CACHE_DIR;
@@ -97,6 +110,17 @@ runs:
               console.log(`Found .cache dir at ${dir}`);
             }
 
+            const globalCacheDir = path.join(os.homedir(), ".codeql", "compile-cache");
+            if (fs.existsSync(globalCacheDir)) {
+              console.log("Found global home dir: " + globalCacheDir);
+              cacheDirs.push(globalCacheDir);
+            }
+
+            if (cacheDirs.length === 0) {
+              console.log("No cache dirs found");
+              return;
+            }
+
             // mkdir -p ${COMBINED_CACHE_DIR}
             fs.mkdirSync(COMBINED_CACHE_DIR, { recursive: true });
 

.github/workflows/compile-queries.yml

@@ -24,14 +24,14 @@ jobs:
         with: 
           key: all-queries
       - name: check formatting
-        run: find */ql -type f \( -name "*.qll" -o -name "*.ql" \) -print0 | xargs -0 codeql query format --check-only
+        run: find */ql -type f \( -name "*.qll" -o -name "*.ql" \) -print0 | xargs -0 -n 3000 -P 10 codeql query format -q --check-only
       - name: compile queries - check-only
         # run with --check-only if running in a PR (github.sha != main)
         if : ${{ github.event_name == 'pull_request' }}
         shell: bash
-        run: codeql query compile -j0 */ql/{src,examples} --keep-going --warnings=error --check-only --compilation-cache "${{ steps.query-cache.outputs.cache-dir }}"
+        run: codeql query compile -q -j0 */ql/{src,examples} --keep-going --warnings=error --check-only --compilation-cache "${{ steps.query-cache.outputs.cache-dir }}"
       - name: compile queries - full
         # do full compile if running on main - this populates the cache
         if : ${{ github.event_name != 'pull_request' }}
         shell: bash
-        run: codeql query compile -j0 */ql/{src,examples} --keep-going --warnings=error --compilation-cache "${{ steps.query-cache.outputs.cache-dir }}"
+        run: codeql query compile -q -j0 */ql/{src,examples} --keep-going --warnings=error --compilation-cache "${{ steps.query-cache.outputs.cache-dir }}"

.github/workflows/ql-for-ql-build.yml

@@ -15,6 +15,8 @@ jobs:
     steps:
       ### Build the queries ###
       - uses: actions/checkout@v3
+        with:
+          fetch-depth: 0
       - name: Find codeql
         id: find-codeql
         uses: github/codeql-action/init@v2
@@ -27,7 +29,9 @@ jobs:
         id: cache-extractor
         uses: actions/cache@v3
         with:
-          path: ql/extractor-pack/
+          path: |
+            ql/extractor-pack/
+            ql/target/release/buramu
           key: ${{ runner.os }}-${{ steps.os_version.outputs.version }}-extractor-${{ hashFiles('ql/**/Cargo.lock') }}-${{ hashFiles('ql/**/*.rs') }}
       - name: Cache cargo
         if: steps.cache-extractor.outputs.cache-hit != 'true'
@@ -50,6 +54,7 @@ jobs:
           key: run-ql-for-ql
       - name: Make database and analyze
         run: |
+          ./ql/target/release/buramu | tee deprecated.blame # Add a blame file for the extractor to parse.
           ${CODEQL} database create -l=ql --search-path ql/extractor-pack ${DB}
           ${CODEQL} database analyze -j0 --format=sarif-latest --output=ql-for-ql.sarif ${DB} ql/ql/src/codeql-suites/ql-code-scanning.qls  --compilation-cache "${{ steps.query-cache.outputs.cache-dir }}"
         env: 
@@ -58,6 +63,7 @@ jobs:
           LGTM_INDEX_FILTERS: |
             exclude:ql/ql/test
             exclude:*/ql/lib/upgrades/
+            exclude:java/ql/integration-tests
       - name: Upload sarif to code-scanning
         uses: github/codeql-action/upload-sarif@v2
         with:

.github/workflows/swift.yml

@@ -5,6 +5,7 @@ on:
     paths:
       - "swift/**"
       - "misc/bazel/**"
+      - "misc/codegen/**"
       - "*.bazel*"
       - .github/workflows/swift.yml
       - .github/actions/**
@@ -19,6 +20,7 @@ on:
     paths:
       - "swift/**"
       - "misc/bazel/**"
+      - "misc/codegen/**"
       - "*.bazel*"
       - .github/workflows/swift.yml
       - .github/actions/**

.pre-commit-config.yaml

@@ -53,5 +53,5 @@ repos:
         name: Run Swift code generation unit tests
         files: ^swift/codegen/.*\.py$
         language: system
-        entry: bazel test //swift/codegen/test
+        entry: bazel test //misc/codegen/test
         pass_filenames: false

CODEOWNERS

@@ -2,10 +2,11 @@
 /csharp/ @github/codeql-csharp
 /go/ @github/codeql-go
 /java/ @github/codeql-java
-/javascript/ @github/codeql-javascript
-/python/ @github/codeql-python
-/ruby/ @github/codeql-ruby
+/javascript/ @github/codeql-dynamic
+/python/ @github/codeql-dynamic
+/ruby/ @github/codeql-dynamic
 /swift/ @github/codeql-swift
+/misc/codegen/ @github/codeql-swift
 /java/kotlin-extractor/ @github/codeql-kotlin
 /java/kotlin-explorer/ @github/codeql-kotlin
 

cpp/ql/lib/CHANGELOG.md

@@ -1,3 +1,7 @@
+## 0.5.3
+
+No user-facing changes.
+
 ## 0.5.2
 
 No user-facing changes.

cpp/ql/lib/change-notes/released/0.5.3.md

@@ -0,0 +1,3 @@
+## 0.5.3
+
+No user-facing changes.

cpp/ql/lib/codeql-pack.release.yml

@@ -1,2 +1,2 @@
 ---
-lastReleaseVersion: 0.5.2
+lastReleaseVersion: 0.5.3

cpp/ql/lib/qlpack.yml

@@ -1,5 +1,5 @@
 name: codeql/cpp-all
-version: 0.5.3-dev
+version: 0.5.4-dev
 groups: cpp
 dbscheme: semmlecode.cpp.dbscheme
 extractor: cpp