Merge branch 'main' into python-clean-up-import-resolution

Author: tausbn

.github/ISSUE_TEMPLATE/lgtm-com---false-positive.md

@@ -0,0 +1,36 @@
+---
+name: CodeQL False positive
+about: Report CodeQL alerts that you think should not have been detected (not applicable, not exploitable, etc.)
+title: False positive
+labels: false-positive
+assignees: ''
+
+---
+
+**Description of the false positive**
+
+<!-- Please explain briefly why you think it shouldn't be included. -->
+
+**Code samples or links to source code**
+
+<!--
+For open source code: file links with line numbers on GitHub, for example:
+https://github.com/github/codeql/blob/dc440aaee6695deb0d9676b87e06ea984e1b4ae5/javascript/ql/test/query-tests/Security/CWE-078/CommandInjection/exec-sh2.js#L10
+
+For closed source code: (redacted) code samples that illustrate the problem, for example:
+
+```
+function execSh(command, options) {
+    return cp.spawn(getShell(), ["-c", command], options) // <- command line injection
+};
+```
+-->
+
+**URL to the alert on GitHub code scanning (optional)**
+
+<!--
+1. Open the project on GitHub.com.
+2. Switch to the `Security` tab.
+3. Browse to the alert that you would like to report.
+4. Copy and paste the page URL here.
+-->

.github/ISSUE_TEMPLATE/ql--false-positive.md

@@ -1,14 +1,22 @@
 name: Fetch CodeQL
 description: Fetches the latest version of CodeQL
+
+inputs:
+  channel:
+    description: 'The CodeQL channel to use'
+    required: false
+    default: 'nightly'
+
 runs:
   using: composite
   steps:
     - name: Fetch CodeQL
       shell: bash
+      env:
+        GITHUB_TOKEN: ${{ github.token }}
+        CHANNEL: ${{ inputs.channel }}
       run: |
         gh extension install github/gh-codeql
-        gh codeql set-channel nightly
+        gh codeql set-channel "$CHANNEL"
         gh codeql version
         gh codeql version --format=json | jq -r .unpackedLocation >> "${GITHUB_PATH}"
-      env:
-        GITHUB_TOKEN: ${{ github.token }}

.github/actions/fetch-codeql/action.yml

@@ -43,3 +43,14 @@ documentation:
 "QL-for-QL":
   - ql/**/*
   - .github/workflows/ql-for-ql*
+
+# Since these are all shared files that need to be synced, just pick _one_ copy of each.
+"DataFlow Library":
+  - "java/ql/lib/semmle/code/java/dataflow/internal/DataFlowImpl.qll"
+  - "java/ql/lib/semmle/code/java/dataflow/internal/DataFlowImplCommon.qll"
+  - "java/ql/lib/semmle/code/java/dataflow/internal/tainttracking1/TaintTrackingImpl.qll"
+  - "java/ql/lib/semmle/code/java/dataflow/internal/DataFlowImplConsistency.qll"
+  - "java/ql/lib/semmle/code/java/dataflow/internal/FlowSummaryImpl.qll"
+
+"ATM":
+  - javascript/ql/experimental/adaptivethreatmodeling/**/*

.github/labeler.yml

@@ -0,0 +1,93 @@
+name: "ATM - Check query suite"
+
+env:
+  QUERY_PACK: javascript/ql/experimental/adaptivethreatmodeling/src
+  QUERY_SUITE: codeql-suites/javascript-atm-code-scanning.qls
+
+on:
+  pull_request:
+    paths:
+      - ".github/workflows/atm-check-query-suite.yml"
+      - "javascript/ql/experimental/adaptivethreatmodeling/**"
+  workflow_dispatch:
+
+jobs:
+  atm-check-query-suite:
+    runs-on: ubuntu-latest
+
+    steps:
+      - uses: actions/checkout@v3
+
+      - name: Setup CodeQL
+        uses: ./.github/actions/fetch-codeql
+        with:
+          channel: release
+
+      - name: Install ATM model
+        run: |
+          set -exu
+
+          # Install dependencies of ATM query pack, i.e. the ATM model
+          codeql pack install "${QUERY_PACK}"
+
+          # Retrieve model checksum
+          model_checksum=$(codeql resolve extensions "${QUERY_PACK}/${QUERY_SUITE}" | jq -r '.models[0].checksum')
+
+          # Trust the model so that we can use it in the ATM boosted queries
+          mkdir -p "$HOME/.config/codeql"
+          echo "--insecurely-execute-ml-model-checksums ${model_checksum}" >> "$HOME/.config/codeql/config"
+
+      - name: Create test DB
+        run: |
+          DB_PATH="${RUNNER_TEMP}/db"
+          echo "DB_PATH=${DB_PATH}" >> "${GITHUB_ENV}"
+
+          codeql database create "${DB_PATH}" --source-root config/atm --language javascript 
+
+      - name: Run ATM query suite
+        run: |
+          SARIF_PATH="${RUNNER_TEMP}/sarif.json"
+          echo "SARIF_PATH=${SARIF_PATH}" >> "${GITHUB_ENV}"
+
+          codeql database analyze \
+            --format sarif-latest \
+            --output "${SARIF_PATH}" \
+            --sarif-group-rules-by-pack \
+            -vv \
+            -- \
+            "${DB_PATH}" \
+            "${QUERY_PACK}/${QUERY_SUITE}"
+
+      - name: Upload SARIF
+        uses: actions/upload-artifact@v3
+        with:
+          name: javascript-ml-powered-queries.sarif
+          path: "${{ env.SARIF_PATH }}"
+          retention-days: 5
+
+      - name: Check results
+        run: |
+          # We should run at least the ML-powered queries in `expected_rules`.
+          expected_rules="js/ml-powered/nosql-injection js/ml-powered/path-injection js/ml-powered/sql-injection js/ml-powered/xss"
+
+          for rule in ${expected_rules}; do
+            found_rule=$(jq --arg rule "${rule}" '[.runs[0].tool.extensions[].rules | select(. != null) |
+              flatten | .[].id] | any(. == $rule)' "${SARIF_PATH}")
+            if [[ "${found_rule}" != "true" ]]; then
+              echo "Expected SARIF output to contain rule '${rule}', but found no such rule."
+              exit 1
+            else
+              echo "Found rule '${rule}'."
+            fi
+          done
+
+          # We should have at least one alert from an ML-powered query.
+          num_alerts=$(jq '[.runs[0].results[] |
+            select(.properties.score != null and (.rule.id | startswith("js/ml-powered/")))] | length' \
+            "${SARIF_PATH}")
+          if [[ "${num_alerts}" -eq 0 ]]; then
+            echo "Expected to find at least one alert from an ML-powered query but found ${num_alerts}."
+            exit 1
+          else
+            echo "Found ${num_alerts} alerts from ML-powered queries.";
+          fi

.github/workflows/atm-check-query-suite.yml

@@ -0,0 +1,12 @@
+name: ATM Model Integration Tests
+
+on:
+  workflow_dispatch:
+
+jobs:
+  hello-world:
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: foo
+        run: echo "Hello world"

.github/workflows/atm-model-integration-tests.yml

@@ -0,0 +1,71 @@
+name: "Compile all queries using the latest stable CodeQL CLI"
+
+on:
+  push:
+    branches:  # makes sure the cache gets populated - running on the branches people tend to merge into.
+      - main
+      - "rc/*"
+      - "codeql-cli-*"
+  pull_request:
+
+jobs:
+  compile-queries:
+    runs-on: ubuntu-latest-xl
+
+    steps:
+      - uses: actions/checkout@v3
+      # calculate the merge-base with main, in a way that works both on PRs and pushes to main. 
+      - name: Calculate merge-base
+        if: ${{ github.event_name == 'pull_request' }}
+        env:
+          BASE_BRANCH: ${{ github.base_ref }}
+        run: |
+          MERGE_BASE=$(git cat-file commit $GITHUB_SHA | grep '^parent ' | head -1 | cut -f 2 -d " ")
+          echo "merge-base=$MERGE_BASE" >> $GITHUB_ENV
+      - name: Read CodeQL query compilation - PR
+        if: ${{ github.event_name == 'pull_request' }}
+        uses: actions/cache@v3
+        with:
+          path: '*/ql/src/.cache'
+          key: codeql-compile-pr-${{ github.sha }} # deliberately not using the `compile-compile-main` keys here.
+          restore-keys: |
+            codeql-compile-${{ github.base_ref }}-${{ env.merge-base }}
+            codeql-compile-${{ github.base_ref }}-
+            codeql-compile-main-
+      - name: Fill CodeQL query compilation cache - main
+        if: ${{ github.event_name != 'pull_request' }}
+        uses: actions/cache@v3
+        with:
+          path: '*/ql/src/.cache'
+          key: codeql-compile-${{ github.ref_name }}-${{ github.sha }} # just fill on main
+          restore-keys: | # restore from another random commit, to speed up compilation.
+            codeql-compile-${{ github.ref_name }}- 
+            codeql-compile-main-
+      - name: Setup CodeQL
+        uses: ./.github/actions/fetch-codeql
+        with:
+          channel: 'release'
+      - name: check formatting
+        run: find */ql -type f \( -name "*.qll" -o -name "*.ql" \) -print0 | xargs -0 codeql query format --check-only
+      - name: compile queries - check-only
+        # run with --check-only if running in a PR (github.sha != main)
+        if : ${{ github.event_name == 'pull_request' }}
+        shell: bash
+        run: codeql query compile -j0 */ql/src --keep-going --warnings=error --check-only
+      - name: compile queries - full
+        # do full compile if running on main - this populates the cache
+        if : ${{ github.event_name != 'pull_request' }}
+        shell: bash
+        run: |
+          # Move all the existing cache into another folder, so we only preserve the cache for the current queries.
+          mkdir -p ${COMBINED_CACHE_DIR}
+          rm */ql/src/.cache/{lock,size}
+          # copy the contents of the .cache folders into the combined cache folder.
+          cp -r */ql/src/.cache/* ${COMBINED_CACHE_DIR}/
+          # clean up the .cache folders
+          rm -rf */ql/src/.cache/*
+
+          # compile the queries
+          codeql query compile -j0 */ql/src --keep-going --warnings=error --compilation-cache ${COMBINED_CACHE_DIR}
+        env: 
+          COMBINED_CACHE_DIR: ${{ github.workspace }}/compilation-dir

.github/workflows/compile-queries.yml

@@ -23,19 +23,6 @@ defaults:
     working-directory: javascript/ql/experimental/adaptivethreatmodeling
 
 jobs:
-  qlformat:
-    name: Check QL formatting
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v3
-
-      - uses: ./.github/actions/fetch-codeql
-
-      - name: Check QL formatting
-        run: |
-          find . "(" -name "*.ql" -or -name "*.qll" ")" -print0 | \
-            xargs -0 codeql query format --check-only
-
   qlcompile:
     name: Check QL compilation
     runs-on: ubuntu-latest

.github/workflows/js-ml-tests.yml

@@ -24,13 +24,13 @@ jobs:
       - uses: actions/checkout@v3
       - name: Find codeql
         id: find-codeql
-        uses: github/codeql-action/init@71a8b35ff4c80fcfcd05bc1cd932fe3c08f943ca
+        uses: github/codeql-action/init@77a8d2d10c0b403a8b4aadbd223dc489ecd22683
         with:
           languages: javascript # does not matter
       - name: Get CodeQL version
         id: get-codeql-version
         run: |
-          echo "::set-output name=version::$("${CODEQL}" --version | head -n 1 | rev | cut -d " " -f 1 | rev)"
+          echo "version=$("${CODEQL}" --version | head -n 1 | rev | cut -d " " -f 1 | rev)" >> $GITHUB_OUTPUT
         shell: bash
         env:
           CODEQL: ${{ steps.find-codeql.outputs.codeql-path }}
@@ -133,7 +133,7 @@ jobs:
         env:
           CONF: ./ql-for-ql-config.yml
       - name: Initialize CodeQL
-        uses: github/codeql-action/init@71a8b35ff4c80fcfcd05bc1cd932fe3c08f943ca
+        uses: github/codeql-action/init@77a8d2d10c0b403a8b4aadbd223dc489ecd22683
         with:
           languages: ql
           db-location: ${{ runner.temp }}/db
@@ -145,7 +145,7 @@ jobs:
           PACK: ${{ runner.temp }}/pack
 
       - name: Perform CodeQL Analysis
-        uses: github/codeql-action/analyze@71a8b35ff4c80fcfcd05bc1cd932fe3c08f943ca
+        uses: github/codeql-action/analyze@77a8d2d10c0b403a8b4aadbd223dc489ecd22683
         with:
           category: "ql-for-ql"
       - name: Copy sarif file to CWD

.github/workflows/ql-for-ql-build.yml

@@ -25,7 +25,7 @@ jobs:
 
       - name: Find codeql
         id: find-codeql
-        uses: github/codeql-action/init@71a8b35ff4c80fcfcd05bc1cd932fe3c08f943ca
+        uses: github/codeql-action/init@77a8d2d10c0b403a8b4aadbd223dc489ecd22683
         with:
           languages: javascript # does not matter
       - uses: actions/cache@v3