Merge pull request github#11118 from github/henrymercer/atm-check-queries-improvements

ATM: Miscellaneous improvements for the check queries workflow

Author: henrymercer

.github/workflows/atm-check-queries-run.yml

@@ -0,0 +1,93 @@
+name: "ATM - Check query suite"
+
+env:
+  QUERY_PACK: javascript/ql/experimental/adaptivethreatmodeling/src
+  QUERY_SUITE: codeql-suites/javascript-atm-code-scanning.qls
+
+on:
+  pull_request:
+    paths:
+      - ".github/workflows/atm-check-query-suite.yml"
+      - "javascript/ql/experimental/adaptivethreatmodeling/**"
+  workflow_dispatch:
+
+jobs:
+  atm-check-query-suite:
+    runs-on: ubuntu-latest
+
+    steps:
+      - uses: actions/checkout@v3
+
+      - name: Setup CodeQL
+        uses: ./.github/actions/fetch-codeql
+        with:
+          channel: release
+
+      - name: Install ATM model
+        run: |
+          set -exu
+
+          # Install dependencies of ATM query pack, i.e. the ATM model
+          codeql pack install "${QUERY_PACK}"
+
+          # Retrieve model checksum
+          model_checksum=$(codeql resolve extensions "${QUERY_PACK}/${QUERY_SUITE}" | jq -r '.models[0].checksum')
+
+          # Trust the model so that we can use it in the ATM boosted queries
+          mkdir -p "$HOME/.config/codeql"
+          echo "--insecurely-execute-ml-model-checksums ${model_checksum}" >> "$HOME/.config/codeql/config"
+
+      - name: Create test DB
+        run: |
+          DB_PATH="${RUNNER_TEMP}/db"
+          echo "DB_PATH=${DB_PATH}" >> "${GITHUB_ENV}"
+
+          codeql database create "${DB_PATH}" --source-root config/atm --language javascript 
+
+      - name: Run ATM query suite
+        run: |
+          SARIF_PATH="${RUNNER_TEMP}/sarif.json"
+          echo "SARIF_PATH=${SARIF_PATH}" >> "${GITHUB_ENV}"
+
+          codeql database analyze \
+            --format sarif-latest \
+            --output "${SARIF_PATH}" \
+            --sarif-group-rules-by-pack \
+            -vv \
+            -- \
+            "${DB_PATH}" \
+            "${QUERY_PACK}/${QUERY_SUITE}"
+
+      - name: Upload SARIF
+        uses: actions/upload-artifact@v3
+        with:
+          name: javascript-ml-powered-queries.sarif
+          path: "${{ env.SARIF_PATH }}"
+          retention-days: 5
+
+      - name: Check results
+        run: |
+          # We should run at least the ML-powered queries in `expected_rules`.
+          expected_rules="js/ml-powered/nosql-injection js/ml-powered/path-injection js/ml-powered/sql-injection js/ml-powered/xss"
+
+          for rule in ${expected_rules}; do
+            found_rule=$(jq --arg rule "${rule}" '[.runs[0].tool.extensions[].rules | select(. != null) |
+              flatten | .[].id] | any(. == $rule)' "${SARIF_PATH}")
+            if [[ "${found_rule}" != "true" ]]; then
+              echo "Expected SARIF output to contain rule '${rule}', but found no such rule."
+              exit 1
+            else
+              echo "Found rule '${rule}'."
+            fi
+          done
+
+          # We should have at least one alert from an ML-powered query.
+          num_alerts=$(jq '[.runs[0].results[] |
+            select(.properties.score != null and (.rule.id | startswith("js/ml-powered/")))] | length' \
+            "${SARIF_PATH}")
+          if [[ "${num_alerts}" -eq 0 ]]; then
+            echo "Expected to find at least one alert from an ML-powered query but found ${num_alerts}."
+            exit 1
+          else
+            echo "Found ${num_alerts} alerts from ML-powered queries.";
+          fi

.github/workflows/atm-check-query-suite.yml

@@ -40,6 +40,7 @@ WORKSPACE.bazel @github/codeql-ci-reviewers
 
 # Workflows
 /.github/workflows/ @github/codeql-ci-reviewers
+/.github/workflows/atm-* @github/codeql-ml-powered-queries-reviewers
 /.github/workflows/go-* @github/codeql-go
 /.github/workflows/js-ml-tests.yml @github/codeql-ml-powered-queries-reviewers
 /.github/workflows/ql-for-ql-* @github/codeql-ql-for-ql-reviewers