ATM: Check the results of the queries too

Author: henrymercer

.github/workflows/atm-check-queries-run.yml

@@ -1,4 +1,4 @@
-name: ATM Check Queries Run
+name: ATM Check Queries
 
 env:
   QUERY_PACK: javascript/ql/experimental/adaptivethreatmodeling/src
@@ -12,7 +12,7 @@ on:
   workflow_dispatch:
 
 jobs:
-  run-atm-queries:
+  atm-check-queries:
     runs-on: ubuntu-latest
 
     steps:
@@ -40,15 +40,54 @@ jobs:
       - name: Create test DB
         run: |
           DB_PATH="${RUNNER_TEMP}/db"
-          codeql database create "${DB_PATH}" --source-root config/atm --language javascript 
           echo "DB_PATH=${DB_PATH}" >> "${GITHUB_ENV}"
 
+          codeql database create "${DB_PATH}" --source-root config/atm --language javascript 
+
       - name: Run ATM query suite
         run: |
+          SARIF_PATH="${RUNNER_TEMP}/sarif.json"
+          echo "SARIF_PATH=${SARIF_PATH}" >> "${GITHUB_ENV}"
+
           codeql database analyze \
             --format sarif-latest \
-            --output "${RUNNER_TEMP}/sarif.json" \
+            --output "${SARIF_PATH}" \
+            --sarif-group-rules-by-pack \
             -vv \
             -- \
             "${DB_PATH}" \
             "${QUERY_PACK}/${QUERY_SUITE}"
+
+      - name: Upload SARIF
+        uses: actions/upload-artifact@v3
+        with:
+          name: javascript-ml-powered-queries.sarif
+          path: "${{ env.SARIF_PATH }}"
+          retention-days: 5
+
+      - name: Check results
+        run: |
+          # We should run at least the ML-powered queries in `expected_rules`.
+          expected_rules="js/ml-powered/nosql-injection js/ml-powered/path-injection js/ml-powered/sql-injection js/ml-powered/xss"
+
+          for rule in ${expected_rules}; do
+            found_rule=$(jq --arg rule "${rule}" '[.runs[0].tool.extensions[].rules | select(. != null) |
+              flatten | .[].id] | any(. == $rule)' "${SARIF_PATH}")
+            if [[ "${found_rule}" != "true" ]]; then
+              echo "Expected SARIF output to contain rule '${rule}', but found no such rule."
+              exit 1
+            else
+              echo "Found rule '${rule}'."
+            fi
+          done
+
+          # We should have at least one alert from an ML-powered query.
+          num_alerts=$(jq '[.runs[0].results[] |
+            select(.properties.score != null and (.rule.id | startswith("js/ml-powered/")))] | length' \
+            "${SARIF_PATH}")
+          if [[ "${num_alerts}" -eq 0 ]]; then
+            echo "Expected to find at least one alert from an ML-powered query but found ${num_alerts}."
+            exit 1
+          else
+            echo "Found ${num_alerts} alerts from ML-powered queries.";
+          fi