Skip to content

Commit 1b30cf8

Browse files
RasmusWLRasmusWL
authored
Merge branch 'main' into call-graph-tests
2 parents 84faf49 + 158ea26 commit 1b30cf8

File tree

1,962 files changed

+140621
-88703
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

1,962 files changed

+140621
-88703
lines changed

‎.github/ISSUE_TEMPLATE/lgtm-com---false-positive.md

Lines changed: 0 additions & 24 deletions
This file was deleted.

‎.github/ISSUE_TEMPLATE/ql--false-positive.md

Lines changed: 36 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,36 @@
1+
---
2+
name: CodeQL False positive
3+
about: Report CodeQL alerts that you think should not have been detected (not applicable, not exploitable, etc.)
4+
title: False positive
5+
labels: false-positive
6+
assignees: ''
7+
8+
---
9+
10+
**Description of the false positive**
11+
12+
<!-- Please explain briefly why you think it shouldn't be included. -->
13+
14+
**Code samples or links to source code**
15+
16+
<!--
17+
For open source code: file links with line numbers on GitHub, for example:
18+
https://github.com/github/codeql/blob/dc440aaee6695deb0d9676b87e06ea984e1b4ae5/javascript/ql/test/query-tests/Security/CWE-078/CommandInjection/exec-sh2.js#L10
19+
20+
For closed source code: (redacted) code samples that illustrate the problem, for example:
21+
22+
```
23+
function execSh(command, options) {
24+
return cp.spawn(getShell(), ["-c", command], options) // <- command line injection
25+
};
26+
```
27+
-->
28+
29+
**URL to the alert on GitHub code scanning (optional)**
30+
31+
<!--
32+
1. Open the project on GitHub.com.
33+
2. Switch to the `Security` tab.
34+
3. Browse to the alert that you would like to report.
35+
4. Copy and paste the page URL here.
36+
-->

‎.github/actions/cache-query-compilation/action.yml

Lines changed: 61 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,61 @@
1+
name: Cache query compilation
2+
description: Caches CodeQL compilation caches - should be run both on PRs and pushes to main.
3+
4+
inputs:
5+
key:
6+
description: 'The cache key to use - should be unique to the workflow'
7+
required: true
8+
9+
outputs:
10+
cache-dir:
11+
description: "The directory where the cache was stored"
12+
value: ${{ steps.fill-compilation-dir.outputs.compdir }}
13+
14+
runs:
15+
using: composite
16+
steps:
17+
# Cache the query compilation caches.
18+
# calculate the merge-base with main, in a way that works both on PRs and pushes to main.
19+
- name: Calculate merge-base
20+
shell: bash
21+
if: ${{ github.event_name == 'pull_request' }}
22+
env:
23+
BASE_BRANCH: ${{ github.base_ref }}
24+
run: |
25+
MERGE_BASE=$(git cat-file commit $GITHUB_SHA | grep '^parent ' | head -1 | cut -f 2 -d " ")
26+
echo "merge_base=$MERGE_BASE" >> $GITHUB_ENV
27+
- name: Read CodeQL query compilation - PR
28+
if: ${{ github.event_name == 'pull_request' }}
29+
uses: erik-krogh/actions-cache@a88d0603fe5fb5606db9f002dfcadeb32b5f84c6
30+
with:
31+
path: '**/.cache'
32+
read-only: true
33+
key: codeql-compile-${{ inputs.key }}-pr-${{ github.sha }} # deliberately not using the `compile-compile-main` keys here.
34+
restore-keys: |
35+
codeql-compile-${{ inputs.key }}-${{ github.base_ref }}-${{ env.merge_base }}
36+
codeql-compile-${{ inputs.key }}-${{ github.base_ref }}-
37+
codeql-compile-${{ inputs.key }}-main-
38+
- name: Fill CodeQL query compilation cache - main
39+
if: ${{ github.event_name != 'pull_request' }}
40+
uses: erik-krogh/actions-cache@a88d0603fe5fb5606db9f002dfcadeb32b5f84c6
41+
with:
42+
path: '**/.cache'
43+
key: codeql-compile-${{ inputs.key }}-${{ github.ref_name }}-${{ github.sha }} # just fill on main
44+
restore-keys: | # restore from another random commit, to speed up compilation.
45+
codeql-compile-${{ inputs.key }}-${{ github.ref_name }}-
46+
codeql-compile-${{ inputs.key }}-main-
47+
- name: Fill compilation cache directory
48+
id: fill-compilation-dir
49+
shell: bash
50+
run: |
51+
# Move all the existing cache into another folder, so we only preserve the cache for the current queries.
52+
mkdir -p ${COMBINED_CACHE_DIR}
53+
rm -f **/.cache/{lock,size} # -f to avoid errors if the cache is empty.
54+
# copy the contents of the .cache folders into the combined cache folder.
55+
cp -r **/.cache/* ${COMBINED_CACHE_DIR}/ || : # ignore missing files
56+
# clean up the .cache folders
57+
rm -rf **/.cache/*
58+
59+
echo "compdir=${COMBINED_CACHE_DIR}" >> $GITHUB_OUTPUT
60+
env:
61+
COMBINED_CACHE_DIR: ${{ github.workspace }}/compilation-dir

‎.github/actions/fetch-codeql/action.yml

Lines changed: 11 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,14 +1,22 @@
11
name: Fetch CodeQL
22
description: Fetches the latest version of CodeQL
3+
4+
inputs:
5+
channel:
6+
description: 'The CodeQL channel to use'
7+
required: false
8+
default: 'nightly'
9+
310
runs:
411
using: composite
512
steps:
613
- name: Fetch CodeQL
714
shell: bash
15+
env:
16+
GITHUB_TOKEN: ${{ github.token }}
17+
CHANNEL: ${{ inputs.channel }}
818
run: |
919
gh extension install github/gh-codeql
10-
gh codeql set-channel nightly
20+
gh codeql set-channel "$CHANNEL"
1121
gh codeql version
1222
gh codeql version --format=json | jq -r .unpackedLocation >> "${GITHUB_PATH}"
13-
env:
14-
GITHUB_TOKEN: ${{ github.token }}

‎.github/labeler.yml

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -43,3 +43,14 @@ documentation:
4343
"QL-for-QL":
4444
- ql/**/*
4545
- .github/workflows/ql-for-ql*
46+
47+
# Since these are all shared files that need to be synced, just pick _one_ copy of each.
48+
"DataFlow Library":
49+
- "java/ql/lib/semmle/code/java/dataflow/internal/DataFlowImpl.qll"
50+
- "java/ql/lib/semmle/code/java/dataflow/internal/DataFlowImplCommon.qll"
51+
- "java/ql/lib/semmle/code/java/dataflow/internal/tainttracking1/TaintTrackingImpl.qll"
52+
- "java/ql/lib/semmle/code/java/dataflow/internal/DataFlowImplConsistency.qll"
53+
- "java/ql/lib/semmle/code/java/dataflow/internal/FlowSummaryImpl.qll"
54+
55+
"ATM":
56+
- javascript/ql/experimental/adaptivethreatmodeling/**/*

‎.github/workflows/atm-check-query-suite.yml

Lines changed: 93 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,93 @@
1+
name: "ATM - Check query suite"
2+
3+
env:
4+
QUERY_PACK: javascript/ql/experimental/adaptivethreatmodeling/src
5+
QUERY_SUITE: codeql-suites/javascript-atm-code-scanning.qls
6+
7+
on:
8+
pull_request:
9+
paths:
10+
- ".github/workflows/atm-check-query-suite.yml"
11+
- "javascript/ql/experimental/adaptivethreatmodeling/**"
12+
workflow_dispatch:
13+
14+
jobs:
15+
atm-check-query-suite:
16+
runs-on: ubuntu-latest
17+
18+
steps:
19+
- uses: actions/checkout@v3
20+
21+
- name: Setup CodeQL
22+
uses: ./.github/actions/fetch-codeql
23+
with:
24+
channel: release
25+
26+
- name: Install ATM model
27+
run: |
28+
set -exu
29+
30+
# Install dependencies of ATM query pack, i.e. the ATM model
31+
codeql pack install "${QUERY_PACK}"
32+
33+
# Retrieve model checksum
34+
model_checksum=$(codeql resolve extensions "${QUERY_PACK}/${QUERY_SUITE}" | jq -r '.models[0].checksum')
35+
36+
# Trust the model so that we can use it in the ATM boosted queries
37+
mkdir -p "$HOME/.config/codeql"
38+
echo "--insecurely-execute-ml-model-checksums ${model_checksum}" >> "$HOME/.config/codeql/config"
39+
40+
- name: Create test DB
41+
run: |
42+
DB_PATH="${RUNNER_TEMP}/db"
43+
echo "DB_PATH=${DB_PATH}" >> "${GITHUB_ENV}"
44+
45+
codeql database create "${DB_PATH}" --source-root config/atm --language javascript
46+
47+
- name: Run ATM query suite
48+
run: |
49+
SARIF_PATH="${RUNNER_TEMP}/sarif.json"
50+
echo "SARIF_PATH=${SARIF_PATH}" >> "${GITHUB_ENV}"
51+
52+
codeql database analyze \
53+
--format sarif-latest \
54+
--output "${SARIF_PATH}" \
55+
--sarif-group-rules-by-pack \
56+
-vv \
57+
-- \
58+
"${DB_PATH}" \
59+
"${QUERY_PACK}/${QUERY_SUITE}"
60+
61+
- name: Upload SARIF
62+
uses: actions/upload-artifact@v3
63+
with:
64+
name: javascript-ml-powered-queries.sarif
65+
path: "${{ env.SARIF_PATH }}"
66+
retention-days: 5
67+
68+
- name: Check results
69+
run: |
70+
# We should run at least the ML-powered queries in `expected_rules`.
71+
expected_rules="js/ml-powered/nosql-injection js/ml-powered/path-injection js/ml-powered/sql-injection js/ml-powered/xss"
72+
73+
for rule in ${expected_rules}; do
74+
found_rule=$(jq --arg rule "${rule}" '[.runs[0].tool.extensions[].rules | select(. != null) |
75+
flatten | .[].id] | any(. == $rule)' "${SARIF_PATH}")
76+
if [[ "${found_rule}" != "true" ]]; then
77+
echo "Expected SARIF output to contain rule '${rule}', but found no such rule."
78+
exit 1
79+
else
80+
echo "Found rule '${rule}'."
81+
fi
82+
done
83+
84+
# We should have at least one alert from an ML-powered query.
85+
num_alerts=$(jq '[.runs[0].results[] |
86+
select(.properties.score != null and (.rule.id | startswith("js/ml-powered/")))] | length' \
87+
"${SARIF_PATH}")
88+
if [[ "${num_alerts}" -eq 0 ]]; then
89+
echo "Expected to find at least one alert from an ML-powered query but found ${num_alerts}."
90+
exit 1
91+
else
92+
echo "Found ${num_alerts} alerts from ML-powered queries.";
93+
fi

‎.github/workflows/atm-check-queries-run.yml renamed to ‎.github/workflows/atm-model-integration-tests.yml

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,5 @@
1-
name: ATM Check Queries Run
1+
name: ATM Model Integration Tests
22

3-
# This check is required, therefore we must run it on all PRs, even if only Markdown has changed.
43
on:
54
workflow_dispatch:
65

‎.github/workflows/compile-queries.yml

Lines changed: 39 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,39 @@
1+
name: "Compile all queries using the latest stable CodeQL CLI"
2+
3+
on:
4+
push:
5+
branches: # makes sure the cache gets populated - running on the branches people tend to merge into.
6+
- main
7+
- "rc/*"
8+
- "codeql-cli-*"
9+
pull_request:
10+
11+
jobs:
12+
compile-queries:
13+
runs-on: ubuntu-latest-xl
14+
15+
steps:
16+
- uses: actions/checkout@v3
17+
- name: Setup CodeQL
18+
uses: ./.github/actions/fetch-codeql
19+
with:
20+
channel: 'release'
21+
- name: Cache compilation cache
22+
id: query-cache
23+
uses: ./.github/actions/cache-query-compilation
24+
with:
25+
key: all-queries
26+
- name: check formatting
27+
run: find */ql -type f \( -name "*.qll" -o -name "*.ql" \) -print0 | xargs -0 codeql query format --check-only
28+
- name: compile queries - check-only
29+
# run with --check-only if running in a PR (github.sha != main)
30+
if : ${{ github.event_name == 'pull_request' }}
31+
shell: bash
32+
run: codeql query compile -j0 */ql/{src,examples} --keep-going --warnings=error --check-only --compilation-cache "${{ steps.query-cache.outputs.cache-dir }}"
33+
- name: compile queries - full
34+
# do full compile if running on main - this populates the cache
35+
if : ${{ github.event_name != 'pull_request' }}
36+
shell: bash
37+
run: codeql query compile -j0 */ql/{src,examples} --keep-going --warnings=error --compilation-cache "${{ steps.query-cache.outputs.cache-dir }}"
38+
env:
39+
COMBINED_CACHE_DIR: ${{ github.workspace }}/compilation-dir

‎.github/workflows/csharp-qltest.yml

Lines changed: 72 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,72 @@
1+
name: "C#: Run QL Tests"
2+
3+
on:
4+
push:
5+
paths:
6+
- "csharp/**"
7+
- "shared/**"
8+
- .github/actions/fetch-codeql/action.yml
9+
- codeql-workspace.yml
10+
branches:
11+
- main
12+
- "rc/*"
13+
pull_request:
14+
paths:
15+
- "csharp/**"
16+
- "shared/**"
17+
- .github/workflows/csharp-qltest.yml
18+
- .github/actions/fetch-codeql/action.yml
19+
- codeql-workspace.yml
20+
branches:
21+
- main
22+
- "rc/*"
23+
24+
defaults:
25+
run:
26+
working-directory: csharp
27+
28+
jobs:
29+
qlupgrade:
30+
runs-on: ubuntu-latest
31+
steps:
32+
- uses: actions/checkout@v3
33+
- uses: ./.github/actions/fetch-codeql
34+
- name: Check DB upgrade scripts
35+
run: |
36+
echo >empty.trap
37+
codeql dataset import -S ql/lib/upgrades/initial/semmlecode.csharp.dbscheme testdb empty.trap
38+
codeql dataset upgrade testdb --additional-packs ql/lib
39+
diff -q testdb/semmlecode.csharp.dbscheme ql/lib/semmlecode.csharp.dbscheme
40+
- name: Check DB downgrade scripts
41+
run: |
42+
echo >empty.trap
43+
rm -rf testdb; codeql dataset import -S ql/lib/semmlecode.csharp.dbscheme testdb empty.trap
44+
codeql resolve upgrades --format=lines --allow-downgrades --additional-packs downgrades \
45+
--dbscheme=ql/lib/semmlecode.csharp.dbscheme --target-dbscheme=downgrades/initial/semmlecode.csharp.dbscheme |
46+
xargs codeql execute upgrades testdb
47+
diff -q testdb/semmlecode.csharp.dbscheme downgrades/initial/semmlecode.csharp.dbscheme
48+
qltest:
49+
runs-on: ubuntu-latest-xl
50+
strategy:
51+
fail-fast: false
52+
matrix:
53+
slice: ["1/2", "2/2"]
54+
steps:
55+
- uses: actions/checkout@v3
56+
- uses: ./.github/actions/fetch-codeql
57+
- uses: ./csharp/actions/create-extractor-pack
58+
- name: Cache compilation cache
59+
id: query-cache
60+
uses: ./.github/actions/cache-query-compilation
61+
with:
62+
key: csharp-qltest-${{ matrix.slice }}
63+
- name: Run QL tests
64+
run: |
65+
CODEQL_PATH=$(gh codeql version --format=json | jq -r .unpackedLocation)
66+
# The legacy ASP extractor is not in this repo, so take the one from the nightly build
67+
mv "$CODEQL_PATH/csharp/tools/extractor-asp.jar" "${{ github.workspace }}/csharp/extractor-pack/tools"
68+
# Safe guard against using the bundled extractor
69+
rm -rf "$CODEQL_PATH/csharp"
70+
codeql test run --threads=0 --ram 52000 --slice ${{ matrix.slice }} --search-path "${{ github.workspace }}/csharp/extractor-pack" --check-databases --check-undefined-labels --check-repeated-labels --check-redefined-labels --consistency-queries ql/consistency-queries ql/test --compilation-cache "${{ steps.query-cache.outputs.cache-dir }}"
71+
env:
72+
GITHUB_TOKEN: ${{ github.token }}

0 commit comments

Comments
 (0)