Skip to content

Commit 56e5f01

Browse files
henrymercerhenrymercer
committed
Merge branch 'main' into codeql-ci/atm/release-0.4.2
2 parents 78d49e4 + adf905d commit 56e5f01

File tree

1,559 files changed

+129174
-81874
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

1,559 files changed

+129174
-81874
lines changed

‎.github/ISSUE_TEMPLATE/lgtm-com---false-positive.md

Lines changed: 0 additions & 24 deletions
This file was deleted.

‎.github/ISSUE_TEMPLATE/ql--false-positive.md

Lines changed: 36 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,36 @@
1+
---
2+
name: CodeQL False positive
3+
about: Report CodeQL alerts that you think should not have been detected (not applicable, not exploitable, etc.)
4+
title: False positive
5+
labels: false-positive
6+
assignees: ''
7+
8+
---
9+
10+
**Description of the false positive**
11+
12+
<!-- Please explain briefly why you think it shouldn't be included. -->
13+
14+
**Code samples or links to source code**
15+
16+
<!--
17+
For open source code: file links with line numbers on GitHub, for example:
18+
https://github.com/github/codeql/blob/dc440aaee6695deb0d9676b87e06ea984e1b4ae5/javascript/ql/test/query-tests/Security/CWE-078/CommandInjection/exec-sh2.js#L10
19+
20+
For closed source code: (redacted) code samples that illustrate the problem, for example:
21+
22+
```
23+
function execSh(command, options) {
24+
return cp.spawn(getShell(), ["-c", command], options) // <- command line injection
25+
};
26+
```
27+
-->
28+
29+
**URL to the alert on GitHub code scanning (optional)**
30+
31+
<!--
32+
1. Open the project on GitHub.com.
33+
2. Switch to the `Security` tab.
34+
3. Browse to the alert that you would like to report.
35+
4. Copy and paste the page URL here.
36+
-->

‎.github/actions/cache-query-compilation/action.yml

Lines changed: 61 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,61 @@
1+
name: Cache query compilation
2+
description: Caches CodeQL compilation caches - should be run both on PRs and pushes to main.
3+
4+
inputs:
5+
key:
6+
description: 'The cache key to use - should be unique to the workflow'
7+
required: true
8+
9+
outputs:
10+
cache-dir:
11+
description: "The directory where the cache was stored"
12+
value: ${{ steps.fill-compilation-dir.outputs.compdir }}
13+
14+
runs:
15+
using: composite
16+
steps:
17+
# Cache the query compilation caches.
18+
# calculate the merge-base with main, in a way that works both on PRs and pushes to main.
19+
- name: Calculate merge-base
20+
shell: bash
21+
if: ${{ github.event_name == 'pull_request' }}
22+
env:
23+
BASE_BRANCH: ${{ github.base_ref }}
24+
run: |
25+
MERGE_BASE=$(git cat-file commit $GITHUB_SHA | grep '^parent ' | head -1 | cut -f 2 -d " ")
26+
echo "merge_base=$MERGE_BASE" >> $GITHUB_ENV
27+
- name: Read CodeQL query compilation - PR
28+
if: ${{ github.event_name == 'pull_request' }}
29+
uses: erik-krogh/actions-cache@a88d0603fe5fb5606db9f002dfcadeb32b5f84c6
30+
with:
31+
path: '**/.cache'
32+
read-only: true
33+
key: codeql-compile-${{ inputs.key }}-pr-${{ github.sha }} # deliberately not using the `compile-compile-main` keys here.
34+
restore-keys: |
35+
codeql-compile-${{ inputs.key }}-${{ github.base_ref }}-${{ env.merge_base }}
36+
codeql-compile-${{ inputs.key }}-${{ github.base_ref }}-
37+
codeql-compile-${{ inputs.key }}-main-
38+
- name: Fill CodeQL query compilation cache - main
39+
if: ${{ github.event_name != 'pull_request' }}
40+
uses: erik-krogh/actions-cache@a88d0603fe5fb5606db9f002dfcadeb32b5f84c6
41+
with:
42+
path: '**/.cache'
43+
key: codeql-compile-${{ inputs.key }}-${{ github.ref_name }}-${{ github.sha }} # just fill on main
44+
restore-keys: | # restore from another random commit, to speed up compilation.
45+
codeql-compile-${{ inputs.key }}-${{ github.ref_name }}-
46+
codeql-compile-${{ inputs.key }}-main-
47+
- name: Fill compilation cache directory
48+
id: fill-compilation-dir
49+
shell: bash
50+
run: |
51+
# Move all the existing cache into another folder, so we only preserve the cache for the current queries.
52+
mkdir -p ${COMBINED_CACHE_DIR}
53+
rm -f **/.cache/{lock,size} # -f to avoid errors if the cache is empty.
54+
# copy the contents of the .cache folders into the combined cache folder.
55+
cp -r **/.cache/* ${COMBINED_CACHE_DIR}/ || : # ignore missing files
56+
# clean up the .cache folders
57+
rm -rf **/.cache/*
58+
59+
echo "compdir=${COMBINED_CACHE_DIR}" >> $GITHUB_OUTPUT
60+
env:
61+
COMBINED_CACHE_DIR: ${{ github.workspace }}/compilation-dir

‎.github/workflows/compile-queries.yml

Lines changed: 13 additions & 31 deletions
Original file line numberDiff line numberDiff line change
@@ -2,56 +2,38 @@ name: "Compile all queries using the latest stable CodeQL CLI"
22

33
on:
44
push:
5-
branches: [main] # makes sure the cache gets populated
6-
pull_request:
7-
branches:
5+
branches: # makes sure the cache gets populated - running on the branches people tend to merge into.
86
- main
97
- "rc/*"
8+
- "codeql-cli-*"
9+
pull_request:
1010

1111
jobs:
1212
compile-queries:
1313
runs-on: ubuntu-latest-xl
1414

1515
steps:
1616
- uses: actions/checkout@v3
17-
with:
18-
fetch-depth: 0
19-
# calculate the merge-base with main, in a way that works both on PRs and pushes to main.
20-
- name: Calculate merge-base
21-
if: ${{ github.event_name == 'pull_request' }}
22-
env:
23-
BASE_BRANCH: ${{ github.base_ref }}
24-
run: |
25-
MERGE_BASE=$(git merge-base --fork-point origin/$BASE_BRANCH)
26-
echo "merge-base=$MERGE_BASE" >> $GITHUB_ENV
27-
- name: Calculate merge-base - branch
28-
if: ${{ github.event_name != 'pull_request' }}
29-
# using github.sha instead, since we're directly on a branch, and not in a PR
30-
run: |
31-
MERGE_BASE=${{ github.sha }}
32-
echo "merge-base=$MERGE_BASE" >> $GITHUB_ENV
33-
- name: Cache CodeQL query compilation
34-
uses: actions/cache@v3
35-
with:
36-
path: '*/ql/src/.cache'
37-
# current GH HEAD first, merge-base second, generic third
38-
key: codeql-stable-compile-${{ github.sha }}
39-
restore-keys: |
40-
codeql-stable-compile-${{ env.merge-base }}
41-
codeql-stable-compile-
4217
- name: Setup CodeQL
4318
uses: ./.github/actions/fetch-codeql
4419
with:
4520
channel: 'release'
21+
- name: Cache compilation cache
22+
id: query-cache
23+
uses: ./.github/actions/cache-query-compilation
24+
with:
25+
key: all-queries
4626
- name: check formatting
47-
run: codeql query format */ql/{src,lib,test}/**/*.{qll,ql} --check-only
27+
run: find */ql -type f \( -name "*.qll" -o -name "*.ql" \) -print0 | xargs -0 codeql query format --check-only
4828
- name: compile queries - check-only
4929
# run with --check-only if running in a PR (github.sha != main)
5030
if : ${{ github.event_name == 'pull_request' }}
5131
shell: bash
52-
run: codeql query compile -j0 */ql/src --keep-going --warnings=error --check-only
32+
run: codeql query compile -j0 */ql/{src,examples} --keep-going --warnings=error --check-only --compilation-cache "${{ steps.query-cache.outputs.cache-dir }}"
5333
- name: compile queries - full
5434
# do full compile if running on main - this populates the cache
5535
if : ${{ github.event_name != 'pull_request' }}
5636
shell: bash
57-
run: codeql query compile -j0 */ql/src --keep-going --warnings=error
37+
run: codeql query compile -j0 */ql/{src,examples} --keep-going --warnings=error --compilation-cache "${{ steps.query-cache.outputs.cache-dir }}"
38+
env:
39+
COMBINED_CACHE_DIR: ${{ github.workspace }}/compilation-dir

‎.github/workflows/csharp-qltest.yml

Lines changed: 86 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,86 @@
1+
name: "C#: Run QL Tests"
2+
3+
on:
4+
push:
5+
paths:
6+
- "csharp/**"
7+
- "shared/**"
8+
- .github/actions/fetch-codeql/action.yml
9+
- codeql-workspace.yml
10+
branches:
11+
- main
12+
- "rc/*"
13+
pull_request:
14+
paths:
15+
- "csharp/**"
16+
- "shared/**"
17+
- .github/workflows/csharp-qltest.yml
18+
- .github/actions/fetch-codeql/action.yml
19+
- codeql-workspace.yml
20+
branches:
21+
- main
22+
- "rc/*"
23+
24+
defaults:
25+
run:
26+
working-directory: csharp
27+
28+
jobs:
29+
qlupgrade:
30+
runs-on: ubuntu-latest
31+
steps:
32+
- uses: actions/checkout@v3
33+
- uses: ./.github/actions/fetch-codeql
34+
- name: Check DB upgrade scripts
35+
run: |
36+
echo >empty.trap
37+
codeql dataset import -S ql/lib/upgrades/initial/semmlecode.csharp.dbscheme testdb empty.trap
38+
codeql dataset upgrade testdb --additional-packs ql/lib
39+
diff -q testdb/semmlecode.csharp.dbscheme ql/lib/semmlecode.csharp.dbscheme
40+
- name: Check DB downgrade scripts
41+
run: |
42+
echo >empty.trap
43+
rm -rf testdb; codeql dataset import -S ql/lib/semmlecode.csharp.dbscheme testdb empty.trap
44+
codeql resolve upgrades --format=lines --allow-downgrades --additional-packs downgrades \
45+
--dbscheme=ql/lib/semmlecode.csharp.dbscheme --target-dbscheme=downgrades/initial/semmlecode.csharp.dbscheme |
46+
xargs codeql execute upgrades testdb
47+
diff -q testdb/semmlecode.csharp.dbscheme downgrades/initial/semmlecode.csharp.dbscheme
48+
qltest:
49+
runs-on: ubuntu-latest-xl
50+
strategy:
51+
fail-fast: false
52+
matrix:
53+
slice: ["1/2", "2/2"]
54+
steps:
55+
- uses: actions/checkout@v3
56+
- uses: ./.github/actions/fetch-codeql
57+
- uses: ./csharp/actions/create-extractor-pack
58+
- name: Cache compilation cache
59+
id: query-cache
60+
uses: ./.github/actions/cache-query-compilation
61+
with:
62+
key: csharp-qltest-${{ matrix.slice }}
63+
- name: Run QL tests
64+
run: |
65+
CODEQL_PATH=$(gh codeql version --format=json | jq -r .unpackedLocation)
66+
# The legacy ASP extractor is not in this repo, so take the one from the nightly build
67+
mv "$CODEQL_PATH/csharp/tools/extractor-asp.jar" "${{ github.workspace }}/csharp/extractor-pack/tools"
68+
# Safe guard against using the bundled extractor
69+
rm -rf "$CODEQL_PATH/csharp"
70+
codeql test run --threads=0 --ram 52000 --slice ${{ matrix.slice }} --search-path "${{ github.workspace }}/csharp/extractor-pack" --check-databases --check-undefined-labels --check-repeated-labels --check-redefined-labels --consistency-queries ql/consistency-queries ql/test --compilation-cache "${{ steps.query-cache.outputs.cache-dir }}"
71+
env:
72+
GITHUB_TOKEN: ${{ github.token }}
73+
unit-tests:
74+
runs-on: ubuntu-latest
75+
steps:
76+
- uses: actions/checkout@v3
77+
- name: Setup dotnet
78+
uses: actions/setup-dotnet@v3
79+
with:
80+
dotnet-version: 6.0.202
81+
- name: Extractor unit tests
82+
run: |
83+
dotnet test -p:RuntimeFrameworkVersion=6.0.4 "${{ github.workspace }}/csharp/extractor/Semmle.Util.Tests"
84+
dotnet test -p:RuntimeFrameworkVersion=6.0.4 "${{ github.workspace }}/csharp/extractor/Semmle.Extraction.Tests"
85+
dotnet test -p:RuntimeFrameworkVersion=6.0.4 "${{ github.workspace }}/csharp/autobuilder/Semmle.Autobuild.CSharp.Tests"
86+
dotnet test -p:RuntimeFrameworkVersion=6.0.4 "${{ github.workspace }}/cpp/autobuilder/Semmle.Autobuild.Cpp.Tests"

‎.github/workflows/js-ml-tests.yml

Lines changed: 0 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -23,19 +23,6 @@ defaults:
2323
working-directory: javascript/ql/experimental/adaptivethreatmodeling
2424

2525
jobs:
26-
qlformat:
27-
name: Check QL formatting
28-
runs-on: ubuntu-latest
29-
steps:
30-
- uses: actions/checkout@v3
31-
32-
- uses: ./.github/actions/fetch-codeql
33-
34-
- name: Check QL formatting
35-
run: |
36-
find . "(" -name "*.ql" -or -name "*.qll" ")" -print0 | \
37-
xargs -0 codeql query format --check-only
38-
3926
qlcompile:
4027
name: Check QL compilation
4128
runs-on: ubuntu-latest

‎.github/workflows/ql-for-ql-build.yml

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -24,13 +24,13 @@ jobs:
2424
- uses: actions/checkout@v3
2525
- name: Find codeql
2626
id: find-codeql
27-
uses: github/codeql-action/init@71a8b35ff4c80fcfcd05bc1cd932fe3c08f943ca
27+
uses: github/codeql-action/init@77a8d2d10c0b403a8b4aadbd223dc489ecd22683
2828
with:
2929
languages: javascript # does not matter
3030
- name: Get CodeQL version
3131
id: get-codeql-version
3232
run: |
33-
echo "::set-output name=version::$("${CODEQL}" --version | head -n 1 | rev | cut -d " " -f 1 | rev)"
33+
echo "version=$("${CODEQL}" --version | head -n 1 | rev | cut -d " " -f 1 | rev)" >> $GITHUB_OUTPUT
3434
shell: bash
3535
env:
3636
CODEQL: ${{ steps.find-codeql.outputs.codeql-path }}
@@ -133,7 +133,7 @@ jobs:
133133
env:
134134
CONF: ./ql-for-ql-config.yml
135135
- name: Initialize CodeQL
136-
uses: github/codeql-action/init@71a8b35ff4c80fcfcd05bc1cd932fe3c08f943ca
136+
uses: github/codeql-action/init@77a8d2d10c0b403a8b4aadbd223dc489ecd22683
137137
with:
138138
languages: ql
139139
db-location: ${{ runner.temp }}/db
@@ -145,7 +145,7 @@ jobs:
145145
PACK: ${{ runner.temp }}/pack
146146

147147
- name: Perform CodeQL Analysis
148-
uses: github/codeql-action/analyze@71a8b35ff4c80fcfcd05bc1cd932fe3c08f943ca
148+
uses: github/codeql-action/analyze@77a8d2d10c0b403a8b4aadbd223dc489ecd22683
149149
with:
150150
category: "ql-for-ql"
151151
- name: Copy sarif file to CWD

‎.github/workflows/ql-for-ql-dataset_measure.yml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -25,7 +25,7 @@ jobs:
2525

2626
- name: Find codeql
2727
id: find-codeql
28-
uses: github/codeql-action/init@71a8b35ff4c80fcfcd05bc1cd932fe3c08f943ca
28+
uses: github/codeql-action/init@77a8d2d10c0b403a8b4aadbd223dc489ecd22683
2929
with:
3030
languages: javascript # does not matter
3131
- uses: actions/cache@v3

‎.github/workflows/ql-for-ql-tests.yml

Lines changed: 1 addition & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -22,7 +22,7 @@ jobs:
2222
- uses: actions/checkout@v3
2323
- name: Find codeql
2424
id: find-codeql
25-
uses: github/codeql-action/init@71a8b35ff4c80fcfcd05bc1cd932fe3c08f943ca
25+
uses: github/codeql-action/init@77a8d2d10c0b403a8b4aadbd223dc489ecd22683
2626
with:
2727
languages: javascript # does not matter
2828
- uses: actions/cache@v3
@@ -47,8 +47,3 @@ jobs:
4747
find ql/ql/src "(" -name "*.ql" -or -name "*.qll" ")" -print0 | xargs -0 "${CODEQL}" query format --check-only
4848
env:
4949
CODEQL: ${{ steps.find-codeql.outputs.codeql-path }}
50-
- name: Check QL compilation
51-
run: |
52-
"${CODEQL}" query compile --check-only --threads=4 --warnings=error --search-path "${{ github.workspace }}/ql/extractor-pack" "ql/ql/src" "ql/ql/examples"
53-
env:
54-
CODEQL: ${{ steps.find-codeql.outputs.codeql-path }}

0 commit comments

Comments
 (0)