Skip to content

Commit 545ad01

Browse files
grddevgrddev
committed
Merge remote-tracking branch 'origin/main' into accelerate-go-list-option
2 parents 256d854 + 28bf0c9 commit 545ad01

File tree

929 files changed

+42477
-16277
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

929 files changed

+42477
-16277
lines changed

‎.github/ISSUE_TEMPLATE/lgtm-com---false-positive.md

Lines changed: 0 additions & 24 deletions
This file was deleted.

‎.github/ISSUE_TEMPLATE/ql--false-positive.md

Lines changed: 36 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,36 @@
1+
---
2+
name: CodeQL false positive
3+
about: Report CodeQL alerts that you think should not have been detected (not applicable, not exploitable, etc.)
4+
title: False positive
5+
labels: false-positive
6+
assignees: ''
7+
8+
---
9+
10+
**Description of the false positive**
11+
12+
<!-- Please explain briefly why you think it shouldn't be included. -->
13+
14+
**Code samples or links to source code**
15+
16+
<!--
17+
For open source code: file links with line numbers on GitHub, for example:
18+
https://github.com/github/codeql/blob/dc440aaee6695deb0d9676b87e06ea984e1b4ae5/javascript/ql/test/query-tests/Security/CWE-078/CommandInjection/exec-sh2.js#L10
19+
20+
For closed source code: (redacted) code samples that illustrate the problem, for example:
21+
22+
```
23+
function execSh(command, options) {
24+
return cp.spawn(getShell(), ["-c", command], options) // <- command line injection
25+
};
26+
```
27+
-->
28+
29+
**URL to the alert on GitHub code scanning (optional)**
30+
31+
<!--
32+
1. Open the project on GitHub.com.
33+
2. Switch to the `Security` tab.
34+
3. Browse to the alert that you would like to report.
35+
4. Copy and paste the page URL here.
36+
-->

‎.github/actions/cache-query-compilation/action.yml

Lines changed: 61 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,61 @@
1+
name: Cache query compilation
2+
description: Caches CodeQL compilation caches - should be run both on PRs and pushes to main.
3+
4+
inputs:
5+
key:
6+
description: 'The cache key to use - should be unique to the workflow'
7+
required: true
8+
9+
outputs:
10+
cache-dir:
11+
description: "The directory where the cache was stored"
12+
value: ${{ steps.fill-compilation-dir.outputs.compdir }}
13+
14+
runs:
15+
using: composite
16+
steps:
17+
# Cache the query compilation caches.
18+
# calculate the merge-base with main, in a way that works both on PRs and pushes to main.
19+
- name: Calculate merge-base
20+
shell: bash
21+
if: ${{ github.event_name == 'pull_request' }}
22+
env:
23+
BASE_BRANCH: ${{ github.base_ref }}
24+
run: |
25+
MERGE_BASE=$(git cat-file commit $GITHUB_SHA | grep '^parent ' | head -1 | cut -f 2 -d " ")
26+
echo "merge_base=$MERGE_BASE" >> $GITHUB_ENV
27+
- name: Read CodeQL query compilation - PR
28+
if: ${{ github.event_name == 'pull_request' }}
29+
uses: erik-krogh/actions-cache@a88d0603fe5fb5606db9f002dfcadeb32b5f84c6
30+
with:
31+
path: '**/.cache'
32+
read-only: true
33+
key: codeql-compile-${{ inputs.key }}-pr-${{ github.sha }} # deliberately not using the `compile-compile-main` keys here.
34+
restore-keys: |
35+
codeql-compile-${{ inputs.key }}-${{ github.base_ref }}-${{ env.merge_base }}
36+
codeql-compile-${{ inputs.key }}-${{ github.base_ref }}-
37+
codeql-compile-${{ inputs.key }}-main-
38+
- name: Fill CodeQL query compilation cache - main
39+
if: ${{ github.event_name != 'pull_request' }}
40+
uses: erik-krogh/actions-cache@a88d0603fe5fb5606db9f002dfcadeb32b5f84c6
41+
with:
42+
path: '**/.cache'
43+
key: codeql-compile-${{ inputs.key }}-${{ github.ref_name }}-${{ github.sha }} # just fill on main
44+
restore-keys: | # restore from another random commit, to speed up compilation.
45+
codeql-compile-${{ inputs.key }}-${{ github.ref_name }}-
46+
codeql-compile-${{ inputs.key }}-main-
47+
- name: Fill compilation cache directory
48+
id: fill-compilation-dir
49+
shell: bash
50+
run: |
51+
# Move all the existing cache into another folder, so we only preserve the cache for the current queries.
52+
mkdir -p ${COMBINED_CACHE_DIR}
53+
rm -f **/.cache/{lock,size} # -f to avoid errors if the cache is empty.
54+
# copy the contents of the .cache folders into the combined cache folder.
55+
cp -r **/.cache/* ${COMBINED_CACHE_DIR}/ || : # ignore missing files
56+
# clean up the .cache folders
57+
rm -rf **/.cache/*
58+
59+
echo "compdir=${COMBINED_CACHE_DIR}" >> $GITHUB_OUTPUT
60+
env:
61+
COMBINED_CACHE_DIR: ${{ github.workspace }}/compilation-dir

‎.github/workflows/compile-queries.yml

Lines changed: 9 additions & 29 deletions
Original file line numberDiff line numberDiff line change
@@ -14,46 +14,26 @@ jobs:
1414

1515
steps:
1616
- uses: actions/checkout@v3
17-
# calculate the merge-base with main, in a way that works both on PRs and pushes to main.
18-
- name: Calculate merge-base
19-
if: ${{ github.event_name == 'pull_request' }}
20-
env:
21-
BASE_BRANCH: ${{ github.base_ref }}
22-
run: |
23-
MERGE_BASE=$(git cat-file commit $GITHUB_SHA | grep '^parent ' | head -1 | cut -f 2 -d " ")
24-
echo "merge-base=$MERGE_BASE" >> $GITHUB_ENV
25-
- name: Read CodeQL query compilation - PR
26-
if: ${{ github.event_name == 'pull_request' }}
27-
uses: actions/cache@v3
28-
with:
29-
path: '*/ql/src/.cache'
30-
key: codeql-compile-pr-${{ github.sha }} # deliberately not using the `compile-compile-main` keys here.
31-
restore-keys: |
32-
codeql-compile-${{ github.base_ref }}-${{ env.merge-base }}
33-
codeql-compile-${{ github.base_ref }}-
34-
codeql-compile-main-
35-
- name: Fill CodeQL query compilation cache - main
36-
if: ${{ github.event_name != 'pull_request' }}
37-
uses: actions/cache@v3
38-
with:
39-
path: '*/ql/src/.cache'
40-
key: codeql-compile-${{ github.ref_name }}-${{ github.sha }} # just fill on main
41-
restore-keys: | # restore from another random commit, to speed up compilation.
42-
codeql-compile-${{ github.ref_name }}-
43-
codeql-compile-main-
4417
- name: Setup CodeQL
4518
uses: ./.github/actions/fetch-codeql
4619
with:
4720
channel: 'release'
21+
- name: Cache compilation cache
22+
id: query-cache
23+
uses: ./.github/actions/cache-query-compilation
24+
with:
25+
key: all-queries
4826
- name: check formatting
4927
run: find */ql -type f \( -name "*.qll" -o -name "*.ql" \) -print0 | xargs -0 codeql query format --check-only
5028
- name: compile queries - check-only
5129
# run with --check-only if running in a PR (github.sha != main)
5230
if : ${{ github.event_name == 'pull_request' }}
5331
shell: bash
54-
run: codeql query compile -j0 */ql/src --keep-going --warnings=error --check-only
32+
run: codeql query compile -j0 */ql/{src,examples} --keep-going --warnings=error --check-only --compilation-cache "${{ steps.query-cache.outputs.cache-dir }}"
5533
- name: compile queries - full
5634
# do full compile if running on main - this populates the cache
5735
if : ${{ github.event_name != 'pull_request' }}
5836
shell: bash
59-
run: codeql query compile -j0 */ql/src --keep-going --warnings=error
37+
run: codeql query compile -j0 */ql/{src,examples} --keep-going --warnings=error --compilation-cache "${{ steps.query-cache.outputs.cache-dir }}"
38+
env:
39+
COMBINED_CACHE_DIR: ${{ github.workspace }}/compilation-dir

‎.github/workflows/csharp-qltest.yml

Lines changed: 86 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,86 @@
1+
name: "C#: Run QL Tests"
2+
3+
on:
4+
push:
5+
paths:
6+
- "csharp/**"
7+
- "shared/**"
8+
- .github/actions/fetch-codeql/action.yml
9+
- codeql-workspace.yml
10+
branches:
11+
- main
12+
- "rc/*"
13+
pull_request:
14+
paths:
15+
- "csharp/**"
16+
- "shared/**"
17+
- .github/workflows/csharp-qltest.yml
18+
- .github/actions/fetch-codeql/action.yml
19+
- codeql-workspace.yml
20+
branches:
21+
- main
22+
- "rc/*"
23+
24+
defaults:
25+
run:
26+
working-directory: csharp
27+
28+
jobs:
29+
qlupgrade:
30+
runs-on: ubuntu-latest
31+
steps:
32+
- uses: actions/checkout@v3
33+
- uses: ./.github/actions/fetch-codeql
34+
- name: Check DB upgrade scripts
35+
run: |
36+
echo >empty.trap
37+
codeql dataset import -S ql/lib/upgrades/initial/semmlecode.csharp.dbscheme testdb empty.trap
38+
codeql dataset upgrade testdb --additional-packs ql/lib
39+
diff -q testdb/semmlecode.csharp.dbscheme ql/lib/semmlecode.csharp.dbscheme
40+
- name: Check DB downgrade scripts
41+
run: |
42+
echo >empty.trap
43+
rm -rf testdb; codeql dataset import -S ql/lib/semmlecode.csharp.dbscheme testdb empty.trap
44+
codeql resolve upgrades --format=lines --allow-downgrades --additional-packs downgrades \
45+
--dbscheme=ql/lib/semmlecode.csharp.dbscheme --target-dbscheme=downgrades/initial/semmlecode.csharp.dbscheme |
46+
xargs codeql execute upgrades testdb
47+
diff -q testdb/semmlecode.csharp.dbscheme downgrades/initial/semmlecode.csharp.dbscheme
48+
qltest:
49+
runs-on: ubuntu-latest-xl
50+
strategy:
51+
fail-fast: false
52+
matrix:
53+
slice: ["1/2", "2/2"]
54+
steps:
55+
- uses: actions/checkout@v3
56+
- uses: ./.github/actions/fetch-codeql
57+
- uses: ./csharp/actions/create-extractor-pack
58+
- name: Cache compilation cache
59+
id: query-cache
60+
uses: ./.github/actions/cache-query-compilation
61+
with:
62+
key: csharp-qltest-${{ matrix.slice }}
63+
- name: Run QL tests
64+
run: |
65+
CODEQL_PATH=$(gh codeql version --format=json | jq -r .unpackedLocation)
66+
# The legacy ASP extractor is not in this repo, so take the one from the nightly build
67+
mv "$CODEQL_PATH/csharp/tools/extractor-asp.jar" "${{ github.workspace }}/csharp/extractor-pack/tools"
68+
# Safe guard against using the bundled extractor
69+
rm -rf "$CODEQL_PATH/csharp"
70+
codeql test run --threads=0 --ram 52000 --slice ${{ matrix.slice }} --search-path "${{ github.workspace }}/csharp/extractor-pack" --check-databases --check-undefined-labels --check-repeated-labels --check-redefined-labels --consistency-queries ql/consistency-queries ql/test --compilation-cache "${{ steps.query-cache.outputs.cache-dir }}"
71+
env:
72+
GITHUB_TOKEN: ${{ github.token }}
73+
unit-tests:
74+
runs-on: ubuntu-latest
75+
steps:
76+
- uses: actions/checkout@v3
77+
- name: Setup dotnet
78+
uses: actions/setup-dotnet@v3
79+
with:
80+
dotnet-version: 6.0.202
81+
- name: Extractor unit tests
82+
run: |
83+
dotnet test -p:RuntimeFrameworkVersion=6.0.4 "${{ github.workspace }}/csharp/extractor/Semmle.Util.Tests"
84+
dotnet test -p:RuntimeFrameworkVersion=6.0.4 "${{ github.workspace }}/csharp/extractor/Semmle.Extraction.Tests"
85+
dotnet test -p:RuntimeFrameworkVersion=6.0.4 "${{ github.workspace }}/csharp/autobuilder/Semmle.Autobuild.CSharp.Tests"
86+
dotnet test -p:RuntimeFrameworkVersion=6.0.4 "${{ github.workspace }}/cpp/autobuilder/Semmle.Autobuild.Cpp.Tests"

‎.github/workflows/go-tests-other-os.yml

Lines changed: 81 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,81 @@
1+
name: "Go: Run Tests - Other OS"
2+
on:
3+
pull_request:
4+
paths:
5+
- "go/**"
6+
- "!go/ql/**" # don't run other-os if only ql/ files changed
7+
- .github/workflows/go-tests-other-os.yml
8+
- .github/actions/fetch-codeql/action.yml
9+
- .github/actions/cache-query-compilation/action.yml
10+
- codeql-workspace.yml
11+
jobs:
12+
test-mac:
13+
name: Test MacOS
14+
runs-on: macos-latest
15+
steps:
16+
- name: Set up Go 1.19
17+
uses: actions/setup-go@v3
18+
with:
19+
go-version: 1.19
20+
id: go
21+
22+
- name: Check out code
23+
uses: actions/checkout@v2
24+
25+
- name: Set up CodeQL CLI
26+
uses: ./.github/actions/fetch-codeql
27+
28+
- name: Enable problem matchers in repository
29+
shell: bash
30+
run: 'find .github/problem-matchers -name \*.json -exec echo "::add-matcher::{}" \;'
31+
32+
- name: Build
33+
run: |
34+
cd go
35+
make
36+
37+
- name: Cache compilation cache
38+
id: query-cache
39+
uses: ./.github/actions/cache-query-compilation
40+
with:
41+
key: go-qltest
42+
- name: Test
43+
run: |
44+
cd go
45+
make test cache="${{ steps.query-cache.outputs.cache-dir }}"
46+
47+
test-win:
48+
name: Test Windows
49+
runs-on: windows-latest-xl
50+
steps:
51+
- name: Set up Go 1.19
52+
uses: actions/setup-go@v3
53+
with:
54+
go-version: 1.19
55+
id: go
56+
57+
- name: Check out code
58+
uses: actions/checkout@v2
59+
60+
- name: Set up CodeQL CLI
61+
uses: ./.github/actions/fetch-codeql
62+
63+
- name: Enable problem matchers in repository
64+
shell: bash
65+
run: 'find .github/problem-matchers -name \*.json -exec echo "::add-matcher::{}" \;'
66+
67+
- name: Build
68+
run: |
69+
cd go
70+
make
71+
72+
- name: Cache compilation cache
73+
id: query-cache
74+
uses: ./.github/actions/cache-query-compilation
75+
with:
76+
key: go-qltest
77+
78+
- name: Test
79+
run: |
80+
cd go
81+
make test cache="${{ steps.query-cache.outputs.cache-dir }}"

0 commit comments

Comments
 (0)