Skip to content

Commit 37c72ca

Browse files
MathiasVPMathiasVP
committed
Merge branch 'main' into promote-arithmetic-uncontrolled
2 parents 5a38f81 + fdb3cd0 commit 37c72ca

File tree

1,309 files changed

+75504
-6493
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

1,309 files changed

+75504
-6493
lines changed

‎.codeqlmanifest.json

Lines changed: 12 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -10,7 +10,16 @@
1010
"javascript/ql/experimental/adaptivethreatmodeling/src/qlpack.yml",
1111
"misc/legacy-support/*/qlpack.yml",
1212
"misc/suite-helpers/qlpack.yml",
13+
"ruby/extractor-pack/codeql-extractor.yml",
1314
"ruby/ql/consistency-queries/qlpack.yml",
14-
"ruby/extractor-pack/codeql-extractor.yml"
15-
]
16-
}
15+
"ql/ql/consistency-queries/qlpack.yml",
16+
"ql/extractor-pack/codeql-extractor.yml"
17+
],
18+
"versionPolicies": {
19+
"default": {
20+
"requireChangeNotes": true,
21+
"committedPrereleaseSuffix": "dev",
22+
"committedVersion": "nextPatchRelease"
23+
}
24+
}
25+
}

‎.github/labeler.yml

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -26,3 +26,6 @@ documentation:
2626
- "**/*.qhelp"
2727
- "**/*.md"
2828
- docs/**/*
29+
30+
"QL-for-QL":
31+
- ql/**/*

‎.github/workflows/check-change-note.yml

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,7 @@ on:
77
- "*/ql/src/**/*.ql"
88
- "*/ql/src/**/*.qll"
99
- "!**/experimental/**"
10+
- "!ql/**"
1011

1112
jobs:
1213
check-change-note:

‎.github/workflows/ql-for-ql-build.yml

Lines changed: 192 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,192 @@
1+
name: Run QL for QL
2+
3+
on:
4+
push:
5+
branches: [main]
6+
pull_request:
7+
branches: [main]
8+
9+
env:
10+
CARGO_TERM_COLOR: always
11+
12+
jobs:
13+
queries:
14+
runs-on: ubuntu-latest
15+
steps:
16+
- uses: actions/checkout@v2
17+
- name: Find codeql
18+
id: find-codeql
19+
uses: github/codeql-action/init@erik-krogh/ql
20+
with:
21+
languages: javascript # does not matter
22+
- name: Get CodeQL version
23+
id: get-codeql-version
24+
run: |
25+
echo "::set-output name=version::$("${CODEQL}" --version | head -n 1 | rev | cut -d " " -f 1 | rev)"
26+
shell: bash
27+
env:
28+
CODEQL: ${{ steps.find-codeql.outputs.codeql-path }}
29+
- name: Cache queries
30+
id: cache-queries
31+
uses: actions/cache@v2
32+
with:
33+
path: ${{ runner.temp }}/query-pack.zip
34+
key: queries-${{ hashFiles('ql/**/*.ql*') }}-${{ hashFiles('ql/ql/src/ql.dbscheme*') }}-${{ steps.get-codeql-version.outputs.version }}
35+
- name: Build query pack
36+
if: steps.cache-queries.outputs.cache-hit != 'true'
37+
run: |
38+
cd ql/ql/src
39+
"${CODEQL}" pack create
40+
cd .codeql/pack/codeql/ql-all/0.0.0
41+
zip "${PACKZIP}" -r .
42+
env:
43+
CODEQL: ${{ steps.find-codeql.outputs.codeql-path }}
44+
PACKZIP: ${{ runner.temp }}/query-pack.zip
45+
- name: Upload query pack
46+
uses: actions/upload-artifact@v2
47+
with:
48+
name: query-pack-zip
49+
path: ${{ runner.temp }}/query-pack.zip
50+
51+
extractors:
52+
strategy:
53+
fail-fast: false
54+
55+
runs-on: ubuntu-latest
56+
57+
steps:
58+
- uses: actions/checkout@v2
59+
- name: Cache entire extractor
60+
id: cache-extractor
61+
uses: actions/cache@v2
62+
with:
63+
path: |
64+
ql/target/release/ql-autobuilder
65+
ql/target/release/ql-autobuilder.exe
66+
ql/target/release/ql-extractor
67+
ql/target/release/ql-extractor.exe
68+
key: ${{ runner.os }}-extractor-${{ hashFiles('ql/**/Cargo.lock') }}-${{ hashFiles('ql/**/*.rs') }}
69+
- name: Cache cargo
70+
if: steps.cache-extractor.outputs.cache-hit != 'true'
71+
uses: actions/cache@v2
72+
with:
73+
path: |
74+
~/.cargo/registry
75+
~/.cargo/git
76+
ql/target
77+
key: ${{ runner.os }}-rust-cargo-${{ hashFiles('ql/**/Cargo.lock') }}
78+
- name: Check formatting
79+
if: steps.cache-extractor.outputs.cache-hit != 'true'
80+
run: cd ql; cargo fmt --all -- --check
81+
- name: Build
82+
if: steps.cache-extractor.outputs.cache-hit != 'true'
83+
run: cd ql; cargo build --verbose
84+
- name: Run tests
85+
if: steps.cache-extractor.outputs.cache-hit != 'true'
86+
run: cd ql; cargo test --verbose
87+
- name: Release build
88+
if: steps.cache-extractor.outputs.cache-hit != 'true'
89+
run: cd ql; cargo build --release
90+
- name: Generate dbscheme
91+
if: steps.cache-extractor.outputs.cache-hit != 'true'
92+
run: ql/target/release/ql-generator --dbscheme ql/ql/src/ql.dbscheme --library ql/ql/src/codeql_ql/ast/internal/TreeSitter.qll
93+
- uses: actions/upload-artifact@v2
94+
with:
95+
name: extractor-ubuntu-latest
96+
path: |
97+
ql/target/release/ql-autobuilder
98+
ql/target/release/ql-autobuilder.exe
99+
ql/target/release/ql-extractor
100+
ql/target/release/ql-extractor.exe
101+
retention-days: 1
102+
package:
103+
runs-on: ubuntu-latest
104+
105+
needs:
106+
- extractors
107+
- queries
108+
109+
steps:
110+
- uses: actions/checkout@v2
111+
- uses: actions/download-artifact@v2
112+
with:
113+
name: query-pack-zip
114+
path: query-pack-zip
115+
- uses: actions/download-artifact@v2
116+
with:
117+
name: extractor-ubuntu-latest
118+
path: linux64
119+
- run: |
120+
unzip query-pack-zip/*.zip -d pack
121+
cp -r ql/codeql-extractor.yml ql/tools ql/ql/src/ql.dbscheme.stats pack/
122+
mkdir -p pack/tools/linux64
123+
if [[ -f linux64/ql-autobuilder ]]; then
124+
cp linux64/ql-autobuilder pack/tools/linux64/autobuilder
125+
chmod +x pack/tools/linux64/autobuilder
126+
fi
127+
if [[ -f linux64/ql-extractor ]]; then
128+
cp linux64/ql-extractor pack/tools/linux64/extractor
129+
chmod +x pack/tools/linux64/extractor
130+
fi
131+
cd pack
132+
zip -rq ../codeql-ql.zip .
133+
- uses: actions/upload-artifact@v2
134+
with:
135+
name: codeql-ql-pack
136+
path: codeql-ql.zip
137+
retention-days: 1
138+
analyze:
139+
runs-on: ubuntu-latest
140+
strategy:
141+
matrix:
142+
folder: [cpp, csharp, java, javascript, python, ql, ruby]
143+
144+
needs:
145+
- package
146+
147+
steps:
148+
- name: Download pack
149+
uses: actions/download-artifact@v2
150+
with:
151+
name: codeql-ql-pack
152+
path: ${{ runner.temp }}/codeql-ql-pack-artifact
153+
154+
- name: Prepare pack
155+
run: |
156+
unzip "${PACK_ARTIFACT}/*.zip" -d "${PACK}"
157+
env:
158+
PACK_ARTIFACT: ${{ runner.temp }}/codeql-ql-pack-artifact
159+
PACK: ${{ runner.temp }}/pack
160+
- name: Hack codeql-action options
161+
run: |
162+
JSON=$(jq -nc --arg pack "${PACK}" '.resolve.queries=["--search-path", $pack] | .resolve.extractor=["--search-path", $pack] | .database.init=["--search-path", $pack]')
163+
echo "CODEQL_ACTION_EXTRA_OPTIONS=${JSON}" >> ${GITHUB_ENV}
164+
env:
165+
PACK: ${{ runner.temp }}/pack
166+
167+
- name: Checkout repository
168+
uses: actions/checkout@v2
169+
- name: Create CodeQL config file
170+
run: |
171+
echo "paths:" > ${CONF}
172+
echo " - ${FOLDER}" >> ${CONF}
173+
echo "paths-ignore:" >> ${CONF}
174+
echo " - ql/ql/test" >> ${CONF}
175+
echo "Config file: "
176+
cat ${CONF}
177+
env:
178+
CONF: ./ql-for-ql-config.yml
179+
FOLDER: ${{ matrix.folder }}
180+
181+
- name: Initialize CodeQL
182+
uses: github/codeql-action/init@erik-krogh/ql
183+
with:
184+
languages: ql
185+
db-location: ${{ runner.temp }}/db
186+
config-file: ./ql-for-ql-config.yml
187+
188+
- name: Perform CodeQL Analysis
189+
uses: github/codeql-action/analyze@erik-krogh/ql
190+
with:
191+
category: "ql-for-ql-${{ matrix.folder }}"
192+

‎.github/workflows/ql-for-ql-dataset_measure.yml

Lines changed: 84 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,84 @@
1+
name: Collect database stats for QL for QL
2+
3+
on:
4+
push:
5+
branches: [main]
6+
paths:
7+
- ql/ql/src/ql.dbscheme
8+
pull_request:
9+
branches: [main]
10+
paths:
11+
- ql/ql/src/ql.dbscheme
12+
workflow_dispatch:
13+
14+
jobs:
15+
measure:
16+
env:
17+
CODEQL_THREADS: 4 # TODO: remove this once it's set by the CLI
18+
strategy:
19+
matrix:
20+
repo:
21+
- github/codeql
22+
- github/codeql-go
23+
runs-on: ubuntu-latest
24+
steps:
25+
- uses: actions/checkout@v2
26+
27+
- name: Find codeql
28+
id: find-codeql
29+
uses: github/codeql-action/init@erik-krogh/ql
30+
with:
31+
languages: javascript # does not matter
32+
- uses: actions/cache@v2
33+
with:
34+
path: |
35+
~/.cargo/registry
36+
~/.cargo/git
37+
ql/target
38+
key: ${{ runner.os }}-qltest-cargo-${{ hashFiles('**/Cargo.lock') }}
39+
- name: Build Extractor
40+
run: cd ql; env "PATH=$PATH:`dirname ${CODEQL}`" ./create-extractor-pack.sh
41+
env:
42+
CODEQL: ${{ steps.find-codeql.outputs.codeql-path }}
43+
- name: Checkout ${{ matrix.repo }}
44+
uses: actions/checkout@v2
45+
with:
46+
repository: ${{ matrix.repo }}
47+
path: ${{ github.workspace }}/repo
48+
- name: Create database
49+
run: |
50+
"${CODEQL}" database create \
51+
--search-path "ql/extractor-pack" \
52+
--threads 4 \
53+
--language ql --source-root "${{ github.workspace }}/repo" \
54+
"${{ runner.temp }}/database"
55+
env:
56+
CODEQL: ${{ steps.find-codeql.outputs.codeql-path }}
57+
- name: Measure database
58+
run: |
59+
mkdir -p "stats/${{ matrix.repo }}"
60+
"${CODEQL}" dataset measure --threads 4 --output "stats/${{ matrix.repo }}/stats.xml" "${{ runner.temp }}/database/db-ql"
61+
env:
62+
CODEQL: ${{ steps.find-codeql.outputs.codeql-path }}
63+
- uses: actions/upload-artifact@v2
64+
with:
65+
name: measurements
66+
path: stats
67+
retention-days: 1
68+
69+
merge:
70+
runs-on: ubuntu-latest
71+
needs: measure
72+
steps:
73+
- uses: actions/checkout@v2
74+
- uses: actions/download-artifact@v2
75+
with:
76+
name: measurements
77+
path: stats
78+
- run: |
79+
python -m pip install --user lxml
80+
find stats -name 'stats.xml' -print0 | sort -z | xargs -0 python ql/scripts/merge_stats.py --output ql/ql/src/ql.dbscheme.stats --normalise ql_tokeninfo
81+
- uses: actions/upload-artifact@v2
82+
with:
83+
name: ql.dbscheme.stats
84+
path: ql/ql/src/ql.dbscheme.stats

‎.github/workflows/ql-for-ql-tests.yml

Lines changed: 52 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,52 @@
1+
name: Run QL for QL Tests
2+
3+
on:
4+
push:
5+
branches: [main]
6+
paths:
7+
- "ql/**"
8+
pull_request:
9+
branches: [main]
10+
paths:
11+
- "ql/**"
12+
13+
env:
14+
CARGO_TERM_COLOR: always
15+
16+
jobs:
17+
qltest:
18+
runs-on: ubuntu-latest
19+
steps:
20+
- uses: actions/checkout@v2
21+
- name: Find codeql
22+
id: find-codeql
23+
uses: github/codeql-action/init@erik-krogh/ql
24+
with:
25+
languages: javascript # does not matter
26+
- uses: actions/cache@v2
27+
with:
28+
path: |
29+
~/.cargo/registry
30+
~/.cargo/git
31+
ql/target
32+
key: ${{ runner.os }}-qltest-cargo-${{ hashFiles('**/Cargo.lock') }}
33+
- name: Build extractor
34+
run: |
35+
cd ql;
36+
codeqlpath=$(dirname ${{ steps.find-codeql.outputs.codeql-path }});
37+
env "PATH=$PATH:$codeqlpath" ./create-extractor-pack.sh
38+
- name: Run QL tests
39+
run: |
40+
"${CODEQL}" test run --check-databases --check-unused-labels --check-repeated-labels --check-redefined-labels --check-use-before-definition --search-path "${{ github.workspace }}/ql/extractor-pack" --consistency-queries ql/ql/consistency-queries ql/ql/test
41+
env:
42+
CODEQL: ${{ steps.find-codeql.outputs.codeql-path }}
43+
- name: Check QL formatting
44+
run: |
45+
find ql/ql "(" -name "*.ql" -or -name "*.qll" ")" -print0 | xargs -0 "${CODEQL}" query format --check-only
46+
env:
47+
CODEQL: ${{ steps.find-codeql.outputs.codeql-path }}
48+
- name: Check QL compilation
49+
run: |
50+
"${CODEQL}" query compile --check-only --threads=4 --warnings=error --search-path "${{ github.workspace }}/ql/extractor-pack" "ql/ql/src" "ql/ql/examples"
51+
env:
52+
CODEQL: ${{ steps.find-codeql.outputs.codeql-path }}

‎CODEOWNERS

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -25,3 +25,6 @@
2525
/docs/codeql-for-visual-studio-code/ @github/codeql-vscode-reviewers
2626
/docs/ql-language-reference/ @github/codeql-frontend-reviewers
2727
/docs/query-*-style-guide.md @github/codeql-analysis-reviewers
28+
29+
# QL for QL reviewers
30+
/ql/ @erik-krogh @tausbn

‎config/identical-files.json

Lines changed: 8 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -452,9 +452,15 @@
452452
"ruby/ql/lib/codeql/ruby/dataflow/internal/SsaImplCommon.qll",
453453
"cpp/ql/lib/semmle/code/cpp/ir/dataflow/internal/SsaImplCommon.qll"
454454
],
455-
"CryptoAlgorithms Python/JS": [
455+
"CryptoAlgorithms Python/JS/Ruby": [
456456
"javascript/ql/lib/semmle/javascript/security/CryptoAlgorithms.qll",
457-
"python/ql/lib/semmle/python/concepts/CryptoAlgorithms.qll"
457+
"python/ql/lib/semmle/python/concepts/CryptoAlgorithms.qll",
458+
"ruby/ql/lib/codeql/ruby/security/CryptoAlgorithms.qll"
459+
],
460+
"CryptoAlgorithmNames Python/JS/Ruby": [
461+
"javascript/ql/lib/semmle/javascript/security/internal/CryptoAlgorithmNames.qll",
462+
"python/ql/lib/semmle/python/concepts/internal/CryptoAlgorithmNames.qll",
463+
"ruby/ql/lib/codeql/ruby/security/internal/CryptoAlgorithmNames.qll"
458464
],
459465
"SensitiveDataHeuristics Python/JS": [
460466
"javascript/ql/lib/semmle/javascript/security/internal/SensitiveDataHeuristics.qll",

0 commit comments

Comments
 (0)