Skip to content

Commit 0e73862

Browse files
atorralbaatorralba
authored
Merge branch 'main' into atorralba/promote-log-injection
2 parents 6613a98 + d912a98 commit 0e73862

File tree

2,629 files changed

+163497
-41472
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

2,629 files changed

+163497
-41472
lines changed

‎.codeqlmanifest.json

Lines changed: 28 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -1,11 +1,28 @@
1-
{ "provide": [ "ruby/.codeqlmanifest.json",
2-
"*/ql/src/qlpack.yml",
3-
"*/ql/lib/qlpack.yml",
4-
"*/ql/test/qlpack.yml",
5-
"cpp/ql/test/query-tests/Security/CWE/CWE-190/semmle/tainted/qlpack.yml",
6-
"*/ql/examples/qlpack.yml",
7-
"*/upgrades/qlpack.yml",
8-
"javascript/ql/experimental/adaptivethreatmodeling/lib/qlpack.yml",
9-
"javascript/ql/experimental/adaptivethreatmodeling/src/qlpack.yml",
10-
"misc/legacy-support/*/qlpack.yml",
11-
"misc/suite-helpers/qlpack.yml" ] }
1+
{
2+
"provide": [
3+
"*/ql/src/qlpack.yml",
4+
"*/ql/lib/qlpack.yml",
5+
"*/ql/test/qlpack.yml",
6+
"*/ql/examples/qlpack.yml",
7+
"*/upgrades/qlpack.yml",
8+
"cpp/ql/test/query-tests/Security/CWE/CWE-190/semmle/tainted/qlpack.yml",
9+
"javascript/ql/experimental/adaptivethreatmodeling/lib/qlpack.yml",
10+
"javascript/ql/experimental/adaptivethreatmodeling/src/qlpack.yml",
11+
"csharp/ql/campaigns/Solorigate/lib/qlpack.yml",
12+
"csharp/ql/campaigns/Solorigate/src/qlpack.yml",
13+
"csharp/ql/campaigns/Solorigate/test/qlpack.yml",
14+
"misc/legacy-support/*/qlpack.yml",
15+
"misc/suite-helpers/qlpack.yml",
16+
"ruby/extractor-pack/codeql-extractor.yml",
17+
"ruby/ql/consistency-queries/qlpack.yml",
18+
"ql/ql/consistency-queries/qlpack.yml",
19+
"ql/extractor-pack/codeql-extractor.yml"
20+
],
21+
"versionPolicies": {
22+
"default": {
23+
"requireChangeNotes": true,
24+
"committedPrereleaseSuffix": "dev",
25+
"committedVersion": "nextPatchRelease"
26+
}
27+
}
28+
}

‎.github/actions/fetch-codeql/action.yml

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -8,7 +8,7 @@ runs:
88
run: |
99
LATEST=$(gh release list --repo https://github.com/github/codeql-cli-binaries | cut -f 1 | grep -v beta | sort --version-sort | tail -1)
1010
gh release download --repo https://github.com/github/codeql-cli-binaries --pattern codeql-linux64.zip "$LATEST"
11-
unzip -q codeql-linux64.zip
12-
echo "${{ github.workspace }}/codeql" >> $GITHUB_PATH
11+
unzip -q -d "${RUNNER_TEMP}" codeql-linux64.zip
12+
echo "${RUNNER_TEMP}/codeql" >> "${GITHUB_PATH}"
1313
env:
1414
GITHUB_TOKEN: ${{ github.token }}

‎.github/labeler.yml

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -26,3 +26,6 @@ documentation:
2626
- "**/*.qhelp"
2727
- "**/*.md"
2828
- docs/**/*
29+
30+
"QL-for-QL":
31+
- ql/**/*

‎.github/workflows/check-change-note.yml

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,7 @@ on:
77
- "*/ql/src/**/*.ql"
88
- "*/ql/src/**/*.qll"
99
- "!**/experimental/**"
10+
- "!ql/**"
1011

1112
jobs:
1213
check-change-note:

‎.github/workflows/post-pr-comment.yml

Lines changed: 31 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,31 @@
1+
name: Post pull-request comment
2+
on:
3+
workflow_run:
4+
workflows: ["Query help preview"]
5+
types:
6+
- completed
7+
8+
permissions:
9+
pull-requests: write
10+
11+
jobs:
12+
post_comment:
13+
runs-on: ubuntu-latest
14+
steps:
15+
- name: Download artifact
16+
run: gh run download "${WORKFLOW_RUN_ID}" --repo "${GITHUB_REPOSITORY}" --name "comment"
17+
env:
18+
GITHUB_TOKEN: ${{ github.token }}
19+
WORKFLOW_RUN_ID: ${{ github.event.workflow_run.id }}
20+
- run: |
21+
PR="$(grep -o '^[0-9]\+$' pr.txt)"
22+
PR_HEAD_SHA="$(gh api "/repos/${GITHUB_REPOSITORY}/pulls/${PR}" --jq .head.sha)"
23+
# Check that the pull-request head SHA matches the head SHA of the workflow run
24+
if [ "${WORKFLOW_RUN_HEAD_SHA}" != "${PR_HEAD_SHA}" ]; then
25+
echo "PR head SHA ${PR_HEAD_SHA} does not match workflow_run event SHA ${WORKFLOW_RUN_HEAD_SHA}. Stopping." 1>&2
26+
exit 1
27+
fi
28+
gh pr comment "${PR}" --repo "${GITHUB_REPOSITORY}" -F comment.txt
29+
env:
30+
GITHUB_TOKEN: ${{ github.token }}
31+
WORKFLOW_RUN_HEAD_SHA: ${{ github.event.workflow_run.head_commit.id }}

‎.github/workflows/qhelp-pr-preview.yml

Lines changed: 38 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -1,39 +1,63 @@
11
name: Query help preview
22

3+
permissions:
4+
contents: read
5+
36
on:
47
pull_request:
58
branches:
69
- main
7-
- 'rc/*'
10+
- "rc/*"
811
paths:
912
- "ruby/**/*.qhelp"
1013

1114
jobs:
1215
qhelp:
1316
runs-on: ubuntu-latest
1417
steps:
18+
- run: echo "${{ github.event.number }}" > pr.txt
19+
- uses: actions/upload-artifact@v2
20+
with:
21+
name: comment
22+
path: pr.txt
23+
retention-days: 1
1524
- uses: actions/checkout@v2
1625
with:
1726
fetch-depth: 2
27+
persist-credentials: false
28+
- uses: ./.github/actions/fetch-codeql
1829
- name: Determine changed files
1930
id: changes
2031
run: |
21-
echo -n "::set-output name=qhelp_files::"
22-
(git diff --name-only --diff-filter=ACMRT HEAD~1 HEAD | grep .qhelp$ | grep -v .inc.qhelp;
23-
git diff --name-only --diff-filter=ACMRT HEAD~1 HEAD | grep .inc.qhelp$ | xargs -d '\n' -rn1 basename | xargs -d '\n' -rn1 git grep -l) |
24-
sort -u | xargs -d '\n' -n1 printf "'%s' "
25-
26-
- uses: ./.github/actions/fetch-codeql
32+
(git diff -z --name-only --diff-filter=ACMRT HEAD~1 HEAD | grep -z '.qhelp$' | grep -z -v '.inc.qhelp';
33+
git diff -z --name-only --diff-filter=ACMRT HEAD~1 HEAD | grep -z '.inc.qhelp$' | xargs --null -rn1 basename | xargs --null -rn1 git grep -z -l) |
34+
grep -z '.qhelp$' | grep -z -v '^-' | sort -z -u > "${RUNNER_TEMP}/paths.txt"
2735
2836
- name: QHelp preview
29-
if: ${{ steps.changes.outputs.qhelp_files }}
3037
run: |
31-
( echo "QHelp previews:";
32-
for path in ${{ steps.changes.outputs.qhelp_files }} ; do
38+
EXIT_CODE=0
39+
echo "QHelp previews:" > comment.txt
40+
while read -r -d $'\0' path; do
41+
if [ ! -f "${path}" ]; then
42+
exit 1
43+
fi
3344
echo "<details> <summary>${path}</summary>"
3445
echo
35-
codeql generate query-help --format=markdown ${path}
46+
codeql generate query-help --format=markdown -- "./${path}" 2> errors.txt || EXIT_CODE="$?"
47+
if [ -s errors.txt ]; then
48+
echo "# errors/warnings:"
49+
echo '```'
50+
cat errors.txt
51+
cat errors.txt 1>&2
52+
echo '```'
53+
fi
3654
echo "</details>"
37-
done) | gh pr comment "${{ github.event.pull_request.number }}" -F -
38-
env:
39-
GITHUB_TOKEN: ${{ github.token }}
55+
done < "${RUNNER_TEMP}/paths.txt" >> comment.txt
56+
exit "${EXIT_CODE}"
57+
58+
- if: always()
59+
uses: actions/upload-artifact@v2
60+
with:
61+
name: comment
62+
path: comment.txt
63+
retention-days: 1

‎.github/workflows/ql-for-ql-build.yml

Lines changed: 192 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,192 @@
1+
name: Run QL for QL
2+
3+
on:
4+
push:
5+
branches: [main]
6+
pull_request:
7+
branches: [main]
8+
9+
env:
10+
CARGO_TERM_COLOR: always
11+
12+
jobs:
13+
queries:
14+
runs-on: ubuntu-latest
15+
steps:
16+
- uses: actions/checkout@v2
17+
- name: Find codeql
18+
id: find-codeql
19+
uses: github/codeql-action/init@erik-krogh/ql
20+
with:
21+
languages: javascript # does not matter
22+
- name: Get CodeQL version
23+
id: get-codeql-version
24+
run: |
25+
echo "::set-output name=version::$("${CODEQL}" --version | head -n 1 | rev | cut -d " " -f 1 | rev)"
26+
shell: bash
27+
env:
28+
CODEQL: ${{ steps.find-codeql.outputs.codeql-path }}
29+
- name: Cache queries
30+
id: cache-queries
31+
uses: actions/cache@v2
32+
with:
33+
path: ${{ runner.temp }}/query-pack.zip
34+
key: queries-${{ hashFiles('ql/**/*.ql*') }}-${{ hashFiles('ql/ql/src/ql.dbscheme*') }}-${{ steps.get-codeql-version.outputs.version }}
35+
- name: Build query pack
36+
if: steps.cache-queries.outputs.cache-hit != 'true'
37+
run: |
38+
cd ql/ql/src
39+
"${CODEQL}" pack create
40+
cd .codeql/pack/codeql/ql-all/0.0.0
41+
zip "${PACKZIP}" -r .
42+
env:
43+
CODEQL: ${{ steps.find-codeql.outputs.codeql-path }}
44+
PACKZIP: ${{ runner.temp }}/query-pack.zip
45+
- name: Upload query pack
46+
uses: actions/upload-artifact@v2
47+
with:
48+
name: query-pack-zip
49+
path: ${{ runner.temp }}/query-pack.zip
50+
51+
extractors:
52+
strategy:
53+
fail-fast: false
54+
55+
runs-on: ubuntu-latest
56+
57+
steps:
58+
- uses: actions/checkout@v2
59+
- name: Cache entire extractor
60+
id: cache-extractor
61+
uses: actions/cache@v2
62+
with:
63+
path: |
64+
ql/target/release/ql-autobuilder
65+
ql/target/release/ql-autobuilder.exe
66+
ql/target/release/ql-extractor
67+
ql/target/release/ql-extractor.exe
68+
key: ${{ runner.os }}-extractor-${{ hashFiles('ql/**/Cargo.lock') }}-${{ hashFiles('ql/**/*.rs') }}
69+
- name: Cache cargo
70+
if: steps.cache-extractor.outputs.cache-hit != 'true'
71+
uses: actions/cache@v2
72+
with:
73+
path: |
74+
~/.cargo/registry
75+
~/.cargo/git
76+
ql/target
77+
key: ${{ runner.os }}-rust-cargo-${{ hashFiles('ql/**/Cargo.lock') }}
78+
- name: Check formatting
79+
if: steps.cache-extractor.outputs.cache-hit != 'true'
80+
run: cd ql; cargo fmt --all -- --check
81+
- name: Build
82+
if: steps.cache-extractor.outputs.cache-hit != 'true'
83+
run: cd ql; cargo build --verbose
84+
- name: Run tests
85+
if: steps.cache-extractor.outputs.cache-hit != 'true'
86+
run: cd ql; cargo test --verbose
87+
- name: Release build
88+
if: steps.cache-extractor.outputs.cache-hit != 'true'
89+
run: cd ql; cargo build --release
90+
- name: Generate dbscheme
91+
if: steps.cache-extractor.outputs.cache-hit != 'true'
92+
run: ql/target/release/ql-generator --dbscheme ql/ql/src/ql.dbscheme --library ql/ql/src/codeql_ql/ast/internal/TreeSitter.qll
93+
- uses: actions/upload-artifact@v2
94+
with:
95+
name: extractor-ubuntu-latest
96+
path: |
97+
ql/target/release/ql-autobuilder
98+
ql/target/release/ql-autobuilder.exe
99+
ql/target/release/ql-extractor
100+
ql/target/release/ql-extractor.exe
101+
retention-days: 1
102+
package:
103+
runs-on: ubuntu-latest
104+
105+
needs:
106+
- extractors
107+
- queries
108+
109+
steps:
110+
- uses: actions/checkout@v2
111+
- uses: actions/download-artifact@v2
112+
with:
113+
name: query-pack-zip
114+
path: query-pack-zip
115+
- uses: actions/download-artifact@v2
116+
with:
117+
name: extractor-ubuntu-latest
118+
path: linux64
119+
- run: |
120+
unzip query-pack-zip/*.zip -d pack
121+
cp -r ql/codeql-extractor.yml ql/tools ql/ql/src/ql.dbscheme.stats pack/
122+
mkdir -p pack/tools/linux64
123+
if [[ -f linux64/ql-autobuilder ]]; then
124+
cp linux64/ql-autobuilder pack/tools/linux64/autobuilder
125+
chmod +x pack/tools/linux64/autobuilder
126+
fi
127+
if [[ -f linux64/ql-extractor ]]; then
128+
cp linux64/ql-extractor pack/tools/linux64/extractor
129+
chmod +x pack/tools/linux64/extractor
130+
fi
131+
cd pack
132+
zip -rq ../codeql-ql.zip .
133+
- uses: actions/upload-artifact@v2
134+
with:
135+
name: codeql-ql-pack
136+
path: codeql-ql.zip
137+
retention-days: 1
138+
analyze:
139+
runs-on: ubuntu-latest
140+
strategy:
141+
matrix:
142+
folder: [cpp, csharp, java, javascript, python, ql, ruby]
143+
144+
needs:
145+
- package
146+
147+
steps:
148+
- name: Download pack
149+
uses: actions/download-artifact@v2
150+
with:
151+
name: codeql-ql-pack
152+
path: ${{ runner.temp }}/codeql-ql-pack-artifact
153+
154+
- name: Prepare pack
155+
run: |
156+
unzip "${PACK_ARTIFACT}/*.zip" -d "${PACK}"
157+
env:
158+
PACK_ARTIFACT: ${{ runner.temp }}/codeql-ql-pack-artifact
159+
PACK: ${{ runner.temp }}/pack
160+
- name: Hack codeql-action options
161+
run: |
162+
JSON=$(jq -nc --arg pack "${PACK}" '.resolve.queries=["--search-path", $pack] | .resolve.extractor=["--search-path", $pack] | .database.init=["--search-path", $pack]')
163+
echo "CODEQL_ACTION_EXTRA_OPTIONS=${JSON}" >> ${GITHUB_ENV}
164+
env:
165+
PACK: ${{ runner.temp }}/pack
166+
167+
- name: Checkout repository
168+
uses: actions/checkout@v2
169+
- name: Create CodeQL config file
170+
run: |
171+
echo "paths:" > ${CONF}
172+
echo " - ${FOLDER}" >> ${CONF}
173+
echo "paths-ignore:" >> ${CONF}
174+
echo " - ql/ql/test" >> ${CONF}
175+
echo "Config file: "
176+
cat ${CONF}
177+
env:
178+
CONF: ./ql-for-ql-config.yml
179+
FOLDER: ${{ matrix.folder }}
180+
181+
- name: Initialize CodeQL
182+
uses: github/codeql-action/init@erik-krogh/ql
183+
with:
184+
languages: ql
185+
db-location: ${{ runner.temp }}/db
186+
config-file: ./ql-for-ql-config.yml
187+
188+
- name: Perform CodeQL Analysis
189+
uses: github/codeql-action/analyze@erik-krogh/ql
190+
with:
191+
category: "ql-for-ql-${{ matrix.folder }}"
192+

0 commit comments

Comments
 (0)