Skip to content

Commit af58329

Browse files
erik-krogherik-krogh
committed
split saving the compilation cache into a shared workflow
1 parent 78ad9ba commit af58329

File tree

2 files changed

+67
-39
lines changed

2 files changed

+67
-39
lines changed

‎.github/actions/cache-query-compilation/action.yml

Lines changed: 60 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,60 @@
1+
name: Cache query compilation
2+
description: Caches CodeQL compilation caches - should be run both on PRs and pushes to main.
3+
4+
inputs:
5+
key:
6+
description: 'The cache key to use - should be unique to the workflow'
7+
required: true
8+
9+
outputs:
10+
cache-dir:
11+
description: "The directory where the cache was stored"
12+
value: ${{ steps.fill-compilation-dir.outputs.compdir }}
13+
14+
runs:
15+
using: composite
16+
steps:
17+
# Cache the query compilation caches.
18+
# calculate the merge-base with main, in a way that works both on PRs and pushes to main.
19+
- name: Calculate merge-base
20+
shell: bash
21+
if: ${{ github.event_name == 'pull_request' }}
22+
env:
23+
BASE_BRANCH: ${{ github.base_ref }}
24+
run: |
25+
MERGE_BASE=$(git cat-file commit $GITHUB_SHA | grep '^parent ' | head -1 | cut -f 2 -d " ")
26+
echo "merge-base=$MERGE_BASE" >> $GITHUB_ENV
27+
- name: Read CodeQL query compilation - PR
28+
if: ${{ github.event_name == 'pull_request' }}
29+
uses: actions/cache@v3
30+
with:
31+
path: '**/.cache'
32+
key: codeql-compile-${{ inputs.key }}-pr-${{ github.sha }} # deliberately not using the `compile-compile-main` keys here.
33+
restore-keys: |
34+
codeql-compile-${{ inputs.key }}-${{ github.base_ref }}-${{ env.merge-base }}
35+
codeql-compile-${{ inputs.key }}-${{ github.base_ref }}-
36+
codeql-compile-${{ inputs.key }}-main-
37+
- name: Fill CodeQL query compilation cache - main
38+
if: ${{ github.event_name != 'pull_request' }}
39+
uses: actions/cache@v3
40+
with:
41+
path: '**/.cache'
42+
key: codeql-compile-${{ inputs.key }}-${{ github.ref_name }}-${{ github.sha }} # just fill on main
43+
restore-keys: | # restore from another random commit, to speed up compilation.
44+
codeql-compile-${{ inputs.key }}-${{ github.ref_name }}-
45+
codeql-compile-${{ inputs.key }}-main-
46+
- name: Fill compilation cache directory
47+
id: fill-compilation-dir
48+
shell: bash
49+
run: |
50+
# Move all the existing cache into another folder, so we only preserve the cache for the current queries.
51+
mkdir -p ${COMBINED_CACHE_DIR}
52+
rm -f **/.cache/{lock,size} # -f to avoid errors if the cache is empty.
53+
# copy the contents of the .cache folders into the combined cache folder.
54+
cp -r **/.cache/* ${COMBINED_CACHE_DIR}/ || : # ignore missing files
55+
# clean up the .cache folders
56+
rm -rf **/.cache/*
57+
58+
echo "compdir=${COMBINED_CACHE_DIR}" >> $GITHUB_OUTPUT
59+
env:
60+
COMBINED_CACHE_DIR: ${{ github.workspace }}/compilation-dir

‎.github/workflows/compile-queries.yml

Lines changed: 7 additions & 39 deletions
Original file line numberDiff line numberDiff line change
@@ -14,58 +14,26 @@ jobs:
1414

1515
steps:
1616
- uses: actions/checkout@v3
17-
# calculate the merge-base with main, in a way that works both on PRs and pushes to main.
18-
- name: Calculate merge-base
19-
if: ${{ github.event_name == 'pull_request' }}
20-
env:
21-
BASE_BRANCH: ${{ github.base_ref }}
22-
run: |
23-
MERGE_BASE=$(git cat-file commit $GITHUB_SHA | grep '^parent ' | head -1 | cut -f 2 -d " ")
24-
echo "merge-base=$MERGE_BASE" >> $GITHUB_ENV
25-
- name: Read CodeQL query compilation - PR
26-
if: ${{ github.event_name == 'pull_request' }}
27-
uses: actions/cache@v3
28-
with:
29-
path: '**/.cache'
30-
key: codeql-compile-pr-${{ github.sha }} # deliberately not using the `compile-compile-main` keys here.
31-
restore-keys: |
32-
codeql-compile-${{ github.base_ref }}-${{ env.merge-base }}
33-
codeql-compile-${{ github.base_ref }}-
34-
codeql-compile-main-
35-
- name: Fill CodeQL query compilation cache - main
36-
if: ${{ github.event_name != 'pull_request' }}
37-
uses: actions/cache@v3
38-
with:
39-
path: '**/.cache'
40-
key: codeql-compile-${{ github.ref_name }}-${{ github.sha }} # just fill on main
41-
restore-keys: | # restore from another random commit, to speed up compilation.
42-
codeql-compile-${{ github.ref_name }}-
43-
codeql-compile-main-
4417
- name: Setup CodeQL
4518
uses: ./.github/actions/fetch-codeql
4619
with:
4720
channel: 'release'
21+
- name: Cache compilation cache
22+
id: query-cache
23+
uses: ./.github/actions/cache-query-compilation
24+
with:
25+
key: all-queries
4826
- name: check formatting
4927
run: find */ql -type f \( -name "*.qll" -o -name "*.ql" \) -print0 | xargs -0 codeql query format --check-only
5028
- name: compile queries - check-only
5129
# run with --check-only if running in a PR (github.sha != main)
5230
if : ${{ github.event_name == 'pull_request' }}
5331
shell: bash
54-
run: codeql query compile -j0 */ql/{src,examples} --keep-going --warnings=error --check-only
32+
run: codeql query compile -j0 */ql/{src,examples} --keep-going --warnings=error --check-only --compilation-cache "${{ steps.query-cache.outputs.cache-dir }}"
5533
- name: compile queries - full
5634
# do full compile if running on main - this populates the cache
5735
if : ${{ github.event_name != 'pull_request' }}
5836
shell: bash
59-
run: |
60-
# Move all the existing cache into another folder, so we only preserve the cache for the current queries.
61-
mkdir -p ${COMBINED_CACHE_DIR}
62-
rm -f */ql/{src,examples}/.cache/{lock,size} # -f to avoid errors if the cache is empty.
63-
# copy the contents of the .cache folders into the combined cache folder.
64-
cp -r */ql/{src,examples}/.cache/* ${COMBINED_CACHE_DIR}/ || : # ignore missing files
65-
# clean up the .cache folders
66-
rm -rf */ql/{src,examples}/.cache/*
67-
68-
# compile the queries
69-
codeql query compile -j0 */ql/{src,examples} --keep-going --warnings=error --compilation-cache ${COMBINED_CACHE_DIR}
37+
run: codeql query compile -j0 */ql/{src,examples} --keep-going --warnings=error --compilation-cache "${{ steps.query-cache.outputs.cache-dir }}"
7038
env:
7139
COMBINED_CACHE_DIR: ${{ github.workspace }}/compilation-dir

0 commit comments

Comments
 (0)