Skip to content

Commit a5ee89d

Browse files
blackheavenfrasertweedale
blackheaven
authored and
frasertweedale
committed
fix(ci): populate cache when it expired
1 parent 7f345c2 commit a5ee89d

File tree

5 files changed

+187
-172
lines changed

5 files changed

+187
-172
lines changed

‎.github/workflows/call-check-advisories.yml

Lines changed: 94 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,94 @@
1+
name: Check and publish security advisories
2+
on:
3+
workflow_call:
4+
inputs:
5+
fetch-key:
6+
required: true
7+
type: string
8+
is-artifact:
9+
required: true
10+
type: boolean
11+
changed-advisories:
12+
required: false
13+
type: string
14+
default: '[]'
15+
jobs:
16+
check-advisories:
17+
runs-on: ubuntu-20.04
18+
steps:
19+
- uses: actions/checkout@v3
20+
with:
21+
path: source
22+
# We need to retrieve full history to determine the correct
23+
# `published` and `modified` timestamps
24+
fetch-depth: 0
25+
- run: mkdir -p ~/.local/dockerImages
26+
- name: Fetch artifact
27+
if: ${{ inputs.is-artifact }}
28+
uses: actions/download-artifact@v3
29+
with:
30+
name: ${{ inputs.fetch-key }}
31+
path: ~/.local/dockerImages
32+
- name: Fetch cache
33+
id: fetch-binaries
34+
if: ${{ !inputs.is-artifact }}
35+
uses: actions/cache/restore@v3
36+
with:
37+
key: ${{ inputs.fetch-key }}
38+
path: ~/.local/dockerImages
39+
- name: Populate cache on cache miss
40+
if: ${{ !inputs.is-artifact && steps.fetch-binaries.outputs.cache-hit != 'true' }}
41+
uses: ./.github/workflows/call-nix.yml
42+
- name: Fetch cache (second attempt after cache miss)
43+
if: ${{ !inputs.is-artifact && steps.fetch-binaries.outputs.cache-hit != 'true' }}
44+
uses: actions/cache/restore@v3
45+
with:
46+
key: ${{ inputs.fetch-key }}
47+
path: ~/.local/dockerImages
48+
fail-on-cache-miss: true
49+
- run: docker load -i ~/.local/dockerImages/hsec-tools
50+
- name: Run advisory syntax checks
51+
env:
52+
CHANGED_ADVISORIES_JSON: ${{ inputs.changed-advisories }}
53+
run: |
54+
CHANGED_ADVISORIES=( $(printenv CHANGED_ADVISORIES_JSON | jq -r '.[]') )
55+
cd source
56+
RESULT=0
57+
# Remove the begining of the README to extract the example.
58+
(echo '```toml'; sed -e '1,/```toml/d' README.md) > EXAMPLE_README.md
59+
while read FILE ; do
60+
[ "$(dirname "$FILE")" != advisories/reserved ] || continue
61+
echo -n "$FILE: "
62+
docker run --rm -v $PWD:/repo --workdir /repo haskell/hsec-tools:latest /bin/hsec-tools check "$FILE" || RESULT=1
63+
done < <([ ${#CHANGED_ADVISORIES[@]} -gt 0 ] && printf "%s\n" "${CHANGED_ADVISORIES[@]}" || find advisories EXAMPLE_README.md EXAMPLE_ADVISORY.md -type f -name "*.md")
64+
exit $RESULT
65+
- name: Run advisory uniqueness checks
66+
run: |
67+
! find source/advisories -type f -name '*.md' -print0 \
68+
| xargs -0n1 basename | sort | uniq -c | grep -E -v '[[:space:]]*1 '
69+
- name: Publish OSV data
70+
if: ${{ github.event_name == 'push' && github.ref_name == 'main' && github.repository == 'haskell/security-advisories' }}
71+
env:
72+
GITHUB_SHA: ${{ github.sha }}
73+
run: |
74+
DATA_DIR=$PWD/osv
75+
mkdir "$DATA_DIR"
76+
cd source
77+
while read FILE ; do
78+
ID=$(basename "$FILE" .md)
79+
YEAR=$(echo "$ID" | cut -d - -f 2)
80+
mkdir -p $DATA_DIR/$YEAR
81+
docker run --rm -v $PWD:/repo --workdir /repo haskell/hsec-tools:latest /bin/hsec-tools osv "$FILE" > $DATA_DIR/$YEAR/$ID.json
82+
done < <(find advisories -type f -name "*.md")
83+
BRANCH=generated/osv-export
84+
REF=refs/remotes/origin/$BRANCH
85+
export GIT_WORK_TREE=$DATA_DIR
86+
git read-tree "$REF"
87+
git add --all --intent-to-add
88+
git diff --quiet && exit
89+
git add --all
90+
TREE=$(git write-tree)
91+
git config user.email [email protected]
92+
git config user.name "Haskell Security Response Team"
93+
COMMIT=$(git commit-tree "$TREE" -p "$REF" -m "$(date --utc --rfc-3339=seconds) ($GITHUB_SHA)")
94+
git push origin $COMMIT:$BRANCH

‎.github/workflows/call-nix.yml

Lines changed: 38 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,38 @@
1+
name: nix build
2+
on:
3+
workflow_call:
4+
jobs:
5+
check_nix:
6+
name: Check nix build
7+
runs-on: ubuntu-22.04
8+
steps:
9+
- name: git checkout
10+
uses: actions/checkout@v3
11+
- name: Install Nix
12+
uses: DeterminateSystems/nix-installer-action@main
13+
with:
14+
extra-conf: system-features = nixos-test benchmark big-parallel kvm
15+
- uses: DeterminateSystems/magic-nix-cache-action@main
16+
- name: Check Nix flake inputs
17+
uses: DeterminateSystems/flake-checker-action@v4
18+
- name: Build executable
19+
run: nix -L build
20+
- name: Build docker image
21+
run: nix build -L '.#packages.x86_64-linux.hsec-tools-image'
22+
- run: mkdir -p ~/.local/dockerImages
23+
- run: cp result ~/.local/dockerImages/hsec-tools
24+
- id: code-hash
25+
name: Compute code directory hash
26+
run: |
27+
code_hash=$(git rev-parse HEAD:code)
28+
echo "code-hash=$code_hash" >> "$GITHUB_OUTPUT"
29+
- uses: actions/cache/save@v3
30+
if: ${{ github.event_name == 'push' && github.ref_name == 'main' }}
31+
with:
32+
key: hsec-tools-${{ steps.code-hash.outputs.code-hash}}
33+
path: ~/.local/dockerImages
34+
- name: upload executable
35+
uses: actions/upload-artifact@v3
36+
with:
37+
name: hsec-tools-${{ github.sha }}
38+
path: ~/.local/dockerImages

‎.github/workflows/check-advisories-standalone.yml

Lines changed: 0 additions & 61 deletions
This file was deleted.

‎.github/workflows/check-advisories.yml

Lines changed: 53 additions & 76 deletions
Original file line numberDiff line numberDiff line change
@@ -1,84 +1,61 @@
1-
name: Check and publish security advisories
1+
name: Check advisories
22
on:
3-
workflow_call:
4-
inputs:
5-
fetch-key:
6-
required: true
7-
type: string
8-
is-artifact:
9-
required: true
10-
type: boolean
11-
changed-advisories:
12-
required: false
13-
type: string
14-
default: '[]'
3+
- pull_request
154
jobs:
16-
check-advisories:
17-
runs-on: ubuntu-20.04
5+
tools_changed:
6+
continue-on-error: true
7+
runs-on: ubuntu-22.04
8+
outputs:
9+
should_skip: ${{ steps.skip_check.outputs.should_skip }}
1810
steps:
19-
- uses: actions/checkout@v3
11+
- id: skip_check
12+
uses: fkirc/[email protected]
2013
with:
21-
path: source
22-
# We need to retrieve full history to determine the correct
23-
# `published` and `modified` timestamps
24-
fetch-depth: 0
25-
- run: mkdir -p ~/.local/dockerImages
26-
- name: Fetch artifact
27-
if: ${{ inputs.is-artifact }}
28-
uses: actions/download-artifact@v3
29-
with:
30-
name: ${{ inputs.fetch-key }}
31-
path: ~/.local/dockerImages
32-
- name: Fetch cache
33-
if: ${{ !inputs.is-artifact }}
34-
uses: actions/cache/restore@v3
14+
concurrent_skipping: "never"
15+
skip_after_successful_duplicate: "true"
16+
paths: '["code/**"]'
17+
do_not_skip: '["push", "workflow_dispatch", "schedule"]'
18+
advisories_changed:
19+
continue-on-error: true
20+
runs-on: ubuntu-22.04
21+
outputs:
22+
should_skip: ${{ steps.skip_check.outputs.should_skip }}
23+
changed_files: ${{ steps.process-changed-files.outputs.out }}
24+
steps:
25+
- id: skip_check
26+
uses: fkirc/[email protected]
3527
with:
36-
key: ${{ inputs.fetch-key }}
37-
path: ~/.local/dockerImages
38-
fail-on-cache-miss: true
39-
- run: docker load -i ~/.local/dockerImages/hsec-tools
40-
- name: Run advisory syntax checks
28+
concurrent_skipping: "never"
29+
skip_after_successful_duplicate: "true"
30+
paths: '["advisories/**", "EXAMPLE_ADVISORY.md"]'
31+
do_not_skip: '["push", "workflow_dispatch", "schedule"]'
32+
- id: process-changed-files
33+
name: Extract matched files list
4134
env:
42-
CHANGED_ADVISORIES_JSON: ${{ inputs.changed-advisories }}
43-
run: |
44-
CHANGED_ADVISORIES=( $(printenv CHANGED_ADVISORIES_JSON | jq -r '.[]') )
45-
cd source
46-
RESULT=0
47-
# Remove the begining of the README to extract the example.
48-
(echo '```toml'; sed -e '1,/```toml/d' README.md) > EXAMPLE_README.md
49-
while read FILE ; do
50-
[ "$(dirname "$FILE")" != advisories/reserved ] || continue
51-
echo -n "$FILE: "
52-
docker run --rm -v $PWD:/repo --workdir /repo haskell/hsec-tools:latest /bin/hsec-tools check "$FILE" || RESULT=1
53-
done < <([ ${#CHANGED_ADVISORIES[@]} -gt 0 ] && printf "%s\n" "${CHANGED_ADVISORIES[@]}" || find advisories EXAMPLE_README.md EXAMPLE_ADVISORY.md -type f -name "*.md")
54-
exit $RESULT
55-
- name: Run advisory uniqueness checks
35+
PATHS_RESULT: ${{ steps.skip_check.outputs.paths_result }}
5636
run: |
57-
! find source/advisories -type f -name '*.md' -print0 \
58-
| xargs -0n1 basename | sort | uniq -c | grep -E -v '[[:space:]]*1 '
59-
- name: Publish OSV data
60-
if: ${{ github.event_name == 'push' && github.ref_name == 'main' && github.repository == 'haskell/security-advisories' }}
61-
env:
62-
GITHUB_SHA: ${{ github.sha }}
37+
echo -n 'out=' >> "$GITHUB_OUTPUT"
38+
# See https://github.com/fkirc/skip-duplicate-actions#paths_result
39+
printenv PATHS_RESULT \
40+
| jq --compact-output .global.matched_files >> "$GITHUB_OUTPUT"
41+
code_hash:
42+
name: Compute code directory hash
43+
runs-on: ubuntu-22.04
44+
outputs:
45+
code_hash: ${{ steps.code-hash.outputs.code-hash }}
46+
steps:
47+
- name: git checkout
48+
uses: actions/checkout@v3
49+
- id: code-hash
6350
run: |
64-
DATA_DIR=$PWD/osv
65-
mkdir "$DATA_DIR"
66-
cd source
67-
while read FILE ; do
68-
ID=$(basename "$FILE" .md)
69-
YEAR=$(echo "$ID" | cut -d - -f 2)
70-
mkdir -p $DATA_DIR/$YEAR
71-
docker run --rm -v $PWD:/repo --workdir /repo haskell/hsec-tools:latest /bin/hsec-tools osv "$FILE" > $DATA_DIR/$YEAR/$ID.json
72-
done < <(find advisories -type f -name "*.md")
73-
BRANCH=generated/osv-export
74-
REF=refs/remotes/origin/$BRANCH
75-
export GIT_WORK_TREE=$DATA_DIR
76-
git read-tree "$REF"
77-
git add --all --intent-to-add
78-
git diff --quiet && exit
79-
git add --all
80-
TREE=$(git write-tree)
81-
git config user.email [email protected]
82-
git config user.name "Haskell Security Response Team"
83-
COMMIT=$(git commit-tree "$TREE" -p "$REF" -m "$(date --utc --rfc-3339=seconds) ($GITHUB_SHA)")
84-
git push origin $COMMIT:$BRANCH
51+
code_hash=$(git rev-parse HEAD:code)
52+
echo "code-hash=$code_hash" >> "$GITHUB_OUTPUT"
53+
check_advisories:
54+
name: Invoke check-advisories workflow
55+
needs: [tools_changed, advisories_changed, code_hash]
56+
if: ${{ needs.tools_changed.outputs.should_skip == 'true' && needs.advisories_changed.outputs.should_skip != 'true' }}
57+
uses: ./.github/workflows/call-check-advisories.yml
58+
with:
59+
fetch-key: hsec-tools-${{ needs.code_hash.outputs.code_hash }}
60+
is-artifact: false
61+
changed-advisories: ${{ needs.advisories_changed.outputs.changed_files }}

‎.github/workflows/nix.yml

Lines changed: 2 additions & 35 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,4 @@
1-
# Our desired pipeline using only a Nix shell environment
21
name: nix build
3-
42
on:
53
- push
64
- pull_request
@@ -22,43 +20,12 @@ jobs:
2220
name: Check nix build
2321
needs: tools_changed
2422
if: ${{ needs.tools_changed.outputs.should_skip != 'true' }}
25-
runs-on: ubuntu-22.04
26-
steps:
27-
- name: git checkout
28-
uses: actions/checkout@v3
29-
- name: Install Nix
30-
uses: DeterminateSystems/nix-installer-action@main
31-
with:
32-
extra-conf: system-features = nixos-test benchmark big-parallel kvm
33-
- uses: DeterminateSystems/magic-nix-cache-action@main
34-
- name: Check Nix flake inputs
35-
uses: DeterminateSystems/flake-checker-action@v4
36-
- name: Build executable
37-
run: nix -L build
38-
- name: Bild docker image
39-
run: nix build -L '.#packages.x86_64-linux.hsec-tools-image'
40-
- run: mkdir -p ~/.local/dockerImages
41-
- run: cp result ~/.local/dockerImages/hsec-tools
42-
- id: code-hash
43-
name: Compute code directory hash
44-
run: |
45-
code_hash=$(git rev-parse HEAD:code)
46-
echo "code-hash=$code_hash" >> "$GITHUB_OUTPUT"
47-
- uses: actions/cache/save@v3
48-
if: ${{ github.event_name == 'push' && github.ref_name == 'main' }}
49-
with:
50-
key: hsec-tools-${{ steps.code-hash.outputs.code-hash}}
51-
path: ~/.local/dockerImages
52-
- name: upload executable
53-
uses: actions/upload-artifact@v3
54-
with:
55-
name: hsec-tools-${{ github.sha }}
56-
path: ~/.local/dockerImages
23+
uses: ./.github/workflows/call-nix.yml
5724
check-advisories:
5825
name: Invoke check-advisories workflow
5926
if: ${{ needs.tools_changed.outputs.should_skip != 'true' }}
6027
needs: check_nix
61-
uses: ./.github/workflows/check-advisories.yml
28+
uses: ./.github/workflows/call-check-advisories.yml
6229
with:
6330
fetch-key: hsec-tools-${{ github.sha }}
6431
is-artifact: true

0 commit comments

Comments
 (0)