Skip to content

Commit 60b1b52

Browse files
strtgbbzvonand
strtgbb
authored and
zvonand
committed
add grype scanning
1 parent fbabb52 commit 60b1b52

File tree

5 files changed

+213
-0
lines changed

5 files changed

+213
-0
lines changed

‎.github/grype/parse_vulnerabilities_grype.py‎

Lines changed: 32 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,32 @@
1+
#!/usr/bin/env python3
2+
import json
3+
4+
from testflows.core import *
5+
6+
xfails = {}
7+
8+
9+
@Name("docker vulnerabilities")
10+
@XFails(xfails)
11+
@TestModule
12+
def docker_vulnerabilities(self):
13+
with Given("I gather grype scan results"):
14+
with open("./result.json", "r") as f:
15+
results = json.load(f)
16+
17+
for vulnerability in results["matches"]:
18+
with Test(
19+
f"{vulnerability['vulnerability']['id']}@{vulnerability['vulnerability']['namespace']},{vulnerability['vulnerability']['severity']}",
20+
flags=TE,
21+
):
22+
note(vulnerability)
23+
critical_levels = set(["HIGH", "CRITICAL"])
24+
if vulnerability['vulnerability']["severity"].upper() in critical_levels:
25+
with Then(
26+
f"Found vulnerability of {vulnerability['vulnerability']['severity']} severity"
27+
):
28+
result(Fail)
29+
30+
31+
if main():
32+
docker_vulnerabilities()

‎.github/grype/run_grype_scan.sh‎

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,18 @@
1+
set -x
2+
set -e
3+
4+
IMAGE=$1
5+
6+
GRYPE_VERSION="v0.80.1"
7+
8+
docker pull $IMAGE
9+
docker pull anchore/grype:${GRYPE_VERSION}
10+
11+
docker run \
12+
--rm --volume /var/run/docker.sock:/var/run/docker.sock \
13+
--name Grype anchore/grype:${GRYPE_VERSION} \
14+
--scope all-layers \
15+
-o json \
16+
$IMAGE > result.json
17+
18+
ls -sh

‎.github/grype/transform_and_upload_results_s3.sh‎

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,13 @@
1+
DOCKER_IMAGE=$(echo "$DOCKER_IMAGE" | sed 's/[\/:]/_/g')
2+
3+
S3_PATH="s3://$S3_BUCKET/$PR_NUMBER/$COMMIT_SHA/grype/$DOCKER_IMAGE"
4+
HTTPS_S3_PATH="https://s3.amazonaws.com/$S3_BUCKET/$PR_NUMBER/$COMMIT_SHA/grype/$DOCKER_IMAGE"
5+
echo "https_s3_path=$HTTPS_S3_PATH" >> $GITHUB_OUTPUT
6+
7+
tfs --no-colors transform nice raw.log nice.log.txt
8+
tfs --no-colors report results -a $HTTPS_S3_PATH raw.log - --copyright "Altinity LTD" | tfs --no-colors document convert > results.html
9+
10+
aws s3 cp --no-progress nice.log.txt $S3_PATH/nice.log.txt --content-type "text/plain; charset=utf-8" || echo "nice log file not found".
11+
aws s3 cp --no-progress results.html $S3_PATH/results.html || echo "results file not found".
12+
aws s3 cp --no-progress raw.log $S3_PATH/raw.log || echo "raw.log file not found".
13+
aws s3 cp --no-progress result.json $S3_PATH/result.json --content-type "text/plain; charset=utf-8" || echo "result.json not found".

‎.github/workflows/grype_scan.yml‎

Lines changed: 132 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,132 @@
1+
name: Grype Scan
2+
run-name: Grype Scan ${{ inputs.docker_image }}
3+
4+
on:
5+
workflow_dispatch:
6+
# Inputs for manual run
7+
inputs:
8+
docker_image:
9+
description: 'Docker image. If no tag, it will be determined by version_helper.py'
10+
required: true
11+
workflow_call:
12+
# Inputs for workflow call
13+
inputs:
14+
docker_image:
15+
description: 'Docker image. If no tag, it will be determined by version_helper.py'
16+
required: true
17+
type: string
18+
env:
19+
PYTHONUNBUFFERED: 1
20+
AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
21+
AWS_DEFAULT_REGION: ${{ secrets.AWS_DEFAULT_REGION }}
22+
AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
23+
24+
jobs:
25+
grype_scan:
26+
name: Grype Scan
27+
runs-on: [self-hosted, altinity-on-demand, altinity-func-tester-aarch64]
28+
steps:
29+
- name: Checkout repository
30+
uses: actions/checkout@v4
31+
32+
- name: Set up Docker
33+
uses: docker/setup-buildx-action@v3
34+
35+
- name: Set up Python
36+
run: |
37+
export TESTFLOWS_VERSION="2.4.19"
38+
sudo apt-get update
39+
sudo apt-get install -y python3-pip python3-venv
40+
python3 -m venv venv
41+
source venv/bin/activate
42+
pip install --upgrade requests chardet urllib3
43+
pip install testflows==$TESTFLOWS_VERSION awscli==1.33.28
44+
echo PATH=$PATH >>$GITHUB_ENV
45+
46+
- name: Set image tag if not given
47+
if: ${{ !contains(inputs.docker_image, ':') }}
48+
id: set_version
49+
run: |
50+
python3 ./tests/ci/version_helper.py | tee /tmp/version_info
51+
source /tmp/version_info
52+
echo "docker_image=${{ inputs.docker_image }}:${{ github.event.pull_request.number || 0 }}-$CLICKHOUSE_VERSION_STRING" >> $GITHUB_OUTPUT
53+
echo "commit_sha=$CLICKHOUSE_VERSION_GITHASH" >> $GITHUB_OUTPUT
54+
55+
- name: Run Grype Scan
56+
run: |
57+
DOCKER_IMAGE=${{ steps.set_version.outputs.docker_image || inputs.docker_image }}
58+
./.github/grype/run_grype_scan.sh $DOCKER_IMAGE
59+
60+
- name: Parse grype results
61+
run: |
62+
python3 -u ./.github/grype/parse_vulnerabilities_grype.py -o nice --no-colors --log raw.log --test-to-end
63+
64+
- name: Transform and Upload Grype Results
65+
if: always()
66+
id: upload_results
67+
env:
68+
S3_BUCKET: "altinity-build-artifacts"
69+
COMMIT_SHA: ${{ steps.set_version.outputs.commit_sha || github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }}
70+
PR_NUMBER: ${{ github.event.pull_request.number || 0 }}
71+
DOCKER_IMAGE: ${{ steps.set_version.outputs.docker_image || inputs.docker_image }}
72+
run: |
73+
./.github/grype/transform_and_upload_results_s3.sh
74+
75+
- name: Create step summary
76+
if: always()
77+
id: create_summary
78+
run: |
79+
jq -r '"**Image**: \(.source.target.userInput)"' result.json >> $GITHUB_STEP_SUMMARY
80+
jq -r '.distro | "**Distro**: \(.name):\(.version)"' result.json >> $GITHUB_STEP_SUMMARY
81+
if jq -e '.matches | length == 0' result.json > /dev/null; then
82+
echo "No CVEs" >> $GITHUB_STEP_SUMMARY
83+
else
84+
echo "| Severity | Count |" >> $GITHUB_STEP_SUMMARY
85+
echo "|------------|-------|" >> $GITHUB_STEP_SUMMARY
86+
jq -r '
87+
.matches |
88+
map(.vulnerability.severity) |
89+
group_by(.) |
90+
map({severity: .[0], count: length}) |
91+
sort_by(.severity) |
92+
map("| \(.severity) | \(.count) |") |
93+
.[]
94+
' result.json >> $GITHUB_STEP_SUMMARY
95+
fi
96+
97+
HIGH_COUNT=$(jq -r '.matches | map(.vulnerability.severity) | map(select(. == "High")) | length' result.json)
98+
CRITICAL_COUNT=$(jq -r '.matches | map(.vulnerability.severity) | map(select(. == "Critical")) | length' result.json)
99+
TOTAL_HIGH_CRITICAL=$((HIGH_COUNT + CRITICAL_COUNT))
100+
echo "total_high_critical=$TOTAL_HIGH_CRITICAL" >> $GITHUB_OUTPUT
101+
102+
if [ $TOTAL_HIGH_CRITICAL -gt 0 ]; then
103+
echo '## High and Critical vulnerabilities found' >> $GITHUB_STEP_SUMMARY
104+
echo '```' >> $GITHUB_STEP_SUMMARY
105+
cat raw.log | tfs --no-colors show tests | grep -Pi 'High|Critical' >> $GITHUB_STEP_SUMMARY
106+
echo '```' >> $GITHUB_STEP_SUMMARY
107+
fi
108+
109+
- name: Set commit status
110+
if: always()
111+
uses: actions/github-script@v7
112+
with:
113+
github-token: ${{ secrets.GITHUB_TOKEN }}
114+
script: |
115+
github.rest.repos.createCommitStatus({
116+
owner: context.repo.owner,
117+
repo: context.repo.repo,
118+
sha: '${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }}',
119+
state: '${{ steps.create_summary.outputs.total_high_critical > 0 && 'failure' || 'success' }}',
120+
target_url: '${{ steps.upload_results.outputs.https_s3_path }}/results.html',
121+
description: 'Grype Scan Completed with ${{ steps.create_summary.outputs.total_high_critical }} high/critical vulnerabilities',
122+
context: 'Grype Scan ${{ steps.set_version.outputs.docker_image || inputs.docker_image }}'
123+
})
124+
125+
- name: Upload artifacts
126+
if: always()
127+
uses: actions/upload-artifact@v4
128+
with:
129+
name: grype-results-${{ hashFiles('raw.log') }}
130+
path: |
131+
result.json
132+
nice.log.txt

‎.github/workflows/release_branches.yml‎

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -577,6 +577,23 @@ jobs:
577577
test_name: Sign aarch64
578578
runner_type: altinity-on-demand, altinity-func-tester
579579
data: ${{ needs.RunConfig.outputs.data }}
580+
GrypeScan:
581+
needs: [RunConfig, DockerServerImage, DockerKeeperImage]
582+
if: ${{ !failure() && !cancelled() }}
583+
strategy:
584+
fail-fast: false
585+
matrix:
586+
include:
587+
- image: server
588+
suffix: ''
589+
- image: server
590+
suffix: '-alpine'
591+
- image: keeper
592+
suffix: ''
593+
uses: ./.github/workflows/grype_scan.yml
594+
secrets: inherit
595+
with:
596+
docker_image: altinityinfra/clickhouse-${{ matrix.image }}:${{ github.event.pull_request.number || 0 }}-${{ fromJson(needs.RunConfig.outputs.data).version }}${{ matrix.suffix }}
580597
FinishCheck:
581598
if: ${{ !cancelled() }}
582599
needs:
@@ -611,6 +628,7 @@ jobs:
611628
- CompatibilityCheckAarch64
612629
- RegressionTestsRelease
613630
- RegressionTestsAarch64
631+
- GrypeScan
614632
- SignRelease
615633
runs-on: [self-hosted, altinity-on-demand, altinity-style-checker-aarch64]
616634
steps:

0 commit comments

Comments
 (0)