Skip to content

Commit 36c5a99

Browse files
puehringerpuehringer
authored
feat: new build-docker-artifacts.yml (#151)
1 parent b26c651 commit 36c5a99

File tree

2 files changed

+340
-0
lines changed

2 files changed

+340
-0
lines changed

‎.github/workflows/build-docker-artifacts-config.schema.json‎

Lines changed: 50 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,50 @@
1+
{
2+
"$schema": "http://json-schema.org/draft-07/schema#",
3+
"type": "object",
4+
"additionalProperties": false,
5+
"properties": {
6+
"flavors": {
7+
"type": "array",
8+
"description": "List of flavors to build",
9+
"items": {
10+
"type": "object",
11+
"additionalProperties": false,
12+
"properties": {
13+
"directory": {
14+
"type": "string",
15+
"description": "Directory of the flavor containing the components to build"
16+
},
17+
"skip": {
18+
"type": "boolean",
19+
"default": false,
20+
"description": "Skip building the flavor"
21+
},
22+
"components": {
23+
"type": "array",
24+
"items": {
25+
"type": "object",
26+
"additionalProperties": false,
27+
"properties": {
28+
"directory": {
29+
"type": "string",
30+
"description": "Directory of the component to build, relative to the flavor directory"
31+
},
32+
"ecr_repository": {
33+
"type": "string",
34+
"description": "ECR repository to push the image to"
35+
},
36+
"skip_image_scan": {
37+
"type": "boolean",
38+
"description": "Skip scanning the image for vulnerabilities"
39+
}
40+
},
41+
"required": ["directory", "ecr_repository"]
42+
}
43+
}
44+
},
45+
"required": ["directory", "components"]
46+
}
47+
}
48+
},
49+
"required": ["flavors"]
50+
}

‎.github/workflows/build-docker-artifacts.yml‎

Lines changed: 290 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,290 @@
1+
name: Build docker artifacts
2+
3+
on:
4+
workflow_call:
5+
inputs:
6+
branch:
7+
type: string
8+
required: false
9+
# When using github.ref || github.head_ref, it would contain the full path, including /, which breaks the postgres hostname
10+
default: ${{ github.sha }}
11+
runs_on:
12+
type: string
13+
required: false
14+
default: "ubuntu-22.04"
15+
secrets:
16+
DATAVISYN_BOT_REPO_TOKEN:
17+
required: false
18+
CHECKOUT_TOKEN:
19+
required: false
20+
description: "Token to use for the checkout actions to access private repositories"
21+
22+
concurrency:
23+
group: "${{ github.workflow }}-${{ github.ref || github.head_ref }}"
24+
cancel-in-progress: true
25+
26+
env:
27+
WORKFLOW_BRANCH: "mp/build_docker"
28+
PYTHON_BASE_IMAGE: "python:3.10.8-slim-bullseye"
29+
DATAVISYN_PYTHON_BASE_IMAGE: "188237246440.dkr.ecr.eu-central-1.amazonaws.com/datavisyn/base/python:main"
30+
NODE_BASE_IMAGE: "node:20.9-bullseye"
31+
DATAVISYN_NGINX_BASE_IMAGE: "188237246440.dkr.ecr.eu-central-1.amazonaws.com/datavisyn/base/nginx:main"
32+
33+
permissions:
34+
contents: read
35+
id-token: write
36+
37+
jobs:
38+
get-flavors:
39+
name: Get flavors from config.json
40+
outputs:
41+
result: ${{ steps.get-flavors.outputs.result }}
42+
runs-on: ${{ inputs.runs_on || 'ubuntu-22.04' }}
43+
steps:
44+
- name: Checkout repository
45+
uses: actions/checkout@v4
46+
with:
47+
ref: ${{ inputs.branch }}
48+
token: ${{ secrets.CHECKOUT_TOKEN || github.event.repository.private == true && secrets.DATAVISYN_BOT_REPO_TOKEN || github.token }}
49+
50+
- name: Checkout github-workflows repository
51+
uses: actions/checkout@v4
52+
with:
53+
repository: datavisyn/github-workflows
54+
ref: ${{ env.WORKFLOW_BRANCH }}
55+
path: ./tmp/github-workflows
56+
57+
- name: Validate ./deploy/build/config.json
58+
shell: bash
59+
run: |
60+
# Validate the config with the schema
61+
python -m venv .venv
62+
source .venv/bin/activate
63+
pip install jsonschema
64+
jsonschema -i ./deploy/build/config.json ./tmp/github-workflows/.github/workflows/build-docker-artifacts-config.schema.json
65+
deactivate
66+
rm -rf .venv
67+
68+
- name: Get all flavors and components from ./deploy/build/config.json
69+
uses: actions/github-script@v7
70+
id: get-flavors
71+
with:
72+
script: |
73+
const fs = require('fs');
74+
const path = require('path');
75+
const config = require('./deploy/build/config.json');
76+
77+
const buildTime = new Date().toISOString().replace(/:/g, '').replace(/\..+/, 'Z');
78+
const imageTagBranchName = "${{ github.ref }}".replace('refs/heads/', '').replace(/[^a-zA-Z0-9._-]/g, '-');
79+
const imageTag = `tagged-${imageTagBranchName}-${buildTime}`;
80+
81+
const flavors = config.flavors.filter(flavor => flavor.skip !== true).map(flavor => {
82+
return {
83+
...flavor,
84+
// Add metadata to the flavor object (will be used as matrix input)
85+
build_time: buildTime,
86+
image_tag: imageTag,
87+
image_tag_branch_name: imageTagBranchName,
88+
ecr_respositories: flavor.components.map(component => component.ecr_repository),
89+
components: flavor.components.map(component => {
90+
return {
91+
...component,
92+
// Add metadata to the component object (will be used as matrix input),
93+
flavor,
94+
flavor_directory: `./deploy/build/${flavor.directory}`,
95+
build_time: buildTime,
96+
image_tag: imageTag,
97+
image_tag_branch_name: imageTagBranchName,
98+
};
99+
}),
100+
};
101+
});
102+
103+
const flattenedComponents = flavors.flatMap(flavor => flavor.components);
104+
105+
const result = {
106+
flavors,
107+
components: flattenedComponents,
108+
};
109+
console.log(result);
110+
return result;
111+
112+
build-flavors:
113+
name: Build ${{ matrix.component.directory }} of ${{ matrix.component.flavor.directory }} (${{ matrix.component.ecr_repository }}:${{ matrix.component.image_tag }})
114+
needs: get-flavors
115+
strategy:
116+
fail-fast: true
117+
matrix:
118+
component: ${{ fromJson(needs.get-flavors.outputs.result).components }}
119+
runs-on: ${{ inputs.runs_on || 'ubuntu-22.04' }}
120+
steps:
121+
- name: View flavor and component
122+
shell: bash
123+
run: |
124+
echo "Component ${{ toJson(matrix.component) }}"
125+
- name: Remove unnecessary files
126+
run: |
127+
sudo rm -rf /usr/share/dotnet
128+
sudo rm -rf /usr/local/lib/android
129+
sudo rm -rf /opt/ghc
130+
# TODO: Support arbitrary repositories, not just the current one?
131+
- name: Checkout repository
132+
uses: actions/checkout@v4
133+
with:
134+
ref: ${{ inputs.branch }}
135+
token: ${{ secrets.CHECKOUT_TOKEN || github.event.repository.private == true && secrets.DATAVISYN_BOT_REPO_TOKEN || github.token }}
136+
# This is required such that yarn install can access private repositories, i.e. visyn_pro
137+
# https://github.com/yarnpkg/yarn/issues/2614#issuecomment-2148174789
138+
persist-credentials: false
139+
- name: Checkout github-workflows repository
140+
uses: actions/checkout@v4
141+
with:
142+
repository: datavisyn/github-workflows
143+
ref: ${{ env.WORKFLOW_BRANCH }}
144+
path: ./tmp/github-workflows
145+
# This is required such that yarn install can access private repositories, i.e. visyn_pro
146+
# https://github.com/yarnpkg/yarn/issues/2614#issuecomment-2148174789
147+
persist-credentials: false
148+
- name: Copy _base folder and .env
149+
shell: bash
150+
run: |
151+
if [[ -d "./deploy/build/_base" ]]; then
152+
echo "copy _base directory into flavor"
153+
cp -r -n "./deploy/build/_base/." "${{ matrix.component.flavor_directory }}"
154+
tree "${{ matrix.component.flavor_directory }}"
155+
fi
156+
if [[ -f "${{ matrix.component.flavor_directory }}/${{ matrix.component.directory }}/.env" ]]; then
157+
echo "copy .env into repo root"
158+
cp "${{ matrix.component.flavor_directory }}/${{ matrix.component.directory }}/.env" "./"
159+
fi
160+
161+
# Required for build secrets to work: https://docs.docker.com/build/ci/github-actions/secrets/#secret-mounts
162+
- name: Set up QEMU
163+
uses: docker/setup-qemu-action@v3
164+
- name: Set up Docker Buildx
165+
uses: docker/setup-buildx-action@v3
166+
167+
- name: Configure AWS Credentials
168+
uses: aws-actions/[email protected]
169+
with:
170+
role-to-assume: ${{ vars.DV_AWS_ECR_ROLE }}
171+
aws-region: ${{ vars.DV_AWS_REGION }}
172+
173+
- name: Login to Amazon ECR
174+
id: login-ecr
175+
uses: aws-actions/[email protected]
176+
177+
- name: Build image
178+
uses: docker/build-push-action@v6
179+
with:
180+
context: .
181+
file: ${{ matrix.component.flavor_directory }}/${{ matrix.component.directory }}/Dockerfile
182+
push: true
183+
# Disable provenance as it creates weird multi-arch images: https://github.com/docker/build-push-action/issues/755
184+
provenance: false
185+
build-args: |
186+
DOCKERFILE_DIRECTORY=${{ matrix.component.flavor_directory }}/${{ matrix.component.directory }}
187+
PYTHON_BASE_IMAGE=${{ env.PYTHON_BASE_IMAGE }}
188+
DATAVISYN_PYTHON_BASE_IMAGE=${{ env.DATAVISYN_PYTHON_BASE_IMAGE }}
189+
NODE_BASE_IMAGE=${{ env.NODE_BASE_IMAGE }}
190+
DATAVISYN_NGINX_BASE_IMAGE=${{ env.DATAVISYN_NGINX_BASE_IMAGE }}
191+
secrets:
192+
# Mount the token as secret mount: https://docs.docker.com/build/ci/github-actions/secrets/#secret-mounts
193+
"github_token=${{ secrets.CHECKOUT_TOKEN || github.event.repository.private == true && secrets.DATAVISYN_BOT_REPO_TOKEN || github.token }}"
194+
# TODO: As soon as we only have a single tag, we can push the same image to multiple repositories: https://docs.docker.com/build/ci/github-actions/push-multi-registries/
195+
# This will be useful for the images which don't change between flavors, e.g. the backend images
196+
tags: |
197+
${{ vars.DV_AWS_ECR_REGISTRY }}/${{ matrix.component.ecr_repository }}:${{ matrix.component.image_tag }}
198+
labels: |
199+
name=${{ matrix.component.ecr_repository }}
200+
version=${{ matrix.component.image_tag_branch_name }}
201+
org.opencontainers.image.description=Image for ${{ matrix.component.ecr_repository }}
202+
org.opencontainers.image.source=${{ github.event.repository.html_url }}
203+
org.opencontainers.image.url=${{ github.event.repository.html_url }}
204+
org.opencontainers.image.title=${{ matrix.component.ecr_repository }}
205+
org.opencontainers.image.version=${{ matrix.component.image_tag_branch_name }}
206+
org.opencontainers.image.created=${{ matrix.component.build_time }}
207+
org.opencontainers.image.revision=${{ github.sha }}
208+
env:
209+
# Disable the build summary for now as it leads to "Failed to export build record: .../export/rec.dockerbuild not found"
210+
# https://github.com/docker/build-push-action/issues/1156#issuecomment-2437227730
211+
DOCKER_BUILD_SUMMARY: false
212+
213+
- name: Log out from Amazon ECR
214+
shell: bash
215+
run: docker logout ${{ steps.login-ecr.outputs.registry }}
216+
217+
- name: Scan image
218+
if: ${{ matrix.component.skip_image_scan != true }}
219+
id: get-ecr-scan-result
220+
uses: ./tmp/github-workflows/.github/actions/get-ecr-scan-result
221+
with:
222+
aws_role: ${{ vars.DV_AWS_ECR_ROLE }}
223+
aws_region: ${{ vars.DV_AWS_REGION }}
224+
ecr_registry: ${{ vars.DV_AWS_ECR_REGISTRY }}
225+
ecr_repository: ${{ matrix.component.ecr_repository }}
226+
image_tag: ${{ matrix.component.image_tag }}
227+
- name: Check scan results
228+
if: ${{ matrix.component.skip_image_scan != true }}
229+
run: |
230+
if [ "${{ steps.get-ecr-scan-result.outputs.critical }}" != "null" ] || [ "${{ steps.get-ecr-scan-result.outputs.high }}" != "null" ]; then
231+
echo "Docker image contains vulnerabilities at critical or high level"
232+
exit 1 #exit execution due to docker image vulnerabilities
233+
fi
234+
235+
post-build:
236+
name: Retag images of flavor ${{ matrix.flavor || 'default' }}
237+
needs: [get-flavors, build-flavors]
238+
strategy:
239+
fail-fast: false
240+
matrix:
241+
flavor: ${{ fromJson(needs.get-flavors.outputs.result).flavors }}
242+
runs-on: ${{ inputs.runs_on || 'ubuntu-22.04' }}
243+
steps:
244+
- name: Checkout repository
245+
uses: actions/checkout@v4
246+
with:
247+
ref: ${{ inputs.branch }}
248+
token: ${{ secrets.CHECKOUT_TOKEN || github.event.repository.private == true && secrets.DATAVISYN_BOT_REPO_TOKEN || github.token }}
249+
250+
- name: Checkout github-workflows repository
251+
uses: actions/checkout@v4
252+
with:
253+
repository: datavisyn/github-workflows
254+
ref: ${{ env.WORKFLOW_BRANCH }}
255+
path: ./tmp/github-workflows
256+
257+
- name: Configure AWS Credentials
258+
uses: aws-actions/[email protected]
259+
with:
260+
role-to-assume: ${{ vars.DV_AWS_ECR_ROLE }}
261+
aws-region: ${{ vars.DV_AWS_REGION }}
262+
263+
- name: Login to Amazon ECR
264+
id: login-ecr
265+
uses: aws-actions/[email protected]
266+
267+
- name: Retag images
268+
shell: bash
269+
run: |
270+
image_tag="${{ matrix.flavor.image_tag }}"
271+
image_tag_branch_name="${{ matrix.flavor.image_tag_branch_name }}"
272+
273+
echo "image_tag=$image_tag"
274+
echo "image_tag_branch_name=$image_tag_branch_name"
275+
276+
for repository_name in $(jq -r '.ecr_respositories[]' <<< "$FLAVOR"); do
277+
IMAGE_META=$(aws ecr describe-images --repository-name "$repository_name" --image-ids imageTag="$image_tag" --output json | jq --arg var "${image_tag_branch_name}" '.imageDetails[0].imageTags | index( $var )')
278+
if [[ -z "${IMAGE_META}" || ${IMAGE_META} == "null" ]]; then
279+
MANIFEST=$(aws ecr batch-get-image --repository-name "$repository_name" --image-ids imageTag="$image_tag" --output json | jq --raw-output --join-output '.images[0].imageManifest')
280+
aws ecr put-image --repository-name "$repository_name" --image-tag "$image_tag_branch_name" --image-manifest "$MANIFEST"
281+
else
282+
echo "image already tagged!"
283+
fi
284+
done;
285+
env:
286+
FLAVOR: ${{ toJSON(matrix.flavor) }}
287+
288+
- name: Log out from Amazon ECR
289+
shell: bash
290+
run: docker logout ${{ steps.login-ecr.outputs.registry }}

0 commit comments

Comments
 (0)