Skip to content

Commit 4aef4f6

Browse files
JuniorJPDJJuniorJPDJ
authored
feat(actions): run build parallel on multiple runners and build arm natively
also adds reusable workflow for this building way
1 parent 1f96211 commit 4aef4f6

File tree

2 files changed

+256
-70
lines changed

2 files changed

+256
-70
lines changed

‎.github/workflows/docker-parallel-multiarch-build.yml‎

Lines changed: 242 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,242 @@
1+
name: Build and push docker container (parallelized multi-arch build with native arm builders)
2+
3+
on:
4+
workflow_call:
5+
inputs:
6+
context:
7+
default: "."
8+
required: false
9+
type: string
10+
file:
11+
required: false
12+
type: string
13+
14+
title:
15+
required: true
16+
type: string
17+
description:
18+
required: false
19+
type: string
20+
21+
app-version:
22+
default: '0.1.0'
23+
required: false
24+
type: string
25+
revision:
26+
default: 1
27+
required: false
28+
type: number
29+
30+
platforms:
31+
default: 'linux/amd64,linux/arm64'
32+
required: false
33+
type: string
34+
buildkit-mount-caches:
35+
description: "JSON object: `{id1: target1, id2: target2}`."
36+
required: false
37+
type: string
38+
buildkit-mount-cache-ids-append-platform:
39+
description: "If true, the workflow will append `-{{ platform }}` to the end of cache mount id (eg. `-linux/amd64`)."
40+
default: false
41+
type: boolean
42+
43+
registry:
44+
default: ghcr.io
45+
required: false
46+
type: string
47+
image-name:
48+
required: true
49+
type: string
50+
secrets:
51+
registry-username:
52+
required: true
53+
registry-password:
54+
required: true
55+
56+
jobs:
57+
vars:
58+
name: Preprocess variables
59+
runs-on: ubuntu-24.04
60+
steps:
61+
- name: Preprocess variables
62+
id: vars
63+
run: |
64+
platforms="${{ inputs.platforms }}"
65+
{
66+
echo "version=${{ inputs.app-version }}-r${{ inputs.revision }}"
67+
echo "platforms<<9743a66f914cc249efca164485a19c5c"
68+
echo "[\"${platforms//,/\",\"}\"]"
69+
echo "9743a66f914cc249efca164485a19c5c"
70+
} >> "$GITHUB_OUTPUT"
71+
outputs:
72+
version: ${{ steps.vars.outputs.version }}
73+
platforms: ${{ steps.vars.outputs.platforms }}
74+
75+
build:
76+
name: Build container for ${{ matrix.platform }}
77+
runs-on: ${{ startsWith(matrix.platform, 'linux/arm') && 'ubuntu-24.04-arm' || 'ubuntu-24.04' }}
78+
needs: vars
79+
permissions:
80+
contents: read
81+
packages: write
82+
# This is used to complete the identity challenge
83+
# with sigstore/fulcio when running outside of PRs.
84+
id-token: write
85+
strategy:
86+
fail-fast: false
87+
matrix:
88+
platform: ${{ fromJSON(needs.vars.outputs.platforms) }}
89+
steps:
90+
- name: Checkout repository
91+
uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
92+
93+
# Workaround: https://github.com/docker/build-push-action/issues/461
94+
- name: Setup Docker buildx
95+
uses: docker/setup-buildx-action@f7ce87c1d6bead3e36075b2ce75da1f6cc28aaca # v3.9.0
96+
97+
- name: Log into registry ${{ inputs.registry }}
98+
uses: docker/login-action@9780b0c442fbb1117ed29e0efdff1e18412f7567 # v3.3.0
99+
with:
100+
registry: ${{ inputs.registry }}
101+
username: ${{ secrets.registry-username }}
102+
password: ${{ secrets.registry-password }}
103+
104+
- name: Extract Docker metadata
105+
id: meta
106+
uses: docker/metadata-action@369eb591f429131d6889c46b94e711f089e6ca96 # v5.6.1
107+
with:
108+
images: ${{ inputs.registry }}/${{ inputs.image-name }}
109+
labels: |
110+
org.opencontainers.image.title=${{ inputs.title }}
111+
org.opencontainers.image.description=${{ inputs.description }}
112+
113+
- name: Prepare variables
114+
id: vars
115+
run: |
116+
platform='${{ matrix.platform }}'
117+
image='${{ steps.meta.outputs.tags }}'
118+
119+
[ '${{ inputs.buildkit-mount-cache-ids-append-platform }}' = 'true' ] && id_suffix='-${{ matrix.platform }}'
120+
121+
{
122+
echo "platform-pair=${platform//\//-}"
123+
echo "platform-pair-un=${platform//\//_}"
124+
echo "image=${image%%:*}"
125+
126+
echo "cache-map<<9743a66f914cc249efca164485a19c5c"
127+
jq --arg 'id_suffix' "$id_suffix" 'to_entries | map({(.key): {target: .value, id: "\(.key)\($id_suffix)"}}) | add' << 9743a66f914cc249efca164485a19c5c
128+
${{ inputs.buildkit-mount-caches }}
129+
9743a66f914cc249efca164485a19c5c
130+
echo "9743a66f914cc249efca164485a19c5c"
131+
132+
echo "cache-paths<<9743a66f914cc249efca164485a19c5c"
133+
jq -r 'to_entries | .[].key' << 9743a66f914cc249efca164485a19c5c
134+
${{ inputs.buildkit-mount-caches }}
135+
9743a66f914cc249efca164485a19c5c
136+
echo "9743a66f914cc249efca164485a19c5c"
137+
} >> $GITHUB_OUTPUT
138+
139+
cat $GITHUB_OUTPUT
140+
141+
- name: Setup buildkit mount cache
142+
uses: actions/cache@1bd1e32a3bdc45362d1e726936510720a7c30a57 # v4.2.0
143+
if: steps.vars.outputs.cache-paths
144+
with:
145+
path: |
146+
${{ steps.vars.outputs.cache-paths }}
147+
key: buildkit-mount-${{ inputs.title }}-${{ steps.vars.outputs.platform-pair }}-${{ github.sha }}
148+
restore-keys: |
149+
buildkit-mount-${{ inputs.title }}-${{ steps.vars.outputs.platform-pair }}-
150+
151+
- name: Inject buildkit mount cache into docker
152+
uses: reproducible-containers/buildkit-cache-dance@5b6db76d1da5c8b307d5d2e0706d266521b710de # v3.1.2
153+
if: steps.vars.outputs.cache-paths
154+
with:
155+
cache-map: |
156+
${{ steps.vars.outputs.cache-map }}
157+
158+
- name: Build and push Docker image
159+
id: build-and-push
160+
uses: docker/build-push-action@ca877d9245402d1537745e0e356eab47c3520991 # v6.13.0
161+
with:
162+
context: ${{ inputs.context }}
163+
file: ${{ inputs.file }}
164+
push: true
165+
platforms: ${{ matrix.platform }}
166+
outputs: type=image,"name=${{ steps.vars.outputs.image }}",push-by-digest=true,name-canonical=true,push=true
167+
labels: ${{ steps.meta.outputs.labels }}
168+
cache-from: type=gha,scope=buildkit-layer-${{ inputs.title }}-${{ steps.vars.outputs.platform-pair }}
169+
cache-to: type=gha,mode=max,scope=buildkit-layer-${{ inputs.title }}-${{ steps.vars.outputs.platform-pair }}
170+
171+
# Workaround for https://github.com/actions/runner/pull/2477
172+
- name: Export digest
173+
run: |
174+
mkdir -p ${{ runner.temp }}/digests
175+
digest="${{ steps.build-and-push.outputs.digest }}"
176+
touch "${{ runner.temp }}/digests/${digest#sha256:}"
177+
178+
- name: Upload digest as an artifact
179+
uses: actions/upload-artifact@65c4c4a1ddee5b72f698fdd19549f0f0fb45cf08 # v4.6.0
180+
with:
181+
name: digests-${{ inputs.title }}-${{ steps.vars.outputs.platform-pair }}
182+
path: ${{ runner.temp }}/digests/*
183+
if-no-files-found: error
184+
retention-days: 1
185+
186+
outputs:
187+
image: ${{ steps.vars.outputs.image }}
188+
# https://github.com/actions/runner/pull/2477 :<
189+
# ${{ steps.vars.outputs.platform-pair-un }}-digest: ${{ steps.build-and-push.outputs.digest }}
190+
191+
merge:
192+
name: Merge images to multi-arch image
193+
runs-on: ubuntu-24.04
194+
permissions:
195+
contents: read
196+
packages: write
197+
# This is used to complete the identity challenge
198+
# with sigstore/fulcio when running outside of PRs.
199+
id-token: write
200+
needs:
201+
- build
202+
steps:
203+
- name: Download digests
204+
uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8
205+
with:
206+
path: ${{ runner.temp }}/digests
207+
pattern: digests-${{ inputs.title }}-*
208+
merge-multiple: true
209+
210+
# Workaround: https://github.com/docker/build-push-action/issues/461
211+
- name: Setup Docker buildx
212+
uses: docker/setup-buildx-action@f7ce87c1d6bead3e36075b2ce75da1f6cc28aaca # v3.9.0
213+
214+
- name: Log into registry ${{ inputs.registry }}
215+
uses: docker/login-action@9780b0c442fbb1117ed29e0efdff1e18412f7567 # v3.3.0
216+
with:
217+
registry: ${{ inputs.registry }}
218+
username: ${{ secrets.registry-username }}
219+
password: ${{ secrets.registry-password }}
220+
221+
- name: Extract Docker metadata
222+
id: meta
223+
uses: docker/metadata-action@369eb591f429131d6889c46b94e711f089e6ca96 # v5.6.1
224+
with:
225+
images: ${{ inputs.registry }}/${{ inputs.image-name }}
226+
labels: |
227+
org.opencontainers.image.title=${{ inputs.title }}
228+
org.opencontainers.image.description=${{ inputs.description }}
229+
tags: |
230+
type=ref,event=branch,priority=1990
231+
type=ref,event=pr,priority=2999
232+
type=raw,value=latest,enable={{is_default_branch}}
233+
type=raw,enable={{is_default_branch}},value=${{ needs.vars.outputs.version }},priority=1999
234+
type=raw,enable={{is_default_branch}},value=${{ inputs.app-version }},priority=1998
235+
type=semver,enable={{is_default_branch}},value=${{ inputs.app-version }},pattern={{major}}.{{minor}},priority=1997
236+
type=semver,enable={{is_default_branch}},value=${{ inputs.app-version }},pattern={{major}},priority=1996
237+
238+
- name: Create manifest list and push
239+
working-directory: ${{ runner.temp }}/digests
240+
run: |
241+
docker buildx imagetools create $(jq -cr '.tags | map("-t " + .) | join(" ")' <<< "$DOCKER_METADATA_OUTPUT_JSON") \
242+
$(printf '${{ needs.build.outputs.image }}@sha256:%s ' *)

‎.github/workflows/docker-publish.yml‎

Lines changed: 14 additions & 70 deletions
Original file line numberDiff line numberDiff line change
@@ -8,13 +8,10 @@ on:
88
types: [opened, reopened, synchronize]
99
workflow_dispatch:
1010

11-
env:
12-
REGISTRY: ghcr.io
13-
1411
jobs:
1512
list_containers:
1613
name: List containers to build
17-
runs-on: ubuntu-latest
14+
runs-on: ubuntu-24.04
1815
outputs:
1916
matrix: ${{ steps.set-matrix.outputs.MATRIX }}
2017
steps:
@@ -46,7 +43,6 @@ jobs:
4643
4744
build:
4845
name: "Build container: ${{ matrix.container }}"
49-
runs-on: ubuntu-latest
5046
needs: list_containers
5147
permissions:
5248
contents: read
@@ -59,68 +55,16 @@ jobs:
5955
fail-fast: false
6056
matrix:
6157
include: ${{ fromJSON(needs.list_containers.outputs.matrix) }}
62-
63-
steps:
64-
- name: Checkout repository
65-
uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
66-
67-
- name: Generate properties
68-
id: props
69-
run: |
70-
PLATFORMS_DEFAULT="linux/amd64,linux/arm64"
71-
APP_VERSION_DEFAULT="0.1.0"
72-
REVISION_DEFAULT="1"
73-
74-
PLATFORMS="${{ matrix.platforms }}"
75-
APP_VERSION="${{ matrix.app_version }}"
76-
REVISION="${{ matrix.revision }}"
77-
78-
platforms="${PLATFORMS:-"${PLATFORMS_DEFAULT}"}"
79-
app_version="${APP_VERSION:-"${APP_VERSION_DEFAULT}"}"
80-
revision="${REVISION:-"${REVISION_DEFAULT}"}"
81-
{
82-
echo "platforms=$platforms"
83-
echo "app_version=$app_version"
84-
echo "revision=$revision"
85-
echo "version=${app_version}-r${revision}"
86-
} >> "$GITHUB_OUTPUT"
87-
88-
# Workaround: https://github.com/docker/build-push-action/issues/461
89-
- name: Setup Docker buildx
90-
uses: docker/setup-buildx-action@f7ce87c1d6bead3e36075b2ce75da1f6cc28aaca # v3.9.0
91-
92-
- name: Log into registry ${{ env.REGISTRY }}
93-
uses: docker/login-action@9780b0c442fbb1117ed29e0efdff1e18412f7567 # v3.3.0
94-
with:
95-
registry: ${{ env.REGISTRY }}
96-
username: ${{ github.actor }}
97-
password: ${{ secrets.GITHUB_TOKEN }}
98-
99-
- name: Extract Docker metadata
100-
id: meta
101-
uses: docker/metadata-action@369eb591f429131d6889c46b94e711f089e6ca96 # v5.6.1
102-
with:
103-
images: ${{ env.REGISTRY }}/${{ github.repository }}/${{ matrix.container }}
104-
labels: |
105-
org.opencontainers.image.title=${{ matrix.container }}
106-
org.opencontainers.image.description=${{ matrix.description }}
107-
tags: |
108-
type=ref,event=branch,priority=1990
109-
type=ref,event=pr,priority=2999
110-
type=raw,value=latest,enable={{is_default_branch}}
111-
type=raw,enable={{is_default_branch}},value=${{ steps.props.outputs.version }},priority=1999
112-
type=raw,enable={{is_default_branch}},value=${{ steps.props.outputs.app_version }},priority=1998
113-
type=semver,enable={{is_default_branch}},value=${{ steps.props.outputs.app_version }},pattern={{major}}.{{minor}},priority=1997
114-
type=semver,enable={{is_default_branch}},value=${{ steps.props.outputs.app_version }},pattern={{major}},priority=1996
115-
116-
- name: Build and push Docker image
117-
id: build-and-push
118-
uses: docker/build-push-action@ca877d9245402d1537745e0e356eab47c3520991 # v6.13.0
119-
with:
120-
context: containers/${{ matrix.container }}
121-
push: true
122-
platforms: ${{ steps.props.outputs.platforms }}
123-
tags: ${{ steps.meta.outputs.tags }}
124-
labels: ${{ steps.meta.outputs.labels }}
125-
cache-from: type=gha
126-
cache-to: type=gha,mode=max
58+
uses: ./.github/workflows/docker-parallel-multiarch-build.yml
59+
with:
60+
context: containers/${{ matrix.container }}
61+
title: ${{ matrix.container }}
62+
description: ${{ matrix.description }}
63+
app-version: ${{ matrix.app_version }}
64+
revision: ${{ fromJSON(matrix.revision) }}
65+
platforms: ${{ matrix.platforms }}
66+
registry: ghcr.io
67+
image-name: ${{ github.repository }}/${{ matrix.container }}
68+
secrets:
69+
registry-username: ${{ github.actor }}
70+
registry-password: ${{ secrets.GITHUB_TOKEN }}

0 commit comments

Comments
 (0)