Skip to content

Commit 80eb032

Browse files
noaccOSnoaccOS
authored
ci: Fix snapshot platform build (astarte-platform#1810)
snapshots are currently built and uploaded for both amd64 and arm64, but the images are built and push separately and the last build overrides the first. to solve this, we use the method described in the [docker documentation] to make the snapshot build faster, caching is also introduced, using the same logic we use when building images for the e2e test [docker documentation]: https://docs.docker.com/build/ci/github-actions/multi-platform/#distribute-build-across-multiple-runners Signed-off-by: Francesco Noacco <francesco.noacco@secomind.com>
1 parent 9167c11 commit 80eb032

File tree

4 files changed

+297
-58
lines changed

4 files changed

+297
-58
lines changed

‎.github/workflows/publish-release-to-dockerhub-workflow.yaml‎

Lines changed: 69 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -45,9 +45,9 @@ jobs:
4545
e2e_tests:
4646
uses: ./.github/workflows/astarte-end-to-end-test-workflow.yaml
4747

48-
push_release_to_registry:
48+
build_release:
4949
name: Push Docker images to Docker Hub
50-
runs-on: ${{ matrix.platform }}
50+
runs-on: ${{ matrix.platform.os }}
5151
permissions:
5252
packages: write
5353
contents: read
@@ -57,8 +57,12 @@ jobs:
5757
fail-fast: true
5858
matrix:
5959
platform:
60-
- ubuntu-22.04
61-
- ubuntu-22.04-arm
60+
- os: ubuntu-22.04
61+
platform: linux/amd64
62+
platform-pair: linux-amd64
63+
- os: ubuntu-22.04-arm
64+
platform: linux/arm64
65+
platform-pair: linux-arm64
6266
app:
6367
- astarte_appengine_api
6468
- astarte_data_updater_plant
@@ -87,6 +91,62 @@ jobs:
8791
- name: Set up Docker Buildx
8892
uses: docker/setup-buildx-action@v3
8993

94+
- name: Build Docker image
95+
uses: docker/build-push-action@v6.18.0
96+
id: build
97+
with:
98+
context: .
99+
file: apps/${{ matrix.app }}/Dockerfile
100+
push: true
101+
platforms: ${{ matrix.platform.platform }}
102+
outputs: type=image,push-by-digest=true,name-canonical=true,push=true
103+
104+
- name: Export digest
105+
run: |
106+
mkdir -p ${{ runner.temp }}/digests/${{ matrix.app }}
107+
digest="${{ steps.build.outputs.digest }}"
108+
touch "${{ runner.temp }}/digests/${{ matrix.app }}/${digest#sha256:}"
109+
110+
- name: Upload digest
111+
uses: actions/upload-artifact@v4
112+
with:
113+
name: digests-${{ matrix.platform.platform-pair }}
114+
path: ${{ runner.temp }}/digests/${{ matrix.app }}/*
115+
if-no-files-found: error
116+
retention-days: 1
117+
118+
merge_images:
119+
name: Publish release
120+
runs-on: ubuntu-22.04
121+
needs:
122+
- build_release
123+
strategy:
124+
fail-fast: true
125+
matrix:
126+
app:
127+
- astarte_appengine_api
128+
- astarte_data_updater_plant
129+
- astarte_housekeeping
130+
- astarte_pairing
131+
- astarte_realm_management
132+
- astarte_trigger_engine
133+
steps:
134+
- name: Download digests
135+
uses: actions/download-artifact@v4
136+
with:
137+
path: ${{ runner.temp }}/digests/${{ matrix.app }}
138+
pattern: digests-*
139+
merge-multiple: true
140+
141+
- name: Login to Docker Hub
142+
uses: docker/login-action@v3
143+
with:
144+
username: ${{ secrets.DOCKER_USERNAME }}
145+
password: ${{ secrets.DOCKER_PASSWORD }}
146+
147+
- name: Set up Docker Buildx
148+
uses: docker/setup-buildx-action@v3
149+
90150
- name: Extract metadata (tags, labels) for Docker
91151
id: meta
92152
uses: docker/metadata-action@9ec57ed1fcdbf14dcef7dfbe97b2010124a938b7
@@ -96,12 +156,8 @@ jobs:
96156
tags: |
97157
type=semver,pattern={{version}}
98158
99-
- name: Build and push tagged Docker image
100-
id: push
101-
uses: docker/build-push-action@3b5e8027fcad23fda98b2e3ac259d8d67585f671
102-
with:
103-
context: .
104-
file: apps/${{ matrix.app }}/Dockerfile
105-
push: true
106-
tags: ${{ steps.meta.outputs.tags }}
107-
labels: ${{ steps.meta.outputs.labels }}
159+
- name: Create manifest list and push
160+
working-directory: ${{ runner.temp }}/digests/${{ matrix.app }}
161+
run: |
162+
docker buildx imagetools create $(jq -cr '.tags | map("-t " + .) | join(" ")' <<< "$DOCKER_METADATA_OUTPUT_JSON") \
163+
$(printf '${{ matrix.app }}@sha256:%s ' *)

‎.github/workflows/publish-snapshot-to-dockerhub-workflow.yaml‎

Lines changed: 77 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -55,7 +55,7 @@ jobs:
5555
e2e_tests:
5656
uses: ./.github/workflows/astarte-end-to-end-test-workflow.yaml
5757

58-
push_snapshot_to_registry:
58+
build_snapshot:
5959
name: Push Docker images to Docker Hub
6060
runs-on: ${{ matrix.platform }}
6161
permissions:
@@ -75,8 +75,12 @@ jobs:
7575
fail-fast: true
7676
matrix:
7777
platform:
78-
- ubuntu-22.04
79-
- ubuntu-22.04-arm
78+
- os: ubuntu-22.04
79+
platform: linux/amd64
80+
platform-pair: linux-amd64
81+
- os: ubuntu-22.04-arm
82+
platform: linux/arm64
83+
platform-pair: linux-arm64
8084
app:
8185
- astarte_appengine_api
8286
- astarte_data_updater_plant
@@ -88,13 +92,76 @@ jobs:
8892
- name: Check out the repo
8993
uses: actions/checkout@v4
9094

95+
- name: Log in to Docker Hub
96+
uses: docker/login-action@v3.4.0
97+
with:
98+
username: ${{ secrets.DOCKER_USERNAME }}
99+
password: ${{ secrets.DOCKER_PASSWORD }}
100+
101+
- name: Build branch slug env variable (job-scoped)
102+
run: |
103+
# Slugify branch/tag name for Docker tag safety and consistency
104+
echo "BRANCH_SLUG=$(echo "${GITHUB_REF_NAME}" | sha1sum | cut -f 1 -d ' ')" >> $GITHUB_ENV
105+
106+
- name: Set up Docker Buildx
107+
uses: docker/setup-buildx-action@v3
108+
109+
- name: Build Docker image
110+
id: build
111+
uses: docker/build-push-action@v6.18.0
112+
with:
113+
context: .
114+
file: apps/${{ matrix.app }}/Dockerfile
115+
push: true
116+
platforms: ${{ matrix.platform.platform }}
117+
cache-from: type=gha,scope=${{ matrix.app.name }}-${{ env.BRANCH_SLUG }}
118+
cache-to: type=gha,mode=max,scope=${{ matrix.app.name }}-${{ env.BRANCH_SLUG }}
119+
outputs: type=image,push-by-digest=true,name-canonical=true,push=true
120+
121+
- name: Export digest
122+
run: |
123+
mkdir -p ${{ runner.temp }}/digests/${{ matrix.app }}
124+
digest="${{ steps.build.outputs.digest }}"
125+
touch "${{ runner.temp }}/digests/${{ matrix.app }}/${digest#sha256:}"
126+
127+
- name: Upload digest
128+
uses: actions/upload-artifact@v4
129+
with:
130+
name: digests-${{ matrix.platform.platform-pair }}
131+
path: ${{ runner.temp }}/digests/${{ matrix.app }}/*
132+
if-no-files-found: error
133+
retention-days: 1
134+
135+
merge_snapshots:
136+
name: Publish snapshot
137+
runs-on: ubuntu-22.04
138+
needs:
139+
- build_snapshot
140+
strategy:
141+
fail-fast: true
142+
matrix:
143+
app:
144+
- astarte_appengine_api
145+
- astarte_data_updater_plant
146+
- astarte_housekeeping
147+
- astarte_pairing
148+
- astarte_realm_management
149+
- astarte_trigger_engine
150+
steps:
151+
- name: Download digests
152+
uses: actions/download-artifact@v4
153+
with:
154+
path: ${{ runner.temp }}/digests/${{ matrix.app }}
155+
pattern: digests-*
156+
merge-multiple: true
157+
91158
- name: Get current datetime
92159
id: datetime
93160
run: |
94161
echo "BUILD_DATE=$(date -u +'%Y-%m-%dT%H:%M:%SZ')" >> $GITHUB_OUTPUT
95162
96-
- name: Log in to Docker Hub
97-
uses: docker/login-action@v3.4.0
163+
- name: Login to Docker Hub
164+
uses: docker/login-action@v3
98165
with:
99166
username: ${{ secrets.DOCKER_USERNAME }}
100167
password: ${{ secrets.DOCKER_PASSWORD }}
@@ -122,13 +189,8 @@ jobs:
122189
org.opencontainers.image.revision=${{ github.sha }}
123190
org.opencontainers.image.source=https://github.com/${{ github.repository }}
124191
125-
- name: Build and push tagged Docker image
126-
id: push
127-
uses: docker/build-push-action@v6.18.0
128-
with:
129-
context: .
130-
file: apps/${{ matrix.app }}/Dockerfile
131-
push: true
132-
tags: ${{ steps.meta.outputs.tags }}
133-
labels: ${{ steps.meta.outputs.labels }}
134-
annotations: ${{ steps.meta.outputs.labels }}
192+
- name: Create manifest list and push
193+
working-directory: ${{ runner.temp }}/digests/${{ matrix.app }}
194+
run: |
195+
docker buildx imagetools create $(jq -cr '.tags | map("-t " + .) | join(" ")' <<< "$DOCKER_METADATA_OUTPUT_JSON") \
196+
$(printf '${{ matrix.app }}@sha256:%s ' *)

‎.github/workflows/publish-tool-release-to-dockerhub-workflow.yaml‎

Lines changed: 68 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -10,9 +10,9 @@ on:
1010
workflow_dispatch:
1111

1212
jobs:
13-
push_tool_release_to_registry:
13+
build_release:
1414
name: Push Docker images of Astarte tools to Docker Hub
15-
runs-on: ${{ matrix.platform }}
15+
runs-on: ${{ matrix.platform.os }}
1616
permissions:
1717
packages: write
1818
contents: read
@@ -22,8 +22,12 @@ jobs:
2222
fail-fast: true
2323
matrix:
2424
platform:
25-
- ubuntu-22.04
26-
- ubuntu-22.04-arm
25+
- os: ubuntu-22.04
26+
platform: linux/amd64
27+
platform-pair: linux-amd64
28+
- os: ubuntu-22.04-arm
29+
platform: linux/arm64
30+
platform-pair: linux-arm64
2731
app:
2832
- tool: astarte_device_fleet_simulator
2933
context: tools/astarte_device_fleet_simulator
@@ -47,24 +51,71 @@ jobs:
4751
username: ${{ secrets.DOCKER_USERNAME }}
4852
password: ${{ secrets.DOCKER_PASSWORD }}
4953

54+
- name: Build Docker image
55+
uses: docker/build-push-action@v6.18.0
56+
id: build
57+
with:
58+
context: ${{ matrix.app.context }}
59+
file: ${{ matrix.app.file }}
60+
push: true
61+
platforms: ${{ matrix.platform.platform }}
62+
outputs: type=image,push-by-digest=true,name-canonical=true,push=true
63+
64+
- name: Export digest
65+
run: |
66+
mkdir -p ${{ runner.temp }}/digests/${{ matrix.app.tool }}
67+
digest="${{ steps.build.outputs.digest }}"
68+
touch "${{ runner.temp }}/digests/${{ matrix.app.tool }}/${digest#sha256:}"
69+
70+
- name: Upload digest
71+
uses: actions/upload-artifact@v4
72+
with:
73+
name: digests-${{ matrix.platform.platform-pair }}
74+
path: ${{ runner.temp }}/digests/${{ matrix.app.tool }}/*
75+
if-no-files-found: error
76+
retention-days: 1
77+
78+
merge_images:
79+
name: Publish release
80+
runs-on: ubuntu-22.04
81+
needs:
82+
- build_release
83+
strategy:
84+
fail-fast: true
85+
matrix:
86+
app:
87+
- astarte_device_fleet_simulator
88+
- astarte_e2e
89+
- astarte_export
90+
- astarte_import
91+
steps:
92+
- name: Download digests
93+
uses: actions/download-artifact@v4
94+
with:
95+
path: ${{ runner.temp }}/digests/${{ matrix.app }}
96+
pattern: digests-*
97+
merge-multiple: true
98+
99+
- name: Login to Docker Hub
100+
uses: docker/login-action@v3
101+
with:
102+
username: ${{ secrets.DOCKER_USERNAME }}
103+
password: ${{ secrets.DOCKER_PASSWORD }}
104+
105+
- name: Set up Docker Buildx
106+
uses: docker/setup-buildx-action@v3
107+
50108
- name: Extract metadata (tags, labels) of Astarte tools for Docker
51109
id: meta
52110
uses: docker/metadata-action@9ec57ed1fcdbf14dcef7dfbe97b2010124a938b7
53111
with:
54112
images: |
55-
astarte/${{ matrix.app.tool }}
113+
astarte/${{ matrix.app }}
56114
tags: |
57115
type=semver,pattern={{version}}
58116
59-
- name: Set up Docker Buildx
60-
uses: docker/setup-buildx-action@v3
61-
62-
- name: Build and push tagged Docker image of Astarte tools
63-
id: push
64-
uses: docker/build-push-action@3b5e8027fcad23fda98b2e3ac259d8d67585f671
65-
with:
66-
context: ${{ matrix.app.context }}
67-
file: ${{ matrix.app.file }}
68-
push: true
69-
tags: ${{ steps.meta.outputs.tags }}
70-
labels: ${{ steps.meta.outputs.labels }}
117+
- name: Create manifest list and push
118+
working-directory: ${{ runner.temp }}/digests/${{ matrix.app }}
119+
run: |
120+
docker buildx imagetools create $(jq -cr '.tags | map("-t " + .) | join(" ")' <<< "$DOCKER_METADATA_OUTPUT_JSON") \
121+
$(printf '${{ matrix.app }}@sha256:%s ' *)

0 commit comments

Comments
 (0)