Skip to content

Commit 4e558ba

Browse files
opajonkopajonk
authored
Use ARM64 and AMD64 Builders (#56)
Cross-building the arm64 container on amd64 is really slow, and the tests have not been executed on arm64 until now. Here, we start using an arm64 builder to avoid cross-building and to allow for running tests on both image variants before publication.
1 parent f2e4b70 commit 4e558ba

File tree

6 files changed

+204
-72
lines changed

6 files changed

+204
-72
lines changed

‎.github/workflows/ci.yaml‎

Lines changed: 55 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -12,8 +12,19 @@ concurrency:
1212

1313
jobs:
1414
build:
15-
name: 'Check, Build, Test, Publish DevContainer'
16-
runs-on: ubuntu-latest
15+
strategy:
16+
matrix:
17+
os: [arm64, amd64]
18+
include:
19+
- os: amd64
20+
name: 'DevContainer (amd64)'
21+
runner: ubuntu-24.04
22+
- os: arm64
23+
name: 'DevContainer (arm64)'
24+
runner: ubuntu-24.04-arm
25+
26+
name: '${{ matrix.name }}'
27+
runs-on: ${{ matrix.runner }}
1728
permissions:
1829
contents: read
1930
packages: write
@@ -38,14 +49,16 @@ jobs:
3849
# We want to only use it for building and testing the actual container, which resides in src/s-core-devcontainer.
3950
push: "never"
4051
runCmd: |
52+
set -eux pipefail
53+
4154
# Check
4255
pre-commit run --show-diff-on-failure --color=always --all-files || exit -1
4356
4457
# Create builder for multi-arch builds
4558
./scripts/create_builder.sh
4659
4760
# Build
48-
./scripts/build.sh
61+
./scripts/build.sh --${{ matrix.os }} "main"
4962
5063
# Test
5164
./scripts/test.sh
@@ -56,5 +69,43 @@ jobs:
5669
if [ "${{ github.ref }}" = "refs/heads/main" ]; then
5770
# manually login to ghcr.io for publishing
5871
echo "${{ secrets.GITHUB_TOKEN }}" | docker login ghcr.io -u ${{ github.actor }} --password-stdin
59-
./scripts/publish.sh "main"
72+
./scripts/publish.sh --${{ matrix.os }} "main"
6073
fi
74+
75+
merge:
76+
name: 'Merge Labels (main only)'
77+
needs: ["build"]
78+
runs-on: ubuntu-24.04
79+
if: github.ref == 'refs/heads/main'
80+
permissions:
81+
contents: read
82+
packages: write
83+
id-token: write
84+
85+
steps:
86+
- name: Checkout (GitHub)
87+
uses: actions/checkout@v3
88+
89+
- name: Login to GitHub Container Registry
90+
uses: docker/login-action@v2
91+
with:
92+
registry: ghcr.io
93+
username: ${{ github.actor }}
94+
password: ${{ secrets.GITHUB_TOKEN }}
95+
96+
# Use .devcontainer from THIS repo for building and testing
97+
- name: Merge
98+
uses: devcontainers/[email protected]
99+
with:
100+
# The .devcontainer is never published as pre-built container.
101+
# We want to only use it for building and testing the actual container, which resides in src/s-core-devcontainer.
102+
push: "never"
103+
runCmd: |
104+
set -eux pipefail
105+
106+
# Merge
107+
# We do not use the push feature of devcontainers/ci here, since that would push the wrong container.
108+
# Instead, we use the publish script which pushes the correct container (residing in src/s-core-devcontainer).
109+
# manually login to ghcr.io for publishing
110+
echo "${{ secrets.GITHUB_TOKEN }}" | docker login ghcr.io -u ${{ github.actor }} --password-stdin
111+
./scripts/merge.sh "main"

‎.github/workflows/release.yaml‎

Lines changed: 53 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -4,11 +4,21 @@ on:
44
push:
55
tags:
66
- '[0-9]+.[0-9]+.[0-9]+'
7-
87
jobs:
98
build:
10-
name: 'Check, Build, Test, Publish DevContainer'
11-
runs-on: ubuntu-latest
9+
strategy:
10+
matrix:
11+
os: [arm64, amd64]
12+
include:
13+
- os: amd64
14+
name: 'DevContainer (amd64)'
15+
runner: ubuntu-24.04
16+
- os: arm64
17+
name: 'DevContainer (arm64)'
18+
runner: ubuntu-24.04-arm
19+
20+
name: '${{ matrix.name }}'
21+
runs-on: ${{ matrix.runner }}
1222
permissions:
1323
contents: read
1424
packages: write
@@ -33,24 +43,60 @@ jobs:
3343
# We want to only use it for building and testing the actual container, which resides in src/s-core-devcontainer.
3444
push: "never"
3545
runCmd: |
46+
set -eux pipefail
47+
3648
# Check
3749
pre-commit run --show-diff-on-failure --color=always --all-files || exit -1
3850
3951
# Create builder for multi-arch builds
4052
./scripts/create_builder.sh
4153
4254
# Build
43-
./scripts/build.sh
55+
./scripts/build.sh --${{ matrix.os }} "${{ github.ref_name }}" "latest"
4456
4557
# Test
4658
./scripts/test.sh
4759
4860
# Publish
4961
# We do not use the push feature of devcontainers/ci here, since that would push the wrong container.
5062
# Instead, we use the publish script which pushes the correct container (residing in src/s-core-devcontainer).
51-
5263
# manually login to ghcr.io for publishing
5364
echo "${{ secrets.GITHUB_TOKEN }}" | docker login ghcr.io -u ${{ github.actor }} --password-stdin
65+
./scripts/publish.sh --${{ matrix.os }} "${{ github.ref_name }}" "latest"
66+
67+
merge:
68+
name: 'Merge Labels'
69+
needs: ["build"]
70+
runs-on: ubuntu-24.04
71+
permissions:
72+
contents: read
73+
packages: write
74+
id-token: write
75+
76+
steps:
77+
- name: Checkout (GitHub)
78+
uses: actions/checkout@v3
79+
80+
- name: Login to GitHub Container Registry
81+
uses: docker/login-action@v2
82+
with:
83+
registry: ghcr.io
84+
username: ${{ github.actor }}
85+
password: ${{ secrets.GITHUB_TOKEN }}
86+
87+
# Use .devcontainer from THIS repo for building and testing
88+
- name: Merge
89+
uses: devcontainers/[email protected]
90+
with:
91+
# The .devcontainer is never published as pre-built container.
92+
# We want to only use it for building and testing the actual container, which resides in src/s-core-devcontainer.
93+
push: "never"
94+
runCmd: |
95+
set -eux pipefail
5496
55-
# Note: "${{ github.ref_name }}" will be the tag name, e.g., "1.0.0"
56-
./scripts/publish.sh "${{ github.ref_name }}" "latest"
97+
# Merge
98+
# We do not use the push feature of devcontainers/ci here, since that would push the wrong container.
99+
# Instead, we use the publish script which pushes the correct container (residing in src/s-core-devcontainer).
100+
# manually login to ghcr.io for publishing
101+
echo "${{ secrets.GITHUB_TOKEN }}" | docker login ghcr.io -u ${{ github.actor }} --password-stdin
102+
./scripts/merge.sh "${{ github.ref_name }}" "latest"

‎README.md‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -106,9 +106,9 @@ It is very simple to develop the development container.
106106
You can change files related to the container and then simply run the `scripts/*`.
107107
They are used by the CI, but especially the build and test scripts can be run also locally out of the box:
108108
````console
109-
$ ./scripts/build.sh
109+
$ ./scripts/build.sh --amd64 local
110110
[... build output..]
111-
{"outcome":"success","imageName":["ghcr.io/eclipse-score/devcontainer"]}
111+
{"outcome":"success","imageName":["ghcr.io/eclipse-score/devcontainer:local-amd64"]}
112112

113113
$ ./scripts/test.sh
114114
[... test output...]
@@ -133,9 +133,9 @@ So in order to execute `S-CORE DevContainer` on your host (and test it as part o
133133

134134
Concretely, this can be done as follows:
135135

136-
* Run `docker save "ghcr.io/eclipse-score/devcontainer" > export.img` in `Development Container A`.
136+
* Run `docker save "ghcr.io/eclipse-score/devcontainer:local-amd64" > export.img` in `Development Container A`.
137137
* On your **host machine** (!!), open a console and run `docker load < /path/to/export.img`.
138-
* In the working copy of the targeted S-CORE module, edit the file `.devcontainer/devcontainer.json` and change the `"image": "..."` entry to `"image": "ghcr.io/eclipse-score/devcontainer:latest"` (if not already set like this).
138+
* In the working copy of the targeted S-CORE module, edit the file `.devcontainer/devcontainer.json` and change the `"image": "..."` entry to `"image": "ghcr.io/eclipse-score/devcontainer:local-amd64"`.
139139
The Visual Studio Code instance related to the targeted S-CORE module will now ask you to rebuild the DevContainer.
140140
If not, press <kbd>Ctrl</kbd> + <kbd>Shift</kbd> + <kbd>p</kbd> and run from there "Dev Containers: Rebuilt Container Without Cache".
141141
Do so, and you have a running instance of `S-CORE DevContainer` related to the targeted S-CORE module.

‎scripts/build.sh‎

Lines changed: 29 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -1,37 +1,44 @@
11
#!/usr/bin/env bash
22
set -euxo pipefail
33

4-
if [ "$#" -eq 0 ]; then
5-
echo "Error: At least one parameter (label) must be provided."
4+
if [[ "$#" -lt 1 || "$1" != "--arm64" && "$1" != "--amd64" ]]; then
5+
echo "Error: First parameter must be --arm64 or --amd64."
66
exit 1
77
fi
88

9+
if [ "$#" -lt 2 ]; then
10+
echo "Error: At least one label must be provided after the architecture option."
11+
exit 1
12+
fi
13+
14+
ARCH_OPTION="$1"
15+
shift
16+
17+
ARCH="amd64"
18+
if [[ "$ARCH_OPTION" == "--arm64" ]]; then
19+
ARCH="arm64"
20+
fi
21+
922
LABELS=()
1023
for LABEL in "$@"; do
1124
LABELS+=("${LABEL}")
1225
done
1326

14-
# Define target architectures
15-
ARCHITECTURES=("amd64" "arm64")
16-
17-
# Build for each architecture, creating all requested tags
18-
for ARCH in "${ARCHITECTURES[@]}"; do
19-
echo "Building all labels (${LABELS[@]}) for architecture: ${ARCH}"
20-
21-
# Prepare image names with tags (each tag includes a label and an architecture)
22-
IMAGES=()
23-
for LABEL in "${LABELS[@]}"; do
24-
IMAGES+=("--image-name \"ghcr.io/eclipse-score/devcontainer:${LABEL}-${ARCH}\"")
25-
done
27+
echo "Building all labels (${LABELS[@]}) for architecture: ${ARCH}"
2628

27-
# Prepare devcontainer build command
28-
DEVCONTAINER_CALL="devcontainer build --workspace-folder src/s-core-devcontainer --cache-from ghcr.io/eclipse-score/devcontainer"
29+
# Prepare image names with tags (each tag includes a label and the architecture)
30+
IMAGES=()
31+
for LABEL in "${LABELS[@]}"; do
32+
IMAGES+=("--image-name \"ghcr.io/eclipse-score/devcontainer:${LABEL}-${ARCH}\"")
33+
done
2934

30-
# Append image names to the build command
31-
for IMAGE in "${IMAGES[@]}"; do
32-
DEVCONTAINER_CALL+=" $IMAGE"
33-
done
35+
# Prepare devcontainer build command
36+
DEVCONTAINER_CALL="devcontainer build --workspace-folder src/s-core-devcontainer --cache-from ghcr.io/eclipse-score/devcontainer"
3437

35-
# Execute the build for the specific architecture
36-
eval "$DEVCONTAINER_CALL --platform linux/${ARCH}"
38+
# Append image names to the build command
39+
for IMAGE in "${IMAGES[@]}"; do
40+
DEVCONTAINER_CALL+=" $IMAGE"
3741
done
42+
43+
# Execute the build for the specific architecture
44+
eval "$DEVCONTAINER_CALL --platform linux/${ARCH}"

‎scripts/merge.sh‎

Lines changed: 35 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,35 @@
1+
#!/usr/bin/env bash
2+
set -euxo pipefail
3+
4+
if [ "$#" -eq 0 ]; then
5+
echo "Error: At least one parameter (label) must be provided."
6+
exit 1
7+
fi
8+
9+
LABELS=()
10+
for LABEL in "$@"; do
11+
LABELS+=("${LABEL}")
12+
done
13+
14+
# Define target architectures
15+
ARCHITECTURES=("amd64" "arm64")
16+
17+
# Pull all architecture-specific images for each label
18+
for LABEL in "${LABELS[@]}"; do
19+
for ARCH in "${ARCHITECTURES[@]}"; do
20+
docker pull --platform "linux/${ARCH}" "ghcr.io/eclipse-score/devcontainer:${LABEL}-${ARCH}"
21+
done
22+
done
23+
24+
# Create and push the merged multiarch manifest for each tag; each tag combines all architecture-specific tags into one tag
25+
for LABEL in "${LABELS[@]}"; do
26+
echo "Merging all architectures (${ARCHITECTURES[@]}) into single tag: ${LABEL}"
27+
28+
MANIFEST_MERGE_CALL="docker buildx imagetools create -t ghcr.io/eclipse-score/devcontainer:${LABEL}"
29+
30+
for ARCH in "${ARCHITECTURES[@]}"; do
31+
MANIFEST_MERGE_CALL+=" ghcr.io/eclipse-score/devcontainer:${LABEL}-${ARCH}"
32+
done
33+
34+
eval "$MANIFEST_MERGE_CALL"
35+
done

‎scripts/publish.sh‎

Lines changed: 28 additions & 35 deletions
Original file line numberDiff line numberDiff line change
@@ -1,50 +1,43 @@
11
#!/usr/bin/env bash
22
set -euxo pipefail
33

4-
if [ "$#" -eq 0 ]; then
5-
echo "Error: At least one parameter (label) must be provided."
4+
if [[ "$#" -lt 1 || "$1" != "--arm64" && "$1" != "--amd64" ]]; then
5+
echo "Error: First parameter must be --arm64 or --amd64."
66
exit 1
77
fi
88

9-
LABELS=()
10-
for LABEL in "$@"; do
11-
LABELS+=("${LABEL}")
12-
done
13-
14-
# Define target architectures
15-
ARCHITECTURES=("amd64" "arm64")
16-
17-
# Build and push for each architecture, creating all requested tags
18-
for ARCH in "${ARCHITECTURES[@]}"; do
19-
echo "Building all tags (${LABELS[@]}) for architecture: ${ARCH}"
20-
21-
# Prepare image names with tags (each tag includes a label and an architecture)
22-
IMAGES=()
23-
for LABEL in "${LABELS[@]}"; do
24-
IMAGES+=("--image-name \"ghcr.io/eclipse-score/devcontainer:${LABEL}-${ARCH}\"")
25-
done
9+
if [ "$#" -lt 2 ]; then
10+
echo "Error: At least one label must be provided after the architecture option."
11+
exit 1
12+
fi
2613

27-
# Prepare devcontainer build command
28-
DEVCONTAINER_CALL="devcontainer build --push --workspace-folder src/s-core-devcontainer --cache-from ghcr.io/eclipse-score/devcontainer"
14+
ARCH_OPTION="$1"
15+
shift
2916

30-
# Append image names to the build command
31-
for IMAGE in "${IMAGES[@]}"; do
32-
DEVCONTAINER_CALL+=" $IMAGE"
33-
done
17+
ARCH="amd64"
18+
if [[ "$ARCH_OPTION" == "--arm64" ]]; then
19+
ARCH="arm64"
20+
fi
3421

35-
# Execute the build and push all tags for the specific architecture
36-
eval "$DEVCONTAINER_CALL --platform linux/${ARCH}"
22+
LABELS=()
23+
for LABEL in "$@"; do
24+
LABELS+=("${LABEL}")
3725
done
3826

39-
# Create and push the merged multiarch manifest for each tag; each tag combines all architecture-specific tags into one tag
27+
echo "Building all tags (${LABELS[@]}) for architecture: ${ARCH}"
28+
# Prepare image names with tags (each tag includes a label and an architecture)
29+
IMAGES=()
4030
for LABEL in "${LABELS[@]}"; do
41-
echo "Merging all architectures (${ARCHITECTURES[@]}) into single tag: ${LABEL}"
42-
43-
MANIFEST_MERGE_CALL="docker buildx imagetools create -t ghcr.io/eclipse-score/devcontainer:${LABEL}"
31+
IMAGES+=("--image-name \"ghcr.io/eclipse-score/devcontainer:${LABEL}-${ARCH}\"")
32+
done
4433

45-
for ARCH in "${ARCHITECTURES[@]}"; do
46-
MANIFEST_MERGE_CALL+=" ghcr.io/eclipse-score/devcontainer:${LABEL}-${ARCH}"
47-
done
34+
# Prepare devcontainer build command
35+
DEVCONTAINER_CALL="devcontainer build --push --workspace-folder src/s-core-devcontainer --cache-from ghcr.io/eclipse-score/devcontainer"
4836

49-
eval "$MANIFEST_MERGE_CALL"
37+
# Append image names to the build command
38+
for IMAGE in "${IMAGES[@]}"; do
39+
DEVCONTAINER_CALL+=" $IMAGE"
5040
done
41+
42+
# Execute the build and push all tags for the specific architecture
43+
eval "$DEVCONTAINER_CALL --platform linux/${ARCH}"

0 commit comments

Comments
 (0)