Skip to content

Commit 17c58f7

Browse files
authored
Merge pull request #18010 from MinaProtocol/ci_fix_prom
CI fix (promotion and arm64)
2 parents c86c6af + 2b45006 commit 17c58f7

File tree

6 files changed

+90
-32
lines changed

6 files changed

+90
-32
lines changed

buildkite/scripts/release/manager.sh

Lines changed: 10 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -145,20 +145,19 @@ function get_suffix() {
145145

146146
case $__profile in
147147
lightnet)
148-
echo "-lightnet"
148+
__profile_part="-lightnet"
149149
;;
150150
instrumented)
151-
echo "-instrumented"
151+
__profile_part="-instrumented"
152152
;;
153-
154153
*)
155-
echo ""
154+
__profile_part=""
156155
;;
157156
esac
158157

159158
case $__artifact in
160159
mina-daemon)
161-
echo "-$__network$__profile"
160+
echo "-$__network$__profile_part"
162161
;;
163162
mina-rosetta)
164163
echo "-$__network"
@@ -256,9 +255,10 @@ function calculate_docker_tag() {
256255
local __target_version=$3
257256
local __codename=$4
258257
local __network=$5
258+
local __profile=$6
259259

260260
local __network_suffix
261-
__network_suffix=$(get_suffix $__artifact $__network)
261+
__network_suffix=$(get_suffix $__artifact $__network "$__profile")
262262

263263
local __arch_suffix
264264
__arch_suffix=$(get_arch_suffix $__arch)
@@ -518,7 +518,7 @@ function promote_and_verify_docker() {
518518
fi
519519

520520
echo " 🐋 Publishing $__artifact docker for '$__network' network and '$__codename' codename with '$__target_version' version and '$__arch' "
521-
echo " 📦 Target version: $(calculate_docker_tag $__publish_to_docker_io $__artifact $__target_version $__codename "$__network ")"
521+
echo " 📦 Target version: $(calculate_docker_tag $__publish_to_docker_io $__artifact $__target_version $__codename $__network $__profile )"
522522
echo ""
523523
if [[ $__dry_run == 0 ]]; then
524524
prefix_cmd "$SUBCOMMAND_TAB" $SCRIPTPATH/../../../scripts/docker/promote.sh \
@@ -1202,7 +1202,7 @@ function promote(){
12021202
fi
12031203

12041204
if [[ $__only_debians == 0 ]]; then
1205-
promote_and_verify_docker $artifact $__source_version $__target_version $__codename $network $__publish_to_docker_io $__verify $__arch $__dry_run
1205+
promote_and_verify_docker $artifact $__source_version $__target_version $__codename $network $__profile $__publish_to_docker_io $__verify $__arch $__dry_run
12061206
fi
12071207
done
12081208
;;
@@ -1225,7 +1225,7 @@ function promote(){
12251225
fi
12261226

12271227
if [[ $__only_debians == 0 ]]; then
1228-
promote_and_verify_docker $artifact $__source_version $__target_version $__codename $network $__publish_to_docker_io $__verify $__arch $__dry_run
1228+
promote_and_verify_docker $artifact $__source_version $__target_version $__codename $network $__profile $__publish_to_docker_io $__verify $__arch $__dry_run
12291229
fi
12301230
done
12311231
;;
@@ -1247,7 +1247,7 @@ function promote(){
12471247
fi
12481248

12491249
if [[ $__only_debians == 0 ]]; then
1250-
promote_and_verify_docker $artifact $__source_version $__target_version $__codename $network $__publish_to_docker_io $__verify $__arch $__dry_run
1250+
promote_and_verify_docker $artifact $__source_version $__target_version $__codename $network $__profile $__publish_to_docker_io $__verify $__arch $__dry_run
12511251
fi
12521252
done
12531253
;;

buildkite/src/Command/MinaArtifact.dhall

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -113,7 +113,8 @@ let build_artifacts
113113
-> Command.build
114114
Command.Config::{
115115
, commands =
116-
Toolchain.select
116+
[ Cmd.run "./scripts/docker/setup_buildx.sh" ]
117+
# Toolchain.select
117118
spec.toolchainSelectMode
118119
spec.debVersion
119120
spec.arch

buildkite/src/Command/RunInToolchain.dhall

Lines changed: 20 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -7,13 +7,18 @@ let ContainerImages = ../Constants/ContainerImages.dhall
77
let Arch = ../Constants/Arch.dhall
88

99
let runInToolchainImage
10-
: Text -> List Text -> Text -> List Cmd.Type
10+
: Text -> Text -> List Text -> Text -> List Cmd.Type
1111
= \(image : Text)
12+
-> \(platform : Text)
1213
-> \(environment : List Text)
1314
-> \(innerScript : Text)
1415
-> [ Mina.fixPermissionsCommand ]
1516
# [ Cmd.runInDocker
16-
Cmd.Docker::{ image = image, extraEnv = environment }
17+
Cmd.Docker::{
18+
, image = image
19+
, extraEnv = environment
20+
, platform = platform
21+
}
1722
innerScript
1823
]
1924

@@ -29,14 +34,19 @@ let runInToolchainNoble
2934
}
3035
arch
3136

32-
in runInToolchainImage image environment innerScript
37+
in runInToolchainImage
38+
image
39+
(Arch.platform arch)
40+
environment
41+
innerScript
3342

3443
let runInToolchainJammy
3544
: List Text -> Text -> List Cmd.Type
3645
= \(environment : List Text)
3746
-> \(innerScript : Text)
3847
-> runInToolchainImage
3948
ContainerImages.minaToolchainJammy.amd64
49+
(Arch.platform Arch.Type.Amd64)
4050
environment
4151
innerScript
4252

@@ -52,7 +62,11 @@ let runInToolchainBookworm
5262
}
5363
arch
5464

55-
in runInToolchainImage image environment innerScript
65+
in runInToolchainImage
66+
image
67+
(Arch.platform arch)
68+
environment
69+
innerScript
5670

5771
let runInToolchainBullseye
5872
: Arch.Type -> List Text -> Text -> List Cmd.Type
@@ -61,6 +75,7 @@ let runInToolchainBullseye
6175
-> \(innerScript : Text)
6276
-> runInToolchainImage
6377
ContainerImages.minaToolchainBullseye.amd64
78+
(Arch.platform arch)
6479
environment
6580
innerScript
6681

@@ -70,6 +85,7 @@ let runInToolchain
7085
-> \(innerScript : Text)
7186
-> runInToolchainImage
7287
ContainerImages.minaToolchain
88+
(Arch.platform Arch.Type.Amd64)
7389
environment
7490
innerScript
7591

buildkite/src/Constants/Toolchain.dhall

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -27,7 +27,10 @@ let select =
2727
-> merge
2828
{ ByDebianAndArch = runner debVersion arch
2929
, Custom =
30-
\(image : Text) -> RunInToolchain.runInToolchainImage image
30+
\(image : Text)
31+
-> RunInToolchain.runInToolchainImage
32+
image
33+
(Arch.platform arch)
3134
}
3235
mode
3336

buildkite/src/Lib/Cmds.dhall

Lines changed: 7 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -18,12 +18,14 @@ let module =
1818
, privileged : Bool
1919
, useBash : Bool
2020
, useRoot : Bool
21+
, platform : Text
2122
}
2223
, default =
2324
{ extraEnv = [] : List Text
2425
, privileged = False
2526
, useBash = True
2627
, useRoot = False
28+
, platform = "linux/amd64"
2729
}
2830
}
2931

@@ -76,11 +78,11 @@ let module =
7678
= if docker.useBash then "/bin/bash" else "/bin/sh"
7779

7880
in { line =
79-
"docker run -it ${maybeRootOption} --rm --entrypoint ${entrypoint} --init --volume /var/storagebox:/var/storagebox --volume /var/secrets:/var/secrets --volume ${sharedDir}:/shared --volume ${outerDir}:/workdir --workdir /workdir${envVars}${ if docker.privileged
81+
"docker run -it ${maybeRootOption} --platform ${docker.platform} --rm --entrypoint ${entrypoint} --init --volume /var/storagebox:/var/storagebox --volume /var/secrets:/var/secrets --volume ${sharedDir}:/shared --volume ${outerDir}:/workdir --workdir /workdir${envVars}${ if docker.privileged
8082
81-
then " --privileged"
83+
then " --privileged"
8284

83-
else ""} ${docker.image} -c '${inner.line}'"
85+
else ""} ${docker.image} -c '${inner.line}'"
8486
, readable =
8587
Optional/map
8688
Text
@@ -147,7 +149,7 @@ let tests =
147149
let dockerExample =
148150
assert
149151
: { line =
150-
"docker run -it --rm --entrypoint /bin/bash --init --volume /var/storagebox:/var/storagebox --volume /var/secrets:/var/secrets --volume /var/buildkite/shared:/shared --volume \\\$BUILDKITE_BUILD_CHECKOUT_PATH:/workdir --workdir /workdir --env ENV1 --env ENV2 --env TEST foo/bar:tag -c 'echo hello'"
152+
"docker run -it --platform linux/amd64 --rm --entrypoint /bin/bash --init --volume /var/storagebox:/var/storagebox --volume /var/secrets:/var/secrets --volume /var/buildkite/shared:/shared --volume \\\$BUILDKITE_BUILD_CHECKOUT_PATH:/workdir --workdir /workdir --env ENV1 --env ENV2 --env TEST foo/bar:tag -c 'echo hello'"
151153
, readable = Some "Docker@foo/bar:tag ( echo hello )"
152154
}
153155
=== M.inDocker
@@ -159,7 +161,7 @@ let tests =
159161

160162
let cacheExample =
161163
assert
162-
: "./buildkite/scripts/cache-through.sh data.tar \"docker run -it --rm --entrypoint /bin/bash --init --volume /var/storagebox:/var/storagebox --volume /var/secrets:/var/secrets --volume /var/buildkite/shared:/shared --volume \\\$BUILDKITE_BUILD_CHECKOUT_PATH:/workdir --workdir /workdir --env ENV1 --env ENV2 --env TEST foo/bar:tag -c 'echo hello > /tmp/data/foo.txt && tar cvf data.tar /tmp/data'\""
164+
: "./buildkite/scripts/cache-through.sh data.tar \"docker run -it --platform linux/amd64 --rm --entrypoint /bin/bash --init --volume /var/storagebox:/var/storagebox --volume /var/secrets:/var/secrets --volume /var/buildkite/shared:/shared --volume \\\$BUILDKITE_BUILD_CHECKOUT_PATH:/workdir --workdir /workdir --env ENV1 --env ENV2 --env TEST foo/bar:tag -c 'echo hello > /tmp/data/foo.txt && tar cvf data.tar /tmp/data'\""
163165
=== M.format
164166
( M.cacheThrough
165167
M.Docker::{

scripts/docker/setup_buildx.sh

Lines changed: 47 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -33,20 +33,18 @@
3333
# - Handles special case for 'docker' driver (uses default builder)
3434
# - Creates or reuses existing builders for other drivers
3535
# - Bootstraps builder instances
36-
# - Installs binfmt emulation for cross-architecture builds
36+
# - Installs binfmt emulation for cross-architecture builds (now independent of driver)
3737
# - Provides summary of available builders
3838
#
3939
# REQUIREMENTS:
4040
# - Docker with Buildx plugin
4141
# - Privileged access (for binfmt installation)
4242
set -euo pipefail
4343

44-
45-
4644
# Config (override via env or args)
4745
NAME="${1:-${BUILDX_NAME:-xbuilder}}"
4846
DRIVER="${DRIVER:-docker}" # docker | docker-container | kubernetes
49-
ARCHS="${ARCHS:-arm64}" # binfmt architectures to install
47+
ARCHS="${ARCHS:-arm64}" # binfmt architectures to install (comma-separated)
5048
INSTALL_BINFMT="${INSTALL_BINFMT:-1}"
5149

5250
command -v docker >/dev/null || { echo "docker not found"; exit 1; }
@@ -62,12 +60,12 @@ use_builder () {
6260
# Special case: docker driver can only have ONE instance (the implicit 'default')
6361
if [[ "$DRIVER" == "docker" ]]; then
6462
echo "[buildx] Using 'docker' driver -> switching to existing 'default' builder"
65-
use_builder default || {
63+
if ! use_builder default; then
6664
# On some setups 'default' exists but isn't initialized yet; bootstrap via inspect
6765
docker buildx create --name default --driver docker --use || true
6866
docker buildx inspect default --bootstrap >/dev/null || true
6967
docker buildx use default
70-
}
68+
fi
7169
else
7270
# For docker-container (recommended) or other drivers: create or reuse NAME
7371
if docker buildx inspect "$NAME" >/dev/null 2>&1; then
@@ -84,15 +82,53 @@ fi
8482
echo "[buildx] Bootstrapping current builder"
8583
docker buildx inspect --bootstrap >/dev/null
8684

87-
# Install binfmt only when using docker-container (useful for cross-building)
85+
# Snapshot builder/driver for logs
8886
CURRENT_BUILDER="$(docker buildx ls | awk '/\*/{gsub(/\*/, "", $1); print $1}')"
8987
CURRENT_DRIVER="$(docker buildx inspect "$CURRENT_BUILDER" | awk -F': ' '/Driver:/ {print $2}')"
9088

91-
if [[ "$INSTALL_BINFMT" = "1" && "$CURRENT_DRIVER" = "docker-container" ]]; then
92-
echo "[binfmt] Ensuring $ARCHS emulation is installed"
93-
docker run --privileged --rm tonistiigi/binfmt --install "$ARCHS" >/dev/null
89+
# --- NEW: Ensure binfmt (QEMU) regardless of driver when requested ---
90+
if [[ "${INSTALL_BINFMT}" = "1" ]]; then
91+
# Normalize commas/spaces, split into array
92+
IFS=',' read -r -a _ARCHES <<<"$(echo "$ARCHS" | tr -d ' ')"
93+
94+
# Map host arch to docker "platform arch" names for quick decisions
95+
HOST_UNAME="$(uname -m)"
96+
case "$HOST_UNAME" in
97+
x86_64) HOST_ARCH_DOCKER="amd64" ;;
98+
aarch64) HOST_ARCH_DOCKER="arm64" ;;
99+
arm64) HOST_ARCH_DOCKER="arm64" ;;
100+
*) HOST_ARCH_DOCKER="$HOST_UNAME" ;;
101+
esac
102+
103+
echo "[binfmt] Requested arches: ${_ARCHES[*]} (host: $HOST_UNAME -> $HOST_ARCH_DOCKER)"
104+
echo "[binfmt] Ensuring emulators are installed (driver: $CURRENT_DRIVER)"
105+
106+
# Install all requested emulators in one shot (idempotent)
107+
docker run --privileged --rm tonistiigi/binfmt --install "$(IFS=','; echo "${_ARCHES[*]}")" >/dev/null || {
108+
echo "[binfmt] ERROR: Failed to install binfmt for: ${_ARCHES[*]}" >&2
109+
exit 1
110+
}
111+
112+
# Smoke-test non-native arches using busybox where applicable
113+
# Build map from arch -> platform string
114+
for a in "${_ARCHES[@]}"; do
115+
# Skip test for native arch
116+
if [[ "$a" == "$HOST_ARCH_DOCKER" ]]; then
117+
continue
118+
fi
119+
platform="linux/${a}"
120+
echo "[binfmt] Smoke test for ${platform} ..."
121+
if docker run --rm --platform="$platform" busybox uname -m 2>/dev/null | grep -qiE 'aarch64|arm64|x86_64|amd64|ppc64le|s390x|riscv64'; then
122+
echo "[binfmt] OK: ${platform} emulation working"
123+
else
124+
echo "[binfmt] WARNING: ${platform} smoke test did not confirm; emulation may still be fine depending on image availability" >&2
125+
fi
126+
done
127+
else
128+
echo "[binfmt] Skipped (INSTALL_BINFMT=${INSTALL_BINFMT})"
94129
fi
130+
# --------------------------------------------------------------------
95131

96132
echo
97133
echo "[summary]"
98-
docker buildx ls
134+
docker buildx ls

0 commit comments

Comments
 (0)