From 207b8722d167a2c0c6b1b53b40097f344f7e1758 Mon Sep 17 00:00:00 2001 From: dkijania Date: Mon, 17 Nov 2025 18:38:54 +0100 Subject: [PATCH 1/3] un-hardcode gcr.io repo from various scripts --- buildkite/scripts/run-test-executive-local.sh | 8 ++++++-- buildkite/src/Command/DockerImage.dhall | 7 +++++++ buildkite/src/Command/Packages/Publish.dhall | 6 ++++++ .../src/Command/Rosetta/Connectivity.dhall | 7 ++++++- buildkite/src/Command/TestExecutive.dhall | 5 ++++- dockerfiles/Dockerfile-mina-daemon-hardfork | 2 +- scripts/docker/build.sh | 1 + scripts/docker/helper.sh | 11 +++++++++-- scripts/docker/promote.sh | 18 +++++++++--------- scripts/docker/release.sh | 1 + scripts/docker/verify.sh | 17 ++++++++++++++++- scripts/tests/rosetta-connectivity.sh | 9 ++++++++- 12 files changed, 74 insertions(+), 18 deletions(-) diff --git a/buildkite/scripts/run-test-executive-local.sh b/buildkite/scripts/run-test-executive-local.sh index 2fbc7ebdd0f0..081524beb0a3 100755 --- a/buildkite/scripts/run-test-executive-local.sh +++ b/buildkite/scripts/run-test-executive-local.sh @@ -25,9 +25,13 @@ fi cleanup TEST_NAME="$1" +DOCKER_REPO="$2" +MINA_DOCKER_NAME="mina-daemon" +MINA_ARCHIVE_DOCKER_NAME="mina-archive" -MINA_IMAGE="gcr.io/o1labs-192920/mina-daemon:$MINA_DOCKER_TAG-berkeley" -ARCHIVE_IMAGE="gcr.io/o1labs-192920/mina-archive:$MINA_DOCKER_TAG-berkeley" + +MINA_IMAGE="$DOCKER_REPO/$MINA_DOCKER_NAME:$MINA_DOCKER_TAG-berkeley" +ARCHIVE_IMAGE="$DOCKER_REPO/$MINA_ARCHIVE_DOCKER_NAME:$MINA_DOCKER_TAG-berkeley" if [[ "${TEST_NAME:0:15}" == "block-prod-prio" ]] && [[ "$RUN_OPT_TESTS" == "" ]]; then echo "Skipping $TEST_NAME" diff --git a/buildkite/src/Command/DockerImage.dhall b/buildkite/src/Command/DockerImage.dhall index 1c3eff265672..c1361fd96774 100644 --- a/buildkite/src/Command/DockerImage.dhall +++ b/buildkite/src/Command/DockerImage.dhall @@ -20,6 +20,8 @@ let DockerLogin = ../Command/DockerLogin/Type.dhall let DebianRepo = ../Constants/DebianRepo.dhall +let DockerRepo = ../Constants/DockerRepo.dhall + let DebianVersions = ../Constants/DebianVersions.dhall let Network = ../Constants/Network.dhall @@ -51,6 +53,7 @@ let ReleaseSpec = , build_flags : BuildFlags.Type , step_key_suffix : Text , docker_publish : DockerPublish.Type + , docker_repo : DockerRepo.Type , verify : Bool , if_ : Optional B/If } @@ -72,6 +75,7 @@ let ReleaseSpec = , docker_publish = DockerPublish.Type.Essential , no_cache = False , no_debian = False + , docker_repo = DockerRepo.Type.Internal , step_key_suffix = "-docker-image" , verify = False , deb_suffix = None Text @@ -166,6 +170,7 @@ let generateStep = ++ debSuffix ++ " --repo ${spec.repo}" ++ " --platform ${Arch.platform spec.arch}" + ++ " --docker-registry ${DockerRepo.show spec.docker_repo}" let releaseDockerCmd = if DockerPublish.shouldPublish @@ -184,6 +189,8 @@ let generateStep = ++ " --deb-build-flags ${BuildFlags.lowerName spec.build_flags}" ++ " --platform ${Arch.platform spec.arch}" + ++ " --docker-registry ${DockerRepo.show + spec.docker_repo}" else " echo In order to ensure storage optimization, skipping publishing docker as this is not essential one or publishing is disabled . Docker publish setting is set to ${DockerPublish.show spec.docker_publish}." diff --git a/buildkite/src/Command/Packages/Publish.dhall b/buildkite/src/Command/Packages/Publish.dhall index 01cb9357012c..c55d77b8cef3 100644 --- a/buildkite/src/Command/Packages/Publish.dhall +++ b/buildkite/src/Command/Packages/Publish.dhall @@ -26,6 +26,8 @@ let DebianVersions = ../../Constants/DebianVersions.dhall let DebianRepo = ../../Constants/DebianRepo.dhall +let DockerRepo = ../../Constants/DockerRepo.dhall + let ContainerImages = ../../Constants/ContainerImages.dhall let Command = ../Base.dhall @@ -72,6 +74,7 @@ let Spec = , depends_on : List Command.TaggedKey.Type , branch : Text , architectures : List Architecture.Type + , docker_repo : DockerRepo.Type , if_ : Optional Text } , default = @@ -89,6 +92,7 @@ let Spec = , verify = True , branch = "" , architectures = [ Architecture.Type.Amd64, Architecture.Type.Arm64 ] + , docker_repo = DockerRepo.Type.Internal , if_ = None Text } } @@ -251,6 +255,8 @@ let publish ++ "--target-version ${r.value} " ++ "--codenames ${codenames} " ++ "--only-dockers " + ++ "--docker-repo ${DockerRepo.show + spec.docker_repo} " ++ "--force-upload-debians " ) ] diff --git a/buildkite/src/Command/Rosetta/Connectivity.dhall b/buildkite/src/Command/Rosetta/Connectivity.dhall index 3382a0dfa2cc..30c5c46cea30 100644 --- a/buildkite/src/Command/Rosetta/Connectivity.dhall +++ b/buildkite/src/Command/Rosetta/Connectivity.dhall @@ -24,6 +24,8 @@ let Dockers = ../../Constants/DockerVersions.dhall let Profiles = ../../Constants/Profiles.dhall +let DockerRepo = ../../Constants/DockerRepo.dhall + let B/SoftFail = B.definitions/commandStep/properties/soft_fail/Type let B/If = B.definitions/commandStep/properties/if/Type @@ -37,6 +39,7 @@ let Spec = , timeout : Natural , profile : Profiles.Type , scope : List PipelineScope.Type + , repo : DockerRepo.Type , if_ : B/If } , default = @@ -47,6 +50,7 @@ let Spec = , timeout = 1000 , profile = Profiles.Type.Devnet , scope = PipelineScope.Full + , repo = DockerRepo.Type.Internal , if_ = "build.pull_request.base_branch != \"develop\" && build.branch != \"develop\"" } @@ -64,7 +68,8 @@ let command , "source ./buildkite/scripts/export-git-env-vars.sh" , "scripts/tests/rosetta-connectivity.sh --network ${Network.lowerName spec.network} --tag \\\${MINA_DOCKER_TAG} --timeout ${Natural/show - spec.timeout} --run-compatibility-test develop --run-load-test " + spec.timeout} --repo ${DockerRepo.show + spec.repo} --run-compatibility-test develop --run-load-test " ] ] , label = diff --git a/buildkite/src/Command/TestExecutive.dhall b/buildkite/src/Command/TestExecutive.dhall index 82034b4905c6..95864b9f54a2 100644 --- a/buildkite/src/Command/TestExecutive.dhall +++ b/buildkite/src/Command/TestExecutive.dhall @@ -4,6 +4,8 @@ let Size = ./Size.dhall let Cmd = ../Lib/Cmds.dhall +let DockerRepo = ../Constants/DockerRepo.dhall + let SelectFiles = ../Lib/SelectFiles.dhall in { executeLocal = @@ -13,7 +15,8 @@ in { executeLocal = Command.Config::{ , commands = [ Cmd.run - "MINA_DEB_CODENAME=bullseye ; source ./buildkite/scripts/export-git-env-vars.sh && ./buildkite/scripts/run-test-executive-local.sh ${testName}" + "MINA_DEB_CODENAME=bullseye ; source ./buildkite/scripts/export-git-env-vars.sh && ./buildkite/scripts/run-test-executive-local.sh ${testName} ${DockerRepo.show + DockerRepo.Type.Internal}" ] , artifact_paths = [ SelectFiles.exactly "." "${testName}.local.test.log" ] diff --git a/dockerfiles/Dockerfile-mina-daemon-hardfork b/dockerfiles/Dockerfile-mina-daemon-hardfork index dafc828d6be2..f619e132cc65 100644 --- a/dockerfiles/Dockerfile-mina-daemon-hardfork +++ b/dockerfiles/Dockerfile-mina-daemon-hardfork @@ -1,7 +1,7 @@ ARG deb_version=latest ARG deb_codename=bullseye ARG network=mainnet -ARG docker_repo="gcr.io/o1labs-192920" +ARG docker_repo ARG deb_release=unstable ARG deb_legacy_version ARG deb_repo="http://packages.o1test.net" diff --git a/scripts/docker/build.sh b/scripts/docker/build.sh index 2df0adc5497f..317bef2218e8 100755 --- a/scripts/docker/build.sh +++ b/scripts/docker/build.sh @@ -47,6 +47,7 @@ while [[ "$#" -gt 0 ]]; do case $1 in -c|--cache-from) INPUT_CACHE="$2"; shift;; -r|--repo) MINA_REPO="$2"; shift;; -p|--platform) INPUT_PLATFORM="$2"; shift;; + --docker-registry) export DOCKER_REGISTRY="$2"; shift;; --no-cache) NO_CACHE="--no-cache"; ;; --deb-codename) INPUT_CODENAME="$2"; shift;; --deb-release) INPUT_RELEASE="$2"; shift;; diff --git a/scripts/docker/helper.sh b/scripts/docker/helper.sh index e4dcbbd469b1..f85fbea41f9b 100644 --- a/scripts/docker/helper.sh +++ b/scripts/docker/helper.sh @@ -89,11 +89,18 @@ function get_platform_suffix() { esac } +function check_docker_registry() { + if [[ -z "${DOCKER_REGISTRY:-}" ]]; then + echo "ERROR: DOCKER_REGISTRY environment variable is not set" >&2 + exit 1 + fi +} function export_docker_tag() { export_suffixes - - export DOCKER_REGISTRY="gcr.io/o1labs-192920" + + check_docker_registry + export DOCKER_REGISTRY="${DOCKER_REGISTRY}" PLATFORM_SUFFIX="$(get_platform_suffix)" export TAG="${DOCKER_REGISTRY}/${SERVICE}:${VERSION}${BUILD_FLAG_SUFFIX}${PLATFORM_SUFFIX}" diff --git a/scripts/docker/promote.sh b/scripts/docker/promote.sh index bfb96c6e5185..e3413ee26857 100755 --- a/scripts/docker/promote.sh +++ b/scripts/docker/promote.sh @@ -4,15 +4,15 @@ set -eo pipefail CLEAR='\033[0m' RED='\033[0;31m' -PUBLISH=0 -GCR_REPO=gcr.io/o1labs-192920 +REGISTRY="" QUIET="" ARCH=amd64 while [[ "$#" -gt 0 ]]; do case $1 in -n|--name) NAME="$2"; shift;; -v|--version) VERSION="$2"; shift;; -t|--tag) TAG="$2"; shift;; - -p|--publish) PUBLISH=1; ;; + -r|--registry) REGISTRY="$2"; shift;; + -p|--publish_registry) TARGET_REPO="$2"; shift;; -q|--quiet) QUIET="-q"; ;; -a|--arch) ARCH="$2"; shift;; *) echo "Unknown parameter passed: $1"; exit 1;; @@ -26,9 +26,10 @@ function usage() { echo " -n, --name The Docker name (mina-devnet, mina-archive etc.)" echo " -v, --version The Docker version" echo " -t, --tag The Additional tag" - echo " -p, --publish The Publish to docker.io flag. If defined script will publish docker do docker.io. Otherwise it will still resides in gcr.io" + echo " -p, --publish The Publish to docker.io flag. If defined script will publish docker do docker.io. Otherwise it will still resides in source registry" echo " -q, --quiet The Quiet mode. If defined script will output limited logs" echo " -a, --arch The Architecture of docker (amd64, arm64)" + echo " -r, --registry The Docker registry (e.g. gcr.io/o1labs-192920)" echo "" echo "Example: $0 --name mina-archive --version 2.0.0-rc1-48efea4 --tag devnet-latest-nightly-bullseye " exit 1 @@ -37,6 +38,7 @@ function usage() { if [[ -z "$NAME" ]]; then usage "Name is not set!"; fi; if [[ -z "$VERSION" ]]; then usage "Version is not set!"; fi; if [[ -z "$TAG" ]]; then usage "Tag is not set!"; fi; +if [[ -z "$REGISTRY" ]]; then usage "Registry is not set!"; fi; # Sanitize the tag to ensure it is compliant with Docker tag format TAG=$(echo "$TAG" | sed 's/[^a-zA-Z0-9_.-]/-/g') @@ -47,16 +49,14 @@ case $ARCH in *) echo "❌ Unknown architecture passed: $ARCH"; exit 1 ;; esac -SOURCE_TAG="${GCR_REPO}/${NAME}:${VERSION}${DOCKER_ARCH_SUFFIX}" +SOURCE_TAG="${REGISTRY}/${NAME}:${VERSION}${DOCKER_ARCH_SUFFIX}" echo "📎 Adding new tag ($TAG) for docker ${SOURCE_TAG}" echo " 📥 pulling ${SOURCE_TAG}" docker pull $QUIET ${SOURCE_TAG} -if [[ $PUBLISH == 1 ]]; then - TARGET_REPO=docker.io/minaprotocol -else - TARGET_REPO=$GCR_REPO +if [[ -z "$TARGET_REPO" ]]; then + TARGET_REPO=$REGISTRY fi TARGET_TAG="${TARGET_REPO}/${NAME}:${TAG}${DOCKER_ARCH_SUFFIX}" diff --git a/scripts/docker/release.sh b/scripts/docker/release.sh index 723d41aeb1b1..eedf1ded870c 100755 --- a/scripts/docker/release.sh +++ b/scripts/docker/release.sh @@ -39,6 +39,7 @@ while [[ "$#" -gt 0 ]]; do case $1 in --deb-profile) export DEB_PROFILE="$2"; shift;; --deb-build-flags) export DEB_BUILD_FLAGS="$2"; shift;; --platform) export INPUT_PLATFORM="$2"; shift;; + --docker-registry) export DOCKER_REGISTRY="$2"; shift;; --help) usage "$@"; exit 0;; *) echo "Unknown parameter passed: $1"; usage "$@"; exit 1;; esac; shift; done diff --git a/scripts/docker/verify.sh b/scripts/docker/verify.sh index bece33db33a5..f318b64b361a 100755 --- a/scripts/docker/verify.sh +++ b/scripts/docker/verify.sh @@ -7,7 +7,7 @@ set -eo pipefail -REPO=gcr.io/o1labs-192920 +REPO="" VERSION=3.0.0-f872d85 ARCH=amd64 @@ -21,6 +21,21 @@ while [[ "$#" -gt 0 ]]; do case $1 in *) echo "Unknown parameter passed: $1"; exit 1;; esac; shift; done +if [[ -z "$PACKAGE" ]]; then + echo "❌ Package is not set! Use -p or --package to set it." + exit 1 +fi + +if [[ -z "$CODENAME" ]]; then + echo "❌ Codename is not set! Use -c or --codename to set it." + exit 1 +fi + +if [[ -z "$REPO" ]]; then + echo "❌ Repo is not set! Use -r or --repo to set it." + exit 1 +fi + COMMANDS=(--version --help) case $PACKAGE in diff --git a/scripts/tests/rosetta-connectivity.sh b/scripts/tests/rosetta-connectivity.sh index 4f9d759174b6..300f60f258eb 100755 --- a/scripts/tests/rosetta-connectivity.sh +++ b/scripts/tests/rosetta-connectivity.sh @@ -81,6 +81,7 @@ while [[ "$#" -gt 0 ]]; do case $1 in --run-load-test) RUN_LOAD_TEST=true ;; --run-compatibility-test) COMPATIBILITY_BRANCH="$2"; shift;; -t|--tag) TAG="$2"; shift;; + -r|--repo) REPO="$2"; shift;; --timeout) TIMEOUT="$2"; shift;; --upgrade-scripts-workdir) UPGRADE_SCRIPTS_WORKDIR="$2"; shift;; -h|--help) usage; exit 0;; @@ -94,11 +95,17 @@ if [[ "$NETWORK" != "devnet" && "$NETWORK" != "mainnet" && "$NETWORK" != "berkel usage; exit 1; fi +if [[ -z "$REPO" ]]; then + echo "❌ Docker repo is not set! Use -r or --repo to set it." + echo "" + usage; exit 1; +fi + if [[ -z "$TAG" ]]; then usage "Docker tag is not set!"; usage; exit 1; fi; set -x -container_id=$(docker run -v .:/workdir -p 3087:3087 -d --env MINA_NETWORK=$NETWORK gcr.io/o1labs-192920/mina-rosetta:$TAG-$NETWORK ) +container_id=$(docker run -v .:/workdir -p 3087:3087 -d --env MINA_NETWORK=$NETWORK $REPO/mina-rosetta:$TAG-$NETWORK ) stop_docker() { { docker stop "$container_id" ; docker rm "$container_id" ; } || true From 724f911637191921fc6d4d2bf069ae79b1beaefe Mon Sep 17 00:00:00 2001 From: dkijania Date: Mon, 17 Nov 2025 21:21:18 +0100 Subject: [PATCH 2/3] adjust release manager script --- buildkite/scripts/release/manager.sh | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/buildkite/scripts/release/manager.sh b/buildkite/scripts/release/manager.sh index f35970c7e7f8..0c79ed4835ff 100755 --- a/buildkite/scripts/release/manager.sh +++ b/buildkite/scripts/release/manager.sh @@ -527,6 +527,7 @@ function promote_and_verify_docker() { -v $__artifact_full_source_version \ -t $__artifact_full_target_version \ -a $__arch \ + -r "$__repo" \ $__publish_arg echo "" @@ -1013,6 +1014,7 @@ function promote(){ local __debian_repo=$DEBIAN_REPO local __debian_sign_key="" local __arch="$DEFAULT_ARCHITECTURES" + local __profile=$DEFAULT_PROFILE while [ ${#} -gt 0 ]; do @@ -1085,6 +1087,10 @@ function promote(){ __arch=${2:?$error_message} shift 2; ;; + --profile ) + __profile=${2:?$error_message} + shift 2; + ;; * ) echo -e "${RED} !! Unknown option: $1${CLEAR}\n"; echo ""; From 75fd9c9c8f840946fff36cb37fb3f1a41bcb2bea Mon Sep 17 00:00:00 2001 From: dkijania Date: Tue, 18 Nov 2025 13:31:33 +0100 Subject: [PATCH 3/3] fix argument names for docker/promote --- buildkite/scripts/release/manager.sh | 10 ++++------ scripts/docker/promote.sh | 19 +++++++++---------- 2 files changed, 13 insertions(+), 16 deletions(-) diff --git a/buildkite/scripts/release/manager.sh b/buildkite/scripts/release/manager.sh index 0c79ed4835ff..5282cf37b592 100755 --- a/buildkite/scripts/release/manager.sh +++ b/buildkite/scripts/release/manager.sh @@ -510,11 +510,9 @@ function promote_and_verify_docker() { local __artifact_full_target_version=$__target_version-$__codename${__suffix} if [[ $__publish_to_docker_io == 1 ]]; then - local __publish_arg="-p" - local __repo=$DOCKER_IO_REPO + local __push_repo=$DOCKER_IO_REPO else - local __publish_arg="" - local __repo=$GCR_REPO + local __push_repo=$GCR_REPO fi echo " 🐋 Publishing $__artifact docker for '$__network' network and '$__codename' codename with '$__target_version' version and '$__arch' " @@ -527,8 +525,8 @@ function promote_and_verify_docker() { -v $__artifact_full_source_version \ -t $__artifact_full_target_version \ -a $__arch \ - -r "$__repo" \ - $__publish_arg + --pull-registry "$GCR_REPO" \ + --push-registry "$__push_repo" \ echo "" diff --git a/scripts/docker/promote.sh b/scripts/docker/promote.sh index e3413ee26857..699cddeaf732 100755 --- a/scripts/docker/promote.sh +++ b/scripts/docker/promote.sh @@ -4,15 +4,14 @@ set -eo pipefail CLEAR='\033[0m' RED='\033[0;31m' -REGISTRY="" QUIET="" ARCH=amd64 while [[ "$#" -gt 0 ]]; do case $1 in -n|--name) NAME="$2"; shift;; -v|--version) VERSION="$2"; shift;; -t|--tag) TAG="$2"; shift;; - -r|--registry) REGISTRY="$2"; shift;; - -p|--publish_registry) TARGET_REPO="$2"; shift;; + --pull-registry) PULL_REGISTRY="$2"; shift;; + --push-registry) PUSH_REGISTRY="$2"; shift;; -q|--quiet) QUIET="-q"; ;; -a|--arch) ARCH="$2"; shift;; *) echo "Unknown parameter passed: $1"; exit 1;; @@ -26,10 +25,10 @@ function usage() { echo " -n, --name The Docker name (mina-devnet, mina-archive etc.)" echo " -v, --version The Docker version" echo " -t, --tag The Additional tag" - echo " -p, --publish The Publish to docker.io flag. If defined script will publish docker do docker.io. Otherwise it will still resides in source registry" + echo " --pull-registry The Docker pull registry (e.g. gcr.io/o1labs-192920)" + echo " --push-registry The Docker push registry (e.g. gcr.io/o1labs-192920)" echo " -q, --quiet The Quiet mode. If defined script will output limited logs" echo " -a, --arch The Architecture of docker (amd64, arm64)" - echo " -r, --registry The Docker registry (e.g. gcr.io/o1labs-192920)" echo "" echo "Example: $0 --name mina-archive --version 2.0.0-rc1-48efea4 --tag devnet-latest-nightly-bullseye " exit 1 @@ -38,7 +37,7 @@ function usage() { if [[ -z "$NAME" ]]; then usage "Name is not set!"; fi; if [[ -z "$VERSION" ]]; then usage "Version is not set!"; fi; if [[ -z "$TAG" ]]; then usage "Tag is not set!"; fi; -if [[ -z "$REGISTRY" ]]; then usage "Registry is not set!"; fi; +if [[ -z "$PULL_REGISTRY" ]]; then usage "Pull registry is not set!"; fi; # Sanitize the tag to ensure it is compliant with Docker tag format TAG=$(echo "$TAG" | sed 's/[^a-zA-Z0-9_.-]/-/g') @@ -49,17 +48,17 @@ case $ARCH in *) echo "❌ Unknown architecture passed: $ARCH"; exit 1 ;; esac -SOURCE_TAG="${REGISTRY}/${NAME}:${VERSION}${DOCKER_ARCH_SUFFIX}" +SOURCE_TAG="${PULL_REGISTRY}/${NAME}:${VERSION}${DOCKER_ARCH_SUFFIX}" echo "📎 Adding new tag ($TAG) for docker ${SOURCE_TAG}" echo " 📥 pulling ${SOURCE_TAG}" docker pull $QUIET ${SOURCE_TAG} -if [[ -z "$TARGET_REPO" ]]; then - TARGET_REPO=$REGISTRY +if [[ -z "$PUSH_REGISTRY" ]]; then + PUSH_REGISTRY=$PULL_REGISTRY fi -TARGET_TAG="${TARGET_REPO}/${NAME}:${TAG}${DOCKER_ARCH_SUFFIX}" +TARGET_TAG="${PUSH_REGISTRY}/${NAME}:${TAG}${DOCKER_ARCH_SUFFIX}" docker tag "${SOURCE_TAG}" "${TARGET_TAG}" echo " 📤 pushing ${TARGET_TAG}"