Skip to content

Commit 072a325

Browse files
lsierantlsierant
committed
Publish artifacts to staging
1 parent aea65d6 commit 072a325

File tree

4 files changed

+274
-30
lines changed

4 files changed

+274
-30
lines changed

‎scripts/dev/configure_container_auth.sh

Lines changed: 61 additions & 30 deletions
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,11 @@ source scripts/funcs/kubernetes
1010

1111
CONTAINER_RUNTIME="${CONTAINER_RUNTIME-"docker"}"
1212

13+
# Registry URLs
14+
ECR_EU_WEST="268558157000.dkr.ecr.eu-west-1.amazonaws.com"
15+
ECR_US_EAST="268558157000.dkr.ecr.us-east-1.amazonaws.com"
16+
ECR_SEARCH_US_EAST="901841024863.dkr.ecr.us-east-1.amazonaws.com"
17+
1318
setup_validate_container_runtime() {
1419
case "${CONTAINER_RUNTIME}" in
1520
"podman")
@@ -104,26 +109,44 @@ if [[ ! -f "${CONFIG_PATH}" ]]; then
104109
write_file '{}' "${CONFIG_PATH}"
105110
fi
106111

107-
if [[ -f "${CONFIG_PATH}" ]]; then
108-
if [[ "${RUNNING_IN_EVG:-"false"}" != "true" ]]; then
109-
echo "Checking if container registry credentials are valid..."
110-
ecr_auth=$(exec_cmd jq -r '.auths."268558157000.dkr.ecr.us-east-1.amazonaws.com".auth // empty' "${CONFIG_PATH}")
112+
check_if_login_required() {
113+
echo "Checking if container registry credentials are valid..."
114+
115+
check_registry_credentials() {
116+
registry_url=$1
117+
image_path=$1
118+
image_tag=$2
119+
# shellcheck disable=SC2016
120+
ecr_auth=$(exec_cmd jq -r --arg registry "${registry_url}" '.auths.[$registry].auth // empty' "${CONFIG_PATH}")
111121

112122
if [[ -n "${ecr_auth}" ]]; then
113-
http_status=$(curl --head -s -o /dev/null -w "%{http_code}" --max-time 3 "https://268558157000.dkr.ecr.us-east-1.amazonaws.com/v2/dev/mongodb-kubernetes/manifests/latest" \
123+
http_status=$(curl --head -s -o /dev/null -w "%{http_code}" --max-time 3 "https://${registry_url}/v2/${image_path}/manifest/${image_tag}" \
114124
-H "Authorization: Basic ${ecr_auth}" 2>/dev/null || echo "error/timeout")
115125

116126
if [[ "${http_status}" != "401" && "${http_status}" != "403" && "${http_status}" != "error/timeout" ]]; then
117127
echo "Container registry credentials are up to date - not performing the new login!"
118-
exit
128+
return 0
119129
fi
120-
echo "Container login required (HTTP status: ${http_status})"
130+
echo -e "${RED}Container login required (HTTP status: ${http_status})${NO_COLOR}"
121131
else
122-
echo "No ECR credentials found in container config - login required"
132+
echo -e "${RED}No ECR credentials found in container config - login required${NO_COLOR}"
123133
fi
134+
135+
return 0
136+
}
137+
138+
check_registry_credentials "${ECR_EU_WEST}" "mongot/community" "1.47.0" | prepend "${ECR_EU_WEST}" || return 1
139+
check_registry_credentials "${ECR_US_EAST}" "dev/mongodb-kubernetes" "latest" | prepend "${ECR_US_EAST}" || return 1
140+
if [[ "${MDB_SEARCH_AWS_SSO_LOGIN:-"false"}" == "true" ]]; then
141+
check_registry_credentials "${ECR_SEARCH_US_EAST}" "mongot-community/rapid-releases" "latest" | prepend "${ECR_SEARCH_US_EAST}" || return 1
124142
fi
125143

144+
return 0
145+
}
146+
147+
login_to_registries() {
126148
title "Performing container login to ECR registries"
149+
echo "$(aws --version)}"
127150

128151
# There could be some leftovers on Evergreen (Docker-specific, skip for Podman)
129152
if [[ "${CONTAINER_RUNTIME}" == "docker" ]]; then
@@ -134,34 +157,42 @@ if [[ -f "${CONFIG_PATH}" ]]; then
134157
remove_element "credHelpers"
135158
fi
136159
fi
137-
fi
138160

161+
aws ecr get-login-password --region "us-east-1" | registry_login "AWS" "${ECR_US_EAST}"
139162

140-
echo "$(aws --version)}"
163+
if [[ "${MDB_SEARCH_AWS_SSO_LOGIN:"-false"}" == "true" ]]; then
164+
aws sso login --profile devprod-platforms-ecr-user
165+
aws --profile devprod-platforms-ecr-user ecr get-login-password --region us-east-1 | registry_login "AWS" "${ECR_SEARCH_US_EAST}"
166+
fi
141167

142-
aws ecr get-login-password --region "us-east-1" | registry_login "AWS" "268558157000.dkr.ecr.us-east-1.amazonaws.com"
168+
# by default docker tries to store credentials in an external storage (e.g. OS keychain) - not in the config.json
169+
# We need to store it as base64 string in config.json instead so we need to remove the "credsStore" element
170+
# This is Docker-specific behavior, Podman stores credentials directly in auth.json
171+
if [[ "${CONTAINER_RUNTIME}" == "docker" ]] && exec_cmd grep -q "credsStore" "${CONFIG_PATH}"; then
172+
remove_element "credsStore"
143173

144-
# by default docker tries to store credentials in an external storage (e.g. OS keychain) - not in the config.json
145-
# We need to store it as base64 string in config.json instead so we need to remove the "credsStore" element
146-
# This is Docker-specific behavior, Podman stores credentials directly in auth.json
147-
if [[ "${CONTAINER_RUNTIME}" == "docker" ]] && exec_cmd grep -q "credsStore" "${CONFIG_PATH}"; then
148-
remove_element "credsStore"
174+
# login again to store the credentials into the config.json
175+
aws ecr get-login-password --region "us-east-1" | registry_login "AWS" "${ECR_US_EAST}"
176+
fi
149177

150-
# login again to store the credentials into the config.json
151-
aws ecr get-login-password --region "us-east-1" | registry_login "AWS" "268558157000.dkr.ecr.us-east-1.amazonaws.com"
152-
fi
178+
aws ecr get-login-password --region "eu-west-1" | registry_login "AWS" "${ECR_EU_WEST}"
179+
180+
if [[ -n "${PRERELEASE_PULLSECRET_DOCKERCONFIGJSON:-}" ]]; then
181+
# log in to quay.io for the mongodb/mongodb-search-community private repo
182+
# TODO remove once we switch to the official repo in Public Preview
183+
quay_io_auth_file=$(mktemp)
184+
config_tmp=$(mktemp)
185+
echo "${PRERELEASE_PULLSECRET_DOCKERCONFIGJSON}" | base64 -d > "${quay_io_auth_file}"
186+
exec_cmd jq -s '.[0] * .[1]' "${quay_io_auth_file}" "${CONFIG_PATH}" > "${config_tmp}"
187+
exec_cmd mv "${config_tmp}" "${CONFIG_PATH}"
188+
rm "${quay_io_auth_file}"
189+
fi
190+
}
153191

154-
aws ecr get-login-password --region "eu-west-1" | registry_login "AWS" "268558157000.dkr.ecr.eu-west-1.amazonaws.com"
155-
156-
if [[ -n "${PRERELEASE_PULLSECRET_DOCKERCONFIGJSON:-}" ]]; then
157-
# log in to quay.io for the mongodb/mongodb-search-community private repo
158-
# TODO remove once we switch to the official repo in Public Preview
159-
quay_io_auth_file=$(mktemp)
160-
config_tmp=$(mktemp)
161-
echo "${PRERELEASE_PULLSECRET_DOCKERCONFIGJSON}" | base64 -d > "${quay_io_auth_file}"
162-
exec_cmd jq -s '.[0] * .[1]' "${quay_io_auth_file}" "${CONFIG_PATH}" > "${config_tmp}"
163-
exec_cmd mv "${config_tmp}" "${CONFIG_PATH}"
164-
rm "${quay_io_auth_file}"
192+
if [[ "${RUNNING_IN_EVG:-"false"}" != "true" ]]; then
193+
check_if_login_required
194+
else
195+
login_to_registries
165196
fi
166197

167198
create_image_registries_secret

‎scripts/dev/contexts/private-context-template

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -101,3 +101,7 @@ export PRERELEASE_PULLSECRET_DOCKERCONFIGJSON="<dockerconfigjson secret>"
101101

102102
# uncomment to enable license update with pre-commit script
103103
# export MDB_UPDATE_LICENSES=true
104+
105+
# enable only for getting mongodb search rapid-releases from
106+
# 901841024863.dkr.ecr.us-east-1.amazonaws.com/mongot-community/rapid-releases
107+
export MDB_SEARCH_AWS_SSO_LOGIN="false"

‎scripts/dev/publish_artifacts_to_staging_quay.sh

Lines changed: 203 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,203 @@
1+
#!/bin/bash
2+
#
3+
# Script to publish (mirror) container images and helm charts to staging registry.
4+
#
5+
6+
set -euo pipefail
7+
8+
test "${MDB_BASH_DEBUG:-0}" -eq 1 && set -x
9+
10+
source scripts/dev/set_env_context.sh
11+
source scripts/funcs/printing
12+
13+
BASE_REPO_URL="268558157000.dkr.ecr.us-east-1.amazonaws.com/dev"
14+
STAGING_BASE_URL="quay.io/mongodb/staging"
15+
16+
if [[ $# -ne 2 ]]; then
17+
echo "The tool mirrors images built in any given evg patch id (or latest) from ${BASE_REPO_URL} to ${STAGING_BASE_URL}"
18+
echo "It publishes helm oci image of the helm chart with chart version \"<prerelease helm version>-<version_id>\""
19+
echo "Usage: $0 <prerelease helm version> <version_id>"
20+
echo "Example: $0 1.4.0-prerelease 68b1a853973bae0007d5eaa0"
21+
echo ""
22+
exit 1
23+
fi
24+
25+
helm_chart_version_prefix="$1"
26+
operator_version="$2"
27+
search_version=${3:-"latest"}
28+
29+
helm_chart_version="${helm_chart_version_prefix}-${operator_version}"
30+
31+
get_arch_digest() {
32+
local image="$1"
33+
local arch="$2"
34+
local manifest_json
35+
manifest_json=$(docker buildx imagetools inspect --raw "${image}" 2>/dev/null || echo '{}')
36+
37+
local media_type
38+
media_type=$(echo "$manifest_json" | jq -r '.mediaType // empty')
39+
40+
if [[ "$media_type" == *"manifest.list"* ]]; then
41+
# this is a multi-arch manifest
42+
local arch_digest
43+
arch_digest=$(echo "$manifest_json" | jq -r ".manifests[] | select(.platform.architecture == \"${arch}\" and .platform.os == \"linux\") | .digest")
44+
45+
if [[ -n "$arch_digest" && "$arch_digest" != "null" ]]; then
46+
echo "$arch_digest"
47+
return 0
48+
fi
49+
elif [[ "${arch}" == "amd64" ]]; then
50+
# otherwise it must be a single-arch (image) manifest, so we return it only if we ask for amd64
51+
local arch_digest
52+
arch_digest="sha256:$(echo -n "$manifest_json" | sha256)"
53+
echo "$arch_digest"
54+
fi
55+
56+
echo ""
57+
return 0
58+
}
59+
60+
process_image() {
61+
local source_image="$1"
62+
local target_image="$2"
63+
64+
echo " Processing ${source_image}..."
65+
66+
local digest_arm64
67+
local digest_amd64
68+
digest_arm64=$(get_arch_digest "${source_image}" arm64)
69+
digest_amd64=$(get_arch_digest "${source_image}" amd64)
70+
71+
if [[ -n "${digest_amd64}" ]]; then
72+
docker pull "${source_image}@${digest_amd64}"
73+
docker tag "${source_image}@${digest_amd64}" "${target_image}-amd64"
74+
docker push "${target_image}-amd64"
75+
fi
76+
77+
if [[ -n "${digest_arm64}" ]]; then
78+
docker pull "${source_image}@${digest_arm64}"
79+
docker tag "${source_image}@${digest_arm64}" "${target_image}-arm64"
80+
docker push "${target_image}-arm64"
81+
fi
82+
83+
docker manifest create "${target_image}" ${digest_amd64:+--amend ${target_image}-amd64} ${digest_arm64:+--amend ${target_image}-arm64}
84+
docker manifest push "${target_image}"
85+
}
86+
87+
publish_images() {
88+
local names=()
89+
local sources=()
90+
local destinations=()
91+
92+
operator_images=(
93+
"mongodb-kubernetes"
94+
"mongodb-kubernetes-database"
95+
"mongodb-kubernetes-init-appdb"
96+
"mongodb-kubernetes-init-database"
97+
"mongodb-kubernetes-init-ops-manager"
98+
"mongodb-kubernetes-readinessprobe"
99+
"mongodb-kubernetes-operator-version-upgrade-post-start-hook"
100+
)
101+
102+
if [[ -n "${search_version}" ]]; then
103+
names+=("mongodb-search")
104+
sources+=("901841024863.dkr.ecr.us-east-1.amazonaws.com/mongot-community/rapid-releases:${search_version}")
105+
destinations+=("${STAGING_BASE_URL}/mongodb-search:${search_version}")
106+
fi
107+
108+
for image in "${operator_images[@]}"; do
109+
names+=("${image}")
110+
sources+=("${BASE_REPO_URL}/${image}:${operator_version}")
111+
destinations+=("${STAGING_BASE_URL}/${image}:${helm_chart_version}")
112+
done
113+
114+
echo "Starting Docker image re-tagging and publishing to staging..."
115+
echo "Version ID: ${operator_version}"
116+
echo "Source repository: ${BASE_REPO_URL}"
117+
echo "Target repository: ${STAGING_BASE_URL}"
118+
echo ""
119+
120+
for i in "${!names[@]}"; do
121+
process_image "${sources[$i]}" "${destinations[$i]}"
122+
done
123+
124+
echo "=== SUMMARY ==="
125+
echo "All images have been successfully re-tagged and pushed to staging!"
126+
echo ""
127+
echo "Images processed:"
128+
for i in "${!names[@]}"; do
129+
echo " ${names[$i]}: ${sources[$i]} -> ${destinations[$i]}"
130+
done
131+
}
132+
133+
update_helm_values() {
134+
scripts/evergreen/release/update_helm_values_files.py
135+
yq eval ".version = \"${helm_chart_version}\"" -i helm_chart/Chart.yaml
136+
echo "Updated helm_chart/Chart.yaml version to: ${helm_chart_version}"
137+
138+
yq eval \
139+
".registry.operator = \"${STAGING_BASE_URL}\" |
140+
.registry.database = \"${STAGING_BASE_URL}\" |
141+
.registry.initDatabase = \"${STAGING_BASE_URL}\" |
142+
.registry.initOpsManager = \"${STAGING_BASE_URL}\" |
143+
.registry.initAppDb = \"${STAGING_BASE_URL}\" |
144+
.registry.appDb = \"${STAGING_BASE_URL}\" |
145+
.registry.versionUpgradeHook = \"${STAGING_BASE_URL}\" |
146+
.registry.readinessProbe = \"${STAGING_BASE_URL}\"
147+
" -i helm_chart/values.yaml
148+
echo "Updated helm_chart/values.yaml registry to: ${STAGING_BASE_URL}"
149+
}
150+
151+
prepare_helm_oci_image() {
152+
mkdir -p tmp
153+
helm package helm_chart -d tmp/
154+
}
155+
156+
push_helm_oci_image() {
157+
export HELM_REGISTRY_CONFIG=~/.docker/config.json
158+
helm push "tmp/mongodb-kubernetes-${helm_chart_version}.tgz" "oci://${STAGING_BASE_URL}/helm-chart"
159+
}
160+
161+
update_release_json() {
162+
if [[ ! -f "release.json" ]]; then
163+
echo "Error: release.json file not found"
164+
exit 1
165+
fi
166+
167+
echo "Updating release.json with versions..."
168+
169+
# Update operator and init versions
170+
jq --arg version "${helm_chart_version}" \
171+
--arg registry "${STAGING_BASE_URL}" \
172+
'.mongodbOperator = $version |
173+
.initDatabaseVersion = $version |
174+
.initOpsManagerVersion = $version |
175+
.initAppDbVersion = $version |
176+
.databaseImageVersion = $version' \
177+
release.json > release.json.tmp && mv release.json.tmp release.json
178+
179+
# Update search community version
180+
jq --arg searchVersion "${search_version}" \
181+
--arg searchRepo "${STAGING_BASE_URL}" \
182+
--arg searchImageName "mongodb-search" \
183+
'.search.community.repo = $searchRepo |
184+
.search.community.name = $searchImageName |
185+
.search.community.version = $searchVersion' \
186+
release.json > release.json.tmp && mv release.json.tmp release.json
187+
188+
echo "Updated release.json with:"
189+
echo " - Operator versions: ${helm_chart_version}"
190+
echo " - Search community version: ${MDB_SEARCH_COMMUNITY_VERSION}"
191+
}
192+
193+
revert_changes_to_local_files() {
194+
echo "Reverting generated/updated files: helm_chart/ public/ config/ release.json"
195+
git checkout -- helm_chart/ public/ config/ release.json
196+
}
197+
198+
publish_images
199+
update_release_json
200+
update_helm_values
201+
prepare_helm_oci_image
202+
push_helm_oci_image
203+
revert_changes_to_local_files

‎scripts/evergreen/release/update_helm_values_files.py

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -89,6 +89,12 @@ def update_helm_charts(operator_version, release):
8989
set_value_in_yaml_file(
9090
"helm_chart/values.yaml", "search.community.version", release["search"]["community"]["version"]
9191
)
92+
set_value_in_yaml_file(
93+
"helm_chart/values.yaml", "search.community.repo", release["search"]["community"]["repo"]
94+
)
95+
set_value_in_yaml_file(
96+
"helm_chart/values.yaml", "search.community.name", release["search"]["community"]["name"]
97+
)
9298

9399

94100
def update_cluster_service_version(operator_version):

0 commit comments

Comments
 (0)