Merge pull request #17842 from MinaProtocol/dkijana/port_publish_fix_master

dkijania · web-flow · commit 92c7bfa5da25 · 2025-10-09T22:09:51.000+02:00
Port publishing improvements from compatible
diff --git a/buildkite/scripts/release/manager.sh b/buildkite/scripts/release/manager.sh
@@ -371,7 +371,10 @@ function publish_debian() {
     local __dry_run=$8
     local __backend=$9
     local __debian_repo=${10}
-    local __debian_sign_key=${11}
+    local __arch=${11:-DEFAULT_ARCHITECTURE}
+    local __force_upload_debians=${12:-0}
+    local __debian_sign_key=${13}
+    local __new_artifact_name=${14:-""}
 
     get_cached_debian_or_download $__backend $__artifact $__codename "$__network"
     local __artifact_full_name
@@ -405,6 +408,7 @@ function publish_debian() {
             --names "$DEBIAN_CACHE_FOLDER/$__codename/${__artifact_full_name}_${__target_version}.deb" \
             --version $__target_version \
             --bucket $__debian_repo \
+            "$(if [[ $__force_upload_debians == 1 ]]; then echo "--force"; fi)" \
             -c $__codename \
             -r $__channel \
             ${__sign_arg[@]}
@@ -558,6 +562,8 @@ function publish_help(){
     printf "  %-25s %s\n" "--backend" "[string] backend to use for storage. e.g gs,hetzner. default: gs";
     printf "  %-25s %s\n" "--debian-repo" "[string] debian repository to publish to. default: $DEBIAN_REPO";
     printf "  %-25s %s\n" "--debian-sign-key" "[string] debian signing key to use. default: lack of presence = no signing";
+    printf "  %-25s %s\n" "--strip-network-from-archive" "[bool] strip network from archive name. E.g mina-archive-devnet -> mina-archive";
+    printf "  %-25s %s\n" "--force-upload-debians" "[bool] force upload debian packages even if they exist already in the repository";
     echo ""
     echo "Example:"
     echo ""
@@ -588,6 +594,9 @@ function publish(){
     local __backend="gs"
     local __debian_repo=$DEBIAN_REPO
     local __debian_sign_key=""
+    local __strip_network_from_archive=0
+    local __arch=${DEFAULT_ARCHITECTURE}
+    local __force_upload_debians=0
 
     while [ ${#} -gt 0 ]; do
         error_message="❌ Error: a value is needed for '$1'";
@@ -655,6 +664,18 @@ function publish(){
                 __debian_sign_key=${2:?$error_message}
                 shift 2;
             ;;
+            --strip-network-from-archive )
+                __strip_network_from_archive=1
+                shift 1;
+            ;;
+            --arch )
+                __arch=${2:?$error_message}
+                shift 2;
+            ;;
+            --force-upload-debians )
+                __force_upload_debians=1
+                shift 1;
+            ;;
             * )
                 echo -e "❌ ${RED} !! Unknown option: $1${CLEAR}\n";
                 echo "";
@@ -700,6 +721,9 @@ function publish(){
     echo " - Backend: $__backend"
     echo " - Debian repo: $__debian_repo"
     echo " - Debian sign key: $__debian_sign_key"
+    echo " - Strip network from archive: $__strip_network_from_archive"
+    echo " - Architecture: $__arch"
+    echo " - Force upload debians: $__force_upload_debians"
     echo ""
 
     if [[ $__backend != "gs" && $__backend != "hetzner" && $__backend != "local" ]]; then
@@ -739,6 +763,8 @@ function publish(){
                                         $__dry_run \
                                         $__backend \
                                         $__debian_repo \
+                                        "$__arch" \
+                                        "$__force_upload_debians" \
                                         "$__debian_sign_key"
                             fi
 
@@ -749,6 +775,13 @@ function publish(){
                         ;;
                         mina-archive)
                             for network in "${__networks_arr[@]}"; do
+
+                                if [[ $__strip_network_from_archive == 1 ]]; then
+                                    new_name="mina-archive"
+                                else
+                                    new_name=""
+                                fi
+
                                 if [[ $__only_dockers == 0 ]]; then
                                         publish_debian $artifact \
                                             $__codename \
@@ -760,7 +793,10 @@ function publish(){
                                             $__dry_run \
                                             $__backend \
                                             $__debian_repo \
-                                            "$__debian_sign_key"
+                                            "$__arch" \
+                                            "$__force_upload_debians" \
+                                            "$__debian_sign_key" \
+                                            "$new_name"
                                 fi
 
                                 if [[ $__only_debians == 0 ]]; then
@@ -781,6 +817,8 @@ function publish(){
                                             $__dry_run \
                                             $__backend \
                                             $__debian_repo \
+                                            "$__arch" \
+                                            "$__force_upload_debians" \
                                             "$__debian_sign_key"
                                 fi
 
@@ -802,6 +840,8 @@ function publish(){
                                             $__dry_run \
                                             $__backend \
                                             $__debian_repo \
+                                            "$__arch" \
+                                            "$__force_upload_debians" \
                                             "$__debian_sign_key"
                                 fi
 
diff --git a/buildkite/src/Command/Packages/Publish.dhall b/buildkite/src/Command/Packages/Publish.dhall
@@ -229,6 +229,7 @@ let publish
                                   ++  "--target-version ${r.value} "
                                   ++  "--codenames ${codenames} "
                                   ++  "--only-dockers "
+                                  ++  "--force-upload-debians "
                                 )
                             ]
                           , label = "Docker Packages Publishing"
diff --git a/buildkite/src/Jobs/Promote/AutoPromoteNightly.dhall b/buildkite/src/Jobs/Promote/AutoPromoteNightly.dhall
@@ -56,7 +56,8 @@ let specs_for_branch =
       ->  \(channel : DebianChannel.Type)
       ->  PublishPackages.Spec::{
           , artifacts =
-            [ Artifacts.Type.Daemon
+            [ Artifacts.Type.LogProc
+            , Artifacts.Type.Daemon
             , Artifacts.Type.Archive
             , Artifacts.Type.Rosetta
             ]
@@ -95,5 +96,8 @@ in  Pipeline.build
           # PublishPackages.publish
               (specs_for_branch "develop" DebianChannel.Type.Develop)
           # PublishPackages.publish
-              (specs_for_branch "master" DebianChannel.Type.Master)
+              ( specs_for_branch
+                  "dkijana/port_publish_fix_master"
+                  DebianChannel.Type.Master
+              )
       }
diff --git a/dockerfiles/stages/3-toolchain b/dockerfiles/stages/3-toolchain
@@ -19,6 +19,7 @@ RUN apt-get update --yes \
     aptly \
     apt-transport-https \
     apt-utils \
+    dnsutils \
     fakeroot \
     gnupg2 \
     jq \
diff --git a/scripts/debian/publish.sh b/scripts/debian/publish.sh
@@ -7,12 +7,16 @@ RED='\033[0;31m'
 ARCH=amd64
 BUCKET=packages.o1test.net
 
+# Forcing upload debian even if it exists already
+FORCE=0
+
 while [[ "$#" -gt 0 ]]; do case $1 in
   -n|--names) DEB_NAMES="$2"; shift;;
   -r|--release) DEB_RELEASE="$2"; shift;;
   -v|--version) DEB_VERSION="$2"; shift;;
   -c|--codename) DEB_CODENAME="$2"; shift;;
   -b|--bucket) BUCKET="$2"; shift;;
+  -f|--force) FORCE=1;;
   -s|--sign) SIGN="$2"; shift;;
   *) echo "❌  Unknown parameter passed: $1"; exit 1;;
 esac; shift; done
@@ -33,6 +37,45 @@ function usage() {
   exit 1
 }
 
+# Invalidate CloudFront cache for the given bucket or CNAME and paths
+# This is to ensure that after uploading new debs, users don't get stale
+# package lists from CloudFront cache
+# Usage: invalidate_cache [bucket-or-cname] codename
+# Example: invalidate_cache nightly.apt.packages.minaprotocol.com bookworm
+function invalidate_cache() {
+  BUCKET_OR_CNAME="${1:-nightly.apt.packages.minaprotocol.com}"
+  PATHS_TO_INVALIDATE="/dists/$2/*"
+
+  echo "🔎 Resolving ${BUCKET_OR_CNAME}..."
+  CF_DOMAIN=$(dig +short CNAME "${BUCKET_OR_CNAME}" | sed 's/\.$//')
+  CF_DOMAIN=$(dig +short CNAME "${BUCKET_OR_CNAME}" | sed 's/\.$//')
+
+  if [[ -z "$CF_DOMAIN" ]]; then
+    echo "❌ Could not resolve ${BUCKET_OR_CNAME} to a CloudFront domain."
+    exit 1
+  fi
+
+  echo "✅ Found CloudFront domain: ${CF_DOMAIN}"
+
+  echo "📋 Searching for distribution ID in CloudFront..."
+  DIST_ID=$(aws cloudfront list-distributions \
+    --query "DistributionList.Items[?DomainName=='${CF_DOMAIN}'].Id" \
+    --output text)
+
+  if [[ -z "$DIST_ID" ]]; then
+    echo "❌ No CloudFront distribution found for domain ${CF_DOMAIN}"
+    exit 1
+  fi
+
+  echo "✅ Found CloudFront distribution ID: ${DIST_ID}"
+
+  echo "🚀 Creating invalidation for paths: ${PATHS_TO_INVALIDATE}"
+  aws cloudfront create-invalidation \
+    --distribution-id "${DIST_ID}" \
+    --paths "${PATHS_TO_INVALIDATE}"
+}
+
+
 if [[ -z "$DEB_NAMES" ]]; then usage "❌  Debian(s) to upload are not set!"; fi;
 if [[ -z "$DEB_VERSION" ]]; then usage "❌  Version is not set!"; fi;
 if [[ -z "$DEB_CODENAME" ]]; then usage "❌  Codename is not set!"; fi;
@@ -54,8 +97,6 @@ else
   GPG_OPTS=("--gpg-options=\"--batch" "--pinentry-mode=loopback" "--yes")
 fi
 
-
-
 echo "Publishing debs: ${DEB_NAMES} to Release: ${DEB_RELEASE} and Codename: ${DEB_CODENAME}"
 # Upload the deb files to s3.
 # If this fails, attempt to remove the lockfile and retry.
@@ -68,17 +109,19 @@ for _ in {1..10}; do (
 #>> Attempting to obtain a lock
 #/var/lib/gems/2.3.0/gems/deb-s3-0.10.0/lib/deb/s3/lock.rb:24:in `throw': uncaught throw #"Unable to obtain a lock after 60, giving up."
 deb-s3 upload $BUCKET_ARG $S3_REGION_ARG \
-  --fail-if-exists \
+  "$([ "$FORCE" -eq 0 ] && echo "--fail-if-exists")" \
   --lock \
   --preserve-versions \
-  --cache-control=max-age=120 \
+  --cache-control "no-store,no-cache,must-revalidate" \
   $SIGN_ARG \
   --component "${DEB_RELEASE}" \
   --codename "${DEB_CODENAME}" \
   "${GPG_OPTS[@]}" \
   "${DEB_NAMES}"
 ) && break || (MINA_DEB_BUCKET=${BUCKET} scripts/debian/clear-s3-lockfile.sh); done
 
+invalidate_cache "$BUCKET" "$DEB_CODENAME"
+
 for deb in $DEB_NAMES
 do
   # extracting name from debian package path. E.g:

Original file line number	Diff line number	Diff line change
`@@ -229,6 +229,7 @@ let publish`
`229`	`229`	`++ "--target-version ${r.value} "`
`230`	`230`	`++ "--codenames ${codenames} "`
`231`	`231`	`++ "--only-dockers "`
	`232`	`+ ++ "--force-upload-debians "`
`232`	`233`	`)`
`233`	`234`	`]`
`234`	`235`	`, label = "Docker Packages Publishing"`