Merge pull request #18027 from MinaProtocol/dkijania/fix_fork_pipeline

[HF] allow to build on more distros than focal. Allow to override precomp blocks bucket

Author: dkijania

buildkite/scripts/entrypoints/run-hardfork-package-gen.sh

@@ -1,5 +1,7 @@
 #!/bin/bash
 
+set -euox pipefail
+
 # Generate Hardfork Package Build Script
 #
 # This script generates Dhall configuration for creating hardfork packages in the Mina protocol
@@ -10,11 +12,12 @@
 #   ./run-hardfork-package-gen.sh
 #
 # REQUIRED ENVIRONMENT VARIABLES:
-#   CODENAMES           - Comma-separated list of Debian codenames (e.g., "Bullseye,Focal")
-#   NETWORK             - Target network name (e.g., "Devnet", "Mainnet")
-#   GENESIS_TIMESTAMP   - Genesis timestamp in ISO format (e.g., "2024-04-07T11:45:00Z")
-#   CONFIG_JSON_GZ_URL  - URL to the gzipped genesis config JSON file
-#   VERSION             - Version string for the hardfork package
+#   CODENAMES                     - Comma-separated list of Debian codenames (e.g., "Bullseye,Focal")
+#   NETWORK                       - Target network name (e.g., "Devnet", "Mainnet")
+#   GENESIS_TIMESTAMP             - Genesis timestamp in ISO format (e.g., "2024-04-07T11:45:00Z")
+#   CONFIG_JSON_GZ_URL            - URL to the gzipped genesis config JSON file
+#   VERSION                       - Version string for the hardfork package (optional, if not set, defaults to calculated from git)
+#   PRECOMPUTED_FORK_BLOCK_PREFIX - (Optional) Prefix for precomputed fork block URLs (e.g., "gs://mina_network_block_data/mainnet")
 #
 # EXAMPLE:
 #   export CODENAMES="Bullseye,Focal"
@@ -42,20 +45,21 @@ function usage() {
     echo -e "${RED}☞  $1${CLEAR}\n";
   fi
   cat << EOF
-  CODENAMES                   The Debian codenames (Bullseye, Focal etc.)
-  NETWORK                     The Docker and Debian network (Devnet, Mainnet)
-  GENESIS_TIMESTAMP           The Genesis timestamp in ISO format (e.g. 2024-04-07T11:45:00Z)
-  CONFIG_JSON_GZ_URL          The URL to the gzipped genesis config JSON file
-  VERSION                     The version of the hardfork package to generate (e.g. 3.0.0devnet-tooling-dkijania-hardfork-package-gen-in-nightly-b37f50e)
-
+  CODENAMES                     The Debian codenames (Bullseye, Focal etc.)
+  NETWORK                       The Docker and Debian network (Devnet, Mainnet)
+  GENESIS_TIMESTAMP             The Genesis timestamp in ISO format (e.g. 2024-04-07T11:45:00Z)
+  CONFIG_JSON_GZ_URL            The URL to the gzipped genesis config JSON file
+  VERSION                       (Optional) The version of the hardfork package to generate (e.g. 3.0.0devnet-tooling-dkijania-hardfork-package-gen-in-nightly-b37f50e)
+  PRECOMPUTED_FORK_BLOCK_PREFIX (Optional) The prefix for precomputed fork block URLs (e.g. gs://mina_network_block_data/mainnet)
 EOF
   exit 1
 }
 
 function to_dhall_list() {
   local input_str="$1"
   local dhall_type="$2"
-  local arr=("${input_str//,/ }")
+  local arr
+  IFS=',' read -ra arr <<< "$input_str"
   local dhall_list=""
 
   if [[ ${#arr[@]} -eq 0 || -z "${arr[0]}" ]]; then
@@ -72,6 +76,52 @@ function to_dhall_list() {
   echo "$dhall_list"
 }
 
+if [[ -z "$CODENAMES" ]]; then
+  usage "CODENAMES environment variable is required"
+fi
+
+if [[ -z "$NETWORK" ]]; then
+  usage "NETWORK environment variable is required"
+fi
+
+if [[ -z "$CONFIG_JSON_GZ_URL" ]]; then
+  usage "CONFIG_JSON_GZ_URL environment variable is required"
+fi
+
+# Format GENESIS_TIMESTAMP as Optional Text for Dhall
+if [[ -z "$GENESIS_TIMESTAMP" ]]; then
+  GENESIS_TIMESTAMP="None Text"
+else
+  # shellcheck disable=SC2089
+  GENESIS_TIMESTAMP="(Some \"${GENESIS_TIMESTAMP}\")"
+fi
+
+# Format VERSION as Optional Text for Dhall
+if [[ -z "$VERSION" ]]; then
+  VERSION="(None Text)"
+else
+  # shellcheck disable=SC2089
+  VERSION="(Some \"${VERSION}\")"
+fi
+
+# Format PRECOMPUTED_FORK_BLOCK_PREFIX as Optional Text for Dhall
+if [[ -z "$PRECOMPUTED_FORK_BLOCK_PREFIX" ]]; then
+  PRECOMPUTED_FORK_BLOCK_PREFIX="(None Text)"
+else
+  # shellcheck disable=SC2089
+  PRECOMPUTED_FORK_BLOCK_PREFIX="(Some \"${PRECOMPUTED_FORK_BLOCK_PREFIX}\")"
+fi
+
 DHALL_CODENAMES=$(to_dhall_list "$CODENAMES" "$DEBIAN_VERSION_DHALL_DEF.DebVersion")
 
-echo $GENERATE_HARDFORK_PACKAGE_DHALL_DEF'.generate_hardfork_package '"$DHALL_CODENAMES"' '$NETWORK_DHALL_DEF'.Type.'"${NETWORK}"' (None Text) "'"${CONFIG_JSON_GZ_URL}"'" "'""'" ' | dhall-to-yaml --quoted
+# shellcheck disable=SC2089
+printf '%s.generate_hardfork_package %s %s.Type.%s %s "%s" "%s" %s %s\n' \
+  "$GENERATE_HARDFORK_PACKAGE_DHALL_DEF" \
+  "$DHALL_CODENAMES" \
+  "$NETWORK_DHALL_DEF" \
+  "$NETWORK" \
+  "$GENESIS_TIMESTAMP" \
+  "$CONFIG_JSON_GZ_URL" \
+  "" \
+  "$VERSION" \
+  "$PRECOMPUTED_FORK_BLOCK_PREFIX" | dhall-to-yaml --quoted

buildkite/src/Constants/Network.dhall

@@ -4,7 +4,7 @@ let List/any = Prelude.List.any
 
 let Network
     : Type
-    = < Devnet | Mainnet | Berkeley | DevnetLegacy | MainnetLegacy >
+    = < Devnet | Mainnet | Berkeley | DevnetLegacy | MainnetLegacy | PreMesa1 >
 
 let capitalName =
           \(network : Network)
@@ -14,6 +14,7 @@ let capitalName =
             , Berkeley = "Berkeley"
             , DevnetLegacy = "DevnetLegacy"
             , MainnetLegacy = "MainnetLegacy"
+            , PreMesa1 = "PreMesa1"
             }
             network
 
@@ -25,6 +26,7 @@ let lowerName =
             , Berkeley = "berkeley"
             , DevnetLegacy = "devnet_legacy"
             , MainnetLegacy = "mainnet_legacy"
+            , PreMesa1 = "hetzner-pre-mesa-1"
             }
             network
 
@@ -36,6 +38,7 @@ let requiresMainnetBuild =
             , Berkeley = False
             , DevnetLegacy = True
             , MainnetLegacy = True
+            , PreMesa1 = False
             }
             network
 

buildkite/src/Constants/Profiles.dhall

@@ -42,6 +42,7 @@ let fromNetwork =
             , Berkeley = Profile.Devnet
             , DevnetLegacy = Profile.Devnet
             , MainnetLegacy = Profile.Mainnet
+            , PreMesa1 = Profile.Devnet
             }
             network
 

buildkite/src/Entrypoints/GenerateHardforkPackage.dhall

@@ -1,6 +1,11 @@
 -- Autogenerates any pre-reqs for monorepo triage execution
 -- Keep these rules lean! They have to run unconditionally.
 
+
+let Prelude = ../External/Prelude.dhall
+
+let List/concatMap = Prelude.List.concatMap
+
 let SelectFiles = ../Lib/SelectFiles.dhall
 
 let Cmd = ../Lib/Cmds.dhall
@@ -37,6 +42,7 @@ let Spec =
           , config_json_gz_url : Text
           , version : Text
           , suffix : Text
+          , precomputed_block_prefix : Optional Text
           }
       , default =
           { codenames = [ DebianVersions.DebVersion.Bullseye ]
@@ -45,6 +51,8 @@ let Spec =
           , config_json_gz_url =
               "https://storage.googleapis.com/o1labs-gitops-infrastructure/devnet/devnet-state-dump-3NK4eDgbkCjKj9fFUXVkrJXsfpfXzJySoAvrFJVCropPW7LLF14F-676026c4d4d2c18a76b357d6422a06f932c3ef4667a8fd88717f68b53fd6b2d7.json.gz"
           , suffix = ""
+          , version = "\\\$MINA_DEB_VERSION"
+          , precomputed_block_prefix = None Text
           }
       }
 
@@ -55,8 +63,6 @@ let generateDockerForCodename =
       ->  let image =
                 Artifacts.fullDockerTag
                   Artifacts.Tag::{
-                  , version =
-                      "${spec.version}-${DebianVersions.lowerName codename}"
                   , remove_profile_from_name = True
                   , network = spec.network
                   }
@@ -76,13 +82,24 @@ let generateDockerForCodename =
                 , deb_profile = profile
                 , deb_repo = DebianRepo.Type.Local
                 , deb_suffix = Some "hardfork"
+                , step_key_suffix =
+                    "-${DebianVersions.lowerName codename}-docker-image"
                 }
 
           let dockerDaemonStep = DockerImage.stepKey dockerDaemonSpec
 
           let dependsOnTest =
                 [ { name = pipelineName, key = dockerDaemonStep } ]
 
+          let precomputed_block_prefix_arg =
+                merge
+                  { Some =
+                          \(prefix : Text)
+                      ->  "--precomputed-block-prefix " ++ prefix
+                  , None = ""
+                  }
+                  spec.precomputed_block_prefix
+
           in  [ MinaArtifact.buildArtifacts
                   MinaArtifact.MinaBuildSpec::{
                   , artifacts =
@@ -118,6 +135,8 @@ let generateDockerForCodename =
                   , deb_codename = codename
                   , deb_profile = profile
                   , deb_repo = DebianRepo.Type.Local
+                  , step_key_suffix =
+                      "-${DebianVersions.lowerName codename}-docker-image"
                   }
               , DockerImage.generateStep
                   DockerImage.ReleaseSpec::{
@@ -133,6 +152,8 @@ let generateDockerForCodename =
                   , deb_repo = DebianRepo.Type.Local
                   , deb_codename = codename
                   , deb_suffix = Some "hardfork"
+                  , step_key_suffix =
+                      "-${DebianVersions.lowerName codename}-docker-image"
                   }
               , Command.build
                   Command.Config::{
@@ -161,11 +182,13 @@ let generateDockerForCodename =
                                                       codename} && source ./buildkite/scripts/export-git-env-vars.sh"
                     , Cmd.runInDocker
                         Cmd.Docker::{ image = image }
-                        "curl ${spec.config_json_gz_url} > config.json.gz && gunzip config.json.gz && FORKING_FROM_CONFIG_JSON=config.json mina-verify-packaged-fork-config ${Network.lowerName
-                                                                                                                                                                                spec.network} config.json /workdir/verification"
+                        "curl ${spec.config_json_gz_url} > config.json.gz && gunzip config.json.gz && FORKING_FROM_CONFIG_JSON=config.json mina-verify-packaged-fork-config --network ${Network.lowerName
+                                                                                                                                                                                          spec.network} --fork-config config.json --working-dir /workdir/verification ${precomputed_block_prefix_arg}"
                     ]
                   , label = "Verify packaged artifacts"
-                  , key = "verify-packaged-artifacts"
+                  , key =
+                      "verify-packaged-artifacts-${DebianVersions.lowerName
+                                                     codename}"
                   , target = Size.XLarge
                   , depends_on = dependsOnTest
                   }
@@ -188,10 +211,13 @@ let pipeline =
                     ]
                   }
                 , steps =
-                    generateDockerForCodename
-                      spec
-                      DebianVersions.DebVersion.Focal
-                      pipelineName
+                    List/concatMap
+                      DebianVersions.DebVersion
+                      Command.Type
+                      (     \(codename : DebianVersions.DebVersion)
+                        ->  generateDockerForCodename spec codename pipelineName
+                      )
+                      spec.codenames
                 }
 
 let generate_hardfork_package =
@@ -200,14 +226,20 @@ let generate_hardfork_package =
       ->  \(genesis_timestamp : Optional Text)
       ->  \(config_json_gz_url : Text)
       ->  \(suffix : Text)
+      ->  \(version : Optional Text)
+      ->  \(precomputed_block_prefix : Optional Text)
       ->  ( pipeline
               Spec::{
               , codenames = codenames
               , network = network
-              , version = "\\\$MINA_DEB_VERSION"
+              , version =
+                  merge
+                    { Some = \(v : Text) -> v, None = "\\\$MINA_DEB_VERSION" }
+                    version
               , genesis_timestamp = genesis_timestamp
               , config_json_gz_url = config_json_gz_url
               , suffix = suffix
+              , precomputed_block_prefix = precomputed_block_prefix
               }
           ).pipeline