Support mina advanced generate-hard-fork-config in hf test

buildkite/src/Jobs/Test/HardForkTestAdvanced.dhall

@@ -0,0 +1,67 @@
+let ContainerImages = ../../Constants/ContainerImages.dhall
+
+let Cmd = ../../Lib/Cmds.dhall
+
+let S = ../../Lib/SelectFiles.dhall
+
+let Pipeline = ../../Pipeline/Dsl.dhall
+
+let PipelineTag = ../../Pipeline/Tag.dhall
+
+let PipelineScope = ../../Pipeline/Scope.dhall
+
+let JobSpec = ../../Pipeline/JobSpec.dhall
+
+let Command = ../../Command/Base.dhall
+
+let Docker = ../../Command/Docker/Type.dhall
+
+let Size = ../../Command/Size.dhall
+
+let B = ../../External/Buildkite.dhall
+
+let B/SoftFail = B.definitions/commandStep/properties/soft_fail/Type
+
+in  Pipeline.build
+      Pipeline.Config::{
+      , spec = JobSpec::{
+        , dirtyWhen =
+          [ S.strictlyStart (S.contains "src")
+          , S.exactly "buildkite/src/Jobs/Test/HardForkTest" "dhall"
+          , S.strictlyStart (S.contains "scripts/hardfork")
+          , S.strictlyStart (S.contains "nix")
+          , S.exactly "flake" "nix"
+          , S.exactly "flake" "lock"
+          , S.exactly "default" "nix"
+          ]
+        , path = "Test"
+        , name = "HardForkTestAdvanced"
+        , scope = PipelineScope.AllButPullRequest
+        , tags =
+          [ PipelineTag.Type.Long
+          , PipelineTag.Type.Test
+          , PipelineTag.Type.Stable
+          , PipelineTag.Type.Hardfork
+          ]
+        }
+      , steps =
+        [ Command.build
+            Command.Config::{
+            , commands =
+              [ Cmd.runInDocker
+                  Cmd.Docker::{
+                  , image = ContainerImages.nixos
+                  , privileged = True
+                  , useBash = False
+                  }
+                  "./scripts/hardfork/build-and-test.sh --fork-from origin/compatible --fork-method advanced"
+              ]
+            , label = "hard fork test - advanced mode"
+            , key = "hard-fork-test-advanced"
+            , target = Size.Integration
+            , soft_fail = Some (B/SoftFail.Boolean False)
+            , docker = None Docker.Type
+            , timeout_in_minutes = Some +420
+            }
+        ]
+      }

buildkite/src/Jobs/Test/HardForkTest.dhall → buildkite/src/Jobs/Test/HardForkTestLegacy.dhall

@@ -35,7 +35,7 @@ in  Pipeline.build
           , S.exactly "default" "nix"
           ]
         , path = "Test"
-        , name = "HardForkTest"
+        , name = "HardForkTestLegacy"
         , scope = PipelineScope.AllButPullRequest
         , tags =
           [ PipelineTag.Type.Long
@@ -54,10 +54,10 @@ in  Pipeline.build
                   , privileged = True
                   , useBash = False
                   }
-                  "./scripts/hardfork/build-and-test.sh \$BUILDKITE_BRANCH"
+                  "./scripts/hardfork/build-and-test.sh --fork-from origin/master"
               ]
-            , label = "hard fork test"
-            , key = "hard-fork-test"
+            , label = "hard fork test - legacy mode"
+            , key = "hard-fork-test-legacy"
             , target = Size.Integration
             , soft_fail = Some (B/SoftFail.Boolean False)
             , docker = None Docker.Type

scripts/hardfork/build-and-test.sh

@@ -1,8 +1,8 @@
 #!/usr/bin/env bash
 
-# This scripts builds master and current branch with nix
+# This scripts builds a designated PREFORK branch and current branch with nix
 # 0. Prepare environment if needed
-# 1. Build master as a prefork build;
+# 1. Build PREFORK as a prefork build;
 # 2. Upload to nix cache, the reason for not uploading cache for following 2 
 # steps is that they change for each PR. 
 # 3. Build current branch as a postfork build;
@@ -12,6 +12,64 @@
 # Step 0. Prepare environment if needed
 set -eux -o pipefail
 
+PREFORK=""
+FORK_METHOD="legacy"
+
+USAGE="Usage: $0 --fork-from <PREFORK> [--fork-method <FORK_METHOD>]"
+usage() {
+  if (( $# > 0 )); then
+    echo "$1" >&2
+    echo "$USAGE"
+    exit 1
+  else
+    echo "$USAGE"
+    exit 0
+  fi
+}
+
+# ---- argument parsing --------------------------------------------------------
+while [[ $# -gt 0 ]]; do
+  case "$1" in
+    --fork-from)
+      # ensure value exists
+      if [[ $# -lt 2 ]]; then
+        usage "Error: $1 requires an argument."
+      fi
+      PREFORK="$2"
+      shift 2
+      ;;
+    --fork-method)
+      # ensure value exists
+      if [[ $# -lt 2 ]]; then
+        usage "Error: $1 requires an argument."
+      fi
+      case "$2" in
+        legacy|advanced)
+          FORK_METHOD="$2"
+          ;;
+        *)
+          usage "Error: $1 must be either 'legacy' or 'advanced'."
+          ;;
+      esac
+      shift 2
+      ;;
+    --help|-h)
+      usage
+      ;;
+    --*)
+      usage "Unknown option: $1"
+      ;;
+    *)
+      # positional arg — store if needed later
+      usage "Unexpected argument: $1"
+      ;;
+  esac
+done
+
+if [[ -z "$PREFORK" ]]; then
+  usage "Error: --fork-from must be provided."
+fi
+
 NIX_OPTS=( --accept-flake-config --experimental-features 'nix-command flakes' )
 
 if [[ -n "${NIX_CACHE_NAR_SECRET:-}" ]]; then
@@ -70,8 +128,8 @@ if [ -n "${BUILDKITE:-}" ]; then
   git fetch origin
 fi
 
-# 1. Build master as a prefork build;
-git checkout origin/master
+# 1. Build PREFORK as a prefork build;
+git checkout $PREFORK
 git submodule update --init --recursive --depth 1
 nix "${NIX_OPTS[@]}" build "$PWD?submodules=1#devnet" --out-link "prefork-devnet"
 
@@ -100,8 +158,8 @@ nix "${NIX_OPTS[@]}" build "$PWD?submodules=1#hardfork_test" --out-link "hardfor
 
 # 5. Execute hardfork_test on them.
 
-SLOT_TX_END=${SLOT_TX_END:-$((40))}
-SLOT_CHAIN_END=${SLOT_CHAIN_END:-$((SLOT_TX_END+5))}
+SLOT_TX_END=33
+SLOT_CHAIN_END=$((SLOT_TX_END+4))
 
 NETWORK_ROOT=$(mktemp -d --tmpdir hardfork-network.XXXXXXX)
 
@@ -113,6 +171,7 @@ hardfork_test/bin/hardfork_test \
   --slot-tx-end "$SLOT_TX_END" \
   --slot-chain-end "$SLOT_CHAIN_END" \
   --script-dir "$SCRIPT_DIR" \
-  --root "$NETWORK_ROOT"
+  --root "$NETWORK_ROOT" \
+  --fork-method "$FORK_METHOD"
 
 popd

scripts/hardfork/create_runtime_config.sh

@@ -1,17 +1,24 @@
 #!/usr/bin/env bash
 
-set -eo pipefail
+# NOTE: This script patches the genesis timestamp of a fork config so a new 
+# network could be schedueld to genesis in some instant in the future.
 
+set -eux -o pipefail
+
+# ==================== Inputs to this script ====================
+# The fork config we aim to patch on 
 FORK_CONFIG_JSON=${FORK_CONFIG_JSON:=fork_config.json}
+# The hashes file generated by runtime-genesis-ledger
 LEDGER_HASHES_JSON=${LEDGER_HASHES_JSON:=ledger_hashes.json}
+# The "base" config file, needed to know the genesis time of the prefork network
 FORKING_FROM_CONFIG_JSON=${FORKING_FROM_CONFIG_JSON:=genesis_ledgers/mainnet.json}
-
-# If not given, the genesis timestamp is set to 10 mins into the future
+# Genesis timestamp of the postfork network, defaults to 10mins in the future
 GENESIS_TIMESTAMP=${GENESIS_TIMESTAMP:=$(date -u +"%Y-%m-%dT%H:%M:%SZ" -d "10 mins")}
 
 # Pull the original genesis timestamp from the pre-fork config file
 ORIGINAL_GENESIS_TIMESTAMP=$(jq -r '.genesis.genesis_state_timestamp' "$FORKING_FROM_CONFIG_JSON")
 OFFSET=$(jq -r '.proof.fork.global_slot_since_genesis' "$FORKING_FROM_CONFIG_JSON")
+# ===============================================================
 
 if [[ "$OFFSET" == null ]]; then
   OFFSET=0
@@ -28,33 +35,38 @@ SLOT=$((DIFFERENCE_IN_SLOTS+OFFSET))
 # jq expression below could be written with less code,
 # but we aimed for maximum verbosity
 
-jq "{\
-    genesis: {\
-        genesis_state_timestamp: \"$GENESIS_TIMESTAMP\"\
-    },\
-    proof: {\
-        fork: {\
-            state_hash: .proof.fork.state_hash,\
-            blockchain_length: .proof.fork.blockchain_length,\
-            global_slot_since_genesis: $SLOT,\
-        },\
-    },\
-    ledger: {\
-        add_genesis_winner: false,\
-        hash: \$hashes[0].ledger.hash,\
-        s3_data_hash: \$hashes[0].ledger.s3_data_hash\
-    },\
-    epoch_data: {\
-        staking: {\
-            seed: .epoch_data.staking.seed,\
-            hash: \$hashes[0].epoch_data.staking.hash,\
-            s3_data_hash: \$hashes[0].epoch_data.staking.s3_data_hash\
-        },\
-        next: {\
-            seed: .epoch_data.next.seed,\
-            hash: \$hashes[0].epoch_data.next.hash,\
-            s3_data_hash: \$hashes[0].epoch_data.next.s3_data_hash\
-        }\
-    }\
-  }" -M \
-  --slurpfile hashes "$LEDGER_HASHES_JSON" "$FORK_CONFIG_JSON"
+jq -M \
+  --arg genesis_timestamp "$GENESIS_TIMESTAMP" \
+  --argjson slot "$SLOT" \
+  --slurpfile hashes "$LEDGER_HASHES_JSON" \
+  '
+  {
+    genesis: {
+      genesis_state_timestamp: $genesis_timestamp
+    },
+    proof: {
+      fork: {
+        state_hash: .proof.fork.state_hash,
+        blockchain_length: .proof.fork.blockchain_length,
+        global_slot_since_genesis: $slot
+      }
+    },
+    ledger: {
+      add_genesis_winner: false,
+      hash: $hashes[0].ledger.hash,
+      s3_data_hash: $hashes[0].ledger.s3_data_hash
+    },
+    epoch_data: {
+      staking: {
+        seed: .epoch_data.staking.seed,
+        hash: $hashes[0].epoch_data.staking.hash,
+        s3_data_hash: $hashes[0].epoch_data.staking.s3_data_hash
+      },
+      next: {
+        seed: .epoch_data.next.seed,
+        hash: $hashes[0].epoch_data.next.hash,
+        s3_data_hash: $hashes[0].epoch_data.next.s3_data_hash
+      }
+    }
+  }
+  ' "$FORK_CONFIG_JSON"

scripts/mina-local-network/mina-local-network.sh

@@ -55,6 +55,7 @@ DEMO_MODE=false
 
 SLOT_TX_END=
 SLOT_CHAIN_END=
+HARDFORK_GENESIS_SLOT_DELTA=
 
 # ================================================
 # Globals (assigned during execution of script)
@@ -140,6 +141,7 @@ help() {
                                          |   Default: None
 -sce |--slot-chain-end                   | When set, stop producing blocks from this chain on.
                                          |   Default: None
+-hfd |--hardfork-genesis-slot-delta      | When set override the value `hard_fork_genesis_slot_delta` in daemon config. 
 -r   |--root                             | When set, override the root working folder (i.e. the value of ROOT) for this script. WARN: this script will clean up anything inside that folder when initializing any run!
                                          |   Default: ${ROOT}
 -h   |--help                             | Displays this help message
@@ -505,6 +507,10 @@ while [[ "$#" -gt 0 ]]; do
     SLOT_CHAIN_END="${2}"
     shift
     ;;
+  -hfd |--hardfork-genesis-slot-delta)
+    HARDFORK_GENESIS_SLOT_DELTA="${2}"
+    shift
+    ;;
   -r | --root)
     ROOT="${2}"
     shift
@@ -748,6 +754,13 @@ load_config() {
     inherit_with:*)
       local replaced_config_file
       IFS=',' read -r replaced_config_file OVERRIDE_GENSIS_LEDGER <<< "${config_mode#inherit_with:}"
+      if [ ! -f "${replaced_config_file}" ]; then
+        echo "Error: Config file '${replaced_config_file}' does not exist, can't inherit_with." >&2
+        exit 1
+      else
+        echo "Inheriting config at ${replaced_config_file}:"
+        cat "${replaced_config_file}"
+      fi
       cp -f "${replaced_config_file}" "${config_file}"
       ;;
   esac
@@ -782,20 +795,25 @@ update_genesis_timestamp() {
 update_genesis_timestamp "${UPDATE_GENESIS_TIMESTAMP}"
 
 if [ ! -z "${OVERRIDE_SLOT_TIME_MS}" ]; then
-  echo 'Modifying configuration to override slot time...'
+  echo "Setting proof.block_window_duration_ms to ${OVERRIDE_SLOT_TIME_MS}..."
   jq-inplace ".proof.block_window_duration_ms=${OVERRIDE_SLOT_TIME_MS}" "${CONFIG}"
 fi
 
 if [ ! -z "${SLOT_TX_END}" ]; then
-  echo 'Modifying configuration to override slot transaction end...'
+  echo "Setting daemon.slot_tx_end to ${SLOT_TX_END}..."
   jq-inplace ".daemon.slot_tx_end=${SLOT_TX_END}" "${CONFIG}"
 fi
 
 if [ ! -z "${SLOT_CHAIN_END}" ]; then
-  echo 'Modifying configuration to override slot chain end...'
+  echo "Setting daemon.slot_chain_end to ${SLOT_CHAIN_END}..."
   jq-inplace ".daemon.slot_chain_end=${SLOT_CHAIN_END}" "${CONFIG}"
 fi
 
+if [ ! -z "${HARDFORK_GENESIS_SLOT_DELTA}" ]; then
+  echo "Setting daemon.hard_fork_genesis_slot_delta to ${HARDFORK_GENESIS_SLOT_DELTA}..."
+  jq-inplace ".daemon.hard_fork_genesis_slot_delta=${HARDFORK_GENESIS_SLOT_DELTA}" "${CONFIG}"
+fi
+
 # ================================================
 # Launch the Nodes
 

src/app/hardfork_test/src/internal/app/root.go

@@ -84,7 +84,8 @@ func init() {
 	rootCmd.Flags().IntVar(&cfg.UserCommandCheckMaxIterations, "user-command-check-max-iterations", cfg.UserCommandCheckMaxIterations, "Max iterations to check for user commands in blocks")
 	rootCmd.Flags().IntVar(&cfg.ForkEarliestBlockMaxRetries, "fork-earliest-block-max-retries", cfg.ForkEarliestBlockMaxRetries, "Maximum number of retries to wait for earliest block in fork network")
 	rootCmd.Flags().IntVar(&cfg.HTTPClientTimeoutSeconds, "http-timeout", cfg.HTTPClientTimeoutSeconds, "HTTP client timeout in seconds for GraphQL requests")
-	rootCmd.Flags().IntVar(&cfg.GraphQLMaxRetries, "graphql-max-retries", cfg.GraphQLMaxRetries, "Maximum number of retries for GraphQL requests")
+	rootCmd.Flags().IntVar(&cfg.ClientMaxRetries, "client-max-retries", cfg.ClientMaxRetries, "Maximum number of retries for client requests")
+	rootCmd.Flags().Var(&cfg.ForkMethod, "fork-method", "The implementation of fork")
 
 	// Mark required flags
 	rootCmd.MarkFlagRequired("main-mina-exe")