diff --git a/buildkite/src/Jobs/Test/HardForkTestAdvanced.dhall b/buildkite/src/Jobs/Test/HardForkTestAdvanced.dhall new file mode 100644 index 000000000000..f9190c6098e2 --- /dev/null +++ b/buildkite/src/Jobs/Test/HardForkTestAdvanced.dhall @@ -0,0 +1,67 @@ +let ContainerImages = ../../Constants/ContainerImages.dhall + +let Cmd = ../../Lib/Cmds.dhall + +let S = ../../Lib/SelectFiles.dhall + +let Pipeline = ../../Pipeline/Dsl.dhall + +let PipelineTag = ../../Pipeline/Tag.dhall + +let PipelineScope = ../../Pipeline/Scope.dhall + +let JobSpec = ../../Pipeline/JobSpec.dhall + +let Command = ../../Command/Base.dhall + +let Docker = ../../Command/Docker/Type.dhall + +let Size = ../../Command/Size.dhall + +let B = ../../External/Buildkite.dhall + +let B/SoftFail = B.definitions/commandStep/properties/soft_fail/Type + +in Pipeline.build + Pipeline.Config::{ + , spec = JobSpec::{ + , dirtyWhen = + [ S.strictlyStart (S.contains "src") + , S.exactly "buildkite/src/Jobs/Test/HardForkTest" "dhall" + , S.strictlyStart (S.contains "scripts/hardfork") + , S.strictlyStart (S.contains "nix") + , S.exactly "flake" "nix" + , S.exactly "flake" "lock" + , S.exactly "default" "nix" + ] + , path = "Test" + , name = "HardForkTestAdvanced" + , scope = PipelineScope.AllButPullRequest + , tags = + [ PipelineTag.Type.Long + , PipelineTag.Type.Test + , PipelineTag.Type.Stable + , PipelineTag.Type.Hardfork + ] + } + , steps = + [ Command.build + Command.Config::{ + , commands = + [ Cmd.runInDocker + Cmd.Docker::{ + , image = ContainerImages.nixos + , privileged = True + , useBash = False + } + "./scripts/hardfork/build-and-test.sh --fork-from origin/compatible --fork-method advanced" + ] + , label = "hard fork test - advanced mode" + , key = "hard-fork-test-advanced" + , target = Size.Integration + , soft_fail = Some (B/SoftFail.Boolean False) + , docker = None Docker.Type + , timeout_in_minutes = Some +420 + } + ] + } diff --git a/buildkite/src/Jobs/Test/HardForkTest.dhall b/buildkite/src/Jobs/Test/HardForkTestLegacy.dhall similarity index 89% rename from buildkite/src/Jobs/Test/HardForkTest.dhall rename to buildkite/src/Jobs/Test/HardForkTestLegacy.dhall index c26d9e4362ae..12455d3f0583 100644 --- a/buildkite/src/Jobs/Test/HardForkTest.dhall +++ b/buildkite/src/Jobs/Test/HardForkTestLegacy.dhall @@ -35,7 +35,7 @@ in Pipeline.build , S.exactly "default" "nix" ] , path = "Test" - , name = "HardForkTest" + , name = "HardForkTestLegacy" , scope = PipelineScope.AllButPullRequest , tags = [ PipelineTag.Type.Long @@ -54,10 +54,10 @@ in Pipeline.build , privileged = True , useBash = False } - "./scripts/hardfork/build-and-test.sh \$BUILDKITE_BRANCH" + "./scripts/hardfork/build-and-test.sh --fork-from origin/master" ] - , label = "hard fork test" - , key = "hard-fork-test" + , label = "hard fork test - legacy mode" + , key = "hard-fork-test-legacy" , target = Size.Integration , soft_fail = Some (B/SoftFail.Boolean False) , docker = None Docker.Type diff --git a/scripts/hardfork/build-and-test.sh b/scripts/hardfork/build-and-test.sh index c5bc8370cbe7..f33db0aacdda 100755 --- a/scripts/hardfork/build-and-test.sh +++ b/scripts/hardfork/build-and-test.sh @@ -1,8 +1,8 @@ #!/usr/bin/env bash -# This scripts builds master and current branch with nix +# This scripts builds a designated PREFORK branch and current branch with nix # 0. Prepare environment if needed -# 1. Build master as a prefork build; +# 1. Build PREFORK as a prefork build; # 2. Upload to nix cache, the reason for not uploading cache for following 2 # steps is that they change for each PR. # 3. Build current branch as a postfork build; @@ -12,6 +12,64 @@ # Step 0. Prepare environment if needed set -eux -o pipefail +PREFORK="" +FORK_METHOD="legacy" + +USAGE="Usage: $0 --fork-from [--fork-method ]" +usage() { + if (( $# > 0 )); then + echo "$1" >&2 + echo "$USAGE" + exit 1 + else + echo "$USAGE" + exit 0 + fi +} + +# ---- argument parsing -------------------------------------------------------- +while [[ $# -gt 0 ]]; do + case "$1" in + --fork-from) + # ensure value exists + if [[ $# -lt 2 ]]; then + usage "Error: $1 requires an argument." + fi + PREFORK="$2" + shift 2 + ;; + --fork-method) + # ensure value exists + if [[ $# -lt 2 ]]; then + usage "Error: $1 requires an argument." + fi + case "$2" in + legacy|advanced) + FORK_METHOD="$2" + ;; + *) + usage "Error: $1 must be either 'legacy' or 'advanced'." + ;; + esac + shift 2 + ;; + --help|-h) + usage + ;; + --*) + usage "Unknown option: $1" + ;; + *) + # positional arg — store if needed later + usage "Unexpected argument: $1" + ;; + esac +done + +if [[ -z "$PREFORK" ]]; then + usage "Error: --fork-from must be provided." +fi + NIX_OPTS=( --accept-flake-config --experimental-features 'nix-command flakes' ) if [[ -n "${NIX_CACHE_NAR_SECRET:-}" ]]; then @@ -70,8 +128,8 @@ if [ -n "${BUILDKITE:-}" ]; then git fetch origin fi -# 1. Build master as a prefork build; -git checkout origin/master +# 1. Build PREFORK as a prefork build; +git checkout $PREFORK git submodule update --init --recursive --depth 1 nix "${NIX_OPTS[@]}" build "$PWD?submodules=1#devnet" --out-link "prefork-devnet" @@ -100,8 +158,8 @@ nix "${NIX_OPTS[@]}" build "$PWD?submodules=1#hardfork_test" --out-link "hardfor # 5. Execute hardfork_test on them. -SLOT_TX_END=${SLOT_TX_END:-$((40))} -SLOT_CHAIN_END=${SLOT_CHAIN_END:-$((SLOT_TX_END+5))} +SLOT_TX_END=33 +SLOT_CHAIN_END=$((SLOT_TX_END+4)) NETWORK_ROOT=$(mktemp -d --tmpdir hardfork-network.XXXXXXX) @@ -113,6 +171,7 @@ hardfork_test/bin/hardfork_test \ --slot-tx-end "$SLOT_TX_END" \ --slot-chain-end "$SLOT_CHAIN_END" \ --script-dir "$SCRIPT_DIR" \ - --root "$NETWORK_ROOT" + --root "$NETWORK_ROOT" \ + --fork-method "$FORK_METHOD" popd diff --git a/scripts/hardfork/create_runtime_config.sh b/scripts/hardfork/create_runtime_config.sh index 6652fc5f50f1..b4e42c11bb03 100755 --- a/scripts/hardfork/create_runtime_config.sh +++ b/scripts/hardfork/create_runtime_config.sh @@ -1,17 +1,24 @@ #!/usr/bin/env bash -set -eo pipefail +# NOTE: This script patches the genesis timestamp of a fork config so a new +# network could be schedueld to genesis in some instant in the future. +set -eux -o pipefail + +# ==================== Inputs to this script ==================== +# The fork config we aim to patch on FORK_CONFIG_JSON=${FORK_CONFIG_JSON:=fork_config.json} +# The hashes file generated by runtime-genesis-ledger LEDGER_HASHES_JSON=${LEDGER_HASHES_JSON:=ledger_hashes.json} +# The "base" config file, needed to know the genesis time of the prefork network FORKING_FROM_CONFIG_JSON=${FORKING_FROM_CONFIG_JSON:=genesis_ledgers/mainnet.json} - -# If not given, the genesis timestamp is set to 10 mins into the future +# Genesis timestamp of the postfork network, defaults to 10mins in the future GENESIS_TIMESTAMP=${GENESIS_TIMESTAMP:=$(date -u +"%Y-%m-%dT%H:%M:%SZ" -d "10 mins")} # Pull the original genesis timestamp from the pre-fork config file ORIGINAL_GENESIS_TIMESTAMP=$(jq -r '.genesis.genesis_state_timestamp' "$FORKING_FROM_CONFIG_JSON") OFFSET=$(jq -r '.proof.fork.global_slot_since_genesis' "$FORKING_FROM_CONFIG_JSON") +# =============================================================== if [[ "$OFFSET" == null ]]; then OFFSET=0 @@ -28,33 +35,38 @@ SLOT=$((DIFFERENCE_IN_SLOTS+OFFSET)) # jq expression below could be written with less code, # but we aimed for maximum verbosity -jq "{\ - genesis: {\ - genesis_state_timestamp: \"$GENESIS_TIMESTAMP\"\ - },\ - proof: {\ - fork: {\ - state_hash: .proof.fork.state_hash,\ - blockchain_length: .proof.fork.blockchain_length,\ - global_slot_since_genesis: $SLOT,\ - },\ - },\ - ledger: {\ - add_genesis_winner: false,\ - hash: \$hashes[0].ledger.hash,\ - s3_data_hash: \$hashes[0].ledger.s3_data_hash\ - },\ - epoch_data: {\ - staking: {\ - seed: .epoch_data.staking.seed,\ - hash: \$hashes[0].epoch_data.staking.hash,\ - s3_data_hash: \$hashes[0].epoch_data.staking.s3_data_hash\ - },\ - next: {\ - seed: .epoch_data.next.seed,\ - hash: \$hashes[0].epoch_data.next.hash,\ - s3_data_hash: \$hashes[0].epoch_data.next.s3_data_hash\ - }\ - }\ - }" -M \ - --slurpfile hashes "$LEDGER_HASHES_JSON" "$FORK_CONFIG_JSON" +jq -M \ + --arg genesis_timestamp "$GENESIS_TIMESTAMP" \ + --argjson slot "$SLOT" \ + --slurpfile hashes "$LEDGER_HASHES_JSON" \ + ' + { + genesis: { + genesis_state_timestamp: $genesis_timestamp + }, + proof: { + fork: { + state_hash: .proof.fork.state_hash, + blockchain_length: .proof.fork.blockchain_length, + global_slot_since_genesis: $slot + } + }, + ledger: { + add_genesis_winner: false, + hash: $hashes[0].ledger.hash, + s3_data_hash: $hashes[0].ledger.s3_data_hash + }, + epoch_data: { + staking: { + seed: .epoch_data.staking.seed, + hash: $hashes[0].epoch_data.staking.hash, + s3_data_hash: $hashes[0].epoch_data.staking.s3_data_hash + }, + next: { + seed: .epoch_data.next.seed, + hash: $hashes[0].epoch_data.next.hash, + s3_data_hash: $hashes[0].epoch_data.next.s3_data_hash + } + } + } + ' "$FORK_CONFIG_JSON" diff --git a/scripts/mina-local-network/mina-local-network.sh b/scripts/mina-local-network/mina-local-network.sh index ccbb08878b26..74e476c7078f 100755 --- a/scripts/mina-local-network/mina-local-network.sh +++ b/scripts/mina-local-network/mina-local-network.sh @@ -55,6 +55,7 @@ DEMO_MODE=false SLOT_TX_END= SLOT_CHAIN_END= +HARDFORK_GENESIS_SLOT_DELTA= # ================================================ # Globals (assigned during execution of script) @@ -140,6 +141,7 @@ help() { | Default: None -sce |--slot-chain-end | When set, stop producing blocks from this chain on. | Default: None +-hfd |--hardfork-genesis-slot-delta | When set override the value `hard_fork_genesis_slot_delta` in daemon config. -r |--root | When set, override the root working folder (i.e. the value of ROOT) for this script. WARN: this script will clean up anything inside that folder when initializing any run! | Default: ${ROOT} -h |--help | Displays this help message @@ -505,6 +507,10 @@ while [[ "$#" -gt 0 ]]; do SLOT_CHAIN_END="${2}" shift ;; + -hfd |--hardfork-genesis-slot-delta) + HARDFORK_GENESIS_SLOT_DELTA="${2}" + shift + ;; -r | --root) ROOT="${2}" shift @@ -748,6 +754,13 @@ load_config() { inherit_with:*) local replaced_config_file IFS=',' read -r replaced_config_file OVERRIDE_GENSIS_LEDGER <<< "${config_mode#inherit_with:}" + if [ ! -f "${replaced_config_file}" ]; then + echo "Error: Config file '${replaced_config_file}' does not exist, can't inherit_with." >&2 + exit 1 + else + echo "Inheriting config at ${replaced_config_file}:" + cat "${replaced_config_file}" + fi cp -f "${replaced_config_file}" "${config_file}" ;; esac @@ -782,20 +795,25 @@ update_genesis_timestamp() { update_genesis_timestamp "${UPDATE_GENESIS_TIMESTAMP}" if [ ! -z "${OVERRIDE_SLOT_TIME_MS}" ]; then - echo 'Modifying configuration to override slot time...' + echo "Setting proof.block_window_duration_ms to ${OVERRIDE_SLOT_TIME_MS}..." jq-inplace ".proof.block_window_duration_ms=${OVERRIDE_SLOT_TIME_MS}" "${CONFIG}" fi if [ ! -z "${SLOT_TX_END}" ]; then - echo 'Modifying configuration to override slot transaction end...' + echo "Setting daemon.slot_tx_end to ${SLOT_TX_END}..." jq-inplace ".daemon.slot_tx_end=${SLOT_TX_END}" "${CONFIG}" fi if [ ! -z "${SLOT_CHAIN_END}" ]; then - echo 'Modifying configuration to override slot chain end...' + echo "Setting daemon.slot_chain_end to ${SLOT_CHAIN_END}..." jq-inplace ".daemon.slot_chain_end=${SLOT_CHAIN_END}" "${CONFIG}" fi +if [ ! -z "${HARDFORK_GENESIS_SLOT_DELTA}" ]; then + echo "Setting daemon.hard_fork_genesis_slot_delta to ${HARDFORK_GENESIS_SLOT_DELTA}..." + jq-inplace ".daemon.hard_fork_genesis_slot_delta=${HARDFORK_GENESIS_SLOT_DELTA}" "${CONFIG}" +fi + # ================================================ # Launch the Nodes diff --git a/src/app/hardfork_test/src/internal/app/root.go b/src/app/hardfork_test/src/internal/app/root.go index 0108b155697e..641d188943cd 100644 --- a/src/app/hardfork_test/src/internal/app/root.go +++ b/src/app/hardfork_test/src/internal/app/root.go @@ -84,7 +84,8 @@ func init() { rootCmd.Flags().IntVar(&cfg.UserCommandCheckMaxIterations, "user-command-check-max-iterations", cfg.UserCommandCheckMaxIterations, "Max iterations to check for user commands in blocks") rootCmd.Flags().IntVar(&cfg.ForkEarliestBlockMaxRetries, "fork-earliest-block-max-retries", cfg.ForkEarliestBlockMaxRetries, "Maximum number of retries to wait for earliest block in fork network") rootCmd.Flags().IntVar(&cfg.HTTPClientTimeoutSeconds, "http-timeout", cfg.HTTPClientTimeoutSeconds, "HTTP client timeout in seconds for GraphQL requests") - rootCmd.Flags().IntVar(&cfg.GraphQLMaxRetries, "graphql-max-retries", cfg.GraphQLMaxRetries, "Maximum number of retries for GraphQL requests") + rootCmd.Flags().IntVar(&cfg.ClientMaxRetries, "client-max-retries", cfg.ClientMaxRetries, "Maximum number of retries for client requests") + rootCmd.Flags().Var(&cfg.ForkMethod, "fork-method", "The implementation of fork") // Mark required flags rootCmd.MarkFlagRequired("main-mina-exe") diff --git a/src/app/hardfork_test/src/internal/graphql/client.go b/src/app/hardfork_test/src/internal/client/client.go similarity index 93% rename from src/app/hardfork_test/src/internal/graphql/client.go rename to src/app/hardfork_test/src/internal/client/client.go index 981a538e1916..fa272949f409 100644 --- a/src/app/hardfork_test/src/internal/graphql/client.go +++ b/src/app/hardfork_test/src/internal/client/client.go @@ -1,4 +1,4 @@ -package graphql +package client import ( "bytes" @@ -27,8 +27,8 @@ func NewClient(timeoutSeconds int, maxRetries int) *Client { } } -// Query sends a GraphQL query to the specified port with retry logic -func (c *Client) Query(port int, query string) (gjson.Result, error) { +// query sends a GraphQL query to the specified port with retry logic +func (c *Client) query(port int, query string) (gjson.Result, error) { url := fmt.Sprintf("http://localhost:%d/graphql", port) // Format the query payload @@ -99,7 +99,7 @@ func (c *Client) Query(port int, query string) (gjson.Result, error) { // GetHeight gets the current block height func (c *Client) GetHeight(port int) (int, error) { - result, err := c.Query(port, "bestChain(maxLength: 1) { protocolState { consensusState { blockHeight } } }") + result, err := c.query(port, "bestChain(maxLength: 1) { protocolState { consensusState { blockHeight } } }") if err != nil { return 0, err } @@ -110,7 +110,7 @@ func (c *Client) GetHeight(port int) (int, error) { // GetHeightAndSlotOfEarliest gets the height and slot of the earliest block func (c *Client) GetHeightAndSlotOfEarliest(port int) (height, slot int, err error) { - result, err := c.Query(port, "bestChain { protocolState { consensusState { blockHeight slotSinceGenesis } } }") + result, err := c.query(port, "bestChain { protocolState { consensusState { blockHeight slotSinceGenesis } } }") if err != nil { return 0, 0, err } @@ -123,7 +123,7 @@ func (c *Client) GetHeightAndSlotOfEarliest(port int) (height, slot int, err err // GetForkConfig gets the fork configuration func (c *Client) GetForkConfig(port int) (gjson.Result, error) { - result, err := c.Query(port, "fork_config") + result, err := c.query(port, "fork_config") if err != nil { return gjson.Result{}, err } @@ -133,7 +133,7 @@ func (c *Client) GetForkConfig(port int) (gjson.Result, error) { // BlocksWithUserCommands gets the number of blocks with user commands func (c *Client) BlocksWithUserCommands(port int) (int, error) { - result, err := c.Query(port, "bestChain { commandTransactionCount }") + result, err := c.query(port, "bestChain { commandTransactionCount }") if err != nil { return 0, err } @@ -150,7 +150,7 @@ func (c *Client) BlocksWithUserCommands(port int) (int, error) { } // Blocks gets blocks data as in the original shell script -const BlocksQuery = ` +const Blocksquery = ` bestChain { commandTransactionCount protocolState { @@ -197,7 +197,7 @@ type BlockData struct { // GetBlocks retrieves block data from the node func (c *Client) GetBlocks(port int) ([]BlockData, error) { - result, err := c.Query(port, BlocksQuery) + result, err := c.query(port, Blocksquery) if err != nil { return nil, err } diff --git a/src/app/hardfork_test/src/internal/config/config.go b/src/app/hardfork_test/src/internal/config/config.go index 2c2ce8a4b600..99249085f901 100644 --- a/src/app/hardfork_test/src/internal/config/config.go +++ b/src/app/hardfork_test/src/internal/config/config.go @@ -2,6 +2,7 @@ package config import ( "fmt" + "math" "os" "time" ) @@ -60,7 +61,9 @@ type Config struct { UserCommandCheckMaxIterations int // Max iterations to check for user commands in blocks ForkEarliestBlockMaxRetries int // Max retries to wait for earliest block in fork network HTTPClientTimeoutSeconds int // HTTP client timeout for GraphQL requests - GraphQLMaxRetries int // Max number of retries for GraphQL requests + ClientMaxRetries int // Max number of retries for client requests + + ForkMethod ForkMethod } // DefaultConfig returns the default configuration with values @@ -72,12 +75,12 @@ func DefaultConfig() *Config { BestChainQueryFrom: 25, MainSlot: 30, ForkSlot: 30, - MainDelay: 8, - ForkDelay: 8, + MainDelay: 7, + ForkDelay: 9, NumWhales: 2, NumFish: 1, - NumNodes: 1, - PaymentInterval: 10, + NumNodes: 0, + PaymentInterval: 20, ShutdownTimeoutMinutes: 10, PollingIntervalSeconds: 5, ForkConfigRetryDelaySeconds: 60, @@ -87,13 +90,15 @@ func DefaultConfig() *Config { ForkEarliestBlockMaxRetries: 10, HTTPClientTimeoutSeconds: 600, // ^ fork config take really long time to complete (2-3 minutes) - GraphQLMaxRetries: 5, + ClientMaxRetries: 5, SeedStartPort: 3000, SnarkCoordinatorPort: 7000, WhaleStartPort: 4000, FishStartPort: 5000, NodeStartPort: 6000, + + ForkMethod: Legacy, } } @@ -139,11 +144,16 @@ func (c *Config) Validate() error { return nil } +func (c *Config) HfSlotDelta() int { + + forkDelayDuration := time.Duration(c.ForkDelay) * time.Minute + return int(math.Ceil(forkDelayDuration.Seconds() / float64(c.MainSlot))) +} // FormatTimestamp formats a UNIX timestamp into the format used by the shell script func FormatTimestamp(unixTs int64) string { t := time.Unix(unixTs, 0).UTC() - return t.Format("2006-01-02 15:04:05+00:00") + return t.Format(time.RFC3339) } // validateExecutable checks if a file exists and has executable permissions diff --git a/src/app/hardfork_test/src/internal/config/fork_method.go b/src/app/hardfork_test/src/internal/config/fork_method.go new file mode 100644 index 000000000000..31dabbbc586e --- /dev/null +++ b/src/app/hardfork_test/src/internal/config/fork_method.go @@ -0,0 +1,51 @@ +package config + +import ( + "fmt" + "sort" + "strings" +) + +type ForkMethod int + +const ( + Legacy ForkMethod = iota + Advanced +) + +var forkMethodToString = map[ForkMethod]string{ + Legacy: "legacy", + Advanced: "advanced", +} + +var stringToForkMethod = map[string]ForkMethod{ + "legacy": Legacy, + "advanced": Advanced, +} + +func (m *ForkMethod) String() string { + if s, ok := forkMethodToString[*m]; ok { + return s + } + return "unknown" +} + +func validKeys(m map[string]ForkMethod) string { + keys := make([]string, 0, len(m)) + for k := range m { + keys = append(keys, k) + } + sort.Strings(keys) + return strings.Join(keys, "|") +} + +func (m *ForkMethod) Set(s string) error { + v, ok := stringToForkMethod[s] + if !ok { + return fmt.Errorf("invalid mode %q (valid: %s)", s, validKeys(stringToForkMethod)) + } + *m = v + return nil +} + +func (m *ForkMethod) Type() string { return "fork method" } diff --git a/src/app/hardfork_test/src/internal/hardfork/config_validation.go b/src/app/hardfork_test/src/internal/hardfork/config_validation.go index 037dba4be7b6..b17c8faeead5 100644 --- a/src/app/hardfork_test/src/internal/hardfork/config_validation.go +++ b/src/app/hardfork_test/src/internal/hardfork/config_validation.go @@ -1,20 +1,38 @@ package hardfork import ( + "bytes" + "encoding/json" "fmt" "os" + "time" - "github.com/MinaProtocol/mina/src/app/hardfork_test/src/internal/graphql" + "github.com/MinaProtocol/mina/src/app/hardfork_test/src/internal/client" ) -// ValidatePreforkLedgerHashes validates the generated prefork ledger hashes -func (t *HardforkTest) ValidatePreforkLedgerHashes( - latestNonEmptyBlock graphql.BlockData, +type HashAndS3 struct { + S3DataHash string `json:"s3_data_hash"` + Hash string `json:"hash"` +} + +type RuntimeGenesisLedgerHashes struct { + EpochData struct { + Staking HashAndS3 `json:"staking"` + Next HashAndS3 `json:"next"` + } `json:"epoch_data"` + Ledger HashAndS3 `json:"ledger"` +} + +func (t *HardforkTest) ValidateRuntimeGenesisLedgerHashes( + latestNonEmptyBlock client.BlockData, genesisEpochStaking string, genesisEpochNext string, latestSnarkedHashPerEpoch map[int]string, - preforkHashesFile string, + ledgerHashesFile string, ) error { + + t.Logger.Info("Validating ledger hashes file generated by runtime-genesis-ledger") + // Calculate slot_tx_end_epoch // 48 as specififed by mina-local-network.sh slotTxEndEpoch := latestNonEmptyBlock.Slot / 48 @@ -31,163 +49,161 @@ func (t *HardforkTest) ValidatePreforkLedgerHashes( } // Read prefork hashes from file - preforkHashesData, err := os.ReadFile(preforkHashesFile) + ledgerHashesBytes, err := os.ReadFile(ledgerHashesFile) if err != nil { - return fmt.Errorf("failed to read prefork hashes file: %w", err) + return fmt.Errorf("failed to read ledger hashes file: %w", err) } - preforkHashesJson := string(preforkHashesData) + var hashes RuntimeGenesisLedgerHashes + dec := json.NewDecoder(bytes.NewReader(ledgerHashesBytes)) + dec.DisallowUnknownFields() - // Validate field values - if err := validateStringField(preforkHashesJson, "epoch_data.staking.hash", expectedStakingHash); err != nil { - return err + if err := dec.Decode(&hashes); err != nil { + return fmt.Errorf("failed to unmarshal ledger hashes file: %w", err) } - if err := validateStringField(preforkHashesJson, "epoch_data.next.hash", expectedNextHash); err != nil { - return err - } - if err := validateStringField(preforkHashesJson, "ledger.hash", latestNonEmptyBlock.StagedHash); err != nil { - return err - } - - ledger_fields := []string{"hash", "s3_data_hash"} - // Validate object structure - ensure only expected fields are present - if err := t.validateObjectFields(preforkHashesJson, "epoch_data.staking", ledger_fields); err != nil { - return err - } - if err := t.validateObjectFields(preforkHashesJson, "epoch_data.next", ledger_fields); err != nil { - return err - } - if err := t.validateObjectFields(preforkHashesJson, "ledger", ledger_fields); err != nil { - return err + if hashes.EpochData.Staking.Hash != expectedStakingHash { + return fmt.Errorf("Expected epoch_data.staking.hash `%s`, got `%s`", expectedStakingHash, hashes.EpochData.Staking.Hash) } - if err := t.validateObjectFields(preforkHashesJson, "epoch_data", []string{"staking", "next"}); err != nil { - return err + + if hashes.EpochData.Next.Hash != expectedNextHash { + return fmt.Errorf("Expected epoch_data.next.hash `%s`, got `%s`", expectedNextHash, hashes.EpochData.Next.Hash) } - // Validate root object contains only expected top-level fields - if err := t.validateRootObjectFields(preforkHashesJson, []string{"epoch_data", "ledger"}); err != nil { - return err + if hashes.Ledger.Hash != latestNonEmptyBlock.StagedHash { + return fmt.Errorf("Expected ledger.hash `%s`, got `%s`", latestNonEmptyBlock.StagedHash, hashes.Ledger.Hash) } - t.Logger.Info("Prefork ledger hashes validated successfully") + t.Logger.Info("Ledger hashes file validated successfully") return nil } -// ValidateForkConfigData validates the extracted fork config against expected values -func (t *HardforkTest) ValidateForkConfigData(latestNonEmptyBlock graphql.BlockData, forkConfigBytes []byte) error { - forkConfigJson := string(forkConfigBytes) +type EpochDataPrepatch struct { + Seed string `json:"seed"` + Accounts any `json:"accounts"` +} - // Validate field values - if err := validateIntField(forkConfigJson, "proof.fork.blockchain_length", latestNonEmptyBlock.BlockHeight); err != nil { - return err - } - if err := validateIntField(forkConfigJson, "proof.fork.global_slot_since_genesis", latestNonEmptyBlock.Slot); err != nil { - return err - } - if err := validateStringField(forkConfigJson, "proof.fork.state_hash", latestNonEmptyBlock.StateHash); err != nil { - return err - } - if err := validateStringField(forkConfigJson, "epoch_data.next.seed", latestNonEmptyBlock.NextEpochSeed); err != nil { - return err - } - if err := validateStringField(forkConfigJson, "epoch_data.staking.seed", latestNonEmptyBlock.CurEpochSeed); err != nil { - return err - } - if err := validateStringField(forkConfigJson, "ledger.hash", latestNonEmptyBlock.StagedHash); err != nil { - return err - } - if err := validateBoolField(forkConfigJson, "ledger.add_genesis_winner", false); err != nil { - return err +type LegacyPrepatchForkConfigView struct { + Proof struct { + Fork struct { + BlockChainLength int `json:"blockchain_length"` + GlobalSlotSinceGenesis int `json:"global_slot_since_genesis"` + StateHash string `json:"state_hash"` + } `json:"fork"` + } `json:"proof"` + EpochData struct { + Staking EpochDataPrepatch `json:"staking"` + Next EpochDataPrepatch `json:"next"` + } `json:"epoch_data"` + Ledger struct { + Hash string `json:"hash"` + Accounts any `json:"accounts"` + AddGenesisWinner bool `json:"add_genesis_winner"` + } `json:"ledger"` +} + +func (t *HardforkTest) ValidateLegacyPrepatchForkConfig(latestNonEmptyBlock client.BlockData, forkConfigBytes []byte) error { + + t.Logger.Info("Validating legacy prepatch fork config") + + var config LegacyPrepatchForkConfigView + dec := json.NewDecoder(bytes.NewReader(forkConfigBytes)) + dec.DisallowUnknownFields() + + if err := dec.Decode(&config); err != nil { + return fmt.Errorf("failed to unmarshal legacy prepatch fork config: %w", err) } - // Validate object structure - ensure only expected fields are present - if err := t.validateObjectFields(forkConfigJson, "proof.fork", []string{"blockchain_length", "global_slot_since_genesis", "state_hash"}); err != nil { - return err + if config.Proof.Fork.BlockChainLength != latestNonEmptyBlock.BlockHeight { + return fmt.Errorf("Expected proof.fork.blockchain_length to be %d, got %d", latestNonEmptyBlock.BlockHeight, config.Proof.Fork.BlockChainLength) } - if err := t.validateObjectFields(forkConfigJson, "epoch_data.staking", []string{"seed", "accounts"}); err != nil { - return err + + if config.Proof.Fork.GlobalSlotSinceGenesis != latestNonEmptyBlock.Slot { + return fmt.Errorf("Expected proof.fork.global_slot_since_genesis to be %d, got %d", latestNonEmptyBlock.Slot, config.Proof.Fork.GlobalSlotSinceGenesis) } - if err := t.validateObjectFields(forkConfigJson, "epoch_data.next", []string{"seed", "accounts"}); err != nil { - return err + + if config.Proof.Fork.StateHash != latestNonEmptyBlock.StateHash { + return fmt.Errorf("Expected proof.fork.state_hash to be `%s`, got `%s`", latestNonEmptyBlock.StateHash, config.Proof.Fork.StateHash) } - if err := t.validateObjectFields(forkConfigJson, "ledger", []string{"hash", "accounts", "add_genesis_winner"}); err != nil { - return err + + if config.EpochData.Staking.Seed != latestNonEmptyBlock.CurEpochSeed { + return fmt.Errorf("Expected proof.epoch_data.staking.seed to be `%s`, got `%s`", latestNonEmptyBlock.CurEpochSeed, config.EpochData.Staking.Seed) } - if err := t.validateObjectFields(forkConfigJson, "epoch_data", []string{"staking", "next"}); err != nil { - return err + + if config.EpochData.Next.Seed != latestNonEmptyBlock.NextEpochSeed { + return fmt.Errorf("Expected proof.epoch_data.next.seed to be `%s`, got `%s`", latestNonEmptyBlock.NextEpochSeed, config.EpochData.Next.Seed) } - if err := t.validateObjectFields(forkConfigJson, "proof", []string{"fork"}); err != nil { - return err + + if config.Ledger.Hash != latestNonEmptyBlock.StagedHash { + return fmt.Errorf("Expected ledger.hash to be `%s`, got `%s`", latestNonEmptyBlock.StagedHash, config.Ledger.Hash) } - // Validate root object contains only expected top-level fields - if err := t.validateRootObjectFields(forkConfigJson, []string{"proof", "epoch_data", "ledger"}); err != nil { - return err + if config.Ledger.AddGenesisWinner != false { + return fmt.Errorf("Expected ledger.add_genesis_winner to be false, got true") } - t.Logger.Info("Fork config data validated successfully") + t.Logger.Info("Legacy prepatch fork config validated successfully") return nil } -// ValidateForkRuntimeConfig validates that the runtime config has correct fork data -func (t *HardforkTest) ValidateForkRuntimeConfig(latestNonEmptyBlock graphql.BlockData, configData []byte, forkGenesisTs, mainGenesisTs int64) error { +type EpochData struct { + Hash string `json:"hash"` + S3DataHash string `json:"s3_data_hash"` + Seed string `json:"seed"` +} + +type FinalForkConfigView struct { + Proof struct { + Fork struct { + BlockChainLength int `json:"blockchain_length"` + GlobalSlotSinceGenesis int64 `json:"global_slot_since_genesis"` + StateHash string `json:"state_hash"` + } `json:"fork"` + } `json:"proof"` + EpochData struct { + Staking EpochData `json:"staking"` + Next EpochData `json:"next"` + } `json:"epoch_data"` + Genesis struct { + GenesisStateTimeStamp time.Time `json:"genesis_state_timestamp"` + } + Ledger struct { + Hash string `json:"hash"` + S3DataHash string `json:"s3_data_hash"` + AddGenesisWinner bool `json:"add_genesis_winner"` + } `json:"ledger"` +} + +func (t *HardforkTest) ValidateFinalForkConfig(latestNonEmptyBlock client.BlockData, forkConfigBytes []byte, forkGenesisTs, mainGenesisTs int64) error { + + t.Logger.Info("Validating final fork config") // Calculate expected genesis slot expectedGenesisSlot := (forkGenesisTs - mainGenesisTs) / int64(t.Config.MainSlot) - configJson := string(configData) + var config FinalForkConfigView + dec := json.NewDecoder(bytes.NewReader(forkConfigBytes)) + dec.DisallowUnknownFields() - // Validate field values - if err := validateIntField(configJson, "proof.fork.blockchain_length", latestNonEmptyBlock.BlockHeight); err != nil { - return err - } - if err := validateInt64Field(configJson, "proof.fork.global_slot_since_genesis", expectedGenesisSlot); err != nil { - return err - } - if err := validateStringField(configJson, "proof.fork.state_hash", latestNonEmptyBlock.StateHash); err != nil { - return err - } - if err := validateUnixTimestampField(configJson, "genesis.genesis_state_timestamp", forkGenesisTs); err != nil { - return err - } - if err := t.validateObjectFields(configJson, "genesis", []string{"genesis_state_timestamp"}); err != nil { - return err + if err := dec.Decode(&config); err != nil { + return fmt.Errorf("failed to unmarshal fork config: %w", err) } - // Validate object structure - ensure only expected fields are present - if err := t.validateObjectFields(configJson, "proof.fork", []string{"blockchain_length", "global_slot_since_genesis", "state_hash"}); err != nil { - return err - } - if err := t.validateObjectFields(configJson, "proof", []string{"fork"}); err != nil { - return err + if config.Proof.Fork.BlockChainLength != latestNonEmptyBlock.BlockHeight { + return fmt.Errorf("Expected proof.fork.blockchain_length to be %d, got %d", latestNonEmptyBlock.BlockHeight, config.Proof.Fork.BlockChainLength) } - epochFields := []string{"hash", "s3_data_hash", "seed"} - // Validate object structure - ensure only expected fields are present - if err := t.validateObjectFields(configJson, "epoch_data.staking", epochFields); err != nil { - return err - } - if err := t.validateObjectFields(configJson, "epoch_data.next", epochFields); err != nil { - return err - } - if err := t.validateObjectFields(configJson, "epoch_data", []string{"staking", "next"}); err != nil { - return err + if config.Proof.Fork.GlobalSlotSinceGenesis != expectedGenesisSlot { + return fmt.Errorf("Expected proof.fork.global_slot_since_genesis to be %d, got %d", expectedGenesisSlot, config.Proof.Fork.GlobalSlotSinceGenesis) } - // Validate ledger.add_genesis_winner is false - if err := validateBoolField(configJson, "ledger.add_genesis_winner", false); err != nil { - return err + if config.Proof.Fork.StateHash != latestNonEmptyBlock.StateHash { + return fmt.Errorf("Expected proof.fork.state_hash to be `%s`, got `%s`", latestNonEmptyBlock.StateHash, config.Proof.Fork.StateHash) } - ledgerFields := []string{"hash", "s3_data_hash", "add_genesis_winner"} - if err := t.validateObjectFields(configJson, "ledger", ledgerFields); err != nil { - return err + if config.Genesis.GenesisStateTimeStamp.Unix() != forkGenesisTs { + return fmt.Errorf("Expected genesis.genesis_state_timestamp to be `%d`(unix timestamp), got `%s`(RFC3339)", forkGenesisTs, config.Genesis.GenesisStateTimeStamp.Format(time.RFC3339)) } - // Validate root object contains only expected top-level fields - if err := t.validateRootObjectFields(configJson, []string{"proof", "epoch_data", "ledger", "genesis"}); err != nil { - return err - } - t.Logger.Info("Config for the fork is correct, starting a new network") + t.Logger.Info("Final fork config validated successfully") return nil } diff --git a/src/app/hardfork_test/src/internal/hardfork/fork_config.go b/src/app/hardfork_test/src/internal/hardfork/fork_config.go index 7aa175039234..1cca6085ea2b 100644 --- a/src/app/hardfork_test/src/internal/hardfork/fork_config.go +++ b/src/app/hardfork_test/src/internal/hardfork/fork_config.go @@ -43,7 +43,7 @@ func (t *HardforkTest) GetForkConfig(port int) ([]byte, error) { } // CreateRuntimeConfig creates the runtime configuration for the fork -func (t *HardforkTest) CreateRuntimeConfig(forkGenesisTimestamp, forkConfigPath, configFile, baseConfigFile, forkHashesFile string) ([]byte, error) { +func (t *HardforkTest) PatchRuntimeConfigLegacy(forkGenesisTimestamp, forkConfigPath, configFile, baseConfigFile, forkHashesFile string) ([]byte, error) { cmd := exec.Command(filepath.Join(t.ScriptDir, "create_runtime_config.sh")) cmd.Env = append(os.Environ(), "GENESIS_TIMESTAMP="+forkGenesisTimestamp, diff --git a/src/app/hardfork_test/src/internal/hardfork/json_validation.go b/src/app/hardfork_test/src/internal/hardfork/json_validation.go deleted file mode 100644 index 1c3a0bb8d837..000000000000 --- a/src/app/hardfork_test/src/internal/hardfork/json_validation.go +++ /dev/null @@ -1,167 +0,0 @@ -package hardfork - -import ( - "fmt" - "time" - - "github.com/tidwall/gjson" -) - -// validateStringField validates that a string field exists and matches the expected value -func validateStringField(json, path, expected string) error { - result := gjson.Get(json, path) - if !result.Exists() { - return fmt.Errorf("missing field: %s", path) - } - actual := result.String() - if actual != expected { - return fmt.Errorf("%s mismatch: expected %s, got %s", path, expected, actual) - } - return nil -} - -// validateIntField validates that an integer field exists and matches the expected value -func validateIntField(json, path string, expected int) error { - result := gjson.Get(json, path) - if !result.Exists() { - return fmt.Errorf("missing field: %s", path) - } - actual := result.Int() - if actual != int64(expected) { - return fmt.Errorf("%s mismatch: expected %d, got %d", path, expected, actual) - } - return nil -} - -// validateInt64Field validates that an int64 field exists and matches the expected value -func validateInt64Field(json, path string, expected int64) error { - result := gjson.Get(json, path) - if !result.Exists() { - return fmt.Errorf("missing field: %s", path) - } - actual := result.Int() - if actual != expected { - return fmt.Errorf("%s mismatch: expected %d, got %d", path, expected, actual) - } - return nil -} - -// validateBoolField validates that a boolean field exists and matches the expected value -func validateBoolField(json, path string, expected bool) error { - result := gjson.Get(json, path) - if !result.Exists() { - return fmt.Errorf("missing field: %s", path) - } - actual := result.Bool() - if actual != expected { - return fmt.Errorf("%s mismatch: expected %v, got %v", path, expected, actual) - } - return nil -} - -// validateUnixTimestampField validates that a timestamp field matches the expected Unix timestamp -// The field must be stored as an RFC3339 formatted string -func validateUnixTimestampField(json, path string, expectedUnixTs int64) error { - result := gjson.Get(json, path) - if !result.Exists() { - return fmt.Errorf("missing field: %s", path) - } - - if result.Type != gjson.String { - return fmt.Errorf("%s must be a string", path) - } - - timestampStr := result.String() - // Try parsing as RFC3339 timestamp - t, err := time.Parse(time.RFC3339, timestampStr) - if err != nil { - // Also try a common variant with space instead of 'T' - t, err = time.Parse("2006-01-02 15:04:05-07:00", timestampStr) - if err != nil { - return fmt.Errorf("%s is not a valid RFC3339 timestamp: %v", path, err) - } - } - - actualUnixTs := t.Unix() - if actualUnixTs != expectedUnixTs { - return fmt.Errorf("%s mismatch: expected %d, got %d", path, expectedUnixTs, actualUnixTs) - } - return nil -} - -// validateObjectFields validates that an object contains only the expected fields -func (t *HardforkTest) validateObjectFields(json, path string, expectedFields []string) error { - obj := gjson.Get(json, path) - if !obj.Exists() { - return fmt.Errorf("missing object: %s", path) - } - if !obj.IsObject() { - return fmt.Errorf("%s is not an object", path) - } - - // Create a map of expected fields for quick lookup - expectedMap := make(map[string]bool) - for _, field := range expectedFields { - expectedMap[field] = true - } - - // Check all fields in the object - fieldCount := 0 - var unexpectedFields []string - obj.ForEach(func(key, value gjson.Result) bool { - fieldCount++ - fieldName := key.String() - if !expectedMap[fieldName] { - unexpectedFields = append(unexpectedFields, fieldName) - t.Logger.Error("Unexpected field in %s: %s", path, fieldName) - } - return true // continue iteration - }) - - if len(unexpectedFields) > 0 { - return fmt.Errorf("%s contains unexpected fields: %v", path, unexpectedFields) - } - - if fieldCount != len(expectedFields) { - return fmt.Errorf("%s should contain exactly %d field(s), found %d", path, len(expectedFields), fieldCount) - } - - return nil -} - -// validateRootObjectFields validates that the root-level JSON contains only expected fields -func (t *HardforkTest) validateRootObjectFields(json string, expectedFields []string) error { - rootResult := gjson.Parse(json) - if !rootResult.IsObject() { - return fmt.Errorf("root is not an object") - } - - // Create a map of expected fields for quick lookup - expectedMap := make(map[string]bool) - for _, field := range expectedFields { - expectedMap[field] = true - } - - // Check all fields in the root object - fieldCount := 0 - var unexpectedFields []string - rootResult.ForEach(func(key, value gjson.Result) bool { - fieldCount++ - fieldName := key.String() - if !expectedMap[fieldName] { - unexpectedFields = append(unexpectedFields, fieldName) - t.Logger.Error("Unexpected field in root object: %s", fieldName) - } - return true - }) - - if len(unexpectedFields) > 0 { - return fmt.Errorf("root object contains unexpected fields: %v", unexpectedFields) - } - - if fieldCount != len(expectedFields) { - return fmt.Errorf("root object should contain exactly %d field(s), found %d", len(expectedFields), fieldCount) - } - - return nil -} diff --git a/src/app/hardfork_test/src/internal/hardfork/ledger.go b/src/app/hardfork_test/src/internal/hardfork/ledger.go index 36ac1cddb5ec..d4acc927d4d3 100644 --- a/src/app/hardfork_test/src/internal/hardfork/ledger.go +++ b/src/app/hardfork_test/src/internal/hardfork/ledger.go @@ -4,6 +4,7 @@ import ( "fmt" "os" "os/exec" + "strconv" "github.com/MinaProtocol/mina/src/app/hardfork_test/src/internal/config" ) @@ -42,42 +43,57 @@ func (t *HardforkTest) GenerateForkLedgers(executablePath, forkConfigPath, ledge return nil } -// GenerateAndValidatePreforkLedgers validates fork config and ledgers -// Note: Fork config extraction happens in RunMainNetworkPhase before nodes shutdown -func (t *HardforkTest) GenerateAndValidatePreforkLedgers(analysis *BlockAnalysisResult, forkConfigPath, preforkLedgersDir, preforkHashesFile string) error { +func (t *HardforkTest) GenerateAndValidateHashesAndLedgers(analysis *BlockAnalysisResult, forkConfigPath, preforkLedgersDir, prepatchForkConfig string) error { // Generate prefork ledgers using main network executable - if err := t.GenerateForkLedgers(t.Config.MainRuntimeGenesisLedger, forkConfigPath, preforkLedgersDir, preforkHashesFile); err != nil { + if err := t.GenerateForkLedgers(t.Config.MainRuntimeGenesisLedger, forkConfigPath, preforkLedgersDir, prepatchForkConfig); err != nil { return err } - // Validate prefork ledger hashes - if err := t.ValidatePreforkLedgerHashes( + return t.ValidateRuntimeGenesisLedgerHashes( analysis.LatestNonEmptyBlock, analysis.GenesisEpochStaking, analysis.GenesisEpochNext, analysis.LatestSnarkedHashPerEpoch, - preforkHashesFile, - ); err != nil { - return err - } - - return nil + prepatchForkConfig, + ) } -// GenerateForkConfigAndLedgers generates the fork configuration and ledgers -func (t *HardforkTest) GenerateForkConfigAndLedgers(analysis *BlockAnalysisResult, forkConfigPath, forkLedgersDir, forkHashesFile, configFile, preforkGenesisConfigFile string, forkGenesisTs, mainGenesisTs int64) error { +// PatchForkConfigAndGenerateLedgersLegacy does the following: +// 1. generate fork ledgers with runtime-genesis-ledger +// 2. patch the genesis time & slot for fork config with create_runtime_config.sh +// 3. perform some base sanity check on the fork config +func (t *HardforkTest) PatchForkConfigAndGenerateLedgersLegacy(analysis *BlockAnalysisResult, forkConfigPath, forkLedgersDir, forkHashesFile, configFile, preforkGenesisConfigFile string, forkGenesisTs, mainGenesisTs int64) ([]byte, error) { // Generate fork ledgers using fork network executable if err := t.GenerateForkLedgers(t.Config.ForkRuntimeGenesisLedger, forkConfigPath, forkLedgersDir, forkHashesFile); err != nil { - return err + return nil, err } // Create runtime config forkGenesisTimestamp := config.FormatTimestamp(forkGenesisTs) - runtimeConfigBytes, err := t.CreateRuntimeConfig(forkGenesisTimestamp, forkConfigPath, configFile, preforkGenesisConfigFile, forkHashesFile) + return t.PatchRuntimeConfigLegacy(forkGenesisTimestamp, forkConfigPath, configFile, preforkGenesisConfigFile, forkHashesFile) +} + +func (t *HardforkTest) AdvancedGenerateHardForkConfig(configDir string, clientPort int) error { + cmd := exec.Command(t.Config.MainMinaExe, + "advanced", "generate-hardfork-config", + "--hardfork-config-dir", configDir, + "--daemon-port", strconv.Itoa(clientPort), + "--generate-fork-validation", "false", + ) + + cmd.Stdout = os.Stdout + cmd.Stderr = os.Stderr + + if err := cmd.Start(); err != nil { + return fmt.Errorf("failed to call mina CLI: %w", err) + } + + cmd.Wait() + + _, err := os.Stat(fmt.Sprintf("%s/activated", configDir)) if err != nil { - return err + return fmt.Errorf("failed to check on activated file for advanced generate fork config: %w", err) } - // Validate modified fork data - return t.ValidateForkRuntimeConfig(analysis.LatestNonEmptyBlock, runtimeConfigBytes, forkGenesisTs, mainGenesisTs) + return nil } diff --git a/src/app/hardfork_test/src/internal/hardfork/network.go b/src/app/hardfork_test/src/internal/hardfork/network.go index 3064345385ce..e9281f4858e9 100644 --- a/src/app/hardfork_test/src/internal/hardfork/network.go +++ b/src/app/hardfork_test/src/internal/hardfork/network.go @@ -94,19 +94,27 @@ func (t *HardforkTest) RunMainNetwork(mainGenesisTs int64) (*exec.Cmd, error) { mainGenesisTimestamp := config.FormatTimestamp(mainGenesisTs) - return t.startLocalNetwork(t.Config.MainMinaExe, "main", []string{ + args := []string{ "--update-genesis-timestamp", fmt.Sprintf("fixed:%s", mainGenesisTimestamp), "--config", "reset", "--override-slot-time", strconv.Itoa(t.Config.MainSlot * 1000), "--slot-transaction-end", strconv.Itoa(t.Config.SlotTxEnd), "--slot-chain-end", strconv.Itoa(t.Config.SlotChainEnd), - }) + } + + switch t.Config.ForkMethod { + case config.Advanced: + args = append(args, "--hardfork-genesis-slot-delta", strconv.Itoa(t.Config.HfSlotDelta())) + case config.Legacy: + // Do nothing as we'll patch the slot at the end of main network + } + + return t.startLocalNetwork(t.Config.MainMinaExe, "main", args) } // RunForkNetwork starts the fork network with hardfork configuration func (t *HardforkTest) RunForkNetwork(configFile, forkLedgersDir string) (*exec.Cmd, error) { return t.startLocalNetwork(t.Config.ForkMinaExe, "fork", []string{ - "--update-genesis-timestamp", fmt.Sprintf("delay_sec:%d", t.Config.ForkDelay*60), "--config", fmt.Sprintf("inherit_with:%s,%s", configFile, forkLedgersDir), "--override-slot-time", strconv.Itoa(t.Config.ForkSlot * 1000)}, ) diff --git a/src/app/hardfork_test/src/internal/hardfork/phases.go b/src/app/hardfork_test/src/internal/hardfork/phases.go index 9dced89d36e5..fb38b1e80945 100644 --- a/src/app/hardfork_test/src/internal/hardfork/phases.go +++ b/src/app/hardfork_test/src/internal/hardfork/phases.go @@ -6,13 +6,15 @@ import ( "time" ) +type HFHandler func(*HardforkTest, *BlockAnalysisResult) error + // RunMainNetworkPhase runs the main network and validates its operation // and returns the fork config bytes and block analysis result -func (t *HardforkTest) RunMainNetworkPhase(mainGenesisTs int64) ([]byte, *BlockAnalysisResult, error) { +func (t *HardforkTest) RunMainNetworkPhase(mainGenesisTs int64, beforeShutdown HFHandler) (*BlockAnalysisResult, error) { // Start the main network mainNetCmd, err := t.RunMainNetwork(mainGenesisTs) if err != nil { - return nil, nil, err + return nil, err } defer t.gracefulShutdown(mainNetCmd, "Main network") @@ -23,44 +25,42 @@ func (t *HardforkTest) RunMainNetworkPhase(mainGenesisTs int64) ([]byte, *BlockA // Check block height at slot BestChainQueryFrom blockHeight, err := t.Client.GetHeight(t.AnyPortOfType(PORT_REST)) if err != nil { - return nil, nil, err + return nil, err } t.Logger.Info("Block height is %d at slot %d.", blockHeight, t.Config.BestChainQueryFrom) // Validate slot occupancy if err := t.ValidateSlotOccupancy(0, blockHeight); err != nil { - return nil, nil, err + return nil, err } // Analyze blocks and get genesis epoch data analysis, err := t.AnalyzeBlocks() if err != nil { - return nil, nil, err + return nil, err } // Validate max slot if err := t.ValidateLatestOccupiedSlot(analysis.LatestOccupiedSlot); err != nil { - return nil, nil, err + return nil, err } // Validate latest block slot if err := t.ValidateLatestNonEmptyBlockSlot(analysis.LatestNonEmptyBlock); err != nil { - return nil, nil, err + return nil, err } // Validate no new blocks are created after chain end if err := t.ValidateNoNewBlocks(t.AnyPortOfType(PORT_REST)); err != nil { - return nil, nil, err + return nil, err } - // Extract fork config before nodes shutdown - forkConfigBytes, err := t.GetForkConfig(t.AnyPortOfType(PORT_REST)) - if err != nil { - return nil, nil, err + if err := beforeShutdown(t, analysis); err != nil { + return nil, err } - return forkConfigBytes, analysis, nil + return analysis, nil } type ForkData struct { @@ -113,49 +113,84 @@ func (t *HardforkTest) RunForkNetworkPhase(latestPreForkHeight int, forkData For func (t *HardforkTest) LegacyForkPhase(analysis *BlockAnalysisResult, forkConfigBytes []byte, mainGenesisTs int64) (*ForkData, error) { - // Define all localnet file paths - if err := os.MkdirAll("fork_data/prefork", 0755); err != nil { + if err := os.MkdirAll("fork_data/prepatch", 0755); err != nil { return nil, err } // Define all fork_data file paths - preforkConfig := "fork_data/prefork/config.json" + prepatchConfig := "fork_data/prepatch/config.json" // Validate fork config data - if err := t.ValidateForkConfigData(analysis.LatestNonEmptyBlock, forkConfigBytes); err != nil { + if err := t.ValidateLegacyPrepatchForkConfig(analysis.LatestNonEmptyBlock, forkConfigBytes); err != nil { return nil, err } // Write fork config to file - if err := os.WriteFile(preforkConfig, forkConfigBytes, 0644); err != nil { + if err := os.WriteFile(prepatchConfig, forkConfigBytes, 0644); err != nil { return nil, err } - { - preforkLedgersDir := "fork_data/prefork/hf_ledgers" - preforkHashesFile := "fork_data/prefork/hf_ledger_hashes.json" - if err := t.GenerateAndValidatePreforkLedgers(analysis, preforkConfig, preforkLedgersDir, preforkHashesFile); err != nil { - return nil, err - } + + prepatchLedgersDir := "fork_data/prepatch/hf_ledgers" + prepatchHashesFile := "fork_data/prepatch/hf_ledger_hashes.json" + if err := t.GenerateAndValidateHashesAndLedgers(analysis, prepatchConfig, prepatchLedgersDir, prepatchHashesFile); err != nil { + return nil, err } - if err := os.MkdirAll("fork_data/postfork", 0755); err != nil { + if err := os.MkdirAll("fork_data/postpatch", 0755); err != nil { return nil, err } - postforkConfig := "fork_data/postfork/config.json" - forkLedgersDir := "fork_data/postfork/hf_ledgers" + postpatchConfig := "fork_data/postpatch/config.json" + postpatchLedgersDir := "fork_data/postpatch/hf_ledgers" // Calculate fork genesis timestamp relative to now (before starting fork network) forkGenesisTs := time.Now().Unix() + int64(t.Config.ForkDelay*60) - t.Logger.Info("Phase 3: Generating fork configuration and ledgers...") - { - preforkGenesisConfigFile := fmt.Sprintf("%s/daemon.json", t.Config.Root) - forkHashesFile := "fork_data/hf_ledger_hashes.json" - if err := t.GenerateForkConfigAndLedgers(analysis, preforkConfig, forkLedgersDir, forkHashesFile, postforkConfig, preforkGenesisConfigFile, forkGenesisTs, mainGenesisTs); err != nil { - return nil, err - } + preforkGenesisConfigFile := fmt.Sprintf("%s/daemon.json", t.Config.Root) + forkHashesFile := "fork_data/hf_ledger_hashes.json" + + patchedConfigBytes, err := t.PatchForkConfigAndGenerateLedgersLegacy(analysis, prepatchConfig, postpatchLedgersDir, forkHashesFile, postpatchConfig, preforkGenesisConfigFile, forkGenesisTs, mainGenesisTs) + if err != nil { + return nil, err + } + err = t.ValidateFinalForkConfig(analysis.LatestNonEmptyBlock, patchedConfigBytes, forkGenesisTs, mainGenesisTs) + if err != nil { + return nil, err + } + + return &ForkData{config: postpatchConfig, ledgersDir: postpatchLedgersDir, genesis: forkGenesisTs}, nil + +} + +// Uses `mina advanced generate-hardfork-config CLI` +func (t *HardforkTest) AdvancedForkPhase(analysis *BlockAnalysisResult, mainGenesisTs int64) (*ForkData, error) { + + cwd := "" + var err error = nil + if cwd, err = os.Getwd(); err != nil { + return nil, err + } + + forkDataPath := fmt.Sprintf("%s/fork_data", cwd) + + if err := t.AdvancedGenerateHardForkConfig(forkDataPath, t.AnyPortOfType(PORT_CLIENT)); err != nil { + return nil, err } - return &ForkData{config: postforkConfig, ledgersDir: forkLedgersDir, genesis: forkGenesisTs}, nil + forkConfig := fmt.Sprintf("%s/daemon.json", forkDataPath) + + forkConfigBytes, err := os.ReadFile(forkConfig) + if err != nil { + return nil, err + } + + forkGenesisSlot := t.Config.SlotChainEnd + t.Config.HfSlotDelta() + forkGenesisTs := mainGenesisTs + int64(forkGenesisSlot*t.Config.MainSlot) + + err = t.ValidateFinalForkConfig(analysis.LatestNonEmptyBlock, forkConfigBytes, forkGenesisTs, mainGenesisTs) + if err != nil { + return nil, err + } + forkLedgersDir := fmt.Sprintf("%s/genesis", forkDataPath) + return &ForkData{config: forkConfig, ledgersDir: forkLedgersDir, genesis: forkGenesisTs}, nil } diff --git a/src/app/hardfork_test/src/internal/hardfork/test.go b/src/app/hardfork_test/src/internal/hardfork/test.go index 7a4b822d1e37..bb7fe3483633 100644 --- a/src/app/hardfork_test/src/internal/hardfork/test.go +++ b/src/app/hardfork_test/src/internal/hardfork/test.go @@ -9,15 +9,15 @@ import ( "syscall" "time" + "github.com/MinaProtocol/mina/src/app/hardfork_test/src/internal/client" "github.com/MinaProtocol/mina/src/app/hardfork_test/src/internal/config" - "github.com/MinaProtocol/mina/src/app/hardfork_test/src/internal/graphql" "github.com/MinaProtocol/mina/src/app/hardfork_test/src/internal/utils" ) // HardforkTest represents the main hardfork test logic type HardforkTest struct { Config *config.Config - Client *graphql.Client + Client *client.Client Logger *utils.Logger ScriptDir string runningCmds []*exec.Cmd @@ -31,7 +31,7 @@ func NewHardforkTest(cfg *config.Config) *HardforkTest { ctx, cancel := context.WithCancel(context.Background()) return &HardforkTest{ Config: cfg, - Client: graphql.NewClient(cfg.HTTPClientTimeoutSeconds, cfg.GraphQLMaxRetries), + Client: client.NewClient(cfg.HTTPClientTimeoutSeconds, cfg.ClientMaxRetries), Logger: utils.NewLogger(), ScriptDir: cfg.ScriptDir, runningCmds: make([]*exec.Cmd, 0), @@ -118,20 +118,51 @@ func (t *HardforkTest) Run() error { // Phase 1: Run and validate main network t.Logger.Info("Phase 1: Running main network...") - forkConfigBytes, analysis, err := t.RunMainNetworkPhase(mainGenesisTs) - if err != nil { - return err - } + forkDataChan := make(chan ForkData, 1) + + var beforeShutdown HFHandler + switch t.Config.ForkMethod { + case config.Legacy: + + beforeShutdown = func(t *HardforkTest, analysis *BlockAnalysisResult) error { + // TODO: refactor so that: + // 1. stuff here go into LegacyForkPhase + // 2. fork validation is agnostic to fork method and it's checked regardless of fork method + t.Logger.Info("Phase 2: Forking the legacy way...") + + forkConfigBytes, err := t.GetForkConfig(t.AnyPortOfType(PORT_REST)) + if err != nil { + return err + } + var forkData *ForkData + forkData, err = t.LegacyForkPhase(analysis, forkConfigBytes, mainGenesisTs) + if err != nil { + return err + } + forkDataChan <- *forkData + return nil + } - t.Logger.Info("Phase 2: Forking the legacy way...") + case config.Advanced: + beforeShutdown = func(t *HardforkTest, analysis *BlockAnalysisResult) error { + t.Logger.Info("Phase 2: Forking with `mina advanced generate-hardfork-config`...") + + forkData, err := t.AdvancedForkPhase(analysis, mainGenesisTs) + if err != nil { + return err + } + forkDataChan <- *forkData + return nil + } + } - forkData, err := t.LegacyForkPhase(analysis, forkConfigBytes, mainGenesisTs) + analysis, err := t.RunMainNetworkPhase(mainGenesisTs, beforeShutdown) if err != nil { return err } t.Logger.Info("Phase 3: Running fork network...") - if err := t.RunForkNetworkPhase(analysis.LatestNonEmptyBlock.BlockHeight, *forkData, mainGenesisTs); err != nil { + if err := t.RunForkNetworkPhase(analysis.LatestNonEmptyBlock.BlockHeight, <-forkDataChan, mainGenesisTs); err != nil { return err } diff --git a/src/app/hardfork_test/src/internal/hardfork/validation.go b/src/app/hardfork_test/src/internal/hardfork/validation.go index 80cbcbde8a56..323e33b4078a 100644 --- a/src/app/hardfork_test/src/internal/hardfork/validation.go +++ b/src/app/hardfork_test/src/internal/hardfork/validation.go @@ -4,14 +4,14 @@ import ( "fmt" "time" - "github.com/MinaProtocol/mina/src/app/hardfork_test/src/internal/graphql" + "github.com/MinaProtocol/mina/src/app/hardfork_test/src/internal/client" ) // BlockAnalysisResult holds the results of analyzing blocks type BlockAnalysisResult struct { LatestOccupiedSlot int LatestSnarkedHashPerEpoch map[int]string // map from epoch to snarked ledger hash - LatestNonEmptyBlock graphql.BlockData + LatestNonEmptyBlock client.BlockData GenesisEpochStaking string GenesisEpochNext string } @@ -35,7 +35,7 @@ func (t *HardforkTest) ValidateLatestOccupiedSlot(latestOccupiedSlot int) error } // ValidateLatestNonEmptyBlockSlot checks that the latest non-empty block is before tx end slot -func (t *HardforkTest) ValidateLatestNonEmptyBlockSlot(latestNonEmptyBlock graphql.BlockData) error { +func (t *HardforkTest) ValidateLatestNonEmptyBlockSlot(latestNonEmptyBlock client.BlockData) error { t.Logger.Info("Latest non-empty block: %s, height: %d, slot: %d", latestNonEmptyBlock.StateHash, latestNonEmptyBlock.BlockHeight, latestNonEmptyBlock.Slot) @@ -75,8 +75,8 @@ func (t *HardforkTest) ValidateNoNewBlocks(port int) error { } // CollectBlocks gathers blocks from multiple slots across different ports -func (t *HardforkTest) CollectBlocks(startSlot, endSlot int) ([]graphql.BlockData, error) { - var allBlocks []graphql.BlockData +func (t *HardforkTest) CollectBlocks(startSlot, endSlot int) ([]client.BlockData, error) { + var allBlocks []client.BlockData for i := startSlot; i <= endSlot; i++ { @@ -105,7 +105,7 @@ func (t *HardforkTest) AnalyzeBlocks() (*BlockAnalysisResult, error) { } // Find the first non-empty block to get genesis epoch hashes - var firstEpochBlock graphql.BlockData + var firstEpochBlock client.BlockData for _, block := range blocks { if block.NonEmpty && block.Epoch == 0 { firstEpochBlock = block @@ -154,10 +154,10 @@ func (t *HardforkTest) AnalyzeBlocks() (*BlockAnalysisResult, error) { // FindLatestNonEmptyBlock processes block data to find the latest non-empty block // and collects other important information // This function assumes that there is at least one block with non-zero slot -func (t *HardforkTest) FindLatestNonEmptyBlock(blocks []graphql.BlockData) ( +func (t *HardforkTest) FindLatestNonEmptyBlock(blocks []client.BlockData) ( latestOccupiedSlot int, latestSnarkedHashPerEpoch map[int]string, // map from epoch to snarked ledger hash - latestNonEmptyBlock graphql.BlockData, + latestNonEmptyBlock client.BlockData, err error) { if len(blocks) == 0 {