UpgradeToFulu: Respect the specification. (#14821)

https://github.com/ethereum/consensus-specs/blob/188a2ff8183d19dd79d87308bb42906ab1dd3a31/specs/fulu/fork.md#upgrading-the-state

Before this commit, the `UpgradeToFulu` did not really respect the specification. This commit fixes that.

How can we be sure now the specification is really respected?

As long as the equivalent of https://github.com/ethereum/consensus-spec-tests/tree/master/tests/mainnet/electra/fork/fork/pyspec_tests are not released, we cannot be sure.

However, with this commit, Prysm and Lighthouse do agree with the post state after the Fulu fork (which is not the case without this commit).

So either both Prysm and Lighthouse are both right,
either the are both wrong (but in the exact same way, which has a pretty low likelyhood).

Author: nalepae

beacon-chain/core/fulu/BUILD.bazel

@@ -6,15 +6,12 @@ go_library(
     importpath = "github.com/prysmaticlabs/prysm/v5/beacon-chain/core/fulu",
     visibility = ["//visibility:public"],
     deps = [
-        "//beacon-chain/core/helpers:go_default_library",
         "//beacon-chain/core/time:go_default_library",
         "//beacon-chain/state:go_default_library",
         "//beacon-chain/state/state-native:go_default_library",
         "//config/params:go_default_library",
-        "//consensus-types/primitives:go_default_library",
         "//proto/engine/v1:go_default_library",
         "//proto/prysm/v1alpha1:go_default_library",
-        "//time/slots:go_default_library",
         "@com_github_pkg_errors//:go_default_library",
     ],
 )
@@ -24,14 +21,12 @@ go_test(
     srcs = ["upgrade_test.go"],
     deps = [
         ":go_default_library",
-        "//beacon-chain/core/helpers:go_default_library",
         "//beacon-chain/core/time:go_default_library",
         "//config/params:go_default_library",
         "//consensus-types/primitives:go_default_library",
         "//proto/engine/v1:go_default_library",
         "//proto/prysm/v1alpha1:go_default_library",
         "//testing/require:go_default_library",
         "//testing/util:go_default_library",
-        "//time/slots:go_default_library",
     ],
 )

beacon-chain/core/fulu/upgrade.go

@@ -1,18 +1,13 @@
 package fulu
 
 import (
-	"sort"
-
 	"github.com/pkg/errors"
-	"github.com/prysmaticlabs/prysm/v5/beacon-chain/core/helpers"
 	"github.com/prysmaticlabs/prysm/v5/beacon-chain/core/time"
 	"github.com/prysmaticlabs/prysm/v5/beacon-chain/state"
 	state_native "github.com/prysmaticlabs/prysm/v5/beacon-chain/state/state-native"
 	"github.com/prysmaticlabs/prysm/v5/config/params"
-	"github.com/prysmaticlabs/prysm/v5/consensus-types/primitives"
 	enginev1 "github.com/prysmaticlabs/prysm/v5/proto/engine/v1"
 	ethpb "github.com/prysmaticlabs/prysm/v5/proto/prysm/v1alpha1"
-	"github.com/prysmaticlabs/prysm/v5/time/slots"
 )
 
 // UpgradeToFulu updates inputs a generic state to return the version Fulu state.
@@ -74,32 +69,37 @@ func UpgradeToFulu(beaconState state.BeaconState) (state.BeaconState, error) {
 	if err != nil {
 		return nil, err
 	}
-
-	earliestExitEpoch := helpers.ActivationExitEpoch(time.CurrentEpoch(beaconState))
-	preActivationIndices := make([]primitives.ValidatorIndex, 0)
-	compoundWithdrawalIndices := make([]primitives.ValidatorIndex, 0)
-	if err = beaconState.ReadFromEveryValidator(func(index int, val state.ReadOnlyValidator) error {
-		if val.ExitEpoch() != params.BeaconConfig().FarFutureEpoch && val.ExitEpoch() > earliestExitEpoch {
-			earliestExitEpoch = val.ExitEpoch()
-		}
-		if val.ActivationEpoch() == params.BeaconConfig().FarFutureEpoch {
-			preActivationIndices = append(preActivationIndices, primitives.ValidatorIndex(index))
-		}
-		if val.HasCompoundingWithdrawalCredentials() {
-			compoundWithdrawalIndices = append(compoundWithdrawalIndices, primitives.ValidatorIndex(index))
-		}
-		return nil
-	}); err != nil {
+	depositBalanceToConsume, err := beaconState.DepositBalanceToConsume()
+	if err != nil {
 		return nil, err
 	}
-
-	earliestExitEpoch++ // Increment to find the earliest possible exit epoch
-
-	// note: should be the same in prestate and post beaconState.
-	// we are deviating from the specs a bit as it calls for using the post beaconState
-	tab, err := helpers.TotalActiveBalance(beaconState)
+	exitBalanceToConsume, err := beaconState.ExitBalanceToConsume()
 	if err != nil {
-		return nil, errors.Wrap(err, "failed to get total active balance")
+		return nil, err
+	}
+	earliestExitEpoch, err := beaconState.EarliestExitEpoch()
+	if err != nil {
+		return nil, err
+	}
+	consolidationBalanceToConsume, err := beaconState.ConsolidationBalanceToConsume()
+	if err != nil {
+		return nil, err
+	}
+	earliestConsolidationEpoch, err := beaconState.EarliestConsolidationEpoch()
+	if err != nil {
+		return nil, err
+	}
+	pendingDeposits, err := beaconState.PendingDeposits()
+	if err != nil {
+		return nil, err
+	}
+	pendingPartialWithdrawals, err := beaconState.PendingPartialWithdrawals()
+	if err != nil {
+		return nil, err
+	}
+	pendingConsolidations, err := beaconState.PendingConsolidations()
+	if err != nil {
+		return nil, err
 	}
 
 	s := &ethpb.BeaconStateFulu{
@@ -155,25 +155,16 @@ func UpgradeToFulu(beaconState state.BeaconState) (state.BeaconState, error) {
 		HistoricalSummaries:          summaries,
 
 		DepositRequestsStartIndex:     params.BeaconConfig().UnsetDepositRequestsStartIndex,
-		DepositBalanceToConsume:       0,
-		ExitBalanceToConsume:          helpers.ActivationExitChurnLimit(primitives.Gwei(tab)),
+		DepositBalanceToConsume:       depositBalanceToConsume,
+		ExitBalanceToConsume:          exitBalanceToConsume,
 		EarliestExitEpoch:             earliestExitEpoch,
-		ConsolidationBalanceToConsume: helpers.ConsolidationChurnLimit(primitives.Gwei(tab)),
-		EarliestConsolidationEpoch:    helpers.ActivationExitEpoch(slots.ToEpoch(beaconState.Slot())),
-		PendingDeposits:               make([]*ethpb.PendingDeposit, 0),
-		PendingPartialWithdrawals:     make([]*ethpb.PendingPartialWithdrawal, 0),
-		PendingConsolidations:         make([]*ethpb.PendingConsolidation, 0),
+		ConsolidationBalanceToConsume: consolidationBalanceToConsume,
+		EarliestConsolidationEpoch:    earliestConsolidationEpoch,
+		PendingDeposits:               pendingDeposits,
+		PendingPartialWithdrawals:     pendingPartialWithdrawals,
+		PendingConsolidations:         pendingConsolidations,
 	}
 
-	// Sorting preActivationIndices based on a custom criteria
-	sort.Slice(preActivationIndices, func(i, j int) bool {
-		// Comparing based on ActivationEligibilityEpoch and then by index if the epochs are the same
-		if s.Validators[preActivationIndices[i]].ActivationEligibilityEpoch == s.Validators[preActivationIndices[j]].ActivationEligibilityEpoch {
-			return preActivationIndices[i] < preActivationIndices[j]
-		}
-		return s.Validators[preActivationIndices[i]].ActivationEligibilityEpoch < s.Validators[preActivationIndices[j]].ActivationEligibilityEpoch
-	})
-
 	// Need to cast the beaconState to use in helper functions
 	post, err := state_native.InitializeFromProtoUnsafeFulu(s)
 	if err != nil {

beacon-chain/core/fulu/upgrade_test.go

@@ -4,15 +4,13 @@ import (
 	"testing"
 
 	"github.com/prysmaticlabs/prysm/v5/beacon-chain/core/fulu"
-	"github.com/prysmaticlabs/prysm/v5/beacon-chain/core/helpers"
 	"github.com/prysmaticlabs/prysm/v5/beacon-chain/core/time"
 	"github.com/prysmaticlabs/prysm/v5/config/params"
 	"github.com/prysmaticlabs/prysm/v5/consensus-types/primitives"
 	enginev1 "github.com/prysmaticlabs/prysm/v5/proto/engine/v1"
 	ethpb "github.com/prysmaticlabs/prysm/v5/proto/prysm/v1alpha1"
 	"github.com/prysmaticlabs/prysm/v5/testing/require"
 	"github.com/prysmaticlabs/prysm/v5/testing/util"
-	"github.com/prysmaticlabs/prysm/v5/time/slots"
 )
 
 func TestUpgradeToFulu(t *testing.T) {
@@ -33,69 +31,57 @@ func TestUpgradeToFulu(t *testing.T) {
 	require.Equal(t, preForkState.GenesisTime(), mSt.GenesisTime())
 	require.DeepSSZEqual(t, preForkState.GenesisValidatorsRoot(), mSt.GenesisValidatorsRoot())
 	require.Equal(t, preForkState.Slot(), mSt.Slot())
+
+	f := mSt.Fork()
+	require.DeepSSZEqual(t, &ethpb.Fork{
+		PreviousVersion: st.Fork().CurrentVersion,
+		CurrentVersion:  params.BeaconConfig().FuluForkVersion,
+		Epoch:           time.CurrentEpoch(st),
+	}, f)
+
 	require.DeepSSZEqual(t, preForkState.LatestBlockHeader(), mSt.LatestBlockHeader())
 	require.DeepSSZEqual(t, preForkState.BlockRoots(), mSt.BlockRoots())
 	require.DeepSSZEqual(t, preForkState.StateRoots(), mSt.StateRoots())
-	require.DeepSSZEqual(t, preForkState.Validators()[2:], mSt.Validators()[2:])
-	require.DeepSSZEqual(t, preForkState.Balances()[2:], mSt.Balances()[2:])
+
+	hr1, err := preForkState.HistoricalRoots()
+	require.NoError(t, err)
+	hr2, err := mSt.HistoricalRoots()
+	require.NoError(t, err)
+	require.DeepEqual(t, hr1, hr2)
+
 	require.DeepSSZEqual(t, preForkState.Eth1Data(), mSt.Eth1Data())
 	require.DeepSSZEqual(t, preForkState.Eth1DataVotes(), mSt.Eth1DataVotes())
 	require.DeepSSZEqual(t, preForkState.Eth1DepositIndex(), mSt.Eth1DepositIndex())
+	require.DeepSSZEqual(t, preForkState.Validators(), mSt.Validators())
+	require.DeepSSZEqual(t, preForkState.Balances(), mSt.Balances())
 	require.DeepSSZEqual(t, preForkState.RandaoMixes(), mSt.RandaoMixes())
 	require.DeepSSZEqual(t, preForkState.Slashings(), mSt.Slashings())
-	require.DeepSSZEqual(t, preForkState.JustificationBits(), mSt.JustificationBits())
-	require.DeepSSZEqual(t, preForkState.PreviousJustifiedCheckpoint(), mSt.PreviousJustifiedCheckpoint())
-	require.DeepSSZEqual(t, preForkState.CurrentJustifiedCheckpoint(), mSt.CurrentJustifiedCheckpoint())
-	require.DeepSSZEqual(t, preForkState.FinalizedCheckpoint(), mSt.FinalizedCheckpoint())
-
-	require.Equal(t, len(preForkState.Validators()), len(mSt.Validators()))
-
-	preVal, err := preForkState.ValidatorAtIndex(0)
-	require.NoError(t, err)
-	require.Equal(t, params.BeaconConfig().MaxEffectiveBalance, preVal.EffectiveBalance)
-
-	preVal2, err := preForkState.ValidatorAtIndex(1)
-	require.NoError(t, err)
-	require.Equal(t, params.BeaconConfig().MaxEffectiveBalance, preVal2.EffectiveBalance)
-
-	// TODO: Fix this test
-	// mVal, err := mSt.ValidatorAtIndex(0)
-	_, err = mSt.ValidatorAtIndex(0)
-	require.NoError(t, err)
-	// require.Equal(t, uint64(0), mVal.EffectiveBalance)
-
-	mVal2, err := mSt.ValidatorAtIndex(1)
-	require.NoError(t, err)
-	require.Equal(t, params.BeaconConfig().MinActivationBalance, mVal2.EffectiveBalance)
 
 	numValidators := mSt.NumValidators()
+
 	p, err := mSt.PreviousEpochParticipation()
 	require.NoError(t, err)
 	require.DeepSSZEqual(t, make([]byte, numValidators), p)
+
 	p, err = mSt.CurrentEpochParticipation()
 	require.NoError(t, err)
 	require.DeepSSZEqual(t, make([]byte, numValidators), p)
+
+	require.DeepSSZEqual(t, preForkState.JustificationBits(), mSt.JustificationBits())
+	require.DeepSSZEqual(t, preForkState.PreviousJustifiedCheckpoint(), mSt.PreviousJustifiedCheckpoint())
+	require.DeepSSZEqual(t, preForkState.CurrentJustifiedCheckpoint(), mSt.CurrentJustifiedCheckpoint())
+	require.DeepSSZEqual(t, preForkState.FinalizedCheckpoint(), mSt.FinalizedCheckpoint())
+
 	s, err := mSt.InactivityScores()
 	require.NoError(t, err)
 	require.DeepSSZEqual(t, make([]uint64, numValidators), s)
 
-	hr1, err := preForkState.HistoricalRoots()
-	require.NoError(t, err)
-	hr2, err := mSt.HistoricalRoots()
-	require.NoError(t, err)
-	require.DeepEqual(t, hr1, hr2)
-
-	f := mSt.Fork()
-	require.DeepSSZEqual(t, &ethpb.Fork{
-		PreviousVersion: st.Fork().CurrentVersion,
-		CurrentVersion:  params.BeaconConfig().FuluForkVersion,
-		Epoch:           time.CurrentEpoch(st),
-	}, f)
 	csc, err := mSt.CurrentSyncCommittee()
 	require.NoError(t, err)
 	psc, err := preForkState.CurrentSyncCommittee()
 	require.NoError(t, err)
 	require.DeepSSZEqual(t, psc, csc)
+
 	nsc, err := mSt.NextSyncCommittee()
 	require.NoError(t, err)
 	psc, err = preForkState.NextSyncCommittee()
@@ -110,7 +96,6 @@ func TestUpgradeToFulu(t *testing.T) {
 	require.NoError(t, err)
 	txRoot, err := prevHeader.TransactionsRoot()
 	require.NoError(t, err)
-
 	wdRoot, err := prevHeader.WithdrawalsRoot()
 	require.NoError(t, err)
 	wanted := &enginev1.ExecutionPayloadHeaderDeneb{
@@ -144,45 +129,57 @@ func TestUpgradeToFulu(t *testing.T) {
 	require.NoError(t, err)
 	require.Equal(t, 0, len(summaries))
 
-	startIndex, err := mSt.DepositRequestsStartIndex()
+	preDepositRequestsStartIndex, err := preForkState.DepositRequestsStartIndex()
 	require.NoError(t, err)
-	require.Equal(t, params.BeaconConfig().UnsetDepositRequestsStartIndex, startIndex)
-
-	balance, err := mSt.DepositBalanceToConsume()
+	postDepositRequestsStartIndex, err := mSt.DepositRequestsStartIndex()
 	require.NoError(t, err)
-	require.Equal(t, primitives.Gwei(0), balance)
+	require.Equal(t, preDepositRequestsStartIndex, postDepositRequestsStartIndex)
 
-	tab, err := helpers.TotalActiveBalance(mSt)
+	preDepositBalanceToConsume, err := preForkState.DepositBalanceToConsume()
+	require.NoError(t, err)
+	postDepositBalanceToConsume, err := mSt.DepositBalanceToConsume()
 	require.NoError(t, err)
+	require.Equal(t, preDepositBalanceToConsume, postDepositBalanceToConsume)
 
-	ebtc, err := mSt.ExitBalanceToConsume()
+	preExitBalanceToConsume, err := preForkState.ExitBalanceToConsume()
 	require.NoError(t, err)
-	require.Equal(t, helpers.ActivationExitChurnLimit(primitives.Gwei(tab)), ebtc)
+	postExitBalanceToConsume, err := mSt.ExitBalanceToConsume()
+	require.NoError(t, err)
+	require.Equal(t, preExitBalanceToConsume, postExitBalanceToConsume)
 
-	eee, err := mSt.EarliestExitEpoch()
+	preEarliestExitEpoch, err := preForkState.EarliestExitEpoch()
+	require.NoError(t, err)
+	postEarliestExitEpoch, err := mSt.EarliestExitEpoch()
 	require.NoError(t, err)
-	require.Equal(t, helpers.ActivationExitEpoch(primitives.Epoch(1)), eee)
+	require.Equal(t, preEarliestExitEpoch, postEarliestExitEpoch)
 
-	cbtc, err := mSt.ConsolidationBalanceToConsume()
+	preConsolidationBalanceToConsume, err := preForkState.ConsolidationBalanceToConsume()
 	require.NoError(t, err)
-	require.Equal(t, helpers.ConsolidationChurnLimit(primitives.Gwei(tab)), cbtc)
+	postConsolidationBalanceToConsume, err := mSt.ConsolidationBalanceToConsume()
+	require.NoError(t, err)
+	require.Equal(t, preConsolidationBalanceToConsume, postConsolidationBalanceToConsume)
 
-	earliestConsolidationEpoch, err := mSt.EarliestConsolidationEpoch()
+	preEarliesConsolidationEoch, err := preForkState.EarliestConsolidationEpoch()
+	require.NoError(t, err)
+	postEarliestConsolidationEpoch, err := mSt.EarliestConsolidationEpoch()
 	require.NoError(t, err)
-	require.Equal(t, helpers.ActivationExitEpoch(slots.ToEpoch(preForkState.Slot())), earliestConsolidationEpoch)
+	require.Equal(t, preEarliesConsolidationEoch, postEarliestConsolidationEpoch)
 
-	// TODO: Fix this test
-	// pendingDeposits, err := mSt.PendingDeposits()
-	_, err = mSt.PendingDeposits()
+	prePendingDeposits, err := preForkState.PendingDeposits()
 	require.NoError(t, err)
-	// require.Equal(t, 2, len(pendingDeposits))
-	// require.Equal(t, uint64(1000), pendingDeposits[1].Amount)
+	postPendingDeposits, err := mSt.PendingDeposits()
+	require.NoError(t, err)
+	require.DeepSSZEqual(t, prePendingDeposits, postPendingDeposits)
 
-	numPendingPartialWithdrawals, err := mSt.NumPendingPartialWithdrawals()
+	prePendingPartialWithdrawals, err := preForkState.PendingPartialWithdrawals()
+	require.NoError(t, err)
+	postPendingPartialWithdrawals, err := mSt.PendingPartialWithdrawals()
 	require.NoError(t, err)
-	require.Equal(t, uint64(0), numPendingPartialWithdrawals)
+	require.DeepSSZEqual(t, prePendingPartialWithdrawals, postPendingPartialWithdrawals)
 
-	consolidations, err := mSt.PendingConsolidations()
+	prePendingConsolidations, err := preForkState.PendingConsolidations()
+	require.NoError(t, err)
+	postPendingConsolidations, err := mSt.PendingConsolidations()
 	require.NoError(t, err)
-	require.Equal(t, 0, len(consolidations))
+	require.DeepSSZEqual(t, prePendingConsolidations, postPendingConsolidations)
 }

changelog/manu_fulu_fork.md

@@ -0,0 +1,3 @@
+### Fixed
+
+- `UpgradeToFulu`: Respect the specification.