port kaustinen code on top of shapella-rebased branch

activate proof generation on fork  + remove code dups

use go-verkle's post-state API to verify proofs (#262)

use prague as the verkle activation fork (#263)

upgrade to latest go-ipa

activate verkle transition in "miner" (#265)

fix: do not force cancunTime upon verkle activation

workaround: do not use root translation in replay

workaround: deactivate overlay transition for now

fixes from trying to get the devnet to work (#267)

this line was left out from the previous commit

upgrade to go-verkle with fixed newvalue serialization

fix: ensure point cache isn't nil in copy (#268)

fix: dependency cycle in tests (#269)

upgrade to latest go-verkle

fix: write trie preimage data to db (#274)

fix: zero-root in produced block + sync (#275)

upgrade go-ipa

fix build

fix typo

include review feedback

add switch to add proofs to blocks (#278)

add fee recipient to witness (#279)

touch all fields in withdrawal account header (#277)

Author: gballet

beacon/engine/gen_ed.go

@@ -64,6 +64,8 @@ type ExecutableData struct {
 	Withdrawals   []*types.Withdrawal `json:"withdrawals"`
 	BlobGasUsed   *uint64             `json:"blobGasUsed"`
 	ExcessBlobGas *uint64             `json:"excessBlobGas"`
+
+	ExecutionWitness *types.ExecutionWitness `json:"executionWitness"`
 }
 
 // JSON type overrides for executableData.
@@ -208,24 +210,25 @@ func ExecutableDataToBlock(params ExecutableData, versionedHashes []common.Hash)
 		withdrawalsRoot = &h
 	}
 	header := &types.Header{
-		ParentHash:      params.ParentHash,
-		UncleHash:       types.EmptyUncleHash,
-		Coinbase:        params.FeeRecipient,
-		Root:            params.StateRoot,
-		TxHash:          types.DeriveSha(types.Transactions(txs), trie.NewStackTrie(nil)),
-		ReceiptHash:     params.ReceiptsRoot,
-		Bloom:           types.BytesToBloom(params.LogsBloom),
-		Difficulty:      common.Big0,
-		Number:          new(big.Int).SetUint64(params.Number),
-		GasLimit:        params.GasLimit,
-		GasUsed:         params.GasUsed,
-		Time:            params.Timestamp,
-		BaseFee:         params.BaseFeePerGas,
-		Extra:           params.ExtraData,
-		MixDigest:       params.Random,
-		WithdrawalsHash: withdrawalsRoot,
-		ExcessBlobGas:   params.ExcessBlobGas,
-		BlobGasUsed:     params.BlobGasUsed,
+		ParentHash:       params.ParentHash,
+		UncleHash:        types.EmptyUncleHash,
+		Coinbase:         params.FeeRecipient,
+		Root:             params.StateRoot,
+		TxHash:           types.DeriveSha(types.Transactions(txs), trie.NewStackTrie(nil)),
+		ReceiptHash:      params.ReceiptsRoot,
+		Bloom:            types.BytesToBloom(params.LogsBloom),
+		Difficulty:       common.Big0,
+		Number:           new(big.Int).SetUint64(params.Number),
+		GasLimit:         params.GasLimit,
+		GasUsed:          params.GasUsed,
+		Time:             params.Timestamp,
+		BaseFee:          params.BaseFeePerGas,
+		Extra:            params.ExtraData,
+		MixDigest:        params.Random,
+		WithdrawalsHash:  withdrawalsRoot,
+		ExcessBlobGas:    params.ExcessBlobGas,
+		BlobGasUsed:      params.BlobGasUsed,
+		ExecutionWitness: params.ExecutionWitness,
 	}
 	block := types.NewBlockWithHeader(header).WithBody(txs, nil /* uncles */).WithWithdrawals(params.Withdrawals)
 	if block.Hash() != params.BlockHash {
@@ -238,23 +241,24 @@ func ExecutableDataToBlock(params ExecutableData, versionedHashes []common.Hash)
 // fields from the given block. It assumes the given block is post-merge block.
 func BlockToExecutableData(block *types.Block, fees *big.Int, blobs []kzg4844.Blob, commitments []kzg4844.Commitment, proofs []kzg4844.Proof) *ExecutionPayloadEnvelope {
 	data := &ExecutableData{
-		BlockHash:     block.Hash(),
-		ParentHash:    block.ParentHash(),
-		FeeRecipient:  block.Coinbase(),
-		StateRoot:     block.Root(),
-		Number:        block.NumberU64(),
-		GasLimit:      block.GasLimit(),
-		GasUsed:       block.GasUsed(),
-		BaseFeePerGas: block.BaseFee(),
-		Timestamp:     block.Time(),
-		ReceiptsRoot:  block.ReceiptHash(),
-		LogsBloom:     block.Bloom().Bytes(),
-		Transactions:  encodeTransactions(block.Transactions()),
-		Random:        block.MixDigest(),
-		ExtraData:     block.Extra(),
-		Withdrawals:   block.Withdrawals(),
-		BlobGasUsed:   block.BlobGasUsed(),
-		ExcessBlobGas: block.ExcessBlobGas(),
+		BlockHash:        block.Hash(),
+		ParentHash:       block.ParentHash(),
+		FeeRecipient:     block.Coinbase(),
+		StateRoot:        block.Root(),
+		Number:           block.NumberU64(),
+		GasLimit:         block.GasLimit(),
+		GasUsed:          block.GasUsed(),
+		BaseFeePerGas:    block.BaseFee(),
+		Timestamp:        block.Time(),
+		ReceiptsRoot:     block.ReceiptHash(),
+		LogsBloom:        block.Bloom().Bytes(),
+		Transactions:     encodeTransactions(block.Transactions()),
+		Random:           block.MixDigest(),
+		ExtraData:        block.Extra(),
+		Withdrawals:      block.Withdrawals(),
+		BlobGasUsed:      block.BlobGasUsed(),
+		ExcessBlobGas:    block.ExcessBlobGas(),
+		ExecutionWitness: block.ExecutionWitness(),
 	}
 	blobsBundle := BlobsBundleV1{
 		Commitments: make([]hexutil.Bytes, 0),

beacon/engine/types.go

@@ -171,9 +171,9 @@ func makeFullNode(ctx *cli.Context) (*node.Node, ethapi.Backend) {
 		v := ctx.Uint64(utils.OverrideCancun.Name)
 		cfg.Eth.OverrideCancun = &v
 	}
-	if ctx.IsSet(utils.OverrideVerkle.Name) {
-		v := ctx.Uint64(utils.OverrideVerkle.Name)
-		cfg.Eth.OverrideVerkle = &v
+	if ctx.IsSet(utils.OverridePrague.Name) {
+		v := ctx.Uint64(utils.OverridePrague.Name)
+		cfg.Eth.OverridePrague = &v
 	}
 	backend, eth := utils.RegisterEthService(stack, &cfg.Eth)
 

cmd/geth/config.go

@@ -68,7 +68,7 @@ var (
 		utils.USBFlag,
 		utils.SmartCardDaemonPathFlag,
 		utils.OverrideCancun,
-		utils.OverrideVerkle,
+		utils.OverridePrague,
 		utils.EnablePersonal,
 		utils.TxPoolLocalsFlag,
 		utils.TxPoolNoLocalsFlag,

cmd/geth/main.go

@@ -268,8 +268,8 @@ var (
 		Usage:    "Manually specify the Cancun fork timestamp, overriding the bundled setting",
 		Category: flags.EthCategory,
 	}
-	OverrideVerkle = &cli.Uint64Flag{
-		Name:     "override.verkle",
+	OverridePrague = &cli.Uint64Flag{
+		Name:     "override.prague",
 		Usage:    "Manually specify the Verkle fork timestamp, overriding the bundled setting",
 		Category: flags.EthCategory,
 	}

cmd/utils/flags.go

@@ -31,6 +31,7 @@ import (
 	"github.com/ethereum/go-ethereum/rpc"
 	"github.com/ethereum/go-ethereum/trie"
 	"github.com/ethereum/go-ethereum/trie/utils"
+	"github.com/gballet/go-verkle"
 	"github.com/holiman/uint256"
 )
 
@@ -329,14 +330,14 @@ func (beacon *Beacon) verifyHeaders(chain consensus.ChainHeaderReader, headers [
 
 // Prepare implements consensus.Engine, initializing the difficulty field of a
 // header to conform to the beacon protocol. The changes are done inline.
-func (beacon *Beacon) Prepare(chain consensus.ChainHeaderReader, header *types.Header) error {
+func (beacon *Beacon) Prepare(chain consensus.ChainHeaderReader, header *types.Header, statedb *state.StateDB) error {
 	// Transition isn't triggered yet, use the legacy rules for preparation.
 	reached, err := IsTTDReached(chain, header.ParentHash, header.Number.Uint64()-1)
 	if err != nil {
 		return err
 	}
 	if !reached {
-		return beacon.ethone.Prepare(chain, header)
+		return beacon.ethone.Prepare(chain, header, statedb)
 	}
 	header.Difficulty = beaconDifficulty
 	return nil
@@ -356,9 +357,17 @@ func (beacon *Beacon) Finalize(chain consensus.ChainHeaderReader, header *types.
 		state.AddBalance(w.Address, amount)
 
 		// The returned gas is not charged
+		state.Witness().TouchAddressOnWriteAndComputeGas(w.Address[:], uint256.Int{}, utils.VersionLeafKey)
 		state.Witness().TouchAddressOnWriteAndComputeGas(w.Address[:], uint256.Int{}, utils.BalanceLeafKey)
-	}
-	// No block reward which is issued by consensus layer instead.
+		state.Witness().TouchAddressOnWriteAndComputeGas(w.Address[:], uint256.Int{}, utils.NonceLeafKey)
+		state.Witness().TouchAddressOnWriteAndComputeGas(w.Address[:], uint256.Int{}, utils.CodeKeccakLeafKey)
+		state.Witness().TouchAddressOnWriteAndComputeGas(w.Address[:], uint256.Int{}, utils.CodeSizeLeafKey)
+	}
+	state.Witness().TouchAddressOnWriteAndComputeGas(header.Coinbase[:], uint256.Int{}, utils.VersionLeafKey)
+	state.Witness().TouchAddressOnWriteAndComputeGas(header.Coinbase[:], uint256.Int{}, utils.BalanceLeafKey)
+	state.Witness().TouchAddressOnWriteAndComputeGas(header.Coinbase[:], uint256.Int{}, utils.NonceLeafKey)
+	state.Witness().TouchAddressOnWriteAndComputeGas(header.Coinbase[:], uint256.Int{}, utils.CodeKeccakLeafKey)
+	state.Witness().TouchAddressOnWriteAndComputeGas(header.Coinbase[:], uint256.Int{}, utils.CodeSizeLeafKey)
 }
 
 // FinalizeAndAssemble implements consensus.Engine, setting the final state and
@@ -384,8 +393,65 @@ func (beacon *Beacon) FinalizeAndAssemble(chain consensus.ChainHeaderReader, hea
 	// Assign the final state root to header.
 	header.Root = state.IntermediateRoot(true)
 
+	var (
+		p    *verkle.VerkleProof
+		k    verkle.StateDiff
+		keys = state.Witness().Keys()
+	)
+	if chain.Config().IsPrague(header.Number, header.Time) && chain.Config().ProofInBlock {
+		// Open the pre-tree to prove the pre-state against
+		parent := chain.GetHeaderByNumber(header.Number.Uint64() - 1)
+		if parent == nil {
+			return nil, fmt.Errorf("nil parent header for block %d", header.Number)
+		}
+
+		preTrie, err := state.Database().OpenTrie(parent.Root)
+		if err != nil {
+			return nil, fmt.Errorf("error opening pre-state tree root: %w", err)
+		}
+
+		var okpre, okpost bool
+		var vtrpre, vtrpost *trie.VerkleTrie
+		switch pre := preTrie.(type) {
+		case *trie.VerkleTrie:
+			vtrpre, okpre = preTrie.(*trie.VerkleTrie)
+			vtrpost, okpost = state.GetTrie().(*trie.VerkleTrie)
+		case *trie.TransitionTrie:
+			vtrpre = pre.Overlay()
+			okpre = true
+			post, _ := state.GetTrie().(*trie.TransitionTrie)
+			vtrpost = post.Overlay()
+			okpost = true
+		default:
+			panic("invalid tree type")
+		}
+		if okpre && okpost {
+			// Resolve values from the pre state, the post
+			// state should already have the values in memory.
+			// TODO: see if this can be captured at the witness
+			// level, like it used to.
+			for _, key := range keys {
+				_, err := vtrpre.GetWithHashedKey(key)
+				if err != nil {
+					panic(err)
+				}
+			}
+
+			if len(keys) > 0 {
+				p, k, err = trie.ProveAndSerialize(vtrpre, vtrpost, keys, vtrpre.FlatdbNodeResolver)
+				if err != nil {
+					return nil, fmt.Errorf("error generating verkle proof for block %d: %w", header.Number, err)
+				}
+			}
+		}
+	}
+
 	// Assemble and return the final block.
-	return types.NewBlockWithWithdrawals(header, txs, uncles, receipts, withdrawals, trie.NewStackTrie(nil)), nil
+	block := types.NewBlockWithWithdrawals(header, txs, uncles, receipts, withdrawals, trie.NewStackTrie(nil))
+	if chain.Config().IsPrague(header.Number, header.Time) && chain.Config().ProofInBlock {
+		block.SetVerkleProof(p, k)
+	}
+	return block, nil
 }
 
 // Seal generates a new sealing request for the given input block and pushes

consensus/beacon/consensus.go

@@ -499,7 +499,7 @@ func (c *Clique) verifySeal(snap *Snapshot, header *types.Header, parents []*typ
 
 // Prepare implements consensus.Engine, preparing all the consensus fields of the
 // header for running the transactions on top.
-func (c *Clique) Prepare(chain consensus.ChainHeaderReader, header *types.Header) error {
+func (c *Clique) Prepare(chain consensus.ChainHeaderReader, header *types.Header, _ *state.StateDB) error {
 	// If the block isn't a checkpoint, cast a random vote (good enough for now)
 	header.Coinbase = common.Address{}
 	header.Nonce = types.BlockNonce{}

consensus/clique/clique.go

@@ -81,7 +81,7 @@ type Engine interface {
 
 	// Prepare initializes the consensus fields of a block header according to the
 	// rules of a particular engine. The changes are executed inline.
-	Prepare(chain ChainHeaderReader, header *types.Header) error
+	Prepare(chain ChainHeaderReader, header *types.Header, state *state.StateDB) error
 
 	// Finalize runs any post-transaction state modifications (e.g. block rewards
 	// or process withdrawals) but does not assemble the block.

consensus/consensus.go

@@ -479,7 +479,7 @@ var DynamicDifficultyCalculator = makeDifficultyCalculator
 
 // Prepare implements consensus.Engine, initializing the difficulty field of a
 // header to conform to the ethash protocol. The changes are done inline.
-func (ethash *Ethash) Prepare(chain consensus.ChainHeaderReader, header *types.Header) error {
+func (ethash *Ethash) Prepare(chain consensus.ChainHeaderReader, header *types.Header, _ *state.StateDB) error {
 	parent := chain.GetHeader(header.ParentHash, header.Number.Uint64()-1)
 	if parent == nil {
 		return consensus.ErrUnknownAncestor
@@ -568,15 +568,15 @@ func accumulateRewards(config *params.ChainConfig, state *state.StateDB, header
 		r.Div(r, big8)
 
 		// This should not happen, but it's useful for replay tests
-		if config.IsVerkle(header.Number, header.Time) {
+		if config.IsPrague(header.Number, header.Time) {
 			state.Witness().TouchAddressOnReadAndComputeGas(uncle.Coinbase.Bytes(), uint256.Int{}, utils.BalanceLeafKey)
 		}
 		state.AddBalance(uncle.Coinbase, r)
 
 		r.Div(blockReward, big32)
 		reward.Add(reward, r)
 	}
-	if config.IsVerkle(header.Number, header.Time) {
+	if config.IsPrague(header.Number, header.Time) {
 		state.Witness().TouchAddressOnReadAndComputeGas(header.Coinbase.Bytes(), uint256.Int{}, utils.BalanceLeafKey)
 		state.Witness().TouchAddressOnReadAndComputeGas(header.Coinbase.Bytes(), uint256.Int{}, utils.VersionLeafKey)
 		state.Witness().TouchAddressOnReadAndComputeGas(header.Coinbase.Bytes(), uint256.Int{}, utils.NonceLeafKey)