Merge pull request #910 from ethereum/witness-additions

feat(verkle): add parent root to witness

Author: jsign

.github/configs/evm.yaml

@@ -22,6 +22,5 @@ pectra-devnet-3:
 verkle:
   impl: geth
   repo: gballet/go-ethereum
-  # ref: kaustinen-with-shapella
-  ref: jsign-witness-fix
+  ref: kaustinen-with-shapella
   evm-bin: evm

src/ethereum_test_specs/blockchain.py

@@ -106,11 +106,7 @@ def count_blobs(txs: List[Transaction]) -> int:
     Returns the number of blobs in a list of transactions.
     """
     return sum(
-        [
-            len(tx.blob_versioned_hashes)
-            for tx in txs
-            if tx.blob_versioned_hashes is not None
-        ]
+        [len(tx.blob_versioned_hashes) for tx in txs if tx.blob_versioned_hashes is not None]
     )
 
 
@@ -247,10 +243,7 @@ class Block(Header):
     returned by the  `evm_transition_tool`.
     """
     exception: (
-        List[TransactionException | BlockException]
-        | TransactionException
-        | BlockException
-        | None
+        List[TransactionException | BlockException] | TransactionException | BlockException | None
     ) = None
     """
     If set, the block is expected to be rejected by the client.
@@ -271,9 +264,7 @@ class Block(Header):
     """
     List of withdrawals to perform for this block.
     """
-    requests: List[DepositRequest | WithdrawalRequest | ConsolidationRequest] | None = (
-        None
-    )
+    requests: List[DepositRequest | WithdrawalRequest | ConsolidationRequest] | None = None
     """
     Custom list of requests to embed in this block.
     """
@@ -295,9 +286,7 @@ def set_environment(self, env: Environment) -> Environment:
         """
         new_env_values["difficulty"] = self.difficulty
         new_env_values["fee_recipient"] = (
-            self.fee_recipient
-            if self.fee_recipient is not None
-            else Environment().fee_recipient
+            self.fee_recipient if self.fee_recipient is not None else Environment().fee_recipient
         )
         new_env_values["gas_limit"] = (
             self.gas_limit or env.parent_gas_limit or Environment().gas_limit
@@ -322,9 +311,7 @@ def set_environment(self, env: Environment) -> Environment:
             if len(env.block_hashes) == 0:
                 new_env_values["number"] = 0
             else:
-                new_env_values["number"] = (
-                    max([Number(n) for n in env.block_hashes.keys()]) + 1
-                )
+                new_env_values["number"] = max([Number(n) for n in env.block_hashes.keys()]) + 1
 
         if self.timestamp is not None:
             new_env_values["timestamp"] = self.timestamp
@@ -364,9 +351,8 @@ def make_genesis(
         assert (
             env.withdrawals is None or len(env.withdrawals) == 0
         ), "withdrawals must be empty at genesis"
-        assert (
-            env.parent_beacon_block_root is None
-            or env.parent_beacon_block_root == Hash(0)
+        assert env.parent_beacon_block_root is None or env.parent_beacon_block_root == Hash(
+            0
         ), "parent_beacon_block_root must be empty at genesis"
 
         pre_alloc = Alloc.merge(
@@ -403,15 +389,11 @@ def make_genesis(
             blob_gas_used=env.blob_gas_used,
             excess_blob_gas=env.excess_blob_gas,
             withdrawals_root=(
-                Withdrawal.list_root(env.withdrawals)
-                if env.withdrawals is not None
-                else None
+                Withdrawal.list_root(env.withdrawals) if env.withdrawals is not None else None
             ),
             parent_beacon_block_root=env.parent_beacon_block_root,
             requests_root=(
-                Requests(root=[]).trie_root
-                if fork.header_requests_required(0, 0)
-                else None
+                Requests(root=[]).trie_root if fork.header_requests_required(0, 0) else None
             ),
         )
 
@@ -422,9 +404,7 @@ def make_genesis(
                 withdrawals=None if env.withdrawals is None else [],
                 deposit_requests=[] if fork.header_requests_required(0, 0) else None,
                 withdrawal_requests=[] if fork.header_requests_required(0, 0) else None,
-                consolidation_requests=[]
-                if fork.header_requests_required(0, 0)
-                else None,
+                consolidation_requests=[] if fork.header_requests_required(0, 0) else None,
             ).with_rlp(
                 txs=[],
                 requests=Requests() if fork.header_requests_required(0, 0) else None,
@@ -577,9 +557,7 @@ def generate_block_data(
 
         requests = None
         if fork.header_requests_required(header.number, header.timestamp):
-            requests_list: List[
-                DepositRequest | WithdrawalRequest | ConsolidationRequest
-            ] = []
+            requests_list: List[DepositRequest | WithdrawalRequest | ConsolidationRequest] = []
             if transition_tool_output.result.deposit_requests is not None:
                 requests_list += transition_tool_output.result.deposit_requests
             if transition_tool_output.result.withdrawal_requests is not None:
@@ -607,11 +585,13 @@ def generate_block_data(
                 )
             )
             transition_tool_output.alloc = previous_alloc
+            # TODO: hack for now, replace with actual witness output once available from t8n
             if transition_tool_output.result.verkle_conversion_ended:
-                # TODO: hack for now, replace with actual witness output once available from t8n
+                witness_parent_root = transition_tool_output.result.parent_root
                 transition_tool_output.witness = Witness(
                     verkle_proof=transition_tool_output.result.verkle_proof,
                     state_diff=transition_tool_output.result.state_diff,
+                    parent_root=witness_parent_root,
                 )
             else:
                 transition_tool_output.witness = None
@@ -669,8 +649,8 @@ def verify_witness(
         Compares the expected witness check allocation account against the values updated
         in the block execution witness state diff.
         """
-        witness_check_state_diff, witness_check_address_mapping = (
-            t8n.get_witness_check_mapping(witness_check)
+        witness_check_state_diff, witness_check_address_mapping = t8n.get_witness_check_mapping(
+            witness_check
         )
         actual_stem_dict = {sd.stem: sd for sd in state_diff.root}
         expected_stem_dict = {sd.stem: sd for sd in witness_check_state_diff.root}
@@ -694,9 +674,7 @@ def verify_witness(
                 )
 
             actual_suffix_dict = {sd.suffix: sd for sd in stem_state_diff.suffix_diffs}
-            expected_suffix_dict = {
-                sd.suffix: sd for sd in expected_stem_state_diff.suffix_diffs
-            }
+            expected_suffix_dict = {sd.suffix: sd for sd in expected_stem_state_diff.suffix_diffs}
             # Check that all actual suffixes are in expected suffixes
             for actual_suffix, suffix_diff in actual_suffix_dict.items():
                 actual_current_value = suffix_diff.current_value
@@ -715,9 +693,7 @@ def verify_witness(
                             indent=4,
                         )
                     )
-                if str(actual_current_value) != str(
-                    expected_suffix_state_diff.current_value
-                ):
+                if str(actual_current_value) != str(expected_suffix_state_diff.current_value):
                     raise ValueError(
                         "Witness check failed - current value mismatch.\n\n"
                         + pformat(
@@ -726,9 +702,7 @@ def verify_witness(
                                 "stem": str(actual_stem),
                                 "suffix": actual_suffix,
                                 "value_actual": str(actual_current_value),
-                                "value_expected": str(
-                                    expected_suffix_state_diff.current_value
-                                ),
+                                "value_expected": str(expected_suffix_state_diff.current_value),
                             },
                             indent=4,
                         )
@@ -752,12 +726,8 @@ def verify_witness(
                     )
                 )
 
-            actual_suffix_dict = {
-                sd.suffix: sd for sd in actual_stem_state_diff.suffix_diffs
-            }
-            expected_suffix_dict = {
-                sd.suffix: sd for sd in expected_stem_state_diff.suffix_diffs
-            }
+            actual_suffix_dict = {sd.suffix: sd for sd in actual_stem_state_diff.suffix_diffs}
+            expected_suffix_dict = {sd.suffix: sd for sd in expected_stem_state_diff.suffix_diffs}
 
             # Check that all expected suffixes are in actual suffixes
             for (
@@ -772,9 +742,7 @@ def verify_witness(
                                 "test_account_address": str(address),
                                 "stem": str(expected_stem),
                                 "suffix": expected_suffix,
-                                "value_expected": str(
-                                    expected_suffix_state_diff.current_value
-                                ),
+                                "value_expected": str(expected_suffix_state_diff.current_value),
                                 "value_actual": "value not found",
                             },
                             indent=4,
@@ -791,12 +759,8 @@ def verify_witness(
                                 "test_account_address": str(address),
                                 "stem": str(expected_stem),
                                 "suffix": expected_suffix,
-                                "value_actual": str(
-                                    actual_suffix_state_diff.current_value
-                                ),
-                                "value_expected": str(
-                                    expected_suffix_state_diff.current_value
-                                ),
+                                "value_actual": str(actual_suffix_state_diff.current_value),
+                                "value_expected": str(expected_suffix_state_diff.current_value),
                             },
                             indent=4,
                         )
@@ -859,10 +823,7 @@ def make_fixture(
                     txs=[FixtureTransaction.from_transaction(tx) for tx in txs],
                     ommers=[],
                     withdrawals=(
-                        [
-                            FixtureWithdrawal.from_withdrawal(w)
-                            for w in new_env.withdrawals
-                        ]
+                        [FixtureWithdrawal.from_withdrawal(w) for w in new_env.withdrawals]
                         if new_env.withdrawals is not None
                         else None
                     ),
@@ -890,9 +851,7 @@ def make_fixture(
                         if requests is not None
                         else None
                     ),
-                    witness=FixtureWitness.from_witness(witness)
-                    if witness is not None
-                    else None,
+                    witness=FixtureWitness.from_witness(witness) if witness is not None else None,
                 ).with_rlp(txs=txs, requests=requests)
                 if block.exception is None:
                     fixture_blocks.append(fixture_block)
@@ -908,8 +867,7 @@ def make_fixture(
                             expect_exception=block.exception,
                             rlp_decoded=(
                                 None
-                                if BlockException.RLP_STRUCTURES_ENCODING
-                                in block.exception
+                                if BlockException.RLP_STRUCTURES_ENCODING in block.exception
                                 else fixture_block.without_rlp()
                             ),
                         ),
@@ -968,16 +926,14 @@ def make_hive_fixture(
 
         for block in self.blocks:
             # TODO: fix witness for hive fixture? Do we need it?
-            new_env, header, txs, new_alloc, requests, new_vkt, _ = (
-                self.generate_block_data(
-                    t8n=t8n,
-                    fork=fork,
-                    block=block,
-                    previous_env=env,
-                    previous_alloc=alloc,
-                    previous_vkt=vkt,
-                    eips=eips,
-                )
+            new_env, header, txs, new_alloc, requests, new_vkt, _ = self.generate_block_data(
+                t8n=t8n,
+                fork=fork,
+                block=block,
+                previous_env=env,
+                previous_alloc=alloc,
+                previous_vkt=vkt,
+                eips=eips,
             )
             if block.rlp is None:
                 fixture_payloads.append(
@@ -996,9 +952,7 @@ def make_hive_fixture(
                     env = apply_new_parent(new_env, header)
                     head_hash = header.block_hash
                     vkt = new_vkt
-        fcu_version = fork.engine_forkchoice_updated_version(
-            header.number, header.timestamp
-        )
+        fcu_version = fork.engine_forkchoice_updated_version(header.number, header.timestamp)
         assert (
             fcu_version is not None
         ), "A hive fixture was requested but no forkchoice update is defined. The framework should"

src/ethereum_test_types/verkle/tests/test_verkle_witness.py

@@ -152,6 +152,7 @@ def witness_data(state_diff_data, verkle_proof_data):
     return {
         "stateDiff": state_diff_data,
         "verkleProof": verkle_proof_data,
+        "parentRoot": "0x5b5fdfedd6a0e932da408ac7d772a36513d1eee9b9926e52620c43a433aad7",
     }
 
 
@@ -170,3 +171,6 @@ def test_witness_validation(witness_data):
         == 0x647ED3C87A4F764421EA2F5BFC73195812F6B7DD15AC2B8D295730C1DEDE1EDF
     )
     assert witness.state_diff.root[0].suffix_diffs[0].new_value is None
+    assert witness.parent_root == (
+        0x5B5FDFEDD6A0E932DA408AC7D772A36513D1EEE9B9926E52620C43A433AAD7
+    )

src/ethereum_test_types/verkle/types.py

@@ -12,7 +12,7 @@
 from ethereum_test_base_types import (
     Address,
     CamelModel,
-    HexNumber,
+    ZeroPaddedHexNumber,
     PaddedFixedSizeBytes,
 )
 from ethereum_test_forks import Fork, Verkle
@@ -69,7 +69,7 @@ class VerkleProof(CamelModel):
     """
 
     other_stems: List[Stem]
-    depth_extension_present: HexNumber
+    depth_extension_present: ZeroPaddedHexNumber
     commitments_by_path: List[Hash]
     d: Hash
     ipa_proof: IpaProof | None = Field(None)
@@ -143,7 +143,7 @@ class Witness(CamelModel):
 
     state_diff: StateDiff
     verkle_proof: VerkleProof
-    # parent_state_root: Hash
+    parent_root: Hash
 
 
 class VerkleTree(RootModel[Dict[Hash, Hash]]):

src/evm_transition_tool/types.py

@@ -112,6 +112,7 @@ class Result(CamelModel):
 
     verkle_proof: VerkleProof | None = None
     state_diff: StateDiff | None = None
+    parent_root: Hash | None = None
 
 
 class TransitionToolInput(CamelModel):