From d9691a3d371fdf97f8d2b5dc308fb3cf075104a7 Mon Sep 17 00:00:00 2001 From: Cayman Date: Wed, 22 May 2024 18:29:36 -0400 Subject: [PATCH 01/12] feat: add StableContainer and Profile ssz_generic test generators --- setup.py | 2 +- tests/core/pyspec/eth2spec/debug/decode.py | 22 +- tests/core/pyspec/eth2spec/debug/encode.py | 18 +- .../pyspec/eth2spec/debug/random_value.py | 27 ++- .../pyspec/eth2spec/utils/ssz/ssz_typing.py | 1 + tests/generators/ssz_generic/ssz_profile.py | 227 ++++++++++++++++++ .../ssz_generic/ssz_stablecontainer.py | 142 +++++++++++ 7 files changed, 435 insertions(+), 4 deletions(-) create mode 100644 tests/generators/ssz_generic/ssz_profile.py create mode 100644 tests/generators/ssz_generic/ssz_stablecontainer.py diff --git a/setup.py b/setup.py index fe2250f7c0..578551aaac 100644 --- a/setup.py +++ b/setup.py @@ -546,7 +546,7 @@ def run(self): "pycryptodome==3.15.0", "py_ecc==6.0.0", "milagro_bls_binding==1.9.0", - "remerkleable==0.1.27", + "remerkleable @ git+https://github.com/etan-status/remerkleable@dev/etan/sc-7495", "trie==2.0.2", RUAMEL_YAML_VERSION, "lru-dict==1.2.0", diff --git a/tests/core/pyspec/eth2spec/debug/decode.py b/tests/core/pyspec/eth2spec/debug/decode.py index 30bfd487bd..4741ada705 100644 --- a/tests/core/pyspec/eth2spec/debug/decode.py +++ b/tests/core/pyspec/eth2spec/debug/decode.py @@ -2,7 +2,8 @@ from eth2spec.utils.ssz.ssz_impl import hash_tree_root from eth2spec.utils.ssz.ssz_typing import ( uint, Container, List, boolean, - Vector, ByteVector, ByteList, Union, View + Vector, ByteVector, ByteList, Union, View, + Profile, StableContainer, ) @@ -27,6 +28,25 @@ def decode(data: Any, typ): assert (data["hash_tree_root"][2:] == hash_tree_root(ret).hex()) return ret + elif issubclass(typ, (StableContainer, Profile)): + temp = {} + for field_name, [field_type, is_optional] in typ.fields().items(): + if data[field_name] is None: + assert is_optional + temp[field_name] = None + if field_name + "_hash_tree_root" in data: + assert (data[field_name + "_hash_tree_root"][2:] == + '00' * 32) + else: + temp[field_name] = decode(data[field_name], field_type) + if field_name + "_hash_tree_root" in data: + assert (data[field_name + "_hash_tree_root"][2:] == + hash_tree_root(temp[field_name]).hex()) + ret = typ(**temp) + if "hash_tree_root" in data: + assert (data["hash_tree_root"][2:] == + hash_tree_root(ret).hex()) + return ret elif issubclass(typ, Union): selector = int(data["selector"]) options = typ.options() diff --git a/tests/core/pyspec/eth2spec/debug/encode.py b/tests/core/pyspec/eth2spec/debug/encode.py index d93f7cf5ef..6f80092056 100644 --- a/tests/core/pyspec/eth2spec/debug/encode.py +++ b/tests/core/pyspec/eth2spec/debug/encode.py @@ -1,7 +1,8 @@ from eth2spec.utils.ssz.ssz_impl import hash_tree_root, serialize from eth2spec.utils.ssz.ssz_typing import ( uint, boolean, - Bitlist, Bitvector, Container, Vector, List, Union + Bitlist, Bitvector, Container, Vector, List, Union, + Profile, StableContainer, ) @@ -31,6 +32,21 @@ def encode(value, include_hash_tree_roots=False): if include_hash_tree_roots: ret["hash_tree_root"] = '0x' + hash_tree_root(value).hex() return ret + elif isinstance(value, (StableContainer, Profile)): + ret = {} + for field_name in value.fields().keys(): + field_value = getattr(value, field_name) + if field_value is None: + ret[field_name] = None + if include_hash_tree_roots: + ret[field_name + "_hash_tree_root"] = '0x' + '00' * 32 + else: + ret[field_name] = encode(field_value, include_hash_tree_roots) + if include_hash_tree_roots: + ret[field_name + "_hash_tree_root"] = '0x' + hash_tree_root(field_value).hex() + if include_hash_tree_roots: + ret["hash_tree_root"] = '0x' + hash_tree_root(value).hex() + return ret elif isinstance(value, Union): inner_value = value.value() return { diff --git a/tests/core/pyspec/eth2spec/debug/random_value.py b/tests/core/pyspec/eth2spec/debug/random_value.py index ff80ee0f4e..5762914f07 100644 --- a/tests/core/pyspec/eth2spec/debug/random_value.py +++ b/tests/core/pyspec/eth2spec/debug/random_value.py @@ -5,7 +5,8 @@ from eth2spec.utils.ssz.ssz_typing import ( View, BasicView, uint, Container, List, boolean, - Vector, ByteVector, ByteList, Bitlist, Bitvector, Union + Vector, ByteVector, ByteList, Bitlist, Bitvector, Union, + Profile, StableContainer, ) # in bytes @@ -115,6 +116,30 @@ def get_random_ssz_object(rng: Random, get_random_ssz_object(rng, field_type, max_bytes_length, max_list_length, mode, chaos) for field_name, field_type in fields.items() }) + elif issubclass(typ, StableContainer): + fields = typ.fields() + # StableContainer + return typ(**{ + field_name: + rng.choice([ + None, + get_random_ssz_object(rng, field_type, max_bytes_length, max_list_length, mode, chaos) + ]) + # this version of remerkleable allows StableContainer non-optional fields + # the EIP says otherwise, so ignore _ for now + for field_name, [field_type, _] in fields.items() + }) + elif issubclass(typ, Profile): + fields = typ.fields() + # Profile + return typ(**{ + field_name: + rng.choice([ + None if is_optional else get_random_ssz_object(rng, field_type, max_bytes_length, max_list_length, mode, chaos), + get_random_ssz_object(rng, field_type, max_bytes_length, max_list_length, mode, chaos) + ]) + for field_name, [field_type, is_optional] in fields.items() + }) elif issubclass(typ, Union): options = typ.options() selector: int diff --git a/tests/core/pyspec/eth2spec/utils/ssz/ssz_typing.py b/tests/core/pyspec/eth2spec/utils/ssz/ssz_typing.py index 5a1b61d0be..e9ac0485bd 100644 --- a/tests/core/pyspec/eth2spec/utils/ssz/ssz_typing.py +++ b/tests/core/pyspec/eth2spec/utils/ssz/ssz_typing.py @@ -7,6 +7,7 @@ from remerkleable.bitfields import Bitvector, Bitlist from remerkleable.byte_arrays import ByteVector, Bytes1, Bytes4, Bytes8, Bytes32, Bytes48, Bytes96, ByteList from remerkleable.core import BasicView, View, Path +from remerkleable.stable_container import Profile, StableContainer Bytes20 = ByteVector[20] # type: ignore diff --git a/tests/generators/ssz_generic/ssz_profile.py b/tests/generators/ssz_generic/ssz_profile.py new file mode 100644 index 0000000000..042ae4bbc2 --- /dev/null +++ b/tests/generators/ssz_generic/ssz_profile.py @@ -0,0 +1,227 @@ +from ssz_test_case import invalid_test_case, valid_test_case +from eth2spec.utils.ssz.ssz_typing import View, Container, byte, uint8, uint16, \ + uint32, uint64, List, ByteList, Vector, Bitvector, Bitlist, Profile, StableContainer +from eth2spec.utils.ssz.ssz_impl import serialize +from random import Random +from typing import Dict, Tuple, Sequence, Callable, Type, Optional +from eth2spec.debug.random_value import RandomizationMode, get_random_ssz_object +from ssz_stablecontainer import SingleFieldTestStableStruct, SmallTestStableStruct, FixedTestStableStruct, \ + VarTestStableStruct, ComplexTestStableStruct, BitsStableStruct + +class SingleFieldTestProfile(Profile[SingleFieldTestStableStruct]): + A: byte + + +class SmallTestProfile1(Profile[SmallTestStableStruct]): + A: uint16 + B: uint16 + + +class SmallTestProfile2(Profile[SmallTestStableStruct]): + A: uint16 + + +class SmallTestProfile3(Profile[SmallTestStableStruct]): + B: uint16 + + +class FixedTestProfile1(Profile[FixedTestStableStruct]): + A: uint8 + B: uint64 + C: uint32 + + +class FixedTestProfile2(Profile[FixedTestStableStruct]): + A: uint8 + B: uint64 + + +class FixedTestProfile3(Profile[FixedTestStableStruct]): + A: uint8 + C: uint32 + + +class FixedTestProfile4(Profile[FixedTestStableStruct]): + C: uint32 + + +class VarTestProfile1(Profile[VarTestStableStruct]): + A: uint16 + B: List[uint16, 1024] + C: uint8 + + +class VarTestProfile2(Profile[VarTestStableStruct]): + B: List[uint16, 1024] + C: uint8 + + +class VarTestProfile3(Profile[VarTestStableStruct]): + B: List[uint16, 1024] + + +class ComplexTestProfile1(Profile[ComplexTestStableStruct]): + A: uint16 + B: List[uint16, 128] + C: uint8 + D: ByteList[256] + E: VarTestStableStruct + F: Vector[FixedTestStableStruct, 4] + G: Vector[VarTestStableStruct, 2] + + +class ComplexTestProfile2(Profile[ComplexTestStableStruct]): + A: uint16 + B: List[uint16, 128] + C: uint8 + D: ByteList[256] + E: VarTestStableStruct + + +class ComplexTestProfile3(Profile[ComplexTestStableStruct]): + A: uint16 + C: uint8 + E: VarTestStableStruct + G: Vector[VarTestStableStruct, 2] + + +class ComplexTestProfile4(Profile[ComplexTestStableStruct]): + B: List[uint16, 128] + D: ByteList[256] + F: Vector[FixedTestStableStruct, 4] + + +class ComplexTestProfile5(Profile[ComplexTestStableStruct]): + E: VarTestStableStruct + F: Vector[FixedTestStableStruct, 4] + G: Vector[VarTestStableStruct, 2] + + +class BitsProfile1(Profile[BitsStableStruct]): + A: Bitlist[5] + B: Bitvector[2] + C: Bitvector[1] + D: Bitlist[6] + E: Bitvector[8] + + +class BitsProfile2(Profile[BitsStableStruct]): + A: Bitlist[5] + B: Bitvector[2] + C: Bitvector[1] + D: Bitlist[6] + + +class BitsProfile3(Profile[BitsStableStruct]): + A: Bitlist[5] + D: Bitlist[6] + E: Bitvector[8] + + +def container_case_fn(rng: Random, mode: RandomizationMode, typ: Type[View], chaos: bool=False): + return get_random_ssz_object(rng, typ, + max_bytes_length=2000, + max_list_length=2000, + mode=mode, chaos=chaos) + + +PRESET_CONTAINERS: Dict[str, Tuple[Type[View], Sequence[int]]] = { + 'SingleFieldTestProfile': (SingleFieldTestProfile, []), + 'SmallTestProfile1': (SmallTestProfile1, []), + 'SmallTestProfile2': (SmallTestProfile2, []), + 'SmallTestProfile3': (SmallTestProfile3, []), + 'FixedTestProfile1': (FixedTestProfile1, []), + 'FixedTestProfile2': (FixedTestProfile2, []), + 'FixedTestProfile3': (FixedTestProfile3, []), + 'FixedTestProfile4': (FixedTestProfile4, []), + 'VarTestProfile1': (VarTestProfile1, [2]), + 'VarTestProfile2': (VarTestProfile2, [2]), + 'VarTestProfile3': (VarTestProfile3, [2]), + 'ComplexTestProfile1': (ComplexTestProfile1, [2, 2 + 4 + 1, 2 + 4 + 1 + 4]), + 'ComplexTestProfile2': (ComplexTestProfile2, [2, 2 + 4 + 1, 2 + 4 + 1 + 4]), + 'ComplexTestProfile3': (ComplexTestProfile3, [2, 2 + 4 + 1, 2 + 4 + 1 + 4]), + 'ComplexTestProfile4': (ComplexTestProfile4, [2, 2 + 4 + 1, 2 + 4 + 1 + 4]), + 'ComplexTestProfile5': (ComplexTestProfile5, [2, 2 + 4 + 1, 2 + 4 + 1 + 4]), + 'BitsProfile1': (BitsProfile1, [0, 4 + 1 + 1, 4 + 1 + 1 + 4]), + 'BitsProfile2': (BitsProfile2, [0, 4 + 1 + 1, 4 + 1 + 1 + 4]), + 'BitsProfile3': (BitsProfile3, [0, 4 + 1 + 1, 4 + 1 + 1 + 4]), +} + + +def valid_cases(): + rng = Random(1234) + for (name, (typ, offsets)) in PRESET_CONTAINERS.items(): + for mode in [RandomizationMode.mode_zero, RandomizationMode.mode_max]: + yield f'{name}_{mode.to_name()}', valid_test_case(lambda: container_case_fn(rng, mode, typ)) + + if len(offsets) == 0: + modes = [RandomizationMode.mode_random, RandomizationMode.mode_zero, RandomizationMode.mode_max] + else: + modes = list(RandomizationMode) + + for mode in modes: + for variation in range(3): + yield f'{name}_{mode.to_name()}_chaos_{variation}', \ + valid_test_case(lambda: container_case_fn(rng, mode, typ, chaos=True)) + # Notes: Below is the second wave of iteration, and only the random mode is selected + # for container without offset since ``RandomizationMode.mode_zero`` and ``RandomizationMode.mode_max`` + # are deterministic. + modes = [RandomizationMode.mode_random] if len(offsets) == 0 else list(RandomizationMode) + for mode in modes: + for variation in range(10): + yield f'{name}_{mode.to_name()}_{variation}', \ + valid_test_case(lambda: container_case_fn(rng, mode, typ)) + + +def mod_offset(b: bytes, offset_index: int, change: Callable[[int], int]): + return b[:offset_index] + \ + (change(int.from_bytes(b[offset_index:offset_index + 4], byteorder='little')) & 0xffffffff) \ + .to_bytes(length=4, byteorder='little') + \ + b[offset_index + 4:] + + +def invalid_cases(): + rng = Random(1234) + for (name, (typ, offsets)) in PRESET_CONTAINERS.items(): + # using mode_max_count, so that the extra byte cannot be picked up as normal list content + yield f'{name}_extra_byte', \ + invalid_test_case(lambda: serialize( + container_case_fn(rng, RandomizationMode.mode_max_count, typ)) + b'\xff') + + if len(offsets) != 0: + # Note: there are many more ways to have invalid offsets, + # these are just example to get clients started looking into hardening ssz. + for mode in [RandomizationMode.mode_random, + RandomizationMode.mode_nil_count, + RandomizationMode.mode_one_count, + RandomizationMode.mode_max_count]: + for index, offset_index in enumerate(offsets): + yield f'{name}_{mode.to_name()}_offset_{offset_index}_plus_one', \ + invalid_test_case(lambda: mod_offset( + b=serialize(container_case_fn(rng, mode, typ)), + offset_index=offset_index, + change=lambda x: x + 1 + )) + yield f'{name}_{mode.to_name()}_offset_{offset_index}_zeroed', \ + invalid_test_case(lambda: mod_offset( + b=serialize(container_case_fn(rng, mode, typ)), + offset_index=offset_index, + change=lambda x: 0 + )) + if index == 0: + yield f'{name}_{mode.to_name()}_offset_{offset_index}_minus_one', \ + invalid_test_case(lambda: mod_offset( + b=serialize(container_case_fn(rng, mode, typ)), + offset_index=offset_index, + change=lambda x: x - 1 + )) + if mode == RandomizationMode.mode_max_count: + serialized = serialize(container_case_fn(rng, mode, typ)) + serialized = serialized + serialized[:2] + yield f'{name}_{mode.to_name()}_last_offset_{offset_index}_overflow', \ + invalid_test_case(lambda: serialized) + if mode == RandomizationMode.mode_one_count: + serialized = serialize(container_case_fn(rng, mode, typ)) + serialized = serialized + serialized[:1] + yield f'{name}_{mode.to_name()}_last_offset_{offset_index}_wrong_byte_length', \ + invalid_test_case(lambda: serialized) diff --git a/tests/generators/ssz_generic/ssz_stablecontainer.py b/tests/generators/ssz_generic/ssz_stablecontainer.py new file mode 100644 index 0000000000..d013450fb7 --- /dev/null +++ b/tests/generators/ssz_generic/ssz_stablecontainer.py @@ -0,0 +1,142 @@ +from ssz_test_case import invalid_test_case, valid_test_case +from eth2spec.utils.ssz.ssz_typing import View, Container, byte, uint8, uint16, \ + uint32, uint64, List, ByteList, Vector, Bitvector, Bitlist, StableContainer +from eth2spec.utils.ssz.ssz_impl import serialize +from random import Random +from typing import Dict, Tuple, Sequence, Callable, Type, Optional +from eth2spec.debug.random_value import RandomizationMode, get_random_ssz_object + + +class SingleFieldTestStableStruct(StableContainer): + A: Optional[byte] + + +class SmallTestStableStruct(StableContainer): + A: Optional[uint16] + B: Optional[uint16] + + +class FixedTestStableStruct(StableContainer): + A: Optional[uint8] + B: Optional[uint64] + C: Optional[uint32] + + +class VarTestStableStruct(StableContainer): + A: Optional[uint16] + B: Optional[List[uint16, 1024]] + C: Optional[uint8] + + +class ComplexTestStableStruct(StableContainer): + A: Optional[uint16] + B: Optional[List[uint16, 128]] + C: Optional[uint8] + D: Optional[ByteList[256]] + E: Optional[VarTestStableStruct] + F: Optional[Vector[FixedTestStableStruct, 4]] + G: Optional[Vector[VarTestStableStruct, 2]] + + +class BitsStableStruct(StableContainer): + A: Optional[Bitlist[5]] + B: Optional[Bitvector[2]] + C: Optional[Bitvector[1]] + D: Optional[Bitlist[6]] + E: Optional[Bitvector[8]] + + +def container_case_fn(rng: Random, mode: RandomizationMode, typ: Type[View], chaos: bool=False): + return get_random_ssz_object(rng, typ, + max_bytes_length=2000, + max_list_length=2000, + mode=mode, chaos=chaos) + + +PRESET_CONTAINERS: Dict[str, Tuple[Type[View], Sequence[int]]] = { + 'SingleFieldTestStableStruct': (SingleFieldTestStableStruct, []), + 'SmallTestStableStruct': (SmallTestStableStruct, []), + 'FixedTestStableStruct': (FixedTestStableStruct, []), + 'VarTestStableStruct': (VarTestStableStruct, [2]), + 'ComplexTestStableStruct': (ComplexTestStableStruct, [2, 2 + 4 + 1, 2 + 4 + 1 + 4]), + 'BitsStableStruct': (BitsStableStruct, [0, 4 + 1 + 1, 4 + 1 + 1 + 4]), +} + + +def valid_cases(): + rng = Random(1234) + for (name, (typ, offsets)) in PRESET_CONTAINERS.items(): + for mode in [RandomizationMode.mode_zero, RandomizationMode.mode_max]: + yield f'{name}_{mode.to_name()}', valid_test_case(lambda: container_case_fn(rng, mode, typ)) + + if len(offsets) == 0: + modes = [RandomizationMode.mode_random, RandomizationMode.mode_zero, RandomizationMode.mode_max] + else: + modes = list(RandomizationMode) + + for mode in modes: + for variation in range(3): + yield f'{name}_{mode.to_name()}_chaos_{variation}', \ + valid_test_case(lambda: container_case_fn(rng, mode, typ, chaos=True)) + # Notes: Below is the second wave of iteration, and only the random mode is selected + # for container without offset since ``RandomizationMode.mode_zero`` and ``RandomizationMode.mode_max`` + # are deterministic. + modes = [RandomizationMode.mode_random] if len(offsets) == 0 else list(RandomizationMode) + for mode in modes: + for variation in range(10): + yield f'{name}_{mode.to_name()}_{variation}', \ + valid_test_case(lambda: container_case_fn(rng, mode, typ)) + + +def mod_offset(b: bytes, offset_index: int, change: Callable[[int], int]): + return b[:offset_index] + \ + (change(int.from_bytes(b[offset_index:offset_index + 4], byteorder='little')) & 0xffffffff) \ + .to_bytes(length=4, byteorder='little') + \ + b[offset_index + 4:] + + +def invalid_cases(): + rng = Random(1234) + for (name, (typ, offsets)) in PRESET_CONTAINERS.items(): + # using mode_max_count, so that the extra byte cannot be picked up as normal list content + yield f'{name}_extra_byte', \ + invalid_test_case(lambda: serialize( + container_case_fn(rng, RandomizationMode.mode_max_count, typ)) + b'\xff') + + if len(offsets) != 0: + # Note: there are many more ways to have invalid offsets, + # these are just example to get clients started looking into hardening ssz. + for mode in [RandomizationMode.mode_random, + RandomizationMode.mode_nil_count, + RandomizationMode.mode_one_count, + RandomizationMode.mode_max_count]: + for index, offset_index in enumerate(offsets): + yield f'{name}_{mode.to_name()}_offset_{offset_index}_plus_one', \ + invalid_test_case(lambda: mod_offset( + b=serialize(container_case_fn(rng, mode, typ)), + offset_index=offset_index, + change=lambda x: x + 1 + )) + yield f'{name}_{mode.to_name()}_offset_{offset_index}_zeroed', \ + invalid_test_case(lambda: mod_offset( + b=serialize(container_case_fn(rng, mode, typ)), + offset_index=offset_index, + change=lambda x: 0 + )) + if index == 0: + yield f'{name}_{mode.to_name()}_offset_{offset_index}_minus_one', \ + invalid_test_case(lambda: mod_offset( + b=serialize(container_case_fn(rng, mode, typ)), + offset_index=offset_index, + change=lambda x: x - 1 + )) + if mode == RandomizationMode.mode_max_count: + serialized = serialize(container_case_fn(rng, mode, typ)) + serialized = serialized + serialized[:2] + yield f'{name}_{mode.to_name()}_last_offset_{offset_index}_overflow', \ + invalid_test_case(lambda: serialized) + if mode == RandomizationMode.mode_one_count: + serialized = serialize(container_case_fn(rng, mode, typ)) + serialized = serialized + serialized[:1] + yield f'{name}_{mode.to_name()}_last_offset_{offset_index}_wrong_byte_length', \ + invalid_test_case(lambda: serialized) From 1193fcd780ae6552356dc3d0c1718d693ae33479 Mon Sep 17 00:00:00 2001 From: Cayman Date: Wed, 22 May 2024 18:43:02 -0400 Subject: [PATCH 02/12] fix stable container types and add to test generators --- tests/generators/ssz_generic/main.py | 4 ++++ tests/generators/ssz_generic/ssz_stablecontainer.py | 12 ++++++------ 2 files changed, 10 insertions(+), 6 deletions(-) diff --git a/tests/generators/ssz_generic/main.py b/tests/generators/ssz_generic/main.py index 2e96ce2e87..eeb79a4c0f 100644 --- a/tests/generators/ssz_generic/main.py +++ b/tests/generators/ssz_generic/main.py @@ -6,6 +6,8 @@ import ssz_boolean import ssz_uints import ssz_container +import ssz_stablecontainer +import ssz_profile from eth2spec.test.helpers.constants import PHASE0 @@ -43,4 +45,6 @@ def cases_fn() -> Iterable[gen_typing.TestCase]: create_provider("uints", "invalid", ssz_uints.invalid_cases), create_provider("containers", "valid", ssz_container.valid_cases), create_provider("containers", "invalid", ssz_container.invalid_cases), + create_provider("stablecontainers", "valid", ssz_stablecontainer.valid_cases), + create_provider("profiles", "valid", ssz_profile.valid_cases), ]) diff --git a/tests/generators/ssz_generic/ssz_stablecontainer.py b/tests/generators/ssz_generic/ssz_stablecontainer.py index d013450fb7..8d1fda4c4f 100644 --- a/tests/generators/ssz_generic/ssz_stablecontainer.py +++ b/tests/generators/ssz_generic/ssz_stablecontainer.py @@ -7,28 +7,28 @@ from eth2spec.debug.random_value import RandomizationMode, get_random_ssz_object -class SingleFieldTestStableStruct(StableContainer): +class SingleFieldTestStableStruct(StableContainer[4]): A: Optional[byte] -class SmallTestStableStruct(StableContainer): +class SmallTestStableStruct(StableContainer[4]): A: Optional[uint16] B: Optional[uint16] -class FixedTestStableStruct(StableContainer): +class FixedTestStableStruct(StableContainer[4]): A: Optional[uint8] B: Optional[uint64] C: Optional[uint32] -class VarTestStableStruct(StableContainer): +class VarTestStableStruct(StableContainer[4]): A: Optional[uint16] B: Optional[List[uint16, 1024]] C: Optional[uint8] -class ComplexTestStableStruct(StableContainer): +class ComplexTestStableStruct(StableContainer[8]): A: Optional[uint16] B: Optional[List[uint16, 128]] C: Optional[uint8] @@ -38,7 +38,7 @@ class ComplexTestStableStruct(StableContainer): G: Optional[Vector[VarTestStableStruct, 2]] -class BitsStableStruct(StableContainer): +class BitsStableStruct(StableContainer[8]): A: Optional[Bitlist[5]] B: Optional[Bitvector[2]] C: Optional[Bitvector[1]] From 7c5e0a52124b407c8acab9000c7448da011300f7 Mon Sep 17 00:00:00 2001 From: Etan Kissling Date: Thu, 23 May 2024 12:41:56 +0200 Subject: [PATCH 03/12] Fix lint --- tests/core/pyspec/eth2spec/debug/random_value.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/core/pyspec/eth2spec/debug/random_value.py b/tests/core/pyspec/eth2spec/debug/random_value.py index 5762914f07..bb7bfc8ab5 100644 --- a/tests/core/pyspec/eth2spec/debug/random_value.py +++ b/tests/core/pyspec/eth2spec/debug/random_value.py @@ -135,7 +135,8 @@ def get_random_ssz_object(rng: Random, return typ(**{ field_name: rng.choice([ - None if is_optional else get_random_ssz_object(rng, field_type, max_bytes_length, max_list_length, mode, chaos), + None if is_optional else get_random_ssz_object( + rng, field_type, max_bytes_length, max_list_length, mode, chaos), get_random_ssz_object(rng, field_type, max_bytes_length, max_list_length, mode, chaos) ]) for field_name, [field_type, is_optional] in fields.items() From c75402c5ff333a432b548f6c6762c020eef849fa Mon Sep 17 00:00:00 2001 From: Etan Kissling Date: Thu, 23 May 2024 14:22:48 +0200 Subject: [PATCH 04/12] Somehow, `Profile` is subclass from `StableContainer` in remerkleable --- .../pyspec/eth2spec/debug/random_value.py | 22 +++++++++---------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/tests/core/pyspec/eth2spec/debug/random_value.py b/tests/core/pyspec/eth2spec/debug/random_value.py index bb7bfc8ab5..25da4df967 100644 --- a/tests/core/pyspec/eth2spec/debug/random_value.py +++ b/tests/core/pyspec/eth2spec/debug/random_value.py @@ -116,30 +116,30 @@ def get_random_ssz_object(rng: Random, get_random_ssz_object(rng, field_type, max_bytes_length, max_list_length, mode, chaos) for field_name, field_type in fields.items() }) - elif issubclass(typ, StableContainer): + elif issubclass(typ, Profile): fields = typ.fields() - # StableContainer + # Profile return typ(**{ field_name: rng.choice([ - None, + None if is_optional else get_random_ssz_object( + rng, field_type, max_bytes_length, max_list_length, mode, chaos), get_random_ssz_object(rng, field_type, max_bytes_length, max_list_length, mode, chaos) ]) - # this version of remerkleable allows StableContainer non-optional fields - # the EIP says otherwise, so ignore _ for now - for field_name, [field_type, _] in fields.items() + for field_name, [field_type, is_optional] in fields.items() }) - elif issubclass(typ, Profile): + elif issubclass(typ, StableContainer): fields = typ.fields() - # Profile + # StableContainer return typ(**{ field_name: rng.choice([ - None if is_optional else get_random_ssz_object( - rng, field_type, max_bytes_length, max_list_length, mode, chaos), + None, get_random_ssz_object(rng, field_type, max_bytes_length, max_list_length, mode, chaos) ]) - for field_name, [field_type, is_optional] in fields.items() + # this version of remerkleable allows StableContainer non-optional fields + # the EIP says otherwise, so ignore _ for now + for field_name, [field_type, _] in fields.items() }) elif issubclass(typ, Union): options = typ.options() From bb542bbdfcf0ab2ab11d486bb93148061d9735b4 Mon Sep 17 00:00:00 2001 From: Etan Kissling Date: Thu, 23 May 2024 20:32:47 +0200 Subject: [PATCH 05/12] Remove unused imports --- tests/generators/ssz_generic/ssz_profile.py | 7 ++++--- tests/generators/ssz_generic/ssz_stablecontainer.py | 2 +- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/tests/generators/ssz_generic/ssz_profile.py b/tests/generators/ssz_generic/ssz_profile.py index 042ae4bbc2..30651ae514 100644 --- a/tests/generators/ssz_generic/ssz_profile.py +++ b/tests/generators/ssz_generic/ssz_profile.py @@ -1,13 +1,14 @@ from ssz_test_case import invalid_test_case, valid_test_case -from eth2spec.utils.ssz.ssz_typing import View, Container, byte, uint8, uint16, \ - uint32, uint64, List, ByteList, Vector, Bitvector, Bitlist, Profile, StableContainer +from eth2spec.utils.ssz.ssz_typing import View, byte, uint8, uint16, \ + uint32, uint64, List, ByteList, Vector, Bitvector, Bitlist, Profile from eth2spec.utils.ssz.ssz_impl import serialize from random import Random -from typing import Dict, Tuple, Sequence, Callable, Type, Optional +from typing import Dict, Tuple, Sequence, Callable, Type from eth2spec.debug.random_value import RandomizationMode, get_random_ssz_object from ssz_stablecontainer import SingleFieldTestStableStruct, SmallTestStableStruct, FixedTestStableStruct, \ VarTestStableStruct, ComplexTestStableStruct, BitsStableStruct + class SingleFieldTestProfile(Profile[SingleFieldTestStableStruct]): A: byte diff --git a/tests/generators/ssz_generic/ssz_stablecontainer.py b/tests/generators/ssz_generic/ssz_stablecontainer.py index 8d1fda4c4f..b4d782fee6 100644 --- a/tests/generators/ssz_generic/ssz_stablecontainer.py +++ b/tests/generators/ssz_generic/ssz_stablecontainer.py @@ -1,5 +1,5 @@ from ssz_test_case import invalid_test_case, valid_test_case -from eth2spec.utils.ssz.ssz_typing import View, Container, byte, uint8, uint16, \ +from eth2spec.utils.ssz.ssz_typing import View, byte, uint8, uint16, \ uint32, uint64, List, ByteList, Vector, Bitvector, Bitlist, StableContainer from eth2spec.utils.ssz.ssz_impl import serialize from random import Random From bd02507fc41bd68ee75b8d7e7a242b30fc6a0ad9 Mon Sep 17 00:00:00 2001 From: Etan Kissling Date: Sat, 1 Jun 2024 23:42:25 +0200 Subject: [PATCH 06/12] Update for new `remerkleable` version --- .../pyspec/eth2spec/debug/random_value.py | 20 +++++++++---------- 1 file changed, 9 insertions(+), 11 deletions(-) diff --git a/tests/core/pyspec/eth2spec/debug/random_value.py b/tests/core/pyspec/eth2spec/debug/random_value.py index 25da4df967..f507bf3c33 100644 --- a/tests/core/pyspec/eth2spec/debug/random_value.py +++ b/tests/core/pyspec/eth2spec/debug/random_value.py @@ -116,30 +116,28 @@ def get_random_ssz_object(rng: Random, get_random_ssz_object(rng, field_type, max_bytes_length, max_list_length, mode, chaos) for field_name, field_type in fields.items() }) - elif issubclass(typ, Profile): + elif issubclass(typ, StableContainer): fields = typ.fields() - # Profile + # StableContainer return typ(**{ field_name: rng.choice([ - None if is_optional else get_random_ssz_object( - rng, field_type, max_bytes_length, max_list_length, mode, chaos), + None, get_random_ssz_object(rng, field_type, max_bytes_length, max_list_length, mode, chaos) ]) - for field_name, [field_type, is_optional] in fields.items() + for field_name, field_type in fields.items() }) - elif issubclass(typ, StableContainer): + elif issubclass(typ, Profile): fields = typ.fields() - # StableContainer + # Profile return typ(**{ field_name: rng.choice([ - None, + None if is_optional else get_random_ssz_object( + rng, field_type, max_bytes_length, max_list_length, mode, chaos), get_random_ssz_object(rng, field_type, max_bytes_length, max_list_length, mode, chaos) ]) - # this version of remerkleable allows StableContainer non-optional fields - # the EIP says otherwise, so ignore _ for now - for field_name, [field_type, _] in fields.items() + for field_name, [field_type, is_optional] in fields.items() }) elif issubclass(typ, Union): options = typ.options() From e86cb66e9d07e1d85d4fc6ed96c97f541cd8a6c6 Mon Sep 17 00:00:00 2001 From: Etan Kissling Date: Wed, 5 Jun 2024 15:32:26 +0200 Subject: [PATCH 07/12] Update decoder, `StableContainer` fields now always optional --- tests/core/pyspec/eth2spec/debug/decode.py | 22 ++++++++++++++++++++-- 1 file changed, 20 insertions(+), 2 deletions(-) diff --git a/tests/core/pyspec/eth2spec/debug/decode.py b/tests/core/pyspec/eth2spec/debug/decode.py index 4741ada705..dcdc786d15 100644 --- a/tests/core/pyspec/eth2spec/debug/decode.py +++ b/tests/core/pyspec/eth2spec/debug/decode.py @@ -28,9 +28,27 @@ def decode(data: Any, typ): assert (data["hash_tree_root"][2:] == hash_tree_root(ret).hex()) return ret - elif issubclass(typ, (StableContainer, Profile)): + elif issubclass(typ, StableContainer): temp = {} - for field_name, [field_type, is_optional] in typ.fields().items(): + for field_name, field_type in typ.fields().items(): + if data[field_name] is None: + temp[field_name] = None + if field_name + "_hash_tree_root" in data: + assert (data[field_name + "_hash_tree_root"][2:] == + '00' * 32) + else: + temp[field_name] = decode(data[field_name], field_type) + if field_name + "_hash_tree_root" in data: + assert (data[field_name + "_hash_tree_root"][2:] == + hash_tree_root(temp[field_name]).hex()) + ret = typ(**temp) + if "hash_tree_root" in data: + assert (data["hash_tree_root"][2:] == + hash_tree_root(ret).hex()) + return ret + elif issubclass(typ, Profile): + temp = {} + for field_name, (field_type, is_optional) in typ.fields().items(): if data[field_name] is None: assert is_optional temp[field_name] = None From c5b236a9aa64e6fef5336e45d8dc9e1d56eaf6b3 Mon Sep 17 00:00:00 2001 From: Etan Kissling Date: Wed, 5 Jun 2024 15:36:53 +0200 Subject: [PATCH 08/12] List syntax instead of tuple for consistency --- tests/core/pyspec/eth2spec/debug/decode.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/core/pyspec/eth2spec/debug/decode.py b/tests/core/pyspec/eth2spec/debug/decode.py index dcdc786d15..a1a3cd9c24 100644 --- a/tests/core/pyspec/eth2spec/debug/decode.py +++ b/tests/core/pyspec/eth2spec/debug/decode.py @@ -48,7 +48,7 @@ def decode(data: Any, typ): return ret elif issubclass(typ, Profile): temp = {} - for field_name, (field_type, is_optional) in typ.fields().items(): + for field_name, [field_type, is_optional] in typ.fields().items(): if data[field_name] is None: assert is_optional temp[field_name] = None From 2ac04223a3a080cc3328fd1704b2b35c54f3560e Mon Sep 17 00:00:00 2001 From: Etan Kissling Date: Fri, 20 Sep 2024 00:52:10 +0200 Subject: [PATCH 09/12] Pyspec support for `StableContainer` and `Profile` usage --- pysetup/helpers.py | 4 ++++ setup.py | 4 ++-- tests/core/pyspec/eth2spec/utils/ssz/ssz_typing.py | 1 + tests/generators/ssz_static/main.py | 7 ++++++- 4 files changed, 13 insertions(+), 3 deletions(-) diff --git a/pysetup/helpers.py b/pysetup/helpers.py index 212eb98c10..f935979a19 100644 --- a/pysetup/helpers.py +++ b/pysetup/helpers.py @@ -203,6 +203,10 @@ def dependency_order_class_objects(objects: Dict[str, str], custom_types: Dict[s for key, value in items: dependencies = [] for line in value.split('\n'): + profile_match = re.match(r'class\s+\w+\s*\(Profile\[\s*(\w+)\s*\]\s*\)\s*:', line) + if profile_match is not None: + dependencies.append(profile_match.group(1)) # SSZ `Profile` base + continue if not re.match(r'\s+\w+: .+', line): continue # skip whitespace etc. line = line[line.index(':') + 1:] # strip of field name diff --git a/setup.py b/setup.py index a06a9ffdb4..790384a098 100644 --- a/setup.py +++ b/setup.py @@ -173,7 +173,7 @@ def _update_constant_vars_with_kzg_setups(constant_vars, preset_name): constant_vars['KZG_SETUP_G1_MONOMIAL'] = VariableDefinition(constant_vars['KZG_SETUP_G1_MONOMIAL'].value, str(kzg_setups[0]), comment, None) constant_vars['KZG_SETUP_G1_LAGRANGE'] = VariableDefinition(constant_vars['KZG_SETUP_G1_LAGRANGE'].value, str(kzg_setups[1]), comment, None) constant_vars['KZG_SETUP_G2_MONOMIAL'] = VariableDefinition(constant_vars['KZG_SETUP_G2_MONOMIAL'].value, str(kzg_setups[2]), comment, None) - + def get_spec(file_name: Path, preset: Dict[str, str], config: Dict[str, str], preset_name=str) -> SpecObject: functions: Dict[str, str] = {} @@ -227,7 +227,7 @@ def get_spec(file_name: Path, preset: Dict[str, str], config: Dict[str, str], pr raise if parent_class: - assert parent_class == "Container" + assert parent_class in ["Container", "StableContainer", "Profile"] # NOTE: trim whitespace from spec ssz_objects[current_name] = "\n".join(line.rstrip() for line in source.splitlines()) else: diff --git a/tests/core/pyspec/eth2spec/utils/ssz/ssz_typing.py b/tests/core/pyspec/eth2spec/utils/ssz/ssz_typing.py index 054c20e034..c10487ddfb 100644 --- a/tests/core/pyspec/eth2spec/utils/ssz/ssz_typing.py +++ b/tests/core/pyspec/eth2spec/utils/ssz/ssz_typing.py @@ -2,6 +2,7 @@ # Ignore linter: This module makes importing SSZ types easy, and hides away the underlying library from the spec. from remerkleable.complex import Container, Vector, List +from remerkleable.stable_container import StableContainer, Profile from remerkleable.union import Union from remerkleable.basic import boolean, bit, uint, byte, uint8, uint16, uint32, uint64, uint128, uint256 from remerkleable.bitfields import Bitvector, Bitlist diff --git a/tests/generators/ssz_static/main.py b/tests/generators/ssz_static/main.py index 3f894ea796..6272cfb355 100644 --- a/tests/generators/ssz_static/main.py +++ b/tests/generators/ssz_static/main.py @@ -32,7 +32,12 @@ def create_test_case(rng: Random, typ, def get_spec_ssz_types(spec): return [ (name, value) for (name, value) in getmembers(spec, isclass) - if issubclass(value, Container) and value != Container # only the subclasses, not the imported base class + if ( + # only the subclasses, not the imported base class + issubclass(value, Container) and value != Container + or issubclass(value, StableContainer) and value != StableContainer + or issubclass(value, Profile) and value != Profile + ) ] From add6f167d3fb2c8a89f8cf9449f84143d6f05503 Mon Sep 17 00:00:00 2001 From: Etan Kissling Date: Fri, 20 Sep 2024 00:53:18 +0200 Subject: [PATCH 10/12] Fix import --- tests/generators/ssz_static/main.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/generators/ssz_static/main.py b/tests/generators/ssz_static/main.py index 6272cfb355..db4d3f8d5b 100644 --- a/tests/generators/ssz_static/main.py +++ b/tests/generators/ssz_static/main.py @@ -7,7 +7,7 @@ from eth2spec.debug import random_value, encode from eth2spec.test.helpers.constants import TESTGEN_FORKS, MINIMAL, MAINNET from eth2spec.test.context import spec_targets -from eth2spec.utils.ssz.ssz_typing import Container +from eth2spec.utils.ssz.ssz_typing import Container, Profile, StableContainer from eth2spec.utils.ssz.ssz_impl import ( hash_tree_root, serialize, From 4318f7b367496811f6df76bae9eed0950b487476 Mon Sep 17 00:00:00 2001 From: Etan Kissling Date: Fri, 20 Sep 2024 00:57:12 +0200 Subject: [PATCH 11/12] Fix lint --- tests/generators/ssz_generic/ssz_profile.py | 34 +++++++++---------- .../ssz_generic/ssz_stablecontainer.py | 34 +++++++++---------- 2 files changed, 34 insertions(+), 34 deletions(-) diff --git a/tests/generators/ssz_generic/ssz_profile.py b/tests/generators/ssz_generic/ssz_profile.py index 30651ae514..e35608d055 100644 --- a/tests/generators/ssz_generic/ssz_profile.py +++ b/tests/generators/ssz_generic/ssz_profile.py @@ -163,7 +163,7 @@ def valid_cases(): for mode in modes: for variation in range(3): yield f'{name}_{mode.to_name()}_chaos_{variation}', \ - valid_test_case(lambda: container_case_fn(rng, mode, typ, chaos=True)) + valid_test_case(lambda: container_case_fn(rng, mode, typ, chaos=True)) # Notes: Below is the second wave of iteration, and only the random mode is selected # for container without offset since ``RandomizationMode.mode_zero`` and ``RandomizationMode.mode_max`` # are deterministic. @@ -171,7 +171,7 @@ def valid_cases(): for mode in modes: for variation in range(10): yield f'{name}_{mode.to_name()}_{variation}', \ - valid_test_case(lambda: container_case_fn(rng, mode, typ)) + valid_test_case(lambda: container_case_fn(rng, mode, typ)) def mod_offset(b: bytes, offset_index: int, change: Callable[[int], int]): @@ -186,8 +186,8 @@ def invalid_cases(): for (name, (typ, offsets)) in PRESET_CONTAINERS.items(): # using mode_max_count, so that the extra byte cannot be picked up as normal list content yield f'{name}_extra_byte', \ - invalid_test_case(lambda: serialize( - container_case_fn(rng, RandomizationMode.mode_max_count, typ)) + b'\xff') + invalid_test_case(lambda: serialize( + container_case_fn(rng, RandomizationMode.mode_max_count, typ)) + b'\xff') if len(offsets) != 0: # Note: there are many more ways to have invalid offsets, @@ -198,23 +198,23 @@ def invalid_cases(): RandomizationMode.mode_max_count]: for index, offset_index in enumerate(offsets): yield f'{name}_{mode.to_name()}_offset_{offset_index}_plus_one', \ - invalid_test_case(lambda: mod_offset( - b=serialize(container_case_fn(rng, mode, typ)), - offset_index=offset_index, - change=lambda x: x + 1 - )) + invalid_test_case(lambda: mod_offset( + b=serialize(container_case_fn(rng, mode, typ)), + offset_index=offset_index, + change=lambda x: x + 1 + )) yield f'{name}_{mode.to_name()}_offset_{offset_index}_zeroed', \ - invalid_test_case(lambda: mod_offset( - b=serialize(container_case_fn(rng, mode, typ)), - offset_index=offset_index, - change=lambda x: 0 - )) + invalid_test_case(lambda: mod_offset( + b=serialize(container_case_fn(rng, mode, typ)), + offset_index=offset_index, + change=lambda x: 0 + )) if index == 0: yield f'{name}_{mode.to_name()}_offset_{offset_index}_minus_one', \ invalid_test_case(lambda: mod_offset( - b=serialize(container_case_fn(rng, mode, typ)), - offset_index=offset_index, - change=lambda x: x - 1 + b=serialize(container_case_fn(rng, mode, typ)), + offset_index=offset_index, + change=lambda x: x - 1 )) if mode == RandomizationMode.mode_max_count: serialized = serialize(container_case_fn(rng, mode, typ)) diff --git a/tests/generators/ssz_generic/ssz_stablecontainer.py b/tests/generators/ssz_generic/ssz_stablecontainer.py index b4d782fee6..fa30816b7c 100644 --- a/tests/generators/ssz_generic/ssz_stablecontainer.py +++ b/tests/generators/ssz_generic/ssz_stablecontainer.py @@ -77,7 +77,7 @@ def valid_cases(): for mode in modes: for variation in range(3): yield f'{name}_{mode.to_name()}_chaos_{variation}', \ - valid_test_case(lambda: container_case_fn(rng, mode, typ, chaos=True)) + valid_test_case(lambda: container_case_fn(rng, mode, typ, chaos=True)) # Notes: Below is the second wave of iteration, and only the random mode is selected # for container without offset since ``RandomizationMode.mode_zero`` and ``RandomizationMode.mode_max`` # are deterministic. @@ -85,7 +85,7 @@ def valid_cases(): for mode in modes: for variation in range(10): yield f'{name}_{mode.to_name()}_{variation}', \ - valid_test_case(lambda: container_case_fn(rng, mode, typ)) + valid_test_case(lambda: container_case_fn(rng, mode, typ)) def mod_offset(b: bytes, offset_index: int, change: Callable[[int], int]): @@ -100,8 +100,8 @@ def invalid_cases(): for (name, (typ, offsets)) in PRESET_CONTAINERS.items(): # using mode_max_count, so that the extra byte cannot be picked up as normal list content yield f'{name}_extra_byte', \ - invalid_test_case(lambda: serialize( - container_case_fn(rng, RandomizationMode.mode_max_count, typ)) + b'\xff') + invalid_test_case(lambda: serialize( + container_case_fn(rng, RandomizationMode.mode_max_count, typ)) + b'\xff') if len(offsets) != 0: # Note: there are many more ways to have invalid offsets, @@ -112,23 +112,23 @@ def invalid_cases(): RandomizationMode.mode_max_count]: for index, offset_index in enumerate(offsets): yield f'{name}_{mode.to_name()}_offset_{offset_index}_plus_one', \ - invalid_test_case(lambda: mod_offset( - b=serialize(container_case_fn(rng, mode, typ)), - offset_index=offset_index, - change=lambda x: x + 1 - )) + invalid_test_case(lambda: mod_offset( + b=serialize(container_case_fn(rng, mode, typ)), + offset_index=offset_index, + change=lambda x: x + 1 + )) yield f'{name}_{mode.to_name()}_offset_{offset_index}_zeroed', \ - invalid_test_case(lambda: mod_offset( - b=serialize(container_case_fn(rng, mode, typ)), - offset_index=offset_index, - change=lambda x: 0 - )) + invalid_test_case(lambda: mod_offset( + b=serialize(container_case_fn(rng, mode, typ)), + offset_index=offset_index, + change=lambda x: 0 + )) if index == 0: yield f'{name}_{mode.to_name()}_offset_{offset_index}_minus_one', \ invalid_test_case(lambda: mod_offset( - b=serialize(container_case_fn(rng, mode, typ)), - offset_index=offset_index, - change=lambda x: x - 1 + b=serialize(container_case_fn(rng, mode, typ)), + offset_index=offset_index, + change=lambda x: x - 1 )) if mode == RandomizationMode.mode_max_count: serialized = serialize(container_case_fn(rng, mode, typ)) From 2d851643619e3e8732ef76db99582836977805a9 Mon Sep 17 00:00:00 2001 From: Etan Kissling Date: Wed, 19 Mar 2025 14:27:17 +0100 Subject: [PATCH 12/12] Adopt EIP-7688: Forward compatible consensus data structures EIP-4788 exposes the beacon root to smart contracts, but smart contracts using it need to be redeployed / upgraded whenever the indexing changes during a fork, even if that fork does not touch any used functionality. This problem expands further to bridges on other blockchains, or even into wallet apps on a phone that verify data from the beacon chain instead of trusting the server. It is quite unrealistic to expect such projects to all align their release cadence with Ethereum's forks. EIP-7688 fixes this by defining forward compatibility for beacon chain data structures. Electra `Profile` retain their Merkleization even when rebased to `StableContainer` definitions from future forks, enabling decentralized protocols to drop the requirement for trusted parties to periodically upgrade beacon state proof verifiers. --- presets/mainnet/electra.yaml | 5 + presets/mainnet/fulu.yaml | 2 +- presets/minimal/electra.yaml | 5 + presets/minimal/fulu.yaml | 2 +- pyproject.toml | 2 +- pysetup/spec_builders/deneb.py | 6 +- pysetup/spec_builders/eip7732.py | 4 +- pysetup/spec_builders/electra.py | 16 +- specs/_features/eip7732/beacon-chain.md | 108 +++++- specs/capella/light-client/full-node.md | 2 +- specs/capella/light-client/sync-protocol.md | 16 +- specs/deneb/light-client/full-node.md | 2 +- specs/deneb/light-client/sync-protocol.md | 5 +- specs/deneb/p2p-interface.md | 15 +- specs/electra/beacon-chain.md | 317 ++++++++++++++++-- specs/electra/fork.md | 21 +- specs/electra/light-client/fork.md | 23 +- specs/electra/light-client/full-node.md | 74 ++++ specs/electra/light-client/sync-protocol.md | 51 ++- specs/electra/p2p-interface.md | 22 ++ .../light_client/test_single_merkle_proof.py | 5 +- .../deneb/unittests/test_config_invariants.py | 8 +- .../eth2spec/test/helpers/electra/fork.py | 4 +- .../eth2spec/test/helpers/light_client.py | 6 + 24 files changed, 653 insertions(+), 68 deletions(-) create mode 100644 specs/electra/light-client/full-node.md diff --git a/presets/mainnet/electra.yaml b/presets/mainnet/electra.yaml index 42afbb233e..0b620f5885 100644 --- a/presets/mainnet/electra.yaml +++ b/presets/mainnet/electra.yaml @@ -48,3 +48,8 @@ MAX_PENDING_PARTIALS_PER_WITHDRAWALS_SWEEP: 8 # --------------------------------------------------------------- # 2**4 ( = 4) pending deposits MAX_PENDING_DEPOSITS_PER_EPOCH: 16 + +# Misc +# --------------------------------------------------------------- +# `floorlog2(get_generalized_index(BeaconBlockBody, 'blob_kzg_commitments')) + 1 + ceillog2(MAX_BLOB_COMMITMENTS_PER_BLOCK)` = 7 + 1 + 12 = 20 +KZG_COMMITMENT_INCLUSION_PROOF_DEPTH_ELECTRA: 20 diff --git a/presets/mainnet/fulu.yaml b/presets/mainnet/fulu.yaml index 84111aba28..20d3e28cdc 100644 --- a/presets/mainnet/fulu.yaml +++ b/presets/mainnet/fulu.yaml @@ -7,4 +7,4 @@ FIELD_ELEMENTS_PER_CELL: 64 # `uint64(2 * 4096)` (= 8192) FIELD_ELEMENTS_PER_EXT_BLOB: 8192 # uint64(floorlog2(get_generalized_index(BeaconBlockBody, 'blob_kzg_commitments')) -KZG_COMMITMENTS_INCLUSION_PROOF_DEPTH: 4 +KZG_COMMITMENTS_INCLUSION_PROOF_DEPTH: 7 diff --git a/presets/minimal/electra.yaml b/presets/minimal/electra.yaml index 44e4769756..318d34fcc8 100644 --- a/presets/minimal/electra.yaml +++ b/presets/minimal/electra.yaml @@ -48,3 +48,8 @@ MAX_PENDING_PARTIALS_PER_WITHDRAWALS_SWEEP: 2 # --------------------------------------------------------------- # 2**4 ( = 4) pending deposits MAX_PENDING_DEPOSITS_PER_EPOCH: 16 + +# Misc +# --------------------------------------------------------------- +# [customized] `floorlog2(get_generalized_index(BeaconBlockBody, 'blob_kzg_commitments')) + 1 + ceillog2(MAX_BLOB_COMMITMENTS_PER_BLOCK)` = 7 + 1 + 5 = 13 +KZG_COMMITMENT_INCLUSION_PROOF_DEPTH_ELECTRA: 13 diff --git a/presets/minimal/fulu.yaml b/presets/minimal/fulu.yaml index 1204822fb8..4db513622a 100644 --- a/presets/minimal/fulu.yaml +++ b/presets/minimal/fulu.yaml @@ -7,4 +7,4 @@ FIELD_ELEMENTS_PER_CELL: 64 # `uint64(2 * 4096)` (= 8192) FIELD_ELEMENTS_PER_EXT_BLOB: 8192 # uint64(floorlog2(get_generalized_index(BeaconBlockBody, 'blob_kzg_commitments')) -KZG_COMMITMENTS_INCLUSION_PROOF_DEPTH: 4 +KZG_COMMITMENTS_INCLUSION_PROOF_DEPTH: 7 diff --git a/pyproject.toml b/pyproject.toml index d9485f2e8c..6b5d1e3142 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -23,7 +23,7 @@ dependencies = [ "py_arkworks_bls12381==0.3.8", "py_ecc==6.0.0", "pycryptodome==3.21.0", - "remerkleable==0.1.28", + "remerkleable @ git+https://github.com/etan-status/remerkleable@dev/etan/sc-default", "ruamel.yaml==0.17.21", "setuptools==75.8.0", "trie==3.0.1", diff --git a/pysetup/spec_builders/deneb.py b/pysetup/spec_builders/deneb.py index d73b8612f7..bfae188d3a 100644 --- a/pysetup/spec_builders/deneb.py +++ b/pysetup/spec_builders/deneb.py @@ -81,7 +81,11 @@ def verify_and_notify_new_payload(self: ExecutionEngine, @classmethod - def hardcoded_func_dep_presets(cls, spec_object) -> Dict[str, str]: + def hardcoded_custom_type_dep_constants(cls, spec_object) -> Dict[str, str]: return { + 'BYTES_PER_FIELD_ELEMENT': spec_object.constant_vars['BYTES_PER_FIELD_ELEMENT'].value, + 'FIELD_ELEMENTS_PER_BLOB': spec_object.preset_vars['FIELD_ELEMENTS_PER_BLOB'].value, + 'MAX_BLOBS_PER_BLOCK': spec_object.config_vars['MAX_BLOBS_PER_BLOCK'].value, + 'MAX_BLOB_COMMITMENTS_PER_BLOCK': spec_object.preset_vars['MAX_BLOB_COMMITMENTS_PER_BLOCK'].value, 'KZG_COMMITMENT_INCLUSION_PROOF_DEPTH': spec_object.preset_vars['KZG_COMMITMENT_INCLUSION_PROOF_DEPTH'].value, } diff --git a/pysetup/spec_builders/eip7732.py b/pysetup/spec_builders/eip7732.py index c9f2ef8496..db3516e81a 100644 --- a/pysetup/spec_builders/eip7732.py +++ b/pysetup/spec_builders/eip7732.py @@ -26,11 +26,11 @@ def concat_generalized_indices(*indices: GeneralizedIndex) -> GeneralizedIndex: @classmethod def deprecate_constants(cls) -> Set[str]: return set([ - 'EXECUTION_PAYLOAD_GINDEX', + 'EXECUTION_PAYLOAD_GINDEX_ELECTRA', ]) @classmethod def deprecate_presets(cls) -> Set[str]: return set([ - 'KZG_COMMITMENT_INCLUSION_PROOF_DEPTH', + 'KZG_COMMITMENT_INCLUSION_PROOF_DEPTH_ELECTRA', ]) diff --git a/pysetup/spec_builders/electra.py b/pysetup/spec_builders/electra.py index f473dbadc3..3a18cd169c 100644 --- a/pysetup/spec_builders/electra.py +++ b/pysetup/spec_builders/electra.py @@ -11,14 +11,22 @@ def imports(cls, preset_name: str): return f''' from eth2spec.deneb import {preset_name} as deneb from eth2spec.utils.ssz.ssz_impl import ssz_serialize, ssz_deserialize +from eth2spec.utils.ssz.ssz_typing import StableContainer, Profile ''' @classmethod def hardcoded_ssz_dep_constants(cls) -> Dict[str, str]: return { - 'FINALIZED_ROOT_GINDEX_ELECTRA': 'GeneralizedIndex(169)', - 'CURRENT_SYNC_COMMITTEE_GINDEX_ELECTRA': 'GeneralizedIndex(86)', - 'NEXT_SYNC_COMMITTEE_GINDEX_ELECTRA': 'GeneralizedIndex(87)', + 'FINALIZED_ROOT_GINDEX_ELECTRA': 'GeneralizedIndex(553)', + 'CURRENT_SYNC_COMMITTEE_GINDEX_ELECTRA': 'GeneralizedIndex(278)', + 'NEXT_SYNC_COMMITTEE_GINDEX_ELECTRA': 'GeneralizedIndex(279)', + 'EXECUTION_PAYLOAD_GINDEX_ELECTRA': 'GeneralizedIndex(137)', + } + + @classmethod + def hardcoded_custom_type_dep_constants(cls, spec_object) -> Dict[str, str]: + return { + 'KZG_COMMITMENT_INCLUSION_PROOF_DEPTH_ELECTRA': spec_object.preset_vars['KZG_COMMITMENT_INCLUSION_PROOF_DEPTH_ELECTRA'].value, } @@ -58,4 +66,4 @@ def verify_and_notify_new_payload(self: ExecutionEngine, return True -EXECUTION_ENGINE = NoopExecutionEngine()""" \ No newline at end of file +EXECUTION_ENGINE = NoopExecutionEngine()""" diff --git a/specs/_features/eip7732/beacon-chain.md b/specs/_features/eip7732/beacon-chain.md index 72e175ae4d..e5537a6c85 100644 --- a/specs/_features/eip7732/beacon-chain.md +++ b/specs/_features/eip7732/beacon-chain.md @@ -25,8 +25,11 @@ - [`ExecutionPayloadEnvelope`](#executionpayloadenvelope) - [`SignedExecutionPayloadEnvelope`](#signedexecutionpayloadenvelope) - [Modified containers](#modified-containers) + - [`StableBeaconBlockBody`](#stablebeaconblockbody) - [`BeaconBlockBody`](#beaconblockbody) + - [`StableExecutionPayloadHeader`](#stableexecutionpayloadheader) - [`ExecutionPayloadHeader`](#executionpayloadheader) + - [`StableBeaconState`](#stablebeaconstate) - [`BeaconState`](#beaconstate) - [Helper functions](#helper-functions) - [Math](#math) @@ -178,12 +181,34 @@ class SignedExecutionPayloadEnvelope(Container): ### Modified containers -#### `BeaconBlockBody` +#### `StableBeaconBlockBody` **Note:** The Beacon Block body is modified to contain a `Signed ExecutionPayloadHeader`. The containers `BeaconBlock` and `SignedBeaconBlock` are modified indirectly. The field `execution_requests` is removed from the beacon block body and moved into the signed execution payload envelope. ```python -class BeaconBlockBody(Container): +class StableBeaconBlockBody(StableContainer[MAX_BEACON_BLOCK_BODY_FIELDS]): + randao_reveal: Optional[BLSSignature] + eth1_data: Optional[Eth1Data] # Eth1 data vote + graffiti: Optional[Bytes32] # Arbitrary data + proposer_slashings: Optional[List[ProposerSlashing, MAX_PROPOSER_SLASHINGS]] + attester_slashings: Optional[List[StableAttesterSlashing, MAX_ATTESTER_SLASHINGS_ELECTRA]] + attestations: Optional[List[Attestation, MAX_ATTESTATIONS_ELECTRA]] + deposits: Optional[List[Deposit, MAX_DEPOSITS]] + voluntary_exits: Optional[List[SignedVoluntaryExit, MAX_VOLUNTARY_EXITS]] + sync_aggregate: Optional[SyncAggregate] + execution_payload: Optional[StableExecutionPayload] # [Removed in EIP-7732] + bls_to_execution_changes: Optional[List[SignedBLSToExecutionChange, MAX_BLS_TO_EXECUTION_CHANGES]] + blob_kzg_commitments: Optional[List[KZGCommitment, MAX_BLOB_COMMITMENTS_PER_BLOCK]] # [Removed in EIP-7732] + execution_requests: Optional[StableExecutionRequests] # [Removed in EIP-7732] + # PBS + signed_execution_payload_header: Optional[SignedExecutionPayloadHeader] # [New in EIP-7732] + payload_attestations: Optional[List[PayloadAttestation, MAX_PAYLOAD_ATTESTATIONS]] # [New in EIP-7732] +``` + +#### `BeaconBlockBody` + +```python +class BeaconBlockBody(Profile[StableBeaconBlockBody]): randao_reveal: BLSSignature eth1_data: Eth1Data # Eth1 data vote graffiti: Bytes32 # Arbitrary data @@ -204,12 +229,26 @@ class BeaconBlockBody(Container): payload_attestations: List[PayloadAttestation, MAX_PAYLOAD_ATTESTATIONS] # [New in EIP-7732] ``` +#### `StableExecutionPayloadHeader` + +```python +class StableExecutionPayloadHeader(StableContainer[MAX_EXECUTION_PAYLOAD_FIELDS]): + parent_block_hash: Optional[Hash32] + parent_block_root: Optional[Root] + block_hash: Optional[Hash32] + gas_limit: Optional[uint64] + builder_index: Optional[ValidatorIndex] + slot: Optional[Slot] + value: Optional[Gwei] + blob_kzg_commitments_root: Optional[Root] +``` + #### `ExecutionPayloadHeader` **Note:** The `ExecutionPayloadHeader` is modified to only contain the block hash of the committed `ExecutionPayload` in addition to the builder's payment information, gas limit and KZG commitments root to verify the inclusion proofs. ```python -class ExecutionPayloadHeader(Container): +class ExecutionPayloadHeader(Profile[StableExecutionPayloadHeader]): parent_block_hash: Hash32 parent_block_root: Root block_hash: Hash32 @@ -220,12 +259,73 @@ class ExecutionPayloadHeader(Container): blob_kzg_commitments_root: Root ``` +#### `StableBeaconState` + +```python +class StableBeaconState(StableContainer[MAX_BEACON_STATE_FIELDS]): + # Versioning + genesis_time: Optional[uint64] + genesis_validators_root: Optional[Root] + slot: Optional[Slot] + fork: Optional[Fork] + # History + latest_block_header: Optional[BeaconBlockHeader] + block_roots: Optional[Vector[Root, SLOTS_PER_HISTORICAL_ROOT]] + state_roots: Optional[Vector[Root, SLOTS_PER_HISTORICAL_ROOT]] + # Frozen in Capella, replaced by historical_summaries + historical_roots: Optional[List[Root, HISTORICAL_ROOTS_LIMIT]] + # Eth1 + eth1_data: Optional[Eth1Data] + eth1_data_votes: Optional[List[Eth1Data, EPOCHS_PER_ETH1_VOTING_PERIOD * SLOTS_PER_EPOCH]] + eth1_deposit_index: Optional[uint64] + # Registry + validators: Optional[List[Validator, VALIDATOR_REGISTRY_LIMIT]] + balances: Optional[List[Gwei, VALIDATOR_REGISTRY_LIMIT]] + # Randomness + randao_mixes: Optional[Vector[Bytes32, EPOCHS_PER_HISTORICAL_VECTOR]] + # Slashings + slashings: Optional[Vector[Gwei, EPOCHS_PER_SLASHINGS_VECTOR]] # Per-epoch sums of slashed effective balances + # Participation + previous_epoch_participation: Optional[List[ParticipationFlags, VALIDATOR_REGISTRY_LIMIT]] + current_epoch_participation: Optional[List[ParticipationFlags, VALIDATOR_REGISTRY_LIMIT]] + # Finality + justification_bits: Optional[Bitvector[JUSTIFICATION_BITS_LENGTH]] # Bit set for every recent justified epoch + previous_justified_checkpoint: Optional[Checkpoint] + current_justified_checkpoint: Optional[Checkpoint] + finalized_checkpoint: Optional[Checkpoint] + # Inactivity + inactivity_scores: Optional[List[uint64, VALIDATOR_REGISTRY_LIMIT]] + # Sync + current_sync_committee: Optional[SyncCommittee] + next_sync_committee: Optional[SyncCommittee] + # Execution + latest_execution_payload_header: Optional[StableExecutionPayloadHeader] + # Withdrawals + next_withdrawal_index: Optional[WithdrawalIndex] + next_withdrawal_validator_index: Optional[ValidatorIndex] + # Deep history valid from Capella onwards + historical_summaries: Optional[List[HistoricalSummary, HISTORICAL_ROOTS_LIMIT]] + deposit_requests_start_index: Optional[uint64] + deposit_balance_to_consume: Optional[Gwei] + exit_balance_to_consume: Optional[Gwei] + earliest_exit_epoch: Optional[Epoch] + consolidation_balance_to_consume: Optional[Gwei] + earliest_consolidation_epoch: Optional[Epoch] + pending_deposits: Optional[List[PendingDeposit, PENDING_DEPOSITS_LIMIT]] + pending_partial_withdrawals: Optional[List[PendingPartialWithdrawal, PENDING_PARTIAL_WITHDRAWALS_LIMIT]] + pending_consolidations: Optional[List[PendingConsolidation, PENDING_CONSOLIDATIONS_LIMIT]] + # PBS + latest_block_hash: Optional[Hash32] # [New in EIP-7732] + latest_full_slot: Optional[Slot] # [New in EIP-7732] + latest_withdrawals_root: Optional[Root] # [New in EIP-7732] +``` + #### `BeaconState` *Note*: The `BeaconState` is modified to track the last withdrawals honored in the CL. The `latest_execution_payload_header` is modified semantically to refer not to a past committed `ExecutionPayload` but instead it corresponds to the state's slot builder's bid. Another addition is to track the last committed block hash and the last slot that was full, that is in which there were both consensus and execution blocks included. ```python -class BeaconState(Container): +class BeaconState(Profile[StableBeaconState]): # Versioning genesis_time: uint64 genesis_validators_root: Root diff --git a/specs/capella/light-client/full-node.md b/specs/capella/light-client/full-node.md index 61e03e8ae6..9ce1c5e7d2 100644 --- a/specs/capella/light-client/full-node.md +++ b/specs/capella/light-client/full-node.md @@ -45,7 +45,7 @@ def block_to_light_client_header(block: SignedBeaconBlock) -> LightClientHeader: withdrawals_root=hash_tree_root(payload.withdrawals), ) execution_branch = ExecutionBranch( - compute_merkle_proof(block.message.body, EXECUTION_PAYLOAD_GINDEX)) + compute_merkle_proof(block.message.body, execution_payload_gindex_at_slot(block.message.slot))) else: # Note that during fork transitions, `finalized_header` may still point to earlier forks. # While Bellatrix blocks also contain an `ExecutionPayload` (minus `withdrawals_root`), diff --git a/specs/capella/light-client/sync-protocol.md b/specs/capella/light-client/sync-protocol.md index abf44051c4..0619ab3607 100644 --- a/specs/capella/light-client/sync-protocol.md +++ b/specs/capella/light-client/sync-protocol.md @@ -12,6 +12,7 @@ - [Containers](#containers) - [Modified `LightClientHeader`](#modified-lightclientheader) - [Helper functions](#helper-functions) + - [`execution_payload_gindex_at_slot`](#execution_payload_gindex_at_slot) - [`get_lc_execution_root`](#get_lc_execution_root) - [Modified `is_valid_light_client_header`](#modified-is_valid_light_client_header) @@ -53,6 +54,16 @@ class LightClientHeader(Container): ## Helper functions +### `execution_payload_gindex_at_slot` + +```python +def execution_payload_gindex_at_slot(slot: Slot) -> GeneralizedIndex: + epoch = compute_epoch_at_slot(slot) + assert epoch >= CAPELLA_FORK_EPOCH + + return EXECUTION_PAYLOAD_GINDEX +``` + ### `get_lc_execution_root` ```python @@ -77,11 +88,10 @@ def is_valid_light_client_header(header: LightClientHeader) -> bool: and header.execution_branch == ExecutionBranch() ) - return is_valid_merkle_branch( + return is_valid_normalized_merkle_branch( leaf=get_lc_execution_root(header), branch=header.execution_branch, - depth=floorlog2(EXECUTION_PAYLOAD_GINDEX), - index=get_subtree_index(EXECUTION_PAYLOAD_GINDEX), + gindex=execution_payload_gindex_at_slot(header.beacon.slot), root=header.beacon.body_root, ) ``` diff --git a/specs/deneb/light-client/full-node.md b/specs/deneb/light-client/full-node.md index 17f4c50071..666121a425 100644 --- a/specs/deneb/light-client/full-node.md +++ b/specs/deneb/light-client/full-node.md @@ -51,7 +51,7 @@ def block_to_light_client_header(block: SignedBeaconBlock) -> LightClientHeader: execution_header.excess_blob_gas = payload.excess_blob_gas execution_branch = ExecutionBranch( - compute_merkle_proof(block.message.body, EXECUTION_PAYLOAD_GINDEX)) + compute_merkle_proof(block.message.body, execution_payload_gindex_at_slot(block.message.slot))) else: # Note that during fork transitions, `finalized_header` may still point to earlier forks. # While Bellatrix blocks also contain an `ExecutionPayload` (minus `withdrawals_root`), diff --git a/specs/deneb/light-client/sync-protocol.md b/specs/deneb/light-client/sync-protocol.md index 53fb6e975f..a6a60cf6d0 100644 --- a/specs/deneb/light-client/sync-protocol.md +++ b/specs/deneb/light-client/sync-protocol.md @@ -75,11 +75,10 @@ def is_valid_light_client_header(header: LightClientHeader) -> bool: and header.execution_branch == ExecutionBranch() ) - return is_valid_merkle_branch( + return is_valid_normalized_merkle_branch( leaf=get_lc_execution_root(header), branch=header.execution_branch, - depth=floorlog2(EXECUTION_PAYLOAD_GINDEX), - index=get_subtree_index(EXECUTION_PAYLOAD_GINDEX), + gindex=execution_payload_gindex_at_slot(header.beacon.slot), root=header.beacon.body_root, ) ``` diff --git a/specs/deneb/p2p-interface.md b/specs/deneb/p2p-interface.md index 99dbd9fb5b..f2bb6ff76f 100644 --- a/specs/deneb/p2p-interface.md +++ b/specs/deneb/p2p-interface.md @@ -11,6 +11,7 @@ - [Constant](#constant) - [Preset](#preset) - [Configuration](#configuration) + - [Custom types](#custom-types) - [Containers](#containers) - [`BlobSidecar`](#blobsidecar) - [`BlobIdentifier`](#blobidentifier) @@ -70,6 +71,12 @@ The specification of these changes continues in the same format as the network s | `MIN_EPOCHS_FOR_BLOB_SIDECARS_REQUESTS` | `2**12` (= 4096 epochs, ~18 days) | The minimum epoch range over which a node must serve blob sidecars | | `BLOB_SIDECAR_SUBNET_COUNT` | `6` | The number of blob sidecar subnets used in the gossipsub protocol. | +### Custom types + +| Name | SSZ equivalent | Description | +| - | - | - | +| `KZGCommitmentInclusionProof` | `Vector[Bytes32, KZG_COMMITMENT_INCLUSION_PROOF_DEPTH]` | Merkle branch of a single `blob_kzg_commitments` list item within `BeaconBlockBody` | + ### Containers #### `BlobSidecar` @@ -83,7 +90,7 @@ class BlobSidecar(Container): kzg_commitment: KZGCommitment kzg_proof: KZGProof # Allows for quick verification of kzg_commitment signed_block_header: SignedBeaconBlockHeader - kzg_commitment_inclusion_proof: Vector[Bytes32, KZG_COMMITMENT_INCLUSION_PROOF_DEPTH] + kzg_commitment_inclusion_proof: KZGCommitmentInclusionProof ``` #### `BlobIdentifier` @@ -102,12 +109,12 @@ class BlobIdentifier(Container): ```python def verify_blob_sidecar_inclusion_proof(blob_sidecar: BlobSidecar) -> bool: - gindex = get_subtree_index(get_generalized_index(BeaconBlockBody, 'blob_kzg_commitments', blob_sidecar.index)) + gindex = get_generalized_index(BeaconBlockBody, 'blob_kzg_commitments', blob_sidecar.index) return is_valid_merkle_branch( leaf=blob_sidecar.kzg_commitment.hash_tree_root(), branch=blob_sidecar.kzg_commitment_inclusion_proof, - depth=KZG_COMMITMENT_INCLUSION_PROOF_DEPTH, - index=gindex, + depth=floorlog2(gindex), + index=get_subtree_index(gindex), root=blob_sidecar.signed_block_header.message.body_root, ) ``` diff --git a/specs/electra/beacon-chain.md b/specs/electra/beacon-chain.md index cbb3528026..560a4a4c2e 100644 --- a/specs/electra/beacon-chain.md +++ b/specs/electra/beacon-chain.md @@ -10,6 +10,7 @@ - [Introduction](#introduction) - [Constants](#constants) + - [`StableContainer` capacities](#stablecontainer-capacities) - [Misc](#misc) - [Withdrawal prefixes](#withdrawal-prefixes) - [Execution layer triggered requests](#execution-layer-triggered-requests) @@ -32,14 +33,25 @@ - [`DepositRequest`](#depositrequest) - [`WithdrawalRequest`](#withdrawalrequest) - [`ConsolidationRequest`](#consolidationrequest) - - [`ExecutionRequests`](#executionrequests) - [`SingleAttestation`](#singleattestation) + - [`StableContainer` definitions](#stablecontainer-definitions) + - [`StableAttestation`](#stableattestation) + - [`StableIndexedAttestation`](#stableindexedattestation) + - [`StableAttesterSlashing`](#stableattesterslashing) + - [`StableExecutionPayload`](#stableexecutionpayload) + - [`StableExecutionPayloadHeader`](#stableexecutionpayloadheader) + - [`StableExecutionRequests`](#stableexecutionrequests) + - [`StableBeaconBlockBody`](#stablebeaconblockbody) + - [`StableBeaconState`](#stablebeaconstate) - [Modified Containers](#modified-containers) - [`AttesterSlashing`](#attesterslashing) - - [`BeaconBlockBody`](#beaconblockbody) - - [Extended Containers](#extended-containers) + - [`Profile` definitions](#profile-definitions) - [`Attestation`](#attestation) - [`IndexedAttestation`](#indexedattestation) + - [`BeaconBlockBody`](#beaconblockbody) + - [`ExecutionPayload`](#executionpayload) + - [`ExecutionPayloadHeader`](#executionpayloadheader) + - [`ExecutionRequests`](#executionrequests) - [`BeaconState`](#beaconstate) - [Helper functions](#helper-functions) - [Predicates](#predicates) @@ -129,6 +141,17 @@ Electra is a consensus-layer upgrade containing a number of features. Including: The following values are (non-configurable) constants used throughout the specification. +### `StableContainer` capacities + +| Name | Value | Description | +| - | - | - | +| `MAX_ATTESTATION_FIELDS` | `uint64(2**3)` (= 8) | Maximum number of fields to which `StableAttestation` can ever grow in the future | +| `MAX_INDEXED_ATTESTATION_FIELDS` | `uint64(2**3)` (= 8) | Maximum number of fields to which `StableIndexedAttestation` can ever grow in the future | +| `MAX_EXECUTION_PAYLOAD_FIELDS` | `uint64(2**6)` (= 64) | Maximum number of fields to which `StableExecutionPayload` can ever grow in the future | +| `MAX_EXECUTION_REQUESTS_FIELDS` | `uint64(2**4)` (= 16) | Maximum number of fields to which `StableExecutionRequests` can ever grow in the future | +| `MAX_BEACON_BLOCK_BODY_FIELDS` | `uint64(2**6)` (= 64) | Maximum number of fields to which `StableBeaconBlockBody` can ever grow in the future | +| `MAX_BEACON_STATE_FIELDS` | `uint64(2**7)` (= 128) | Maximum number of fields to which `StableBeaconState` can ever grow in the future | + ### Misc | Name | Value | Description | @@ -289,27 +312,202 @@ class ConsolidationRequest(Container): target_pubkey: BLSPubkey ``` -#### `ExecutionRequests` +#### `SingleAttestation` + +```python +class SingleAttestation(Container): + committee_index: CommitteeIndex + attester_index: ValidatorIndex + data: AttestationData + signature: BLSSignature +``` + +### `StableContainer` definitions + +These definitions provide EIP-7495 forward-compatibility guarantees. `Profile` based on these `StableContainer` definitions retain their Merkleization when rebased to `StableContainer` definitions of future forks. + +#### `StableAttestation` + +*Note*: The `StableContainer` is new in EIP7688. + +```python +class StableAttestation(StableContainer[MAX_ATTESTATION_FIELDS]): + aggregation_bits: Optional[Bitlist[MAX_VALIDATORS_PER_COMMITTEE * MAX_COMMITTEES_PER_SLOT]] + data: Optional[AttestationData] + signature: Optional[BLSSignature] + committee_bits: Optional[Bitvector[MAX_COMMITTEES_PER_SLOT]] +``` + +#### `StableIndexedAttestation` + +*Note*: The `StableContainer` is new in EIP7688. + +```python +class StableIndexedAttestation(StableContainer[MAX_INDEXED_ATTESTATION_FIELDS]): + attesting_indices: Optional[List[ValidatorIndex, MAX_VALIDATORS_PER_COMMITTEE * MAX_COMMITTEES_PER_SLOT]] + data: Optional[AttestationData] + signature: Optional[BLSSignature] +``` + +#### `StableAttesterSlashing` + +*Note*: The `StableContainer` is new in EIP7688. + +```python +class StableAttesterSlashing(Container): + attestation_1: StableIndexedAttestation + attestation_2: StableIndexedAttestation +``` + +#### `StableExecutionPayload` + +*Note*: The `StableContainer` is new in EIP7688. -*Note*: This container holds requests from the execution layer that are received in [ +```python +class StableExecutionPayload(StableContainer[MAX_EXECUTION_PAYLOAD_FIELDS]): + parent_hash: Optional[Hash32] + fee_recipient: Optional[ExecutionAddress] # 'beneficiary' in the yellow paper + state_root: Optional[Bytes32] + receipts_root: Optional[Bytes32] + logs_bloom: Optional[ByteVector[BYTES_PER_LOGS_BLOOM]] + prev_randao: Optional[Bytes32] # 'difficulty' in the yellow paper + block_number: Optional[uint64] # 'number' in the yellow paper + gas_limit: Optional[uint64] + gas_used: Optional[uint64] + timestamp: Optional[uint64] + extra_data: Optional[ByteList[MAX_EXTRA_DATA_BYTES]] + base_fee_per_gas: Optional[uint256] + block_hash: Optional[Hash32] # Hash of execution block + transactions: Optional[List[Transaction, MAX_TRANSACTIONS_PER_PAYLOAD]] + withdrawals: Optional[List[Withdrawal, MAX_WITHDRAWALS_PER_PAYLOAD]] # [New in Capella] + blob_gas_used: Optional[uint64] # [New in Deneb:EIP4844] + excess_blob_gas: Optional[uint64] # [New in Deneb:EIP4844] +``` + +#### `StableExecutionPayloadHeader` + +*Note*: The `StableContainer` is new in EIP7688. + +```python +class StableExecutionPayloadHeader(StableContainer[MAX_EXECUTION_PAYLOAD_FIELDS]): + parent_hash: Optional[Hash32] + fee_recipient: Optional[ExecutionAddress] + state_root: Optional[Bytes32] + receipts_root: Optional[Bytes32] + logs_bloom: Optional[ByteVector[BYTES_PER_LOGS_BLOOM]] + prev_randao: Optional[Bytes32] + block_number: Optional[uint64] + gas_limit: Optional[uint64] + gas_used: Optional[uint64] + timestamp: Optional[uint64] + extra_data: Optional[ByteList[MAX_EXTRA_DATA_BYTES]] + base_fee_per_gas: Optional[uint256] + block_hash: Optional[Hash32] # Hash of execution block + transactions_root: Optional[Root] + withdrawals_root: Optional[Root] # [New in Capella] + blob_gas_used: Optional[uint64] # [New in Deneb:EIP4844] + excess_blob_gas: Optional[uint64] # [New in Deneb:EIP4844] +``` + +#### `StableExecutionRequests` + +*Note*: This `StableContainer` holds requests from the execution layer that are received in [ `ExecutionPayloadV4`](https://github.com/ethereum/execution-apis/blob/main/src/engine/prague.md#executionpayloadv4) via the Engine API. These requests are required for CL state transition (see `BeaconBlockBody`). ```python -class ExecutionRequests(Container): - deposits: List[DepositRequest, MAX_DEPOSIT_REQUESTS_PER_PAYLOAD] # [New in Electra:EIP6110] - withdrawals: List[WithdrawalRequest, MAX_WITHDRAWAL_REQUESTS_PER_PAYLOAD] # [New in Electra:EIP7002:EIP7251] - consolidations: List[ConsolidationRequest, MAX_CONSOLIDATION_REQUESTS_PER_PAYLOAD] # [New in Electra:EIP7251] +class StableExecutionRequests(StableContainer[MAX_EXECUTION_REQUESTS_FIELDS]): + # [New in Electra:EIP6110] + deposits: Optional[List[DepositRequest, MAX_DEPOSIT_REQUESTS_PER_PAYLOAD]] + # [New in Electra:EIP7002:EIP7251] + withdrawals: Optional[List[WithdrawalRequest, MAX_WITHDRAWAL_REQUESTS_PER_PAYLOAD]] + # [New in Electra:EIP7251] + consolidations: Optional[List[ConsolidationRequest, MAX_CONSOLIDATION_REQUESTS_PER_PAYLOAD]] ``` -#### `SingleAttestation` +#### `StableBeaconBlockBody` + +*Note*: The `StableContainer` is new in EIP7688. ```python -class SingleAttestation(Container): - committee_index: CommitteeIndex - attester_index: ValidatorIndex - data: AttestationData - signature: BLSSignature +class StableBeaconBlockBody(StableContainer[MAX_BEACON_BLOCK_BODY_FIELDS]): + randao_reveal: Optional[BLSSignature] + eth1_data: Optional[Eth1Data] # Eth1 data vote + graffiti: Optional[Bytes32] # Arbitrary data + proposer_slashings: Optional[List[ProposerSlashing, MAX_PROPOSER_SLASHINGS]] + # [Modified in Electra:EIP7549] + attester_slashings: Optional[List[StableAttesterSlashing, MAX_ATTESTER_SLASHINGS_ELECTRA]] + attestations: Optional[List[StableAttestation, MAX_ATTESTATIONS_ELECTRA]] # [Modified in Electra:EIP7549] + deposits: Optional[List[Deposit, MAX_DEPOSITS]] + voluntary_exits: Optional[List[SignedVoluntaryExit, MAX_VOLUNTARY_EXITS]] + sync_aggregate: Optional[SyncAggregate] # [New in Altair] + execution_payload: Optional[StableExecutionPayload] # [New in Bellatrix] + # [New in Capella] + bls_to_execution_changes: Optional[List[SignedBLSToExecutionChange, MAX_BLS_TO_EXECUTION_CHANGES]] + blob_kzg_commitments: Optional[List[KZGCommitment, MAX_BLOB_COMMITMENTS_PER_BLOCK]] # [New in Deneb:EIP4844] + execution_requests: Optional[StableExecutionRequests] # [New in Electra] +``` + +#### `StableBeaconState` + +*Note*: The `StableContainer` is new in EIP7688. + +```python +class StableBeaconState(StableContainer[MAX_BEACON_STATE_FIELDS]): + # Versioning + genesis_time: Optional[uint64] + genesis_validators_root: Optional[Root] + slot: Optional[Slot] + fork: Optional[Fork] + # History + latest_block_header: Optional[BeaconBlockHeader] + block_roots: Optional[Vector[Root, SLOTS_PER_HISTORICAL_ROOT]] + state_roots: Optional[Vector[Root, SLOTS_PER_HISTORICAL_ROOT]] + # Frozen in Capella, replaced by historical_summaries + historical_roots: Optional[List[Root, HISTORICAL_ROOTS_LIMIT]] + # Eth1 + eth1_data: Optional[Eth1Data] + eth1_data_votes: Optional[List[Eth1Data, EPOCHS_PER_ETH1_VOTING_PERIOD * SLOTS_PER_EPOCH]] + eth1_deposit_index: Optional[uint64] + # Registry + validators: Optional[List[Validator, VALIDATOR_REGISTRY_LIMIT]] + balances: Optional[List[Gwei, VALIDATOR_REGISTRY_LIMIT]] + # Randomness + randao_mixes: Optional[Vector[Bytes32, EPOCHS_PER_HISTORICAL_VECTOR]] + # Slashings + slashings: Optional[Vector[Gwei, EPOCHS_PER_SLASHINGS_VECTOR]] # Per-epoch sums of slashed effective balances + # Participation + previous_epoch_participation: Optional[List[ParticipationFlags, VALIDATOR_REGISTRY_LIMIT]] # [Modified in Altair] + current_epoch_participation: Optional[List[ParticipationFlags, VALIDATOR_REGISTRY_LIMIT]] # [Modified in Altair] + # Finality + justification_bits: Optional[Bitvector[JUSTIFICATION_BITS_LENGTH]] # Bit set for every recent justified epoch + previous_justified_checkpoint: Optional[Checkpoint] + current_justified_checkpoint: Optional[Checkpoint] + finalized_checkpoint: Optional[Checkpoint] + # Inactivity + inactivity_scores: Optional[List[uint64, VALIDATOR_REGISTRY_LIMIT]] # [New in Altair] + # Sync + current_sync_committee: Optional[SyncCommittee] # [New in Altair] + next_sync_committee: Optional[SyncCommittee] # [New in Altair] + # Execution + latest_execution_payload_header: Optional[StableExecutionPayloadHeader] # [New in Bellatrix] + # Withdrawals + next_withdrawal_index: Optional[WithdrawalIndex] # [New in Capella] + next_withdrawal_validator_index: Optional[ValidatorIndex] # [New in Capella] + # Deep history valid from Capella onwards + historical_summaries: Optional[List[HistoricalSummary, HISTORICAL_ROOTS_LIMIT]] # [New in Capella] + deposit_requests_start_index: Optional[uint64] # [New in Electra:EIP6110] + deposit_balance_to_consume: Optional[Gwei] # [New in Electra:EIP7251] + exit_balance_to_consume: Optional[Gwei] # [New in Electra:EIP7251] + earliest_exit_epoch: Optional[Epoch] # [New in Electra:EIP7251] + consolidation_balance_to_consume: Optional[Gwei] # [New in Electra:EIP7251] + earliest_consolidation_epoch: Optional[Epoch] # [New in Electra:EIP7251] + # [New in Electra:EIP7251] + pending_deposits: Optional[List[PendingDeposit, PENDING_DEPOSITS_LIMIT]] + # [New in Electra:EIP7251] + pending_partial_withdrawals: Optional[List[PendingPartialWithdrawal, PENDING_PARTIAL_WITHDRAWALS_LIMIT]] + # [New in Electra:EIP7251] + pending_consolidations: Optional[List[PendingConsolidation, PENDING_CONSOLIDATIONS_LIMIT]] ``` ### Modified Containers @@ -322,10 +520,32 @@ class AttesterSlashing(Container): attestation_2: IndexedAttestation # [Modified in Electra:EIP7549] ``` +### `Profile` definitions + +#### `Attestation` + +```python +class Attestation(Profile[StableAttestation]): + aggregation_bits: Bitlist[MAX_VALIDATORS_PER_COMMITTEE * MAX_COMMITTEES_PER_SLOT] # [Modified in Electra:EIP7549] + data: AttestationData + signature: BLSSignature + committee_bits: Bitvector[MAX_COMMITTEES_PER_SLOT] # [New in Electra:EIP7549] +``` + +#### `IndexedAttestation` + +```python +class IndexedAttestation(Profile[StableIndexedAttestation]): + # [Modified in Electra:EIP7549] + attesting_indices: List[ValidatorIndex, MAX_VALIDATORS_PER_COMMITTEE * MAX_COMMITTEES_PER_SLOT] + data: AttestationData + signature: BLSSignature +``` + #### `BeaconBlockBody` ```python -class BeaconBlockBody(Container): +class BeaconBlockBody(Profile[StableBeaconBlockBody]): randao_reveal: BLSSignature eth1_data: Eth1Data # Eth1 data vote graffiti: Bytes32 # Arbitrary data @@ -343,32 +563,69 @@ class BeaconBlockBody(Container): execution_requests: ExecutionRequests # [New in Electra] ``` -### Extended Containers +#### `ExecutionPayload` -#### `Attestation` +```python +class ExecutionPayload(Profile[StableExecutionPayload]): + # Execution block header fields + parent_hash: Hash32 + fee_recipient: ExecutionAddress + state_root: Bytes32 + receipts_root: Bytes32 + logs_bloom: ByteVector[BYTES_PER_LOGS_BLOOM] + prev_randao: Bytes32 + block_number: uint64 + gas_limit: uint64 + gas_used: uint64 + timestamp: uint64 + extra_data: ByteList[MAX_EXTRA_DATA_BYTES] + base_fee_per_gas: uint256 + # Extra payload fields + block_hash: Hash32 + transactions: List[Transaction, MAX_TRANSACTIONS_PER_PAYLOAD] + withdrawals: List[Withdrawal, MAX_WITHDRAWALS_PER_PAYLOAD] + blob_gas_used: uint64 + excess_blob_gas: uint64 +``` + +#### `ExecutionPayloadHeader` ```python -class Attestation(Container): - aggregation_bits: Bitlist[MAX_VALIDATORS_PER_COMMITTEE * MAX_COMMITTEES_PER_SLOT] # [Modified in Electra:EIP7549] - data: AttestationData - signature: BLSSignature - committee_bits: Bitvector[MAX_COMMITTEES_PER_SLOT] # [New in Electra:EIP7549] +class ExecutionPayloadHeader(Profile[StableExecutionPayloadHeader]): + # Execution block header fields + parent_hash: Hash32 + fee_recipient: ExecutionAddress + state_root: Bytes32 + receipts_root: Bytes32 + logs_bloom: ByteVector[BYTES_PER_LOGS_BLOOM] + prev_randao: Bytes32 + block_number: uint64 + gas_limit: uint64 + gas_used: uint64 + timestamp: uint64 + extra_data: ByteList[MAX_EXTRA_DATA_BYTES] + base_fee_per_gas: uint256 + # Extra payload fields + block_hash: Hash32 + transactions_root: Root + withdrawals_root: Root + blob_gas_used: uint64 + excess_blob_gas: uint64 ``` -#### `IndexedAttestation` +#### `ExecutionRequests` ```python -class IndexedAttestation(Container): - # [Modified in Electra:EIP7549] - attesting_indices: List[ValidatorIndex, MAX_VALIDATORS_PER_COMMITTEE * MAX_COMMITTEES_PER_SLOT] - data: AttestationData - signature: BLSSignature +class ExecutionRequests(Profile[StableExecutionRequests]): + deposits: List[DepositRequest, MAX_DEPOSIT_REQUESTS_PER_PAYLOAD] # [New in Electra:EIP6110] + withdrawals: List[WithdrawalRequest, MAX_WITHDRAWAL_REQUESTS_PER_PAYLOAD] # [New in Electra:EIP7002:EIP7251] + consolidations: List[ConsolidationRequest, MAX_CONSOLIDATION_REQUESTS_PER_PAYLOAD] # [New in Electra:EIP7251] ``` #### `BeaconState` ```python -class BeaconState(Container): +class BeaconState(Profile[StableBeaconState]): # Versioning genesis_time: uint64 genesis_validators_root: Root diff --git a/specs/electra/fork.md b/specs/electra/fork.md index 879e548853..14732b7517 100644 --- a/specs/electra/fork.md +++ b/specs/electra/fork.md @@ -74,6 +74,25 @@ an irregular state change is made to upgrade to Electra. ```python def upgrade_to_electra(pre: deneb.BeaconState) -> BeaconState: epoch = deneb.get_current_epoch(pre) + latest_execution_payload_header = ExecutionPayloadHeader( + parent_hash=pre.latest_execution_payload_header.parent_hash, + fee_recipient=pre.latest_execution_payload_header.fee_recipient, + state_root=pre.latest_execution_payload_header.state_root, + receipts_root=pre.latest_execution_payload_header.receipts_root, + logs_bloom=pre.latest_execution_payload_header.logs_bloom, + prev_randao=pre.latest_execution_payload_header.prev_randao, + block_number=pre.latest_execution_payload_header.block_number, + gas_limit=pre.latest_execution_payload_header.gas_limit, + gas_used=pre.latest_execution_payload_header.gas_used, + timestamp=pre.latest_execution_payload_header.timestamp, + extra_data=pre.latest_execution_payload_header.extra_data, + base_fee_per_gas=pre.latest_execution_payload_header.base_fee_per_gas, + block_hash=pre.latest_execution_payload_header.block_hash, + transactions_root=pre.latest_execution_payload_header.transactions_root, + withdrawals_root=pre.latest_execution_payload_header.withdrawals_root, + blob_gas_used=pre.latest_execution_payload_header.blob_gas_used, + excess_blob_gas=pre.latest_execution_payload_header.excess_blob_gas, + ) earliest_exit_epoch = compute_activation_exit_epoch(get_current_epoch(pre)) for validator in pre.validators: @@ -122,7 +141,7 @@ def upgrade_to_electra(pre: deneb.BeaconState) -> BeaconState: current_sync_committee=pre.current_sync_committee, next_sync_committee=pre.next_sync_committee, # Execution-layer - latest_execution_payload_header=pre.latest_execution_payload_header, + latest_execution_payload_header=latest_execution_payload_header, # Withdrawals next_withdrawal_index=pre.next_withdrawal_index, next_withdrawal_validator_index=pre.next_withdrawal_validator_index, diff --git a/specs/electra/light-client/fork.md b/specs/electra/light-client/fork.md index da44ee4e40..7f73c24058 100644 --- a/specs/electra/light-client/fork.md +++ b/specs/electra/light-client/fork.md @@ -41,8 +41,27 @@ An Electra `LightClientStore` can still process earlier light client data. In or def upgrade_lc_header_to_electra(pre: deneb.LightClientHeader) -> LightClientHeader: return LightClientHeader( beacon=pre.beacon, - execution=pre.execution, - execution_branch=pre.execution_branch, + execution=ExecutionPayloadHeader( + parent_hash=pre.execution.parent_hash, + fee_recipient=pre.execution.fee_recipient, + state_root=pre.execution.state_root, + receipts_root=pre.execution.receipts_root, + logs_bloom=pre.execution.logs_bloom, + prev_randao=pre.execution.prev_randao, + block_number=pre.execution.block_number, + gas_limit=pre.execution.gas_limit, + gas_used=pre.execution.gas_used, + timestamp=pre.execution.timestamp, + extra_data=pre.execution.extra_data, + base_fee_per_gas=pre.execution.base_fee_per_gas, + block_hash=pre.execution.block_hash, + transactions_root=pre.execution.transactions_root, + withdrawals_root=pre.execution.withdrawals_root, + blob_gas_used=pre.execution.blob_gas_used, + excess_blob_gas=pre.execution.excess_blob_gas, + ), + execution_branch=normalize_merkle_branch( + pre.execution_branch, EXECUTION_PAYLOAD_GINDEX_ELECTRA), ) ``` diff --git a/specs/electra/light-client/full-node.md b/specs/electra/light-client/full-node.md new file mode 100644 index 0000000000..8c1d7424b8 --- /dev/null +++ b/specs/electra/light-client/full-node.md @@ -0,0 +1,74 @@ +# Electra Light Client -- Full Node + +**Notice**: This document is a work-in-progress for researchers and implementers. + +## Table of contents + + + + + +- [Introduction](#introduction) +- [Helper functions](#helper-functions) + - [Modified `block_to_light_client_header`](#modified-block_to_light_client_header) + + + + +## Introduction + +Execution payload data is updated to account for the Electra upgrade. + +## Helper functions + +### Modified `block_to_light_client_header` + +```python +def block_to_light_client_header(block: SignedBeaconBlock) -> LightClientHeader: + epoch = compute_epoch_at_slot(block.message.slot) + + if epoch >= CAPELLA_FORK_EPOCH: + payload = block.message.body.execution_payload + execution_header = ExecutionPayloadHeader( + parent_hash=payload.parent_hash, + fee_recipient=payload.fee_recipient, + state_root=payload.state_root, + receipts_root=payload.receipts_root, + logs_bloom=payload.logs_bloom, + prev_randao=payload.prev_randao, + block_number=payload.block_number, + gas_limit=payload.gas_limit, + gas_used=payload.gas_used, + timestamp=payload.timestamp, + extra_data=payload.extra_data, + base_fee_per_gas=payload.base_fee_per_gas, + block_hash=payload.block_hash, + transactions_root=hash_tree_root(payload.transactions), + withdrawals_root=hash_tree_root(payload.withdrawals), + ) + if epoch >= DENEB_FORK_EPOCH: + execution_header.blob_gas_used = payload.blob_gas_used + execution_header.excess_blob_gas = payload.excess_blob_gas + + execution_branch = ExecutionBranch( + compute_merkle_proof(block.message.body, execution_payload_gindex_at_slot(block.message.slot))) + else: + # Note that during fork transitions, `finalized_header` may still point to earlier forks. + # While Bellatrix blocks also contain an `ExecutionPayload` (minus `withdrawals_root`), + # it was not included in the corresponding light client data. To ensure compatibility + # with legacy data going through `upgrade_lc_header_to_capella`, leave out execution data. + execution_header = ExecutionPayloadHeader() + execution_branch = ExecutionBranch() + + return LightClientHeader( + beacon=BeaconBlockHeader( + slot=block.message.slot, + proposer_index=block.message.proposer_index, + parent_root=block.message.parent_root, + state_root=block.message.state_root, + body_root=hash_tree_root(block.message.body), + ), + execution=execution_header, + execution_branch=execution_branch, + ) +``` diff --git a/specs/electra/light-client/sync-protocol.md b/specs/electra/light-client/sync-protocol.md index e3e41cfb79..136705c435 100644 --- a/specs/electra/light-client/sync-protocol.md +++ b/specs/electra/light-client/sync-protocol.md @@ -17,7 +17,9 @@ - [Modified `finalized_root_gindex_at_slot`](#modified-finalized_root_gindex_at_slot) - [Modified `current_sync_committee_gindex_at_slot`](#modified-current_sync_committee_gindex_at_slot) - [Modified `next_sync_committee_gindex_at_slot`](#modified-next_sync_committee_gindex_at_slot) + - [Modified `execution_payload_gindex_at_slot`](#modified-execution_payload_gindex_at_slot) - [Modified `get_lc_execution_root`](#modified-get_lc_execution_root) + - [Modified `is_valid_light_client_header`](#modified-is_valid_light_client_header) @@ -27,6 +29,7 @@ This upgrade updates light client data to include the Electra changes to the [`ExecutionPayload`](../beacon-chain.md) structure and to the generalized indices of surrounding containers. It extends the [Deneb Light Client specifications](../../deneb/light-client/sync-protocol.md). The [fork document](./fork.md) explains how to upgrade existing Deneb based deployments to Electra. Additional documents describes the impact of the upgrade on certain roles: +- [Full node](./full-node.md) - [Networking](./p2p-interface.md) ## Custom types @@ -36,26 +39,29 @@ Additional documents describes the impact of the upgrade on certain roles: | `FinalityBranch` | `Vector[Bytes32, floorlog2(FINALIZED_ROOT_GINDEX_ELECTRA)]` | Merkle branch of `finalized_checkpoint.root` within `BeaconState` | | `CurrentSyncCommitteeBranch` | `Vector[Bytes32, floorlog2(CURRENT_SYNC_COMMITTEE_GINDEX_ELECTRA)]` | Merkle branch of `current_sync_committee` within `BeaconState` | | `NextSyncCommitteeBranch` | `Vector[Bytes32, floorlog2(NEXT_SYNC_COMMITTEE_GINDEX_ELECTRA)]` | Merkle branch of `next_sync_committee` within `BeaconState` | +| `ExecutionBranch` | `Vector[Bytes32, floorlog2(EXECUTION_PAYLOAD_GINDEX_ELECTRA)]` | Merkle branch of `execution_payload` within `BeaconBlockBody` | ## Constants ### Frozen constants -Existing `GeneralizedIndex` constants are frozen at their [Altair](../../altair/light-client/sync-protocol.md#constants) values. +Existing `GeneralizedIndex` constants are frozen at their [Altair](../../altair/light-client/sync-protocol.md#constants) and [Capella](../../capella/light-client/sync-protocol.md#constants) values. | Name | Value | | - | - | | `FINALIZED_ROOT_GINDEX` | `get_generalized_index(altair.BeaconState, 'finalized_checkpoint', 'root')` (= 105) | | `CURRENT_SYNC_COMMITTEE_GINDEX` | `get_generalized_index(altair.BeaconState, 'current_sync_committee')` (= 54) | | `NEXT_SYNC_COMMITTEE_GINDEX` | `get_generalized_index(altair.BeaconState, 'next_sync_committee')` (= 55) | +| `EXECUTION_PAYLOAD_GINDEX` | `get_generalized_index(capella.BeaconBlockBody, 'execution_payload')` (= 25) | ### New constants | Name | Value | | - | - | -| `FINALIZED_ROOT_GINDEX_ELECTRA` | `get_generalized_index(BeaconState, 'finalized_checkpoint', 'root')` (= 169) | -| `CURRENT_SYNC_COMMITTEE_GINDEX_ELECTRA` | `get_generalized_index(BeaconState, 'current_sync_committee')` (= 86) | -| `NEXT_SYNC_COMMITTEE_GINDEX_ELECTRA` | `get_generalized_index(BeaconState, 'next_sync_committee')` (= 87) | +| `FINALIZED_ROOT_GINDEX_ELECTRA` | `get_generalized_index(BeaconState, 'finalized_checkpoint', 'root')` (= 553) | +| `CURRENT_SYNC_COMMITTEE_GINDEX_ELECTRA` | `get_generalized_index(BeaconState, 'current_sync_committee')` (= 278) | +| `NEXT_SYNC_COMMITTEE_GINDEX_ELECTRA` | `get_generalized_index(BeaconState, 'next_sync_committee')` (= 279) | +| `EXECUTION_PAYLOAD_GINDEX_ELECTRA` | `get_generalized_index(BeaconBlockBody, 'execution_payload')` (= 137) | ## Helper functions @@ -95,6 +101,19 @@ def next_sync_committee_gindex_at_slot(slot: Slot) -> GeneralizedIndex: return NEXT_SYNC_COMMITTEE_GINDEX ``` +### Modified `execution_payload_gindex_at_slot` + +```python +def execution_payload_gindex_at_slot(slot: Slot) -> GeneralizedIndex: + epoch = compute_epoch_at_slot(slot) + assert epoch >= CAPELLA_FORK_EPOCH + + # [Modified in Electra] + if epoch >= ELECTRA_FORK_EPOCH: + return EXECUTION_PAYLOAD_GINDEX_ELECTRA + return EXECUTION_PAYLOAD_GINDEX +``` + ### Modified `get_lc_execution_root` ```python @@ -150,3 +169,27 @@ def get_lc_execution_root(header: LightClientHeader) -> Root: return Root() ``` + +### Modified `is_valid_light_client_header` + +```python +def is_valid_light_client_header(header: LightClientHeader) -> bool: + epoch = compute_epoch_at_slot(header.beacon.slot) + + if epoch < DENEB_FORK_EPOCH: + if header.execution.blob_gas_used != uint64(0) or header.execution.excess_blob_gas != uint64(0): + return False + + if epoch < CAPELLA_FORK_EPOCH: + return ( + header.execution == ExecutionPayloadHeader() + and header.execution_branch == ExecutionBranch() + ) + + return is_valid_normalized_merkle_branch( + leaf=get_lc_execution_root(header), + branch=header.execution_branch, + gindex=execution_payload_gindex_at_slot(header.beacon.slot), + root=header.beacon.body_root, + ) +``` diff --git a/specs/electra/p2p-interface.md b/specs/electra/p2p-interface.md index 7a15a88200..ab0b7c4147 100644 --- a/specs/electra/p2p-interface.md +++ b/specs/electra/p2p-interface.md @@ -10,7 +10,9 @@ - [Introduction](#introduction) - [Modifications in Electra](#modifications-in-electra) + - [Preset](#preset) - [Configuration](#configuration) + - [Custom types](#custom-types) - [The gossip domain: gossipsub](#the-gossip-domain-gossipsub) - [Topics and messages](#topics-and-messages) - [Global topics](#global-topics) @@ -37,6 +39,20 @@ The specification of these changes continues in the same format as the network s ## Modifications in Electra +### Preset + +Existing `PROOF_DEPTH` presets are frozen at their [Deneb](../../deneb/p2p-interface.md#preset) values. + +| Name | Value | +|------------------------------------------|-----------------------------------| +| `KZG_COMMITMENT_INCLUSION_PROOF_DEPTH` | `uint64(floorlog2(get_generalized_index(deneb.BeaconBlockBody, 'blob_kzg_commitments')) + 1 + ceillog2(MAX_BLOB_COMMITMENTS_PER_BLOCK))` (= 17) | + +*[New in Electra:EIP7688]* + +| Name | Value | Description | +|------------------------------------------|-----------------------------------|---------------------------------------------------------------------| +| `KZG_COMMITMENT_INCLUSION_PROOF_DEPTH_ELECTRA` | `uint64(floorlog2(get_generalized_index(BeaconBlockBody, 'blob_kzg_commitments')) + 1 + ceillog2(MAX_BLOB_COMMITMENTS_PER_BLOCK))` (= 20) | Merkle proof depth for `blob_kzg_commitments` list item | + ### Configuration *[New in Electra:EIP7691]* @@ -46,6 +62,12 @@ The specification of these changes continues in the same format as the network s | `MAX_REQUEST_BLOB_SIDECARS_ELECTRA` | `MAX_REQUEST_BLOCKS_DENEB * MAX_BLOBS_PER_BLOCK_ELECTRA` | Maximum number of blob sidecars in a single request | | `BLOB_SIDECAR_SUBNET_COUNT_ELECTRA` | `9` | The number of blob sidecar subnets used in the gossipsub protocol | +### Custom types + +| Name | SSZ equivalent | Description | +| - | - | - | +| `KZGCommitmentInclusionProof` | `Vector[Bytes32, KZG_COMMITMENT_INCLUSION_PROOF_DEPTH_ELECTRA]` | Merkle branch of a single `blob_kzg_commitments` list item within `BeaconBlockBody` | + ### The gossip domain: gossipsub Some gossip meshes are upgraded in the fork of Electra to support upgraded types. diff --git a/tests/core/pyspec/eth2spec/test/capella/light_client/test_single_merkle_proof.py b/tests/core/pyspec/eth2spec/test/capella/light_client/test_single_merkle_proof.py index 7f414ab285..cb69f3fdee 100644 --- a/tests/core/pyspec/eth2spec/test/capella/light_client/test_single_merkle_proof.py +++ b/tests/core/pyspec/eth2spec/test/capella/light_client/test_single_merkle_proof.py @@ -6,6 +6,9 @@ from eth2spec.test.helpers.attestations import ( state_transition_with_full_block, ) +from eth2spec.test.helpers.light_client import ( + latest_execution_payload_gindex, +) @with_test_suite_name("BeaconBlockBody") @@ -15,7 +18,7 @@ def test_execution_merkle_proof(spec, state): block = state_transition_with_full_block(spec, state, True, False) yield "object", block.message.body - gindex = spec.EXECUTION_PAYLOAD_GINDEX + gindex = latest_execution_payload_gindex(spec) branch = spec.compute_merkle_proof(block.message.body, gindex) yield "proof", { "leaf": "0x" + block.message.body.execution_payload.hash_tree_root().hex(), diff --git a/tests/core/pyspec/eth2spec/test/deneb/unittests/test_config_invariants.py b/tests/core/pyspec/eth2spec/test/deneb/unittests/test_config_invariants.py index 1f44257856..691ff9aa9f 100644 --- a/tests/core/pyspec/eth2spec/test/deneb/unittests/test_config_invariants.py +++ b/tests/core/pyspec/eth2spec/test/deneb/unittests/test_config_invariants.py @@ -5,6 +5,12 @@ ) +def latest_kzg_commitment_inclusion_proof_depth(spec): + if hasattr(spec, 'KZG_COMMITMENT_INCLUSION_PROOF_DEPTH_ELECTRA'): + return spec.KZG_COMMITMENT_INCLUSION_PROOF_DEPTH_ELECTRA + return spec.KZG_COMMITMENT_INCLUSION_PROOF_DEPTH + + @with_deneb_and_later @spec_test @single_phase @@ -25,4 +31,4 @@ def test_networking(spec): assert spec.config.BLOB_SIDECAR_SUBNET_COUNT == spec.config.MAX_BLOBS_PER_BLOCK for i in range(spec.MAX_BLOB_COMMITMENTS_PER_BLOCK): gindex = spec.get_generalized_index(spec.BeaconBlockBody, 'blob_kzg_commitments', i) - assert spec.floorlog2(gindex) == spec.KZG_COMMITMENT_INCLUSION_PROOF_DEPTH + assert spec.floorlog2(gindex) == latest_kzg_commitment_inclusion_proof_depth(spec) diff --git a/tests/core/pyspec/eth2spec/test/helpers/electra/fork.py b/tests/core/pyspec/eth2spec/test/helpers/electra/fork.py index 2aeaaa995d..3ef54692db 100644 --- a/tests/core/pyspec/eth2spec/test/helpers/electra/fork.py +++ b/tests/core/pyspec/eth2spec/test/helpers/electra/fork.py @@ -38,14 +38,12 @@ def run_fork_test(post_spec, pre_state): 'next_withdrawal_index', 'next_withdrawal_validator_index', # Deep history valid from Capella onwards 'historical_summaries', - 'latest_execution_payload_header' - ] for field in stable_fields: assert getattr(pre_state, field) == getattr(post_state, field) # Modified fields - modified_fields = ['fork'] + modified_fields = ['fork', 'latest_execution_payload_header'] for field in modified_fields: assert getattr(pre_state, field) != getattr(post_state, field) diff --git a/tests/core/pyspec/eth2spec/test/helpers/light_client.py b/tests/core/pyspec/eth2spec/test/helpers/light_client.py index b3c1a914be..76a17d76e2 100644 --- a/tests/core/pyspec/eth2spec/test/helpers/light_client.py +++ b/tests/core/pyspec/eth2spec/test/helpers/light_client.py @@ -32,6 +32,12 @@ def latest_next_sync_committee_gindex(spec): return spec.NEXT_SYNC_COMMITTEE_GINDEX +def latest_execution_payload_gindex(spec): + if hasattr(spec, 'EXECUTION_PAYLOAD_GINDEX_ELECTRA'): + return spec.EXECUTION_PAYLOAD_GINDEX_ELECTRA + return spec.EXECUTION_PAYLOAD_GINDEX + + def latest_normalize_merkle_branch(spec, branch, gindex): if hasattr(spec, 'normalize_merkle_branch'): return spec.normalize_merkle_branch(branch, gindex)