Merge branch 'dev' into consolidation-switch-to-comp

Author: mkalinin

.github/workflows/generate_vectors.yml

@@ -12,17 +12,19 @@ on:
         default: dev
         type: string
         required: true
+  schedule:
+    - cron:  '0 2 * * *'
 
 jobs:
   generate-tests:
-    runs-on: [self-hosted-ghr-custom, size-chungus-x64, profile-consensusSpecs]
+    runs-on: [self-hosted-ghr-custom, size-xl-x64, profile-consensusSpecs]
     steps:
       - name: Checkout repository
         uses: actions/checkout@v4
         with:
           repository: 'ethereum/consensus-specs'
           path: 'consensus-specs'
-          ref: ${{ inputs.source_ref }}
+          ref: ${{ inputs.ref || 'dev' }}
       - name: Checkout consensus-spec-tests repository
         uses: actions/checkout@v4
         with:
@@ -50,6 +52,16 @@ jobs:
           cp -r presets/ ../consensus-spec-tests/presets
           cp -r configs/ ../consensus-spec-tests/configs
           find . -type d -empty -delete
+      - name: Check for errors
+        run: |
+          if grep -q "\[ERROR\]" consensustestgen.log; then
+            echo "There is an error in the log"
+            exit 1
+          fi
+          if find . -type f -name "INCOMPLETE" | grep -q "INCOMPLETE"; then
+            echo "There is an INCOMPLETE file"
+            exit 1
+          fi
       - name: Archive configurations
         run: |
           cd consensus-spec-tests

Makefile

@@ -34,7 +34,7 @@ MARKDOWN_FILES = $(wildcard $(SPEC_DIR)/*/*.md) \
                  $(wildcard $(SPEC_DIR)/_features/*/*/*.md) \
                  $(wildcard $(SSZ_DIR)/*.md)
 
-ALL_EXECUTABLE_SPEC_NAMES = phase0 altair bellatrix capella deneb electra whisk eip6800 eip7732
+ALL_EXECUTABLE_SPEC_NAMES = phase0 altair bellatrix capella deneb electra whisk eip6800 eip7594 eip7732
 # The parameters for commands. Use `foreach` to avoid listing specs again.
 COVERAGE_SCOPE := $(foreach S,$(ALL_EXECUTABLE_SPEC_NAMES), --cov=eth2spec.$S.$(TEST_PRESET_TYPE))
 PYLINT_SCOPE := $(foreach S,$(ALL_EXECUTABLE_SPEC_NAMES), ./eth2spec/$S)
@@ -96,13 +96,16 @@ dist_check:
 dist_upload:
 	python3 -m twine upload dist/*
 
+build_wheel: install_test pyspec
+	. venv/bin/activate && \
+	python3 -m build --no-isolation --outdir ./dist ./
 
 # "make generate_tests" to run all generators
 generate_tests: $(GENERATOR_TARGETS)
 
 # "make pyspec" to create the pyspec for all phases.
 pyspec:
-	python3 -m venv venv; . venv/bin/activate; python3 setup.py pyspecdev
+	@python3 -m venv venv; . venv/bin/activate; python3 setup.py pyspecdev
 
 # check the setup tool requirements
 preinstallation:
@@ -138,13 +141,21 @@ endif
 open_cov:
 	((open "$(COV_INDEX_FILE)" || xdg-open "$(COV_INDEX_FILE)") &> /dev/null) &
 
+# Check all files and error if any ToC were modified.
 check_toc: $(MARKDOWN_FILES:=.toc)
+	@[ "$$(find . -name '*.md.tmp' -print -quit)" ] && exit 1 || exit 0
 
+# Generate ToC sections & save copy of original if modified.
 %.toc:
-	cp $* $*.tmp && \
-	doctoc $* && \
-	diff -q $* $*.tmp && \
-	rm $*.tmp
+	@cp $* $*.tmp; \
+	doctoc $* > /dev/null; \
+	if diff -q $* $*.tmp > /dev/null; then \
+		echo "Good $*"; \
+		rm $*.tmp; \
+	else \
+		echo "\033[1;33m Bad $*\033[0m"; \
+		echo "\033[1;34m See $*.tmp\033[0m"; \
+	fi
 
 codespell:
 	codespell . --skip "./.git,./venv,$(PY_SPEC_DIR)/.mypy_cache" -I .codespell-whitelist
@@ -195,7 +206,8 @@ define run_generator
 	cd $(GENERATOR_DIR)/$(1); \
 	if ! test -d venv; then python3 -m venv venv; fi; \
 	. venv/bin/activate; \
-	pip3 install -r requirements.txt; \
+	pip3 install ../../../dist/eth2spec-*.whl; \
+	pip3 install 'eth2spec[generator]'; \
 	python3 main.py -o $(CURRENT_DIR)/$(TEST_VECTOR_DIR); \
 	echo "generator $(1) finished"
 endef
@@ -217,7 +229,7 @@ gen_kzg_setups:
 
 # For any generator, build it using the run_generator function.
 # (creation of output dir is a dependency)
-gen_%: $(TEST_VECTOR_DIR)
+gen_%: build_wheel $(TEST_VECTOR_DIR)
 	$(call run_generator,$*)
 
 detect_generator_incomplete: $(TEST_VECTOR_DIR)

README.md

@@ -8,7 +8,7 @@ This repository hosts the current Ethereum proof-of-stake specifications. Discus
 
 ## Specs
 
-[![GitHub release](https://img.shields.io/github/v/release/ethereum/eth2.0-specs)](https://github.com/ethereum/eth2.0-specs/releases/) [![PyPI version](https://badge.fury.io/py/eth2spec.svg)](https://badge.fury.io/py/eth2spec)
+[![GitHub release](https://img.shields.io/github/v/release/ethereum/consensus-specs)](https://github.com/ethereum/consensus-specs/releases/) [![PyPI version](https://badge.fury.io/py/eth2spec.svg)](https://badge.fury.io/py/eth2spec) [![testgen](https://github.com/ethereum/consensus-specs/actions/workflows/generate_vectors.yml/badge.svg?branch=dev&event=schedule)](https://github.com/ethereum/consensus-specs/actions/workflows/generate_vectors.yml)
 
 Core specifications for Ethereum proof-of-stake clients can be found in [specs](specs/). These are divided into features.
 Features are researched and developed in parallel, and then consolidated into sequential upgrades when ready.

presets/minimal/electra.yaml

@@ -41,5 +41,5 @@ MAX_WITHDRAWAL_REQUESTS_PER_PAYLOAD: 2
 
 # Withdrawals processing
 # ---------------------------------------------------------------
-# 2**0 ( = 1) pending withdrawals
-MAX_PENDING_PARTIALS_PER_WITHDRAWALS_SWEEP: 1
+# 2**1 ( = 2) pending withdrawals
+MAX_PENDING_PARTIALS_PER_WITHDRAWALS_SWEEP: 2

pysetup/helpers.py

@@ -119,6 +119,7 @@ def format_constant(name: str, vardef: VariableDefinition) -> str:
     hardcoded_func_dep_presets = reduce(lambda obj, builder: {**obj, **builder.hardcoded_func_dep_presets(spec_object)}, builders, {})
     # Concatenate all strings
     imports =              reduce(lambda txt, builder: (txt + "\n\n" + builder.imports(preset_name)  ).strip("\n"), builders, "")
+    classes =              reduce(lambda txt, builder: (txt + "\n\n" + builder.classes()             ).strip("\n"), builders, "")
     preparations =         reduce(lambda txt, builder: (txt + "\n\n" + builder.preparations()        ).strip("\n"), builders, "")
     sundry_functions =     reduce(lambda txt, builder: (txt + "\n\n" + builder.sundry_functions()    ).strip("\n"), builders, "")
     # Keep engine from the most recent fork
@@ -154,6 +155,8 @@ def format_constant(name: str, vardef: VariableDefinition) -> str:
         constant_vars_spec,
         preset_vars_spec,
         config_spec,
+        # Custom classes which are not required to be SSZ containers.
+        classes,
         ordered_class_objects_spec,
         protocols_spec,
         functions_spec,

pysetup/spec_builders/base.py

@@ -15,6 +15,13 @@ def imports(cls, preset_name: str) -> str:
         """
         return ""
 
+    @classmethod
+    def classes(cls) -> str:
+        """
+        Define special classes.
+        """
+        return ""
+
     @classmethod
     def preparations(cls) -> str:
         """

pysetup/spec_builders/deneb.py

@@ -12,6 +12,21 @@ def imports(cls, preset_name: str):
 from eth2spec.capella import {preset_name} as capella
 '''
 
+    @classmethod
+    def classes(cls):
+        return f'''
+class BLSFieldElement(bls.Scalar):
+    pass
+
+
+class Polynomial(list):
+    def __init__(self, evals: Optional[Sequence[BLSFieldElement]] = None):
+        if evals is None:
+            evals = [BLSFieldElement(0)] * FIELD_ELEMENTS_PER_BLOB
+        if len(evals) != FIELD_ELEMENTS_PER_BLOB:
+            raise ValueError("expected FIELD_ELEMENTS_PER_BLOB evals")
+        super().__init__(evals)
+'''
 
     @classmethod
     def preparations(cls):

pysetup/spec_builders/eip7594.py

@@ -12,12 +12,41 @@ def imports(cls, preset_name: str):
         return f'''
 from eth2spec.deneb import {preset_name} as deneb
 '''
-    
+
+
+    @classmethod
+    def classes(cls):
+        return f'''
+class PolynomialCoeff(list):
+    def __init__(self, coeffs: Sequence[BLSFieldElement]):
+        if len(coeffs) > FIELD_ELEMENTS_PER_EXT_BLOB:
+            raise ValueError("expected <= FIELD_ELEMENTS_PER_EXT_BLOB coeffs")
+        super().__init__(coeffs)
+
+
+class Coset(list):
+    def __init__(self, coeffs: Optional[Sequence[BLSFieldElement]] = None):
+        if coeffs is None:
+            coeffs = [BLSFieldElement(0)] * FIELD_ELEMENTS_PER_CELL
+        if len(coeffs) != FIELD_ELEMENTS_PER_CELL:
+            raise ValueError("expected FIELD_ELEMENTS_PER_CELL coeffs")
+        super().__init__(coeffs)
+
+
+class CosetEvals(list):
+    def __init__(self, evals: Optional[Sequence[BLSFieldElement]] = None):
+        if evals is None:
+            evals = [BLSFieldElement(0)] * FIELD_ELEMENTS_PER_CELL
+        if len(evals) != FIELD_ELEMENTS_PER_CELL:
+            raise ValueError("expected FIELD_ELEMENTS_PER_CELL coeffs")
+        super().__init__(evals)
+'''
 
     @classmethod
     def sundry_functions(cls) -> str:
         return """
 def retrieve_column_sidecars(beacon_block_root: Root) -> Sequence[DataColumnSidecar]:
+    # pylint: disable=unused-argument
     return []
 """
 

pysetup/spec_builders/electra.py

@@ -19,3 +19,41 @@ def hardcoded_ssz_dep_constants(cls) -> Dict[str, str]:
             'CURRENT_SYNC_COMMITTEE_GINDEX_ELECTRA': 'GeneralizedIndex(86)',
             'NEXT_SYNC_COMMITTEE_GINDEX_ELECTRA': 'GeneralizedIndex(87)',
         }
+
+
+    @classmethod
+    def execution_engine_cls(cls) -> str:
+        return """
+class NoopExecutionEngine(ExecutionEngine):
+
+    def notify_new_payload(self: ExecutionEngine,
+                           execution_payload: ExecutionPayload,
+                           execution_requests: ExecutionRequests,
+                           parent_beacon_block_root: Root) -> bool:
+        return True
+
+    def notify_forkchoice_updated(self: ExecutionEngine,
+                                  head_block_hash: Hash32,
+                                  safe_block_hash: Hash32,
+                                  finalized_block_hash: Hash32,
+                                  payload_attributes: Optional[PayloadAttributes]) -> Optional[PayloadId]:
+        pass
+
+    def get_payload(self: ExecutionEngine, payload_id: PayloadId) -> GetPayloadResponse:
+        # pylint: disable=unused-argument
+        raise NotImplementedError("no default block production")
+
+    def is_valid_block_hash(self: ExecutionEngine,
+                            execution_payload: ExecutionPayload,
+                            parent_beacon_block_root: Root) -> bool:
+        return True
+
+    def is_valid_versioned_hashes(self: ExecutionEngine, new_payload_request: NewPayloadRequest) -> bool:
+        return True
+
+    def verify_and_notify_new_payload(self: ExecutionEngine,
+                                      new_payload_request: NewPayloadRequest) -> bool:
+        return True
+
+
+EXECUTION_ENGINE = NoopExecutionEngine()"""

requirements_preinstallation.txt

@@ -1,4 +1,5 @@
 pip>=24.0.0
 wheel>=0.44.0
 setuptools>=72.0.0
-pylint>=3.2.0
+pylint>=3.2.0
+build>=1.2.2