Merge pull request #4422 from ethereum/dev

Pull in fixes from dev for v1.6.0-alpha.3 release

Author: jtraglia

tests/core/pyspec/eth2spec/test/fulu/fork_choice/test_on_block.py

@@ -31,7 +31,10 @@ def get_alt_sidecars(spec, state):
     Get alternative sidecars for negative test cases.
     """
     rng = Random(4321)
-    _, _, _, _, alt_sidecars = get_block_with_blob_and_sidecars(spec, state, rng=rng, blob_count=2)
+    state_copy = state.copy()
+    _, _, _, _, alt_sidecars = get_block_with_blob_and_sidecars(
+        spec, state_copy, rng=rng, blob_count=2
+    )
     return alt_sidecars
 
 
@@ -55,20 +58,20 @@ def test_on_block_peerdas__ok(spec, state):
     assert store.time == current_time
 
     # On receiving a block of `GENESIS_SLOT + 1` slot
-    _, blobs, blob_kzg_proofs, signed_block, sidecars = get_block_with_blob_and_sidecars(
+    _, _, _, signed_block, sidecars = get_block_with_blob_and_sidecars(
         spec, state, rng=rng, blob_count=2
     )
-    blob_data = BlobData(blobs, blob_kzg_proofs, sidecars)
+    blob_data = BlobData(sidecars=sidecars)
 
     yield from tick_and_add_block_with_data(spec, store, signed_block, test_steps, blob_data)
 
     assert spec.get_head(store) == signed_block.message.hash_tree_root()
 
     # On receiving a block of next epoch
-    _, blobs, blob_kzg_proofs, signed_block, sidecars = get_block_with_blob_and_sidecars(
+    _, _, _, signed_block, sidecars = get_block_with_blob_and_sidecars(
         spec, state, rng=rng, blob_count=2
     )
-    blob_data = BlobData(blobs, blob_kzg_proofs, sidecars)
+    blob_data = BlobData(sidecars=sidecars)
 
     yield from tick_and_add_block_with_data(spec, store, signed_block, test_steps, blob_data)
 
@@ -93,11 +96,11 @@ def run_on_block_peerdas_invalid_test(spec, state, fn):
     on_tick_and_append_step(spec, store, current_time, test_steps)
     assert store.time == current_time
 
-    _, blobs, blob_kzg_proofs, signed_block, sidecars = get_block_with_blob_and_sidecars(
+    _, _, _, signed_block, sidecars = get_block_with_blob_and_sidecars(
         spec, state, rng=rng, blob_count=2
     )
     sidecars = fn(sidecars)
-    blob_data = BlobData(blobs, blob_kzg_proofs, [])
+    blob_data = BlobData(sidecars=sidecars)
 
     yield from tick_and_add_block_with_data(
         spec, store, signed_block, test_steps, blob_data, valid=False
@@ -148,7 +151,7 @@ def invalid_index(sidecars):
         sidecars[0].index = 128  # Invalid index
         return sidecars
 
-    run_on_block_peerdas_invalid_test(spec, state, invalid_index)
+    yield from run_on_block_peerdas_invalid_test(spec, state, invalid_index)
 
 
 @with_fulu_and_later
@@ -162,7 +165,7 @@ def invalid_index(sidecars):
         sidecars[0].index = 256  # Invalid index
         return sidecars
 
-    run_on_block_peerdas_invalid_test(spec, state, invalid_index)
+    yield from run_on_block_peerdas_invalid_test(spec, state, invalid_index)
 
 
 @with_fulu_and_later
@@ -176,7 +179,7 @@ def invalid_mismatch_len_column(sidecars):
         sidecars[0].column = sidecars[0].column[1:]
         return sidecars
 
-    run_on_block_peerdas_invalid_test(spec, state, invalid_mismatch_len_column)
+    yield from run_on_block_peerdas_invalid_test(spec, state, invalid_mismatch_len_column)
 
 
 @with_fulu_and_later
@@ -190,7 +193,7 @@ def invalid_mismatch_len_column(sidecars):
         sidecars[1].column = sidecars[1].column[1:]
         return sidecars
 
-    run_on_block_peerdas_invalid_test(spec, state, invalid_mismatch_len_column)
+    yield from run_on_block_peerdas_invalid_test(spec, state, invalid_mismatch_len_column)
 
 
 @with_fulu_and_later
@@ -204,7 +207,7 @@ def invalid_mismatch_len_kzg_commitments(sidecars):
         sidecars[0].kzg_commitments = sidecars[0].kzg_commitments[1:]
         return sidecars
 
-    run_on_block_peerdas_invalid_test(spec, state, invalid_mismatch_len_kzg_commitments)
+    yield from run_on_block_peerdas_invalid_test(spec, state, invalid_mismatch_len_kzg_commitments)
 
 
 @with_fulu_and_later
@@ -218,7 +221,7 @@ def invalid_mismatch_len_kzg_commitments(sidecars):
         sidecars[1].kzg_commitments = sidecars[1].kzg_commitments[1:]
         return sidecars
 
-    run_on_block_peerdas_invalid_test(spec, state, invalid_mismatch_len_kzg_commitments)
+    yield from run_on_block_peerdas_invalid_test(spec, state, invalid_mismatch_len_kzg_commitments)
 
 
 @with_fulu_and_later
@@ -232,7 +235,7 @@ def invalid_mismatch_len_kzg_proofs(sidecars):
         sidecars[0].kzg_proofs = sidecars[0].kzg_proofs[1:]
         return sidecars
 
-    run_on_block_peerdas_invalid_test(spec, state, invalid_mismatch_len_kzg_proofs)
+    yield from run_on_block_peerdas_invalid_test(spec, state, invalid_mismatch_len_kzg_proofs)
 
 
 @with_fulu_and_later
@@ -246,7 +249,7 @@ def invalid_mismatch_len_kzg_proofs(sidecars):
         sidecars[1].kzg_proofs = sidecars[1].kzg_proofs[1:]
         return sidecars
 
-    run_on_block_peerdas_invalid_test(spec, state, invalid_mismatch_len_kzg_proofs)
+    yield from run_on_block_peerdas_invalid_test(spec, state, invalid_mismatch_len_kzg_proofs)
 
 
 @with_fulu_and_later
@@ -260,7 +263,7 @@ def invalid_wrong_column(sidecars):
         sidecars[0].column[0] = flip_one_bit_in_bytes(sidecars[0].column[0], 80)
         return sidecars
 
-    run_on_block_peerdas_invalid_test(spec, state, invalid_wrong_column)
+    yield from run_on_block_peerdas_invalid_test(spec, state, invalid_wrong_column)
 
 
 @with_fulu_and_later
@@ -274,7 +277,7 @@ def invalid_wrong_column(sidecars):
         sidecars[1].column[1] = flip_one_bit_in_bytes(sidecars[1].column[1], 20)
         return sidecars
 
-    run_on_block_peerdas_invalid_test(spec, state, invalid_wrong_column)
+    yield from run_on_block_peerdas_invalid_test(spec, state, invalid_wrong_column)
 
 
 @with_fulu_and_later
@@ -285,11 +288,11 @@ def test_on_block_peerdas__invalid_wrong_commitment_1(spec, state):
     """
     alt_sidecars = get_alt_sidecars(spec, state)
 
-    def invalid_wrong_commitment(sidecars):
+    def invalid_wrong_commitment(sidecars, alt_sidecars=alt_sidecars):
         sidecars[0].kzg_commitments[0] = alt_sidecars[0].kzg_commitments[0]
         return sidecars
 
-    run_on_block_peerdas_invalid_test(spec, state, invalid_wrong_commitment)
+    yield from run_on_block_peerdas_invalid_test(spec, state, invalid_wrong_commitment)
 
 
 @with_fulu_and_later
@@ -300,11 +303,11 @@ def test_on_block_peerdas__invalid_wrong_commitment_2(spec, state):
     """
     alt_sidecars = get_alt_sidecars(spec, state)
 
-    def invalid_wrong_commitment(sidecars):
+    def invalid_wrong_commitment(sidecars, alt_sidecars=alt_sidecars):
         sidecars[1].kzg_commitments[1] = alt_sidecars[1].kzg_commitments[1]
         return sidecars
 
-    run_on_block_peerdas_invalid_test(spec, state, invalid_wrong_commitment)
+    yield from run_on_block_peerdas_invalid_test(spec, state, invalid_wrong_commitment)
 
 
 @with_fulu_and_later
@@ -315,11 +318,11 @@ def test_on_block_peerdas__invalid_wrong_proof_1(spec, state):
     """
     alt_sidecars = get_alt_sidecars(spec, state)
 
-    def invalid_wrong_proof(sidecars):
+    def invalid_wrong_proof(sidecars, alt_sidecars=alt_sidecars):
         sidecars[0].kzg_proofs[0] = alt_sidecars[0].kzg_proofs[0]
         return sidecars
 
-    run_on_block_peerdas_invalid_test(spec, state, invalid_wrong_proof)
+    yield from run_on_block_peerdas_invalid_test(spec, state, invalid_wrong_proof)
 
 
 @with_fulu_and_later
@@ -330,8 +333,8 @@ def test_on_block_peerdas__invalid_wrong_proof_2(spec, state):
     """
     alt_sidecars = get_alt_sidecars(spec, state)
 
-    def invalid_wrong_proof(sidecars):
+    def invalid_wrong_proof(sidecars, alt_sidecars=alt_sidecars):
         sidecars[1].kzg_proofs[1] = alt_sidecars[1].kzg_proofs[1]
         return sidecars
 
-    run_on_block_peerdas_invalid_test(spec, state, invalid_wrong_proof)
+    yield from run_on_block_peerdas_invalid_test(spec, state, invalid_wrong_proof)

tests/core/pyspec/eth2spec/test/helpers/fork_choice.py

@@ -30,10 +30,16 @@ class BlobData(NamedTuple):
     The return values of ``retrieve_blobs_and_proofs`` helper.
     """
 
-    blobs: Sequence[Any]
+    blobs: Sequence[Any] | None = None
     proofs: Sequence[bytes] | None = None
     sidecars: Sequence[DataColumnSidecar] | None = None
 
+    def is_post_fulu(self):
+        return self.sidecars is not None and self.blobs is None and self.proofs is None
+
+    def is_pre_fulu(self):
+        return self.blobs is not None and self.proofs is not None and self.sidecars is None
+
 
 def with_blob_data(spec, blob_data: BlobData, func):
     if not is_post_fulu(spec):
@@ -245,7 +251,7 @@ def get_sidecar_file_name(sidecar: DataColumnSidecar) -> str:
     """
     Returns the file name for a single sidecar.
     """
-    return f"sidecar_{encode_hex(sidecar.hash_tree_root())}"
+    return f"column_{encode_hex(sidecar.hash_tree_root())}"
 
 
 def on_tick_and_append_step(spec, store, time, test_steps):
@@ -292,25 +298,32 @@ def add_block(
 
     # Check blob_data
     if blob_data is not None:
-        blobs = spec.List[spec.Blob, spec.MAX_BLOB_COMMITMENTS_PER_BLOCK](blob_data.blobs)
-        blobs_root = blobs.hash_tree_root()
-        yield get_blobs_file_name(blobs_root=blobs_root), blobs
+        assert blob_data.is_pre_fulu() or blob_data.is_post_fulu(), "Integrity fail blob_data"
+
+        if blob_data.is_pre_fulu():
+            blobs = spec.List[spec.Blob, spec.MAX_BLOB_COMMITMENTS_PER_BLOCK](blob_data.blobs)
+            blobs_root = blobs.hash_tree_root()
+            yield get_blobs_file_name(blobs_root=blobs_root), blobs
 
-        if blob_data.sidecars is not None:
+        if blob_data.is_post_fulu():
             for sidecar in blob_data.sidecars:
                 yield get_sidecar_file_name(sidecar), sidecar
 
     def _append_step(valid=True):
         if blob_data is not None:
-            step = {
-                "block": get_block_file_name(signed_block),
-                "blobs": get_blobs_file_name(blobs_root=blobs_root),
-                "valid": valid,
-            }
-            if blob_data.proofs is not None:
-                step["proofs"] = [encode_hex(proof) for proof in blob_data.proofs]
-            if blob_data.sidecars is not None:
-                step["sidecars"] = get_sidecars_file_names(blob_data.sidecars)
+            if blob_data.is_post_fulu():
+                step = {
+                    "block": get_block_file_name(signed_block),
+                    "columns": get_sidecars_file_names(blob_data.sidecars),
+                    "valid": valid,
+                }
+            if blob_data.is_pre_fulu():
+                step = {
+                    "block": get_block_file_name(signed_block),
+                    "blobs": get_blobs_file_name(blobs_root=blobs_root),
+                    "proofs": [encode_hex(proof) for proof in blob_data.proofs],
+                    "valid": valid,
+                }
 
             test_steps.append(step)
         else:

tests/formats/fork_choice/README.md

@@ -92,7 +92,7 @@ The parameter that is required for executing `on_block(store, block)`.
     blobs: string           -- optional, the name of the `blobs_<32-byte-root>.ssz_snappy` file.
                                The blobs file content is a `List[Blob, MAX_BLOB_COMMITMENTS_PER_BLOCK]` SSZ object.
     proofs: array of byte48 hex string -- optional, the proofs of blob commitments.
-    sidecars: string        -- optional, array of the names of the `sidecar_<32-byte-root>.ssz_snappy` files.
+    columns: string        -- optional, array of the names of the `column_<32-byte-root>.ssz_snappy` files.
     valid: bool             -- optional, default to `true`.
                                If it's `false`, this execution step is expected to be invalid.
 }
@@ -105,6 +105,15 @@ the expected values from `retrieve_blobs_and_proofs()` helper inside
 `is_data_available()` helper. If these two fields are not provided,
 `retrieve_blobs_and_proofs()` returns empty lists.
 
+`columns` is a new field in Fulu EIP-7594. This field indicate the expected
+values from `retrieve_column_sidecars` helper inside `is_data_available()`
+helper. If this field is an empty array, `retrieve_column_sidecars` should throw
+an exception (not enough data sampled). If this field is not provided,
+`retrieve_column_sidecars` returns an empty list.
+
+Post-Deneb and pre-Fulu, `columns` should not be present. Post-Fulu `blobs` and
+`proofs` should not be present.
+
 After this step, the `store` object may have been updated.
 
 #### `on_merge_block` execution step