src(partial blob response): Framework changes and test added for unavailable blob situations (#2072)

* wip

* fix(plugins/forks): Remove evm_bin usage from forks plugin

* cherry-picked mario pr for removing evm-bin dependency

* mario feedback

---------

Co-authored-by: Mario Vega <[email protected]>

Author: felix314159

src/ethereum_clis/clis/execution_specs.py

@@ -19,10 +19,12 @@
     TransactionException,
 )
 from ethereum_test_forks import Fork
+from pytest_plugins.logging import get_logger
 
 from ..transition_tool import TransitionTool
 
 DAEMON_STARTUP_TIMEOUT_SECONDS = 5
+logger = get_logger(__name__)
 
 
 class ExecutionSpecsTransitionTool(TransitionTool):
@@ -116,7 +118,10 @@ def is_fork_supported(self, fork: Fork) -> bool:
 
         `ethereum-spec-evm` appends newlines to forks in the help string.
         """
-        return (fork.transition_tool_name() + "\n") in self.help_string
+        fork_is_supported = (fork.transition_tool_name() + "\n") in self.help_string
+        logger.debug(f"EELS supports fork {fork}: {fork_is_supported}")
+
+        return fork_is_supported
 
     def _generate_post_args(
         self, t8n_data: TransitionTool.TransitionToolData

src/ethereum_test_execution/blob_transaction.py

@@ -9,10 +9,15 @@
 from ethereum_test_base_types.base_types import Bytes
 from ethereum_test_forks import Fork
 from ethereum_test_rpc import BlobAndProofV1, BlobAndProofV2, EngineRPC, EthRPC
-from ethereum_test_types import NetworkWrappedTransaction, Transaction, TransactionTestMetadata
+from ethereum_test_rpc.types import GetBlobsResponse
+from ethereum_test_types import NetworkWrappedTransaction, Transaction
+from ethereum_test_types.transaction_types import TransactionTestMetadata
+from pytest_plugins.logging import get_logger
 
 from .base import BaseExecute
 
+logger = get_logger(__name__)
+
 
 def versioned_hashes_with_blobs_and_proofs(
     tx: NetworkWrappedTransaction,
@@ -54,6 +59,7 @@ class BlobTransaction(BaseExecute):
     requires_engine_rpc: ClassVar[bool] = True
 
     txs: List[NetworkWrappedTransaction | Transaction]
+    nonexisting_blob_hashes: List[Hash] | None = None
 
     def execute(
         self, fork: Fork, eth_rpc: EthRPC, engine_rpc: EngineRPC | None, request: FixtureRequest
@@ -87,15 +93,39 @@ def execute(
             )
         version = fork.engine_get_blobs_version()
         assert version is not None, "Engine get blobs version is not supported by the fork."
-        blob_response = engine_rpc.get_blobs(list(versioned_hashes.keys()), version=version)
+
+        # ensure that clients respond 'null' when they have no access to at least one blob
+        list_versioned_hashes = list(versioned_hashes.keys())
+        if self.nonexisting_blob_hashes is not None:
+            list_versioned_hashes.extend(self.nonexisting_blob_hashes)
+
+        blob_response: GetBlobsResponse | None = engine_rpc.get_blobs(
+            list_versioned_hashes, version=version
+        )  # noqa: E501
+
+        # if non-existing blob hashes were request then the response must be 'null'
+        if self.nonexisting_blob_hashes is not None:
+            if blob_response is not None:
+                raise ValueError(
+                    f"Non-existing blob hashes were requested and "
+                    "the client was expected to respond with 'null', but instead it replied: "
+                    f"{blob_response.root}"
+                )
+            else:
+                logger.info(
+                    "Test was passed (partial responses are not allowed and the client "
+                    "correctly returned 'null')"
+                )
+                eth_rpc.wait_for_transactions(sent_txs)
+                return
+
+        assert blob_response is not None
         local_blobs_and_proofs = list(versioned_hashes.values())
-        if len(blob_response) != len(local_blobs_and_proofs):
-            raise ValueError(
-                f"Expected {len(local_blobs_and_proofs)} blobs and proofs, "
-                f"got {len(blob_response)}."
-            )
+        assert len(blob_response) == len(local_blobs_and_proofs), "Expected "
+        f"{len(local_blobs_and_proofs)} blobs and proofs, got {len(blob_response)}."
+
         for expected_blob, received_blob in zip(
-            local_blobs_and_proofs, blob_response.root, strict=False
+            local_blobs_and_proofs, blob_response.root, strict=True
         ):
             if received_blob is None:
                 raise ValueError("Received blob is empty.")

src/ethereum_test_rpc/rpc.py

@@ -11,6 +11,7 @@
 
 from ethereum_test_base_types import Address, Bytes, Hash, to_json
 from ethereum_test_types import Transaction
+from pytest_plugins.logging import get_logger
 
 from .types import (
     EthConfigResponse,
@@ -24,6 +25,8 @@
     TransactionByHashResponse,
 )
 
+logger = get_logger(__name__)
+
 BlockNumberType = int | Literal["latest", "earliest", "pending"]
 
 
@@ -405,13 +408,18 @@ def get_blobs(
         versioned_hashes: List[Hash],
         *,
         version: int,
-    ) -> GetBlobsResponse:
+    ) -> GetBlobsResponse | None:
         """`engine_getBlobsVX`: Retrieves blobs from an execution layers tx pool."""
+        response = self.post_request(
+            f"getBlobsV{version}",
+            [f"{h}" for h in versioned_hashes],
+        )
+        if response is None:  # for tests that request non-existing blobs
+            logger.debug("get_blobs response received but it has value: None")
+            return None
+
         return GetBlobsResponse.model_validate(
-            self.post_request(
-                f"getBlobsV{version}",
-                [f"{h}" for h in versioned_hashes],
-            ),
+            response,
             context=self.response_validation_context,
         )
 

src/ethereum_test_specs/blobs.py

@@ -4,6 +4,7 @@
 
 from ethereum_clis import TransitionTool
 from ethereum_test_base_types import Alloc
+from ethereum_test_base_types.base_types import Hash
 from ethereum_test_execution import BaseExecute, BlobTransaction
 from ethereum_test_fixtures import (
     BaseFixture,
@@ -20,6 +21,7 @@ class BlobsTest(BaseTest):
 
     pre: Alloc
     txs: List[NetworkWrappedTransaction | Transaction]
+    nonexisting_blob_hashes: List[Hash] | None = None
 
     supported_execute_formats: ClassVar[Sequence[LabeledExecuteFormat]] = [
         LabeledExecuteFormat(
@@ -48,7 +50,7 @@ def execute(
         """Generate the list of test fixtures."""
         if execute_format == BlobTransaction:
             return BlobTransaction(
-                txs=self.txs,
+                txs=self.txs, nonexisting_blob_hashes=self.nonexisting_blob_hashes
             )
         raise Exception(f"Unsupported execute format: {execute_format}")
 

src/pytest_plugins/forks/forks.py

@@ -25,6 +25,9 @@
     get_transition_forks,
     transition_fork_to,
 )
+from pytest_plugins.logging import get_logger
+
+logger = get_logger(__name__)
 
 
 def pytest_addoption(parser):
@@ -524,6 +527,7 @@ def get_fork_option(config, option_name: str, parameter_name: str) -> Set[Fork]:
         config.unsupported_forks = frozenset(  # type: ignore
             fork for fork in selected_fork_set if not t8n.is_fork_supported(fork)
         )
+        logger.debug(f"List of unsupported forks: {list(config.unsupported_forks)}")  # type: ignore
 
 
 @pytest.hookimpl(trylast=True)

tests/osaka/eip7594_peerdas/test_get_blobs.py

@@ -3,10 +3,12 @@
     Test get blobs engine endpoint for [EIP-7594: PeerDAS - Peer Data Availability Sampling](https://eips.ethereum.org/EIPS/eip-7594).
 """  # noqa: E501
 
+from hashlib import sha256
 from typing import List, Optional
 
 import pytest
 
+from ethereum_test_base_types.base_types import Hash
 from ethereum_test_forks import Fork
 from ethereum_test_tools import (
     Address,
@@ -320,7 +322,20 @@ def test_get_blobs(
     Test valid blob combinations where one or more txs in the block
     serialized version contain a full blob (network version) tx.
     """
-    blobs_test(
-        pre=pre,
-        txs=txs,
-    )
+    blobs_test(pre=pre, txs=txs)
+
+
+@pytest.mark.parametrize_by_fork(
+    "txs_blobs",
+    generate_valid_blob_tests,
+)
+@pytest.mark.exception_test
+@pytest.mark.valid_from("Cancun")
+def test_get_blobs_nonexisting(
+    blobs_test: BlobsTestFiller,
+    pre: Alloc,
+    txs: List[NetworkWrappedTransaction | Transaction],
+):
+    """Test that ensures clients respond with 'null' when at least one requested blob is not available."""  # noqa: E501
+    nonexisting_blob_hashes = [Hash(sha256(str(i).encode()).digest()) for i in range(5)]
+    blobs_test(pre=pre, txs=txs, nonexisting_blob_hashes=nonexisting_blob_hashes)