feat(tools): Add system contract error tests generator

Author: marioevz

docs/CHANGELOG.md

@@ -23,6 +23,7 @@ Test fixtures for use by clients are available for each release on the [Github r
 #### Tools
 
 - 🔀 `generate_system_contract_deploy_test` test generator has been updated to handle system contracts that are not allowed to be absent when the fork happens ([#1394](https://github.com/ethereum/execution-spec-tests/pull/1394)).
+- ✨ Add `generate_system_contract_error_test` to generate tests on system contracts that invalidate a block in case of error ([#1394](https://github.com/ethereum/execution-spec-tests/pull/1394)).
 
 #### Exceptions
 

src/ethereum_test_tools/__init__.py

@@ -82,7 +82,11 @@
     Yul,
     YulCompiler,
 )
-from .utility.generators import DeploymentTestType, generate_system_contract_deploy_test
+from .utility.generators import (
+    DeploymentTestType,
+    generate_system_contract_deploy_test,
+    generate_system_contract_error_test,
+)
 from .utility.pytest import extend_with_defaults
 
 __all__ = (
@@ -157,6 +161,7 @@
     "compute_eofcreate_address",
     "extend_with_defaults",
     "generate_system_contract_deploy_test",
+    "generate_system_contract_error_test",
     "keccak256",
     "vm",
 )

src/ethereum_test_tools/utility/generators.py

@@ -13,6 +13,8 @@
 from ethereum_test_specs import BlockchainTestFiller
 from ethereum_test_specs.blockchain import Block
 from ethereum_test_types import Alloc, Transaction
+from ethereum_test_vm import Bytecode
+from ethereum_test_vm import Opcodes as Op
 
 
 class DeploymentTestType(Enum):
@@ -23,6 +25,23 @@ class DeploymentTestType(Enum):
     DEPLOY_AFTER_FORK = "deploy_after_fork"
 
 
+class SystemContractTestType(Enum):
+    """Represents the type of system contract test."""
+
+    GAS_LIMIT = "system_contract_reaches_gas_limit"
+    OUT_OF_GAS_ERROR = "system_contract_out_of_gas"
+    REVERT_ERROR = "system_contract_reverts"
+    EXCEPTION_ERROR = "system_contract_throws"
+
+    def param(self):
+        """Return the parameter for the test."""
+        return pytest.param(
+            self,
+            id=self.value,
+            marks=pytest.mark.exception_test if self != SystemContractTestType.GAS_LIMIT else [],
+        )
+
+
 class ContractAddressHasBalance(Enum):
     """Represents whether the target deployment test has a balance before deployment."""
 
@@ -239,3 +258,116 @@ def wrapper(
         return wrapper
 
     return decorator
+
+
+def generate_system_contract_error_test(
+    *,
+    max_gas_limit: int,
+):
+    """
+    Generate a test that verifies the correct behavior when a system contract fails execution.
+
+    Parametrizations required:
+        - system_contract (Address): The address of the system contract to deploy.
+        - valid_from (Fork): The fork from which the test is valid.
+
+    Args:
+        max_gas_limit (int): The maximum gas limit for the system transaction.
+
+    """
+
+    def decorator(func: SystemContractDeployTestFunction):
+        @pytest.mark.parametrize("test_type", [v.param() for v in SystemContractTestType])
+        @pytest.mark.execute(pytest.mark.skip(reason="modifies pre-alloc"))
+        def wrapper(
+            blockchain_test: BlockchainTestFiller,
+            pre: Alloc,
+            test_type: SystemContractTestType,
+            system_contract: Address,
+            fork: Fork,
+        ):
+            modified_system_contract_code = Bytecode()
+
+            # Depending on the test case, we need to modify the system contract code accordingly.
+            if (
+                test_type == SystemContractTestType.GAS_LIMIT
+                or test_type == SystemContractTestType.OUT_OF_GAS_ERROR
+            ):
+                # Run code so that it reaches the gas limit.
+                gas_costs = fork.gas_costs()
+                # The code works by storing N values to storage, and N is calculated based on the
+                # gas costs for the given fork.
+                # This code will only work once, so if the system contract is re-executed
+                # in a subsequent block, it will consume less gas.
+                gas_used_per_storage = (
+                    gas_costs.G_STORAGE_SET + gas_costs.G_COLD_SLOAD + (gas_costs.G_VERY_LOW * 2)
+                )
+                modified_system_contract_code += sum(
+                    Op.SSTORE(i, 1) for i in range(max_gas_limit // gas_used_per_storage)
+                )
+                # If the gas limit is not divisible by the gas used per storage, we need to add
+                # some NO-OP (JUMPDEST) to the code that each consume 1 gas.
+                assert gas_costs.G_JUMPDEST == 1, (
+                    f"JUMPDEST gas cost should be 1, but got {gas_costs.G_JUMPDEST}. "
+                    "Generator `generate_system_contract_error_test` needs to be updated."
+                )
+                modified_system_contract_code += sum(
+                    Op.JUMPDEST for _ in range(max_gas_limit % gas_used_per_storage)
+                )
+
+                if test_type == SystemContractTestType.OUT_OF_GAS_ERROR:
+                    # If the test type is OUT_OF_GAS_ERROR, we need to add a JUMPDEST to the code
+                    # to ensure that we go over the limit by one gas.
+                    modified_system_contract_code += Op.JUMPDEST
+                modified_system_contract_code += Op.STOP
+            elif test_type == SystemContractTestType.REVERT_ERROR:
+                # Run a simple revert.
+                modified_system_contract_code = Op.REVERT(0, 0)
+            elif test_type == SystemContractTestType.EXCEPTION_ERROR:
+                # Run a simple exception.
+                modified_system_contract_code = Op.INVALID()
+            else:
+                raise ValueError(f"Invalid test type: {test_type}")
+
+            pre[system_contract] = Account(
+                code=modified_system_contract_code,
+                nonce=1,
+                balance=0,
+            )
+
+            # Simple test transaction to verify the block failed to modify the state.
+            value_receiver = pre.fund_eoa(amount=0)
+            test_tx = Transaction(
+                to=value_receiver,
+                value=1,
+                gas_limit=100_000,
+                sender=pre.fund_eoa(),
+            )
+            post = Alloc()
+            post[value_receiver] = (
+                Account.NONEXISTENT
+                if test_type != SystemContractTestType.GAS_LIMIT
+                else Account(
+                    balance=1,
+                )
+            )
+
+            blockchain_test(
+                pre=pre,
+                blocks=[
+                    Block(  # Deployment block
+                        txs=[test_tx],
+                        exception=BlockException.SYSTEM_CONTRACT_CALL_FAILED
+                        if test_type != SystemContractTestType.GAS_LIMIT
+                        else None,
+                    )
+                ],
+                post=post,
+            )
+
+        wrapper.__name__ = func.__name__  # type: ignore
+        wrapper.__doc__ = func.__doc__  # type: ignore
+
+        return wrapper
+
+    return decorator