Re-organize the fixture filler helper tools (#1219)

* Re-organize the fixture filler helper tools

* cleanup trinity module

* move rlp util to be not private

* fix docs

Author: pipermerriam

.gitignore

@@ -46,7 +46,7 @@ docs/modules.rst
 .pytest_cache/
 
 # fixtures
-fixtures/**
+./fixtures/**
 
 # profiling
 prof/**

docs/api/eth/api.tools.rst

@@ -0,0 +1,10 @@
+Tools
+=====
+
+
+.. toctree::
+   :maxdepth: 4
+   :name: toc-eth-api-tools
+   :caption: Tools
+
+   tools/api.tools.fixtures

docs/api/eth/index.rst

@@ -8,8 +8,10 @@ This section aims to provide a detailed description of all APIs. If you are look
    :name: toc-api-eth
    :caption: API
 
+   
    api.chain
    api.db
    api.exceptions
    api.rlp
+   api.tools
    api.vm

docs/api/eth/tools/api.tools.fixtures.rst

@@ -0,0 +1,187 @@
+Builder Tools
+=============
+
+
+The JSON test fillers found in `eth.tools.fixtures` is a set of tools which facilitate
+creating standard JSON consensus tests as found in the
+`ethereum/tests repository <https://github.com/ethereum/tests>`_. 
+
+.. note:: Only VM and state tests are supported right now.
+
+
+State Test Fillers
+------------------
+
+Tests are generated in two steps. 
+
+* First, a *test filler* is written that contains a high level description of the test case.
+* Subsequently, the filler is compiled to the actual test in a process called
+  filling, mainly consisting of calculating the resulting state root.
+
+The test builder represents each stage as a nested dictionary. Helper functions are provided to
+assemble the filler file step by step in the correct format. The
+:func:`~eth.tools.fixtures.fillers.fill_test` function handles compilation and
+takes additional parameters that can't be inferred from the filler.
+
+
+Creating a Filler
+~~~~~~~~~~~~~~~~~
+
+Fillers are generated in a functional fashion by piping a dictionary through a
+sequence of functions.
+
+.. code-block:: python
+
+    filler = pipe(
+        setup_main_filler("test"),
+        pre_state(
+            (sender, "balance", 1),
+            (receiver, "balance", 0),
+        ),
+        expect(
+            networks=["Frontier"],
+            transaction={
+                "to": receiver,
+                "value": 1,
+                "secretKey": sender_key,
+            },
+            post_state=[
+                [sender, "balance", 0],
+                [receiver, "balance", 1],
+            ]
+        )
+    )
+
+.. note:: 
+
+    Note that :func:`~eth.tools.fixtures.setup_filler` returns a
+    dictionary, whereas all of the following functions such as
+    :func:`~eth.tools.fixtures.pre_state`,
+    :func:`~eth.tools.fixtures.expect`, expect to be passed a dictionary
+    as their single argument and return an updated version of the dictionary.
+
+
+.. autofunction:: eth.tools.fixtures.fillers.common.setup_main_filler
+
+    This function kicks off the filler generation process by creating the general filler scaffold with
+    a test name and general information about the testing environment.
+
+    For tests for the main chain, the `environment` parameter is expected to be a dictionary with some
+    or all of the following keys:
+
+    +------------------------+---------------------------------+
+    | key                    | description                     |
+    +========================+=================================+
+    | ``"currentCoinbase"``  | the coinbase address            |
+    +------------------------+---------------------------------+
+    | ``"currentNumber"``    | the block number                |
+    +------------------------+---------------------------------+
+    | ``"previousHash"``     | the hash of the parent block    |
+    +------------------------+---------------------------------+
+    | ``"currentDifficulty"``| the block's difficulty          |
+    +------------------------+---------------------------------+
+    | ``"currentGasLimit"``  | the block's gas limit           |
+    +------------------------+---------------------------------+
+    | ``"currentTimestamp"`` | the timestamp of the block      |
+    +------------------------+---------------------------------+
+
+
+.. autofunction:: eth.tools.fixtures.fillers.pre_state
+
+    This function specifies the state prior to the test execution. Multiple invocations don't override
+    the state but extend it instead.
+
+    In general, the elements of `state_definitions` are nested dictionaries of the following form:
+
+    .. code-block:: python
+
+        {
+            address: {
+                "nonce": <account nonce>,
+                "balance": <account balance>,
+                "code": <account code>,
+                "storage": {
+                    <storage slot>: <storage value>
+                }
+            }
+        }
+
+    To avoid unnecessary nesting especially if only few fields per account are specified, the following
+    and similar formats are possible as well:
+
+    .. code-block:: python
+
+        (address, "balance", <account balance>)
+        (address, "storage", <storage slot>, <storage value>)
+        (address, "storage", {<storage slot>: <storage value>})
+        (address, {"balance", <account balance>})
+
+
+.. autofunction:: eth.tools.fixtures.fillers.execution
+
+    For VM tests, this function specifies the code that is being run as well as the current state of
+    the EVM. State tests don't support this object. The parameter is a dictionary specifying some or
+    all of the following keys:
+
+    +--------------------+------------------------------------------------------------+
+    |  key               | description                                                |
+    +====================+============================================================+
+    | ``"address"``      | the address of the account executing the code              |
+    +--------------------+------------------------------------------------------------+
+    | ``"caller"``       | the caller address                                         |
+    +--------------------+------------------------------------------------------------+
+    | ``"origin"``       | the origin address (defaulting to the caller address)      |
+    +--------------------+------------------------------------------------------------+
+    | ``"value"``        | the value of the call                                      |
+    +--------------------+------------------------------------------------------------+
+    | ``"data"``         | the data passed with the call                              |
+    +--------------------+------------------------------------------------------------+
+    | ``"gasPrice"``     | the gas price of the call                                  |
+    +--------------------+------------------------------------------------------------+
+    | ``"gas"``          | the amount of gas allocated for the call                   |
+    +--------------------+------------------------------------------------------------+
+    | ``"code"``         | the bytecode to execute                                    |
+    +--------------------+------------------------------------------------------------+
+    | ``"vyperLLLCode"`` | the code in Vyper LLL (compiled to bytecode automatically) |
+    +--------------------+------------------------------------------------------------+
+
+
+.. autofunction:: eth.tools.fixtures.fillers.expect
+
+    This specifies the expected result of the test.
+
+    For state tests, multiple expectations can be given, differing in the transaction data, gas
+    limit, and value, in the applicable networks, and as a result also in the post state. VM tests
+    support only a single expectation with no specified network and no transaction (here, its role is
+    played by :func:`~eth.tools.fixtures.fillers.execution`).
+
+    * ``post_state`` is a list of state definition in the same form as expected by `pre_state`. State items
+      that are not set explicitly default to their pre state.
+
+    * ``networks`` defines the forks under which the expectation is applicable. It should be a sublist of
+      the following identifiers (also available in `ALL_FORKS`):
+
+      * ``"Frontier"``
+      * ``"Homestead"``
+      * ``"EIP150"``
+      * ``"EIP158"``
+      * ``"Byzantium"``
+
+    * ``transaction`` is a dictionary coming in two variants. For the main shard:
+
+      +----------------+-------------------------------+
+      | key            | description                   |
+      +================+===============================+
+      | ``"data"``     | the transaction data,         |
+      +----------------+-------------------------------+
+      | ``"gasLimit"`` | the transaction gas limit,    |
+      +----------------+-------------------------------+
+      | ``"gasPrice"`` | the gas price,                |
+      +----------------+-------------------------------+
+      | ``"nonce"``    | the transaction nonce,        |
+      +----------------+-------------------------------+
+      | ``"value"``    | the transaction value         |
+      +----------------+-------------------------------+
+
+    In addition, one should specify either the signature itself (via keys ``"v"``, ``"r"``, and ``"s"``) or
+    a private key used for signing (via ``"secretKey"``).

eth/chains/base.py

@@ -95,7 +95,7 @@
     encode_hex,
 )
 from eth.utils.rlp import (
-    ensure_imported_block_unchanged,
+    validate_imported_block_unchanged,
 )
 
 if TYPE_CHECKING:
@@ -635,7 +635,7 @@ def import_block(self,
 
         # Validate the imported block.
         if perform_validation:
-            ensure_imported_block_unchanged(imported_block, block)
+            validate_imported_block_unchanged(imported_block, block)
             self.validate_block(imported_block)
 
         (

eth/tools/test_builder/__init__.py renamed to eth/tools/_utils/__init__.py

@@ -0,0 +1,8 @@
+import subprocess
+
+from eth_utils import to_text
+
+
+def get_version_from_git():
+    version = subprocess.check_output(["git", "describe"]).strip()
+    return to_text(version)

eth/tools/_utils/git.py

@@ -0,0 +1,16 @@
+from eth_hash.auto import keccak
+
+import rlp
+
+from eth.rlp.logs import Log
+
+
+def hash_log_entries(log_entries):
+    """
+    Helper function for computing the RLP hash of the logs from transaction
+    execution.
+    """
+    logs = [Log(*entry) for entry in log_entries]
+    encoded_logs = rlp.encode(logs)
+    logs_hash = keccak(encoded_logs)
+    return logs_hash

eth/tools/_utils/hashing.py

@@ -0,0 +1,43 @@
+from collections.abc import Mapping
+import itertools
+
+from cytoolz import merge_with
+
+
+def merge_if_dicts(values):
+    if all(isinstance(item, Mapping) for item in values):
+        return merge_with(merge_if_dicts, *values)
+    else:
+        return values[-1]
+
+
+def deep_merge(*dicts):
+    return merge_with(merge_if_dicts, *dicts)
+
+
+def is_cleanly_mergable(*dicts):
+    """Check that nothing will be overwritten when dictionaries are merged using `deep_merge`.
+
+    Examples:
+
+        >>> is_cleanly_mergable({"a": 1}, {"b": 2}, {"c": 3})
+        True
+        >>> is_cleanly_mergable({"a": 1}, {"b": 2}, {"a": 0, c": 3})
+        False
+        >>> is_cleanly_mergable({"a": 1, "b": {"ba": 2}}, {"c": 3, {"b": {"bb": 4}})
+        True
+        >>> is_cleanly_mergable({"a": 1, "b": {"ba": 2}}, {"b": {"ba": 4}})
+        False
+
+    """
+    if len(dicts) <= 1:
+        return True
+    elif len(dicts) == 2:
+        if not all(isinstance(d, Mapping) for d in dicts):
+            return False
+        else:
+            shared_keys = set(dicts[0].keys()) & set(dicts[1].keys())
+            return all(is_cleanly_mergable(dicts[0][key], dicts[1][key]) for key in shared_keys)
+    else:
+        dict_combinations = itertools.combinations(dicts, 2)
+        return all(is_cleanly_mergable(*combination) for combination in dict_combinations)

eth/tools/_utils/mappings.py

@@ -0,0 +1,30 @@
+import functools
+
+try:
+    from vyper.compile_lll import (
+        compile_to_assembly,
+        assembly_to_evm,
+    )
+    from vyper.parser.parser_utils import LLLnode
+except ImportError:
+    vyper_available = False
+else:
+    vyper_available = True
+
+
+def require_vyper(fn):
+    @functools.wraps(fn)
+    def inner(*args, **kwargs):
+        if vyper_available:
+            return fn(*args, **kwargs)
+        else:
+            raise ImportError("The `{0}` function requires the vyper compiler.")
+    return inner
+
+
+@require_vyper
+def compile_vyper_lll(vyper_code):
+    lll_node = LLLnode.from_list(vyper_code)
+    assembly = compile_to_assembly(lll_node)
+    code = assembly_to_evm(assembly)
+    return code