fix(unit_tests): fixed blob unit tests when run with xdist (#1946)

* fixed blob unit tests for running with xdist

* codespell is a **********

* Update src/ethereum_test_types/tests/test_blob_types.py

---------

Co-authored-by: Mario Vega <[email protected]>

Author: felix314159

src/ethereum_test_types/blob_types.py

@@ -26,12 +26,19 @@ def clear_blob_cache(cached_blobs_folder_path: Path):
     """Delete all cached blobs."""
     if not cached_blobs_folder_path.is_dir():
         return
-    for f in cached_blobs_folder_path.glob("*.json"):  # only delete .json files
+
+    json_files = list(cached_blobs_folder_path.glob("*.json"))
+
+    for f in json_files:
+        lock_file_path = f.with_suffix(".lock")
+
         try:
-            f.unlink()  # permanently delete this file
+            # get file lock for what you want to delete
+            with FileLock(lock_file_path):
+                f.unlink()
         except Exception as e:
             print(
-                f"Critical error while trying to delete file {f}:{e}.. "
+                f"Error while trying to delete file {f}:{e}. "
                 "Aborting clearing of blob cache folder."
             )
             return
@@ -186,11 +193,15 @@ def get_cells(fork: Fork, data: Bytes) -> List[Bytes] | None:
         # (blob related constants are needed and only available for normal forks)
         fork = fork.fork_at(timestamp=timestamp)
 
-        # if this blob already exists then load from file
+        # if this blob already exists then load from file. use lock
         blob_location: Path = Blob.get_filepath(fork, seed)
-        if blob_location.exists():
-            logger.debug(f"Blob exists already, reading it from file {blob_location}")
-            return Blob.from_file(Blob.get_filename(fork, seed))
+
+        # use lock to avoid race conditions
+        lock_file_path = blob_location.with_suffix(".lock")
+        with FileLock(lock_file_path):
+            if blob_location.exists():
+                logger.debug(f"Blob exists already, reading it from file {blob_location}")
+                return Blob.from_file(Blob.get_filename(fork, seed))
 
         assert fork.supports_blobs(), f"Provided fork {fork.name()} does not support blobs!"
 
@@ -237,17 +248,14 @@ def from_file(file_name: str) -> "Blob":
         # determine path where this blob would be stored if it existed
         blob_file_location = CACHED_BLOBS_DIRECTORY / file_name
 
-        # use lock to avoid race conditions
-        lock_file_path = blob_file_location.with_suffix(".lock")
-        with FileLock(lock_file_path):
-            # check whether blob exists
-            assert blob_file_location.exists(), (
-                f"Tried to load blob from file but {blob_file_location} does not exist"
-            )
+        # check whether blob exists
+        assert blob_file_location.exists(), (
+            f"Tried to load blob from file but {blob_file_location} does not exist"
+        )
 
-            # read blob from file
-            with open(blob_file_location, "r", encoding="utf-8") as f:
-                json_str: str = f.read()
+        # read blob from file
+        with open(blob_file_location, "r", encoding="utf-8") as f:
+            json_str: str = f.read()
 
         # reconstruct and return blob object
         return Blob.model_validate_json(json_str)

src/ethereum_test_types/tests/test_blob_types.py

@@ -1,8 +1,10 @@
 """Test suite for blobs."""
 
 import copy
+import time
 
 import pytest
+from filelock import FileLock
 
 from ethereum_test_forks import (
     Cancun,
@@ -18,6 +20,65 @@
 from ..blob_types import CACHED_BLOBS_DIRECTORY, Blob, clear_blob_cache
 
 
+def increment_counter(timeout: float = 10):
+    """
+    Increment counter in file, creating if doesn't exist.
+
+    This is needed because we require the unit test 'test_transition_fork_blobs' to run
+    at the end without having to include another dependency for ordering tests.
+    That test has to run at the end because it assumes that no json blobs not created
+    by itself are created while it is running.
+
+    The hardcoded counter value in the test above has to be updated if any new blob_related
+    unit tests that create json blobs are added in the future.
+
+    """
+    file_path = CACHED_BLOBS_DIRECTORY / "blob_unit_test_counter.txt"
+    lock_file = file_path.with_suffix(".lock")
+
+    with FileLock(lock_file, timeout=timeout):
+        # Read current value or start at 0
+        if file_path.exists():
+            current_value = int(file_path.read_text().strip())
+        else:
+            current_value = 0
+
+        # Increment and write back
+        new_value = current_value + 1
+        file_path.write_text(str(new_value))
+
+        return new_value
+
+
+def wait_until_counter_reached(target: int, poll_interval: float = 0.1):
+    """Wait until blob unit test counter reaches target value."""
+    file_path = CACHED_BLOBS_DIRECTORY / "blob_unit_test_counter.txt"
+    lock_file = file_path.with_suffix(".lock")  # Add lock file path
+
+    while True:
+        # Use FileLock when reading!
+        with FileLock(lock_file, timeout=10):
+            if file_path.exists():
+                try:
+                    current_value = int(file_path.read_text().strip())
+                    if current_value == target:
+                        # file_path.unlink()  # get rid to effectively reset counter to 0
+                        return current_value
+                    elif current_value > target:
+                        pytest.fail(
+                            f"The blob_unit_test lock counter is too high! "
+                            f"Expected {target}, but got {current_value}. "
+                            f"It probably reused an existing file that was not cleared. "
+                            f"Delete {file_path} manually to fix this."
+                        )
+                except Exception:
+                    current_value = 0
+            else:
+                current_value = 0
+
+        time.sleep(poll_interval)
+
+
 @pytest.mark.parametrize("seed", [0, 10, 100])
 @pytest.mark.parametrize("fork", [Cancun, Prague, Osaka])
 def test_blob_creation_and_writing_and_reading(
@@ -42,6 +103,8 @@ def test_blob_creation_and_writing_and_reading(
     # ensure file you read equals file you wrote
     assert b.model_dump() == restored.model_dump()
 
+    increment_counter()
+
 
 @pytest.mark.parametrize(
     "corruption_mode",
@@ -71,6 +134,8 @@ def test_blob_proof_corruption(
         "proof is unchanged!"
     )
 
+    increment_counter()
+
 
 @pytest.mark.parametrize("timestamp", [14999, 15000])
 @pytest.mark.parametrize(
@@ -81,6 +146,9 @@ def test_transition_fork_blobs(
     timestamp,
 ):
     """Generates blobs for transition forks (time 14999 is old fork, time 15000 is new fork)."""
+    # line below guarantees that this test runs only after the other blob unit tests are done
+    wait_until_counter_reached(21)
+
     clear_blob_cache(CACHED_BLOBS_DIRECTORY)
 
     print(f"Original fork: {fork}, Timestamp: {timestamp}")
@@ -109,3 +177,7 @@ def test_transition_fork_blobs(
             f"Transition fork failure! Fork {fork.name()} at timestamp: {timestamp} should have "
             f"transitioned to {post_transition_fork_at_15k.name()} but is still at {b.fork.name()}"
         )
+
+    # delete counter at last iteration (otherwise re-running all unit tests will fail)
+    if timestamp == 15_000 and pre_transition_fork == Prague:
+        (CACHED_BLOBS_DIRECTORY / "blob_unit_test_counter.txt").unlink()