Add decorator that xfails on select exception(s), retrying otherwise.

Author: fselmo

web3/_utils/module_testing/eth_module.py

@@ -58,6 +58,7 @@
     assert_contains_log,
     async_mock_offchain_lookup_request_response,
     flaky_geth_dev_mining,
+    flaky_with_xfail_on_exception,
     mock_offchain_lookup_request_response,
 )
 from web3._utils.module_testing.utils import (
@@ -2359,10 +2360,9 @@ async def test_async_eth_replace_transaction_gas_price_too_low(
         with pytest.raises(Web3ValueError):
             await async_w3.eth.replace_transaction(txn_hash, txn_params)
 
-    @pytest.mark.xfail(
-        reason="Very flaky on CI runs, hard to reproduce locally",
-        strict=False,
-        raises=(RequestTimedOut, asyncio.TimeoutError, Web3ValueError),
+    @flaky_with_xfail_on_exception(
+        reason="Very flaky on CI runs, hard to reproduce locally.",
+        exception=RequestTimedOut,
     )
     @pytest.mark.asyncio
     async def test_async_eth_replace_transaction_gas_price_defaulting_minimum(
@@ -2387,10 +2387,9 @@ async def test_async_eth_replace_transaction_gas_price_defaulting_minimum(
             gas_price * 1.125
         )  # minimum gas price
 
-    @pytest.mark.xfail(
-        reason="Very flaky on CI runs, hard to reproduce locally",
-        strict=False,
-        raises=(RequestTimedOut, asyncio.TimeoutError, Web3ValueError),
+    @flaky_with_xfail_on_exception(
+        reason="Very flaky on CI runs, hard to reproduce locally.",
+        exception=RequestTimedOut,
     )
     @pytest.mark.asyncio
     async def test_async_eth_replace_transaction_gas_price_defaulting_strategy_higher(
@@ -2420,10 +2419,9 @@ def higher_gas_price_strategy(async_w3: "AsyncWeb3", txn: TxParams) -> Wei:
         )  # Strategy provides higher gas price
         async_w3.eth.set_gas_price_strategy(None)  # reset strategy
 
-    @pytest.mark.xfail(
-        reason="Very flaky on CI runs, hard to reproduce locally",
-        strict=False,
-        raises=(RequestTimedOut, asyncio.TimeoutError, Web3ValueError),
+    @flaky_with_xfail_on_exception(
+        reason="Very flaky on CI runs, hard to reproduce locally.",
+        exception=RequestTimedOut,
     )
     @pytest.mark.asyncio
     async def test_async_eth_replace_transaction_gas_price_defaulting_strategy_lower(

web3/_utils/module_testing/module_testing_utils.py

@@ -1,12 +1,17 @@
 import asyncio
+import functools
+import pytest
 from typing import (
     TYPE_CHECKING,
     Any,
+    Callable,
     Collection,
     Dict,
     Generator,
     Literal,
     Sequence,
+    Tuple,
+    Type,
     Union,
 )
 
@@ -55,7 +60,44 @@
 for the duration of the test. This behavior can be flaky
 due to timing of the test running as a block is mined.
 """
-flaky_geth_dev_mining = flaky(max_runs=3)
+flaky_geth_dev_mining = flaky(max_runs=3, min_passes=1)
+
+
+def flaky_with_xfail_on_exception(
+    reason: str,
+    exception: Union[Type[Exception], Tuple[Type[Exception], ...]],
+    max_runs: int = 3,
+    min_passes: int = 1,
+) -> Callable[[Any], Any]:
+    """
+    Some tests inconsistently fail hard with a particular exception and retrying
+    these tests often times does not get them "unstuck". If we've exhausted all flaky
+    retries and this expected exception is raised, `xfail` the test with the given
+    reason.
+    """
+    runs = max_runs
+
+    def decorator(func: Any) -> Any:
+        @flaky(max_runs=max_runs, min_passes=min_passes)
+        @functools.wraps(func)
+        async def wrapper(self: Any, *args: Any, **kwargs: Any) -> Any:
+            nonlocal runs
+            try:
+                return await func(self, *args, **kwargs)
+            except exception:
+                # xfail the test only if the exception is raised and we have exhausted
+                # all flaky retries
+                if runs == 1:
+                    pytest.xfail(reason)
+                runs -= 1
+                pytest.fail(f"xfailed but {runs} run(s) remaining with flaky...")
+            except Exception as e:
+                # let flaky handle it
+                raise e
+
+        return wrapper
+
+    return decorator
 
 
 def assert_contains_log(