Let geth choose the open port, pass it to fixtures

- Let geth choose the open port and parse this port from the logs
  to pass along to the consuming fixtures.

Author: fselmo

tests/conftest.py

@@ -19,21 +19,12 @@
     RequestMocker,
 )
 
-from .utils import (
-    get_open_port,
-)
-
 
 @pytest.fixture(scope="module", params=[lambda x: to_bytes(hexstr=x), identity])
 def address_conversion_func(request):
     return request.param
 
 
-@pytest.fixture()
-def open_port():
-    return get_open_port()
-
-
 # --- session-scoped constants --- #
 
 

tests/core/providers/test_legacy_websocket_provider.py

@@ -1,19 +1,5 @@
 import pytest
-import asyncio
-from asyncio.exceptions import (
-    TimeoutError,
-)
-from threading import (
-    Thread,
-)
-
-from websockets.legacy.server import (
-    serve,
-)
 
-from tests.utils import (
-    wait_for_ws,
-)
 from web3 import (
     Web3,
 )
@@ -26,39 +12,6 @@
 )
 
 
-@pytest.fixture
-def start_websocket_server(open_port):
-    event_loop = asyncio.new_event_loop()
-
-    def run_server():
-        async def empty_server(websocket, path):
-            data = await websocket.recv()
-            await asyncio.sleep(0.02)
-            await websocket.send(data)
-
-        asyncio.set_event_loop(event_loop)
-        server = serve(empty_server, "127.0.0.1", open_port)
-        event_loop.run_until_complete(server)
-        event_loop.run_forever()
-
-    thd = Thread(target=run_server)
-    thd.start()
-    try:
-        yield
-    finally:
-        event_loop.call_soon_threadsafe(event_loop.stop)
-
-
-@pytest.fixture
-def w3(open_port, start_websocket_server):
-    # need new event loop as the one used by server is already running
-    event_loop = asyncio.new_event_loop()
-    endpoint_uri = f"ws://127.0.0.1:{open_port}"
-    event_loop.run_until_complete(wait_for_ws(endpoint_uri))
-    provider = LegacyWebSocketProvider(endpoint_uri, websocket_timeout=0.01)
-    return Web3(provider)
-
-
 def test_no_args():
     provider = LegacyWebSocketProvider()
     w3 = Web3(provider)
@@ -69,11 +22,6 @@ def test_no_args():
         w3.is_connected(show_traceback=True)
 
 
-def test_websocket_provider_timeout(w3):
-    with pytest.raises(TimeoutError):
-        w3.eth.accounts
-
-
 def test_restricted_websocket_kwargs():
     invalid_kwargs = {"uri": "ws://127.0.0.1:8546"}
     re_exc_message = f".*found: {set(invalid_kwargs)!r}*"

tests/integration/go_ethereum/conftest.py

@@ -4,12 +4,13 @@
 from pathlib import (
     Path,
 )
+import re
 import subprocess
+import time
 import zipfile
 
 from eth_utils import (
     is_dict,
-    to_text,
 )
 import pytest_asyncio
 
@@ -133,36 +134,51 @@ def genesis_file(datadir):
     return genesis_file_path
 
 
+def wait_for_port(proc, timeout=10):
+    start = time.time()
+    wait_time = start + timeout
+
+    while time.time() < wait_time:
+        line = proc.stderr.readline()
+        if not line:
+            continue
+
+        if match := re.compile(r"127\.0\.0\.1:(\d+)").search(line):
+            port = int(match.group(1))
+            if port not in {0, 80}:
+                # remove false positive matches
+                return port
+
+    raise TimeoutError(f"Did not find port in logs within {timeout} seconds")
+
+
 @pytest.fixture
-def geth_process(geth_binary, datadir, genesis_file, geth_command_arguments):
-    init_datadir_command = (
+def start_geth_process_and_yield_port(
+    geth_binary, datadir, genesis_file, geth_command_arguments
+):
+    init_cmd = (
         geth_binary,
         "--datadir",
         str(datadir),
         "init",
         str(genesis_file),
     )
-    subprocess.check_output(
-        init_datadir_command,
-        stdin=subprocess.PIPE,
-        stderr=subprocess.PIPE,
-    )
+    subprocess.check_output(init_cmd, stdin=subprocess.PIPE, stderr=subprocess.PIPE)
     proc = subprocess.Popen(
         geth_command_arguments,
         stdin=subprocess.PIPE,
         stdout=subprocess.PIPE,
         stderr=subprocess.PIPE,
+        text=True,
+        bufsize=1,
     )
-    try:
-        yield proc
-    finally:
-        kill_proc_gracefully(proc)
-        output, errors = proc.communicate()
-        print(
-            "Geth Process Exited:\n"
-            f"stdout:{to_text(output)}\n\n"
-            f"stderr:{to_text(errors)}\n\n"
-        )
+
+    port = wait_for_port(proc)
+    yield port
+
+    kill_proc_gracefully(proc)
+    output, errors = proc.communicate(timeout=5)
+    print("Geth Process Exited:\n" f"stdout: {output}\n\n" f"stderr: {errors}\n\n")
 
 
 @pytest.fixture

tests/integration/go_ethereum/test_goethereum_http.py

@@ -5,9 +5,6 @@
 )
 import pytest_asyncio
 
-from tests.utils import (
-    get_open_port,
-)
 from web3 import (
     AsyncWeb3,
     Web3,
@@ -32,29 +29,15 @@
     GoEthereumTxPoolModuleTest,
     GoEthereumWeb3ModuleTest,
 )
-from .utils import (
-    wait_for_aiohttp,
-    wait_for_http,
-)
-
-
-@pytest.fixture
-def rpc_port():
-    return get_open_port()
-
-
-@pytest.fixture
-def endpoint_uri(rpc_port):
-    return f"http://localhost:{rpc_port}"
 
 
-def _geth_command_arguments(rpc_port, base_geth_command_arguments, geth_version):
+def _geth_command_arguments(base_geth_command_arguments, geth_version):
     yield from base_geth_command_arguments
     if geth_version.major == 1:
         yield from (
             "--http",
             "--http.port",
-            rpc_port,
+            "0",
             "--http.api",
             "admin,debug,eth,net,web3,txpool",
             "--ipcdisable",
@@ -64,26 +47,28 @@ def _geth_command_arguments(rpc_port, base_geth_command_arguments, geth_version)
 
 
 @pytest.fixture
-def geth_command_arguments(rpc_port, base_geth_command_arguments, get_geth_version):
-    return _geth_command_arguments(
-        rpc_port, base_geth_command_arguments, get_geth_version
-    )
+def geth_command_arguments(base_geth_command_arguments, get_geth_version):
+    return _geth_command_arguments(base_geth_command_arguments, get_geth_version)
 
 
 @pytest.fixture
-def w3(geth_process, endpoint_uri):
-    wait_for_http(endpoint_uri)
-    return Web3(Web3.HTTPProvider(endpoint_uri, request_kwargs={"timeout": 10}))
+def w3(start_geth_process_and_yield_port):
+    port = start_geth_process_and_yield_port
+    _w3 = Web3(
+        Web3.HTTPProvider(f"http://127.0.0.1:{port}", request_kwargs={"timeout": 10})
+    )
+    return _w3
 
 
 @pytest.fixture
-def auto_w3(geth_process, endpoint_uri):
-    wait_for_http(endpoint_uri)
-
+def auto_w3(start_geth_process_and_yield_port, monkeypatch):
     from web3.auto import (
         w3,
     )
 
+    port = start_geth_process_and_yield_port
+    monkeypatch.setenv("WEB3_PROVIDER_URI", f"http://127.0.0.1:{port}")
+
     return w3
 
 
@@ -116,13 +101,7 @@ class TestGoEthereumDebugModuleTest(GoEthereumDebugModuleTest):
 
 
 class TestGoEthereumEthModuleTest(GoEthereumEthModuleTest):
-    def test_auto_provider_batching(
-        self,
-        auto_w3: "Web3",
-        monkeypatch,
-        endpoint_uri,
-    ) -> None:
-        monkeypatch.setenv("WEB3_PROVIDER_URI", endpoint_uri)
+    def test_auto_provider_batching(self, auto_w3: "Web3") -> None:
         # test that batch_requests doesn't error out when using the auto provider
         auto_w3.batch_requests()
 
@@ -139,12 +118,15 @@ class TestGoEthereumTxPoolModuleTest(GoEthereumTxPoolModuleTest):
 
 
 @pytest_asyncio.fixture
-async def async_w3(geth_process, endpoint_uri):
-    await wait_for_aiohttp(endpoint_uri)
+async def async_w3(start_geth_process_and_yield_port):
+    port = start_geth_process_and_yield_port
     _w3 = AsyncWeb3(
-        AsyncHTTPProvider(endpoint_uri, request_kwargs={"timeout": ClientTimeout(10)})
+        AsyncHTTPProvider(
+            f"http://127.0.0.1:{port}", request_kwargs={"timeout": ClientTimeout(10)}
+        )
     )
-    return _w3
+    yield _w3
+    await _w3.provider.disconnect()
 
 
 class TestGoEthereumAsyncWeb3ModuleTest(GoEthereumAsyncWeb3ModuleTest):

tests/integration/go_ethereum/test_goethereum_ipc.py

@@ -54,18 +54,19 @@ def geth_ipc_path(datadir):
 
 
 @pytest.fixture
-def auto_w3(geth_process, geth_ipc_path):
-    wait_for_socket(geth_ipc_path)
-
+def auto_w3(start_geth_process_and_yield_port, geth_ipc_path, monkeypatch):
     from web3.auto import (
         w3,
     )
 
+    wait_for_socket(geth_ipc_path)
+    monkeypatch.setenv("WEB3_PROVIDER_URI", f"file:///{geth_ipc_path}")
+
     return w3
 
 
 @pytest.fixture
-def w3(geth_process, geth_ipc_path):
+def w3(start_geth_process_and_yield_port, geth_ipc_path):
     wait_for_socket(geth_ipc_path)
     return Web3(Web3.IPCProvider(geth_ipc_path, timeout=10))
 
@@ -79,13 +80,7 @@ class TestGoEthereumDebugModuleTest(GoEthereumDebugModuleTest):
 
 
 class TestGoEthereumEthModuleTest(GoEthereumEthModuleTest):
-    def test_auto_provider_batching(
-        self,
-        auto_w3: "Web3",
-        monkeypatch,
-        geth_ipc_path,
-    ) -> None:
-        monkeypatch.setenv("WEB3_PROVIDER_URI", f"file:///{geth_ipc_path}")
+    def test_auto_provider_batching(self, auto_w3: "Web3") -> None:
         # test that batch_requests doesn't error out when using the auto provider
         auto_w3.batch_requests()
 
@@ -118,7 +113,7 @@ def test_admin_start_stop_ws(self, w3: "Web3") -> None:
 
 
 @pytest_asyncio.fixture
-async def async_w3(geth_process, geth_ipc_path):
+async def async_w3(start_geth_process_and_yield_port, geth_ipc_path):
     await wait_for_async_socket(geth_ipc_path)
     async with AsyncWeb3(AsyncIPCProvider(geth_ipc_path, request_timeout=10)) as _aw3:
         yield _aw3