Merge branch 'master' into 7634-opentelemetry-instrument-rpc-clients

Author: bisgaard-itis

packages/postgres-database/tests/test_comp_tasks.py

@@ -5,6 +5,7 @@
 
 import asyncio
 import json
+from collections.abc import AsyncIterator
 
 import pytest
 from aiopg.sa.engine import Engine, SAConnection
@@ -25,7 +26,9 @@ async def db_connection(aiopg_engine: Engine) -> SAConnection:
 
 
 @pytest.fixture()
-async def db_notification_queue(db_connection: SAConnection) -> asyncio.Queue:
+async def db_notification_queue(
+    db_connection: SAConnection,
+) -> AsyncIterator[asyncio.Queue]:
     listen_query = f"LISTEN {DB_CHANNEL_NAME};"
     await db_connection.execute(listen_query)
     notifications_queue: asyncio.Queue = db_connection.connection.notifies

packages/pytest-simcore/src/pytest_simcore/db_entries_mocks.py

@@ -7,7 +7,6 @@
 from typing import Any
 from uuid import uuid4
 
-import aiopg.sa
 import pytest
 import sqlalchemy as sa
 from faker import Faker
@@ -20,6 +19,7 @@
     ProjectNodeCreate,
     ProjectNodesRepo,
 )
+from sqlalchemy.ext.asyncio import AsyncEngine
 
 
 @pytest.fixture()
@@ -63,7 +63,7 @@ def creator(**user_kwargs) -> dict[str, Any]:
 
 @pytest.fixture
 async def project(
-    aiopg_engine: aiopg.sa.engine.Engine, faker: Faker
+    sqlalchemy_async_engine: AsyncEngine, faker: Faker
 ) -> AsyncIterator[Callable[..., Awaitable[ProjectAtDB]]]:
     created_project_ids: list[str] = []
 
@@ -86,17 +86,22 @@ async def creator(
             "workbench": {},
         }
         project_config.update(**project_overrides)
-        async with aiopg_engine.acquire() as con, con.begin():
+        async with sqlalchemy_async_engine.connect() as con, con.begin():
             result = await con.execute(
                 projects.insert()
                 .values(**project_config)
                 .returning(sa.literal_column("*"))
             )
 
-            inserted_project = ProjectAtDB.model_validate(await result.first())
+            inserted_project = ProjectAtDB.model_validate(result.one())
             project_nodes_repo = ProjectNodesRepo(project_uuid=project_uuid)
             # NOTE: currently no resources is passed until it becomes necessary
-            default_node_config = {"required_resources": {}, "key": faker.pystr(), "version": faker.pystr(), "label": faker.pystr()}
+            default_node_config = {
+                "required_resources": {},
+                "key": faker.pystr(),
+                "version": faker.pystr(),
+                "label": faker.pystr(),
+            }
             if project_nodes_overrides:
                 default_node_config.update(project_nodes_overrides)
             await project_nodes_repo.add(
@@ -113,17 +118,15 @@ async def creator(
     yield creator
 
     # cleanup
-    async with aiopg_engine.acquire() as con:
+    async with sqlalchemy_async_engine.begin() as con:
         await con.execute(
             projects.delete().where(projects.c.uuid.in_(created_project_ids))
         )
     print(f"<-- delete projects {created_project_ids=}")
 
 
 @pytest.fixture
-def pipeline(
-    postgres_db: sa.engine.Engine,
-) -> Iterator[Callable[..., dict[str, Any]]]:
+def pipeline(postgres_db: sa.engine.Engine) -> Iterator[Callable[..., dict[str, Any]]]:
     created_pipeline_ids: list[str] = []
 
     def creator(**pipeline_kwargs) -> dict[str, Any]:

packages/pytest-simcore/src/pytest_simcore/helpers/playwright.py

@@ -13,7 +13,6 @@
 from dataclasses import dataclass, field
 from datetime import UTC, datetime, timedelta
 from enum import Enum, unique
-from pathlib import Path
 from typing import Any, Final
 
 import arrow
@@ -590,7 +589,6 @@ def expected_service_running(
     press_start_button: bool,
     product_url: AnyUrl,
     is_service_legacy: bool,
-    assertion_output_folder: Path,
 ) -> Generator[ServiceRunning, None, None]:
     started = arrow.utcnow()
     with contextlib.ExitStack() as stack:
@@ -610,6 +608,10 @@ def expected_service_running(
             waiter = SocketIONodeProgressCompleteWaiter(
                 node_id=node_id,
                 logger=ctx.logger,
+                max_idle_timeout=min(
+                    _SOCKET_IO_NODE_PROGRESS_WAITER_MAX_IDLE_TIMEOUT,
+                    timedelta(seconds=timeout / 1000 - 10),
+                ),
             )
             stack.enter_context(
                 websocket.expect_event("framereceived", waiter, timeout=timeout)
@@ -647,7 +649,6 @@ def wait_for_service_running(
     press_start_button: bool,
     product_url: AnyUrl,
     is_service_legacy: bool,
-    assertion_output_folder: Path,
 ) -> FrameLocator:
     """NOTE: if the service was already started this will not work as some of the required websocket events will not be emitted again
     In which case this will need further adjutment"""
@@ -669,6 +670,10 @@ def wait_for_service_running(
             waiter = SocketIONodeProgressCompleteWaiter(
                 node_id=node_id,
                 logger=ctx.logger,
+                max_idle_timeout=min(
+                    _SOCKET_IO_NODE_PROGRESS_WAITER_MAX_IDLE_TIMEOUT,
+                    timedelta(seconds=timeout / 1000 - 10),
+                ),
             )
             stack.enter_context(
                 websocket.expect_event("framereceived", waiter, timeout=timeout)

packages/pytest-simcore/src/pytest_simcore/helpers/playwright_sim4life.py

@@ -2,7 +2,6 @@
 import logging
 import re
 from dataclasses import dataclass
-from pathlib import Path
 from typing import Final, TypedDict
 
 import arrow
@@ -108,7 +107,6 @@ def wait_for_launched_s4l(
     copy_workspace: bool,
     product_url: AnyUrl,
     is_service_legacy: bool,
-    assertion_output_folder: Path,
 ) -> WaitForS4LDict:
     with log_context(logging.INFO, "launch S4L") as ctx:
         predicate = S4LWaitForWebsocket(logger=ctx.logger)
@@ -136,7 +134,6 @@ def wait_for_launched_s4l(
                 press_start_button=False,
                 product_url=product_url,
                 is_service_legacy=is_service_legacy,
-                assertion_output_folder=assertion_output_folder,
             )
         s4l_websocket = ws_info.value
         ctx.logger.info("acquired S4L websocket!")

packages/pytest-simcore/src/pytest_simcore/postgres_service.py

@@ -3,22 +3,23 @@
 # pylint: disable=unused-variable
 
 import json
+import warnings
 from collections.abc import AsyncIterator, Iterator
 from typing import Final
 
 import docker
 import pytest
 import sqlalchemy as sa
 import tenacity
-from pytest_simcore.helpers.monkeypatch_envs import setenvs_from_dict
-from pytest_simcore.helpers.typing_env import EnvVarsDict
 from sqlalchemy.ext.asyncio import AsyncEngine
 from tenacity.stop import stop_after_delay
 from tenacity.wait import wait_fixed
 
 from .helpers.docker import get_service_published_port
 from .helpers.host import get_localhost_ip
+from .helpers.monkeypatch_envs import setenvs_from_dict
 from .helpers.postgres_tools import PostgresTestConfig, migrated_pg_tables_context
+from .helpers.typing_env import EnvVarsDict
 
 _TEMPLATE_DB_TO_RESTORE = "template_simcore_db"
 
@@ -212,6 +213,12 @@ async def aiopg_engine(
 
     engine = await create_engine(str(postgres_db.url))
 
+    warnings.warn(
+        "The 'aiopg_engine' fixture is deprecated and will be removed in a future release. "
+        "Please use 'asyncpg_engine' fixture instead.",
+        DeprecationWarning,
+        stacklevel=2,
+    )
     yield engine
 
     if engine:

scripts/maintenance/computational-clusters/autoscaled_monitor/core.py

@@ -457,7 +457,11 @@ def _print_summary_as_json(
                     "user_id": cluster.primary.user_id,
                     "wallet_id": cluster.primary.wallet_id,
                     "disk_space": cluster.primary.disk_space.human_readable(),
-                    "last_heartbeat": cluster.primary.last_heartbeat.isoformat(),
+                    "last_heartbeat": (
+                        cluster.primary.last_heartbeat.isoformat()
+                        if cluster.primary.last_heartbeat
+                        else "n/a"
+                    ),
                 },
                 "workers": [
                     {
@@ -806,6 +810,7 @@ async def terminate_dynamic_instances(
         rich.print("no instances found")
         raise typer.Exit(1)
 
+    assert state.ec2_resource_autoscaling  # nosec
     _print_dynamic_instances(
         dynamic_autoscaled_instances,
         state.environment,

services/director-v2/requirements/_test.in

@@ -11,7 +11,6 @@
 
 aio_pika
 aioboto3
-aiopg[sa]
 alembic # migration due to pytest_simcore.postgres_service2
 asgi_lifespan
 async-asgi-testclient # replacement for fastapi.testclient.TestClient [see b) below]

services/director-v2/requirements/_test.txt

@@ -21,8 +21,6 @@ aiohttp==3.11.13
     #   aiobotocore
 aioitertools==0.12.0
     # via aiobotocore
-aiopg==1.4.0
-    # via -r requirements/_test.in
 aiormq==6.8.1
     # via
     #   -c requirements/_base.txt
@@ -43,8 +41,6 @@ asgi-lifespan==2.1.0
     # via -r requirements/_test.in
 async-asgi-testclient==1.4.11
     # via -r requirements/_test.in
-async-timeout==4.0.3
-    # via aiopg
 attrs==25.1.0
     # via
     #   -c requirements/_base.txt
@@ -227,11 +223,6 @@ psutil==6.1.0
     # via
     #   -c requirements/_base.txt
     #   distributed
-psycopg2-binary==2.9.10
-    # via
-    #   -c requirements/_base.txt
-    #   aiopg
-    #   sqlalchemy
 pytest==8.3.5
     # via
     #   -r requirements/_test.in
@@ -299,7 +290,6 @@ sqlalchemy==1.4.54
     #   -c requirements/../../../requirements/constraints.txt
     #   -c requirements/_base.txt
     #   -r requirements/_test.in
-    #   aiopg
     #   alembic
 sqlalchemy2-stubs==0.0.2a38
     # via sqlalchemy

tests/e2e-playwright/tests/jupyterlabs/test_jupyterlab.py

@@ -11,7 +11,6 @@
 import re
 from collections.abc import Callable
 from dataclasses import dataclass
-from pathlib import Path
 from typing import Any, Final, Literal
 
 from playwright.sync_api import Page, WebSocket
@@ -73,7 +72,6 @@ def test_jupyterlab(
     large_file_block_size: ByteSize,
     product_url: AnyUrl,
     is_service_legacy: bool,
-    playwright_test_results_dir: Path,
 ):
     # NOTE: this waits for the jupyter to send message, but is not quite enough
     with (
@@ -106,7 +104,6 @@ def test_jupyterlab(
             press_start_button=False,
             product_url=product_url,
             is_service_legacy=is_service_legacy,
-            assertion_output_folder=playwright_test_results_dir,
         )
 
     iframe = page.frame_locator("iframe")

tests/e2e-playwright/tests/sim4life/test_sim4life.py

@@ -8,7 +8,6 @@
 
 
 from collections.abc import Callable
-from pathlib import Path
 from typing import Any
 
 from playwright.sync_api import Page
@@ -35,7 +34,6 @@ def test_sim4life(
     check_videostreaming: bool,
     product_url: AnyUrl,
     is_service_legacy: bool,
-    playwright_test_results_dir: Path,
 ):
     if use_plus_button:
         project_data = create_project_from_new_button(service_key)
@@ -59,7 +57,6 @@ def test_sim4life(
         copy_workspace=False,
         product_url=product_url,
         is_service_legacy=is_service_legacy,
-        assertion_output_folder=playwright_test_results_dir,
     )
     s4l_websocket = resp["websocket"]
     s4l_iframe = resp["iframe"]