Merge remote-tracking branch 'upstream/master' into pr-osparc-s3-zip-stream-worker-code

Author: Andrei Neagu

.github/workflows/ci-testing-deploy.yml

@@ -514,7 +514,7 @@ jobs:
   unit-test-storage:
     needs: changes
     if: ${{ needs.changes.outputs.storage == 'true' || github.event_name == 'push' }}
-    timeout-minutes: 18 # if this timeout gets too small, then split the tests
+    timeout-minutes: 25 # if this timeout gets too small, then split the tests
     name: "[unit] storage"
     runs-on: ${{ matrix.os }}
     strategy:

api/specs/web-server/_auth.py

@@ -15,29 +15,29 @@
 from models_library.rest_error import EnvelopedError, Log
 from pydantic import BaseModel, Field, confloat
 from simcore_service_webserver._meta import API_VTAG
-from simcore_service_webserver.login._2fa_handlers import Resend2faBody
-from simcore_service_webserver.login._auth_handlers import (
+from simcore_service_webserver.login._controller.rest.auth import (
     LoginBody,
     LoginNextPage,
     LoginTwoFactorAuthBody,
     LogoutBody,
 )
-from simcore_service_webserver.login.handlers_change import (
+from simcore_service_webserver.login._controller.rest.change import (
     ChangeEmailBody,
     ChangePasswordBody,
     ResetPasswordBody,
 )
-from simcore_service_webserver.login.handlers_confirmation import (
+from simcore_service_webserver.login._controller.rest.confirmation import (
     PhoneConfirmationBody,
     ResetPasswordConfirmation,
 )
-from simcore_service_webserver.login.handlers_registration import (
+from simcore_service_webserver.login._controller.rest.registration import (
     InvitationCheck,
     InvitationInfo,
     RegisterBody,
     RegisterPhoneBody,
     RegisterPhoneNextPage,
 )
+from simcore_service_webserver.login._controller.rest.twofa import Resend2faBody
 
 router = APIRouter(prefix=f"/{API_VTAG}", tags=["auth"])
 

packages/aws-library/src/aws_library/s3/_client.py

@@ -85,21 +85,30 @@ async def create(
         cls, settings: S3Settings, s3_max_concurrency: int = _S3_MAX_CONCURRENCY_DEFAULT
     ) -> "SimcoreS3API":
         session = aioboto3.Session()
-        session_client = session.client(  # type: ignore[call-overload]
-            "s3",
-            endpoint_url=f"{settings.S3_ENDPOINT}",
-            aws_access_key_id=settings.S3_ACCESS_KEY,
-            aws_secret_access_key=settings.S3_SECRET_KEY,
-            region_name=settings.S3_REGION,
-            config=Config(signature_version="s3v4"),
-        )
-        assert isinstance(session_client, ClientCreatorContext)  # nosec
+        session_client = None
         exit_stack = contextlib.AsyncExitStack()
-        s3_client = cast(S3Client, await exit_stack.enter_async_context(session_client))
-        # NOTE: this triggers a botocore.exception.ClientError in case the connection is not made to the S3 backend
-        await s3_client.list_buckets()
+        try:
+            session_client = session.client(  # type: ignore[call-overload]
+                "s3",
+                endpoint_url=f"{settings.S3_ENDPOINT}",
+                aws_access_key_id=settings.S3_ACCESS_KEY,
+                aws_secret_access_key=settings.S3_SECRET_KEY,
+                region_name=settings.S3_REGION,
+                config=Config(signature_version="s3v4"),
+            )
+            assert isinstance(session_client, ClientCreatorContext)  # nosec
 
-        return cls(s3_client, session, exit_stack, s3_max_concurrency)
+            s3_client = cast(
+                S3Client, await exit_stack.enter_async_context(session_client)
+            )
+            # NOTE: this triggers a botocore.exception.ClientError in case the connection is not made to the S3 backend
+            await s3_client.list_buckets()
+
+            return cls(s3_client, session, exit_stack, s3_max_concurrency)
+        except Exception:
+            await exit_stack.aclose()
+
+            raise
 
     async def close(self) -> None:
         await self._exit_stack.aclose()

packages/models-library/src/models_library/api_schemas_catalog/services.py

@@ -323,11 +323,20 @@ def _update_json_schema_extra(schema: JsonDict) -> None:
     )
 
 
-PageRpcServicesGetV2: TypeAlias = PageRpc[
+PageRpcLatestServiceGet: TypeAlias = PageRpc[
     # WARNING: keep this definition in models_library and not in the RPC interface
+    # otherwise the metaclass PageRpc[*] will create *different* classes in server/client side
+    # and will fail to serialize/deserialize these parameters when transmitted/received
     LatestServiceGet
 ]
 
+PageRpcServiceRelease: TypeAlias = PageRpc[
+    # WARNING: keep this definition in models_library and not in the RPC interface
+    # otherwise the metaclass PageRpc[*] will create *different* classes in server/client side
+    # and will fail to serialize/deserialize these parameters when transmitted/received
+    ServiceRelease
+]
+
 ServiceResourcesGet: TypeAlias = ServiceResourcesDict
 
 
@@ -365,3 +374,6 @@ class MyServiceGet(CatalogOutputSchema):
 
     owner: GroupID | None
     my_access_rights: ServiceGroupAccessRightsV2
+
+
+__all__: tuple[str, ...] = ("ServiceRelease",)

packages/models-library/src/models_library/rest_pagination.py

@@ -28,6 +28,8 @@
 
 PageOffsetInt: TypeAlias = NonNegativeInt
 
+PageTotalCount: TypeAlias = NonNegativeInt
+
 DEFAULT_NUMBER_OF_ITEMS_PER_PAGE: Final[PageLimitInt] = TypeAdapter(
     PageLimitInt
 ).validate_python(20)
@@ -70,7 +72,7 @@ class PageQueryParameters(RequestParameters):
 
 class PageMetaInfoLimitOffset(BaseModel):
     limit: PositiveInt = DEFAULT_NUMBER_OF_ITEMS_PER_PAGE
-    total: NonNegativeInt
+    total: PageTotalCount
     offset: NonNegativeInt = 0
     count: NonNegativeInt
 

packages/pytest-simcore/src/pytest_simcore/docker_compose.py

@@ -2,10 +2,10 @@
 # pylint: disable=unused-argument
 # pylint: disable=unused-variable
 
-""" Fixtures to create docker-compose.yaml configuration files (as in Makefile)
+"""Fixtures to create docker-compose.yaml configuration files (as in Makefile)
 
-    - Basically runs `docker compose config
-    - Services in stack can be selected using 'core_services_selection', 'ops_services_selection' fixtures
+- Basically runs `docker compose config
+- Services in stack can be selected using 'core_services_selection', 'ops_services_selection' fixtures
 
 """
 
@@ -391,6 +391,10 @@ def _filter_services_and_dump(
         if "environment" in service:
             service["environment"] = _minio_fix(service["environment"])
 
+        if name == "postgres":
+            # NOTE: # -c fsync=off is not recommended for production as this disable writing to disk https://pythonspeed.com/articles/faster-db-tests/
+            service["command"] += ["-c", "fsync=off"]
+
     # updates current docker-compose (also versioned ... do not change by hand)
     with docker_compose_path.open("wt") as fh:
         yaml.dump(content, fh, default_flow_style=False)

packages/pytest-simcore/src/pytest_simcore/helpers/catalog_rpc_server.py

@@ -0,0 +1,117 @@
+# pylint: disable=not-context-manager
+# pylint: disable=protected-access
+# pylint: disable=redefined-outer-name
+# pylint: disable=unused-argument
+# pylint: disable=unused-variable
+
+
+from models_library.api_schemas_catalog.services import LatestServiceGet, ServiceGetV2
+from models_library.api_schemas_webserver.catalog import (
+    CatalogServiceUpdate,
+)
+from models_library.products import ProductName
+from models_library.rest_pagination import PageOffsetInt
+from models_library.rpc_pagination import PageLimitInt, PageRpc
+from models_library.services_history import ServiceRelease
+from models_library.services_types import ServiceKey, ServiceVersion
+from models_library.users import UserID
+from pydantic import NonNegativeInt, TypeAdapter
+from servicelib.rabbitmq._client_rpc import RabbitMQRPCClient
+
+
+class CatalogRpcSideEffects:
+    # pylint: disable=no-self-use
+    async def list_services_paginated(
+        self,
+        rpc_client: RabbitMQRPCClient,
+        *,
+        product_name: ProductName,
+        user_id: UserID,
+        limit: PageLimitInt,
+        offset: NonNegativeInt,
+    ):
+        assert rpc_client
+        assert product_name
+        assert user_id
+
+        items = TypeAdapter(list[LatestServiceGet]).validate_python(
+            LatestServiceGet.model_json_schema()["examples"],
+        )
+        total_count = len(items)
+
+        return PageRpc[LatestServiceGet].create(
+            items[offset : offset + limit],
+            total=total_count,
+            limit=limit,
+            offset=offset,
+        )
+
+    async def get_service(
+        self,
+        rpc_client: RabbitMQRPCClient,
+        *,
+        product_name: ProductName,
+        user_id: UserID,
+        service_key: ServiceKey,
+        service_version: ServiceVersion,
+    ):
+        assert rpc_client
+        assert product_name
+        assert user_id
+
+        got = ServiceGetV2.model_validate(
+            ServiceGetV2.model_json_schema()["examples"][0]
+        )
+        got.version = service_version
+        got.key = service_key
+
+        return got
+
+    async def update_service(
+        self,
+        rpc_client: RabbitMQRPCClient,
+        *,
+        product_name: ProductName,
+        user_id: UserID,
+        service_key: ServiceKey,
+        service_version: ServiceVersion,
+        update: CatalogServiceUpdate,
+    ):
+        assert rpc_client
+        assert product_name
+        assert user_id
+
+        got = ServiceGetV2.model_validate(
+            ServiceGetV2.model_json_schema()["examples"][0]
+        )
+        got.version = service_version
+        got.key = service_key
+        return got.model_copy(update=update.model_dump(exclude_unset=True))
+
+    async def list_my_service_history_paginated(
+        self,
+        rpc_client: RabbitMQRPCClient,
+        *,
+        product_name: ProductName,
+        user_id: UserID,
+        service_key: ServiceKey,
+        offset: PageOffsetInt,
+        limit: PageLimitInt,
+    ) -> PageRpc[ServiceRelease]:
+
+        assert rpc_client
+        assert product_name
+        assert user_id
+        assert service_key
+
+        items = TypeAdapter(list[ServiceRelease]).validate_python(
+            ServiceRelease.model_json_schema()["examples"],
+        )
+        total_count = len(items)
+
+        return PageRpc[ServiceRelease].create(
+            items[offset : offset + limit],
+            total=total_count,
+            limit=limit,
+            offset=offset,
+        )

packages/pytest-simcore/src/pytest_simcore/helpers/playwright.py

@@ -11,7 +11,7 @@
 from typing import Any, Final
 
 from playwright._impl._sync_base import EventContextManager
-from playwright.sync_api import APIRequestContext, FrameLocator, Page, Request
+from playwright.sync_api import APIRequestContext, FrameLocator, Locator, Page, Request
 from playwright.sync_api import TimeoutError as PlaywrightTimeoutError
 from playwright.sync_api import WebSocket
 from pydantic import AnyUrl
@@ -137,9 +137,7 @@ def on_framereceived(payload: str | bytes) -> None:
                     ctx.logger.debug("⬆️ Frame received: %s", payload)
 
                 def on_close(_: WebSocket) -> None:
-                    ctx.logger.warning(
-                        "⚠️ WebSocket closed. Attempting to reconnect..."
-                    )
+                    ctx.logger.warning("⚠️ WebSocket closed. Attempting to reconnect...")
                     self._attempt_reconnect(ctx.logger)
 
                 def on_socketerror(error_msg: str) -> None:
@@ -320,9 +318,9 @@ def __call__(self, message: str) -> bool:
                         new_progress
                         != self._current_progress[node_progress_event.progress_type]
                     ):
-                        self._current_progress[
-                            node_progress_event.progress_type
-                        ] = new_progress
+                        self._current_progress[node_progress_event.progress_type] = (
+                            new_progress
+                        )
 
                         self.logger.info(
                             "Current startup progress [expected number of node-progress-types=%d]: %s",
@@ -343,29 +341,30 @@ def __call__(self, message: str) -> bool:
                 url = (
                     f"https://{self.node_id}.services.{self.get_partial_product_url()}"
                 )
-            response = self.api_request_context.get(url, timeout=1000)
-            level = logging.DEBUG
-            if (response.status >= 400) and (response.status not in (502, 503)):
-                level = logging.ERROR
-            self.logger.log(
-                level,
-                "Querying service endpoint in case we missed some websocket messages. Url: %s Response: '%s' TIP: %s",
-                url,
-                f"{response.status}: {response.text()}",
-                (
-                    "We are emulating the frontend; a 5XX response is acceptable if the service is not yet ready."
-                ),
-            )
+            with contextlib.suppress(PlaywrightTimeoutError):
+                response = self.api_request_context.get(url, timeout=1000)
+                level = logging.DEBUG
+                if (response.status >= 400) and (response.status not in (502, 503)):
+                    level = logging.ERROR
+                self.logger.log(
+                    level,
+                    "Querying service endpoint in case we missed some websocket messages. Url: %s Response: '%s' TIP: %s",
+                    url,
+                    f"{response.status}: {response.text()}",
+                    (
+                        "We are emulating the frontend; a 5XX response is acceptable if the service is not yet ready."
+                    ),
+                )
 
-            if response.status <= 400:
-                # NOTE: If the response status is less than 400, it means that the backend is ready (There are some services that respond with a 3XX)
-                if self.got_expected_node_progress_types():
-                    self.logger.warning(
-                        "⚠️ Progress bar didn't receive 100 percent but service is already running: %s. TIP: we missed some websocket messages! ⚠️",  # https://github.com/ITISFoundation/osparc-simcore/issues/6449
-                        self.get_current_progress(),
-                    )
-                return True
-            self._last_poll_timestamp = datetime.now(UTC)
+                if response.status <= 400:
+                    # NOTE: If the response status is less than 400, it means that the backend is ready (There are some services that respond with a 3XX)
+                    if self.got_expected_node_progress_types():
+                        self.logger.warning(
+                            "⚠️ Progress bar didn't receive 100 percent but service is already running: %s. TIP: we missed some websocket messages! ⚠️",  # https://github.com/ITISFoundation/osparc-simcore/issues/6449
+                            self.get_current_progress(),
+                        )
+                    return True
+                self._last_poll_timestamp = datetime.now(UTC)
 
         return False
 
@@ -508,3 +507,16 @@ def app_mode_trigger_next_app(page: Page) -> None:
     ):
         # Move to next step (this auto starts the next service)
         page.get_by_test_id("AppMode_NextBtn").click()
+
+
+def wait_for_label_text(
+    page: Page, locator: str, substring: str, timeout: int = 10000
+) -> Locator:
+    page.locator(locator).wait_for(state="visible", timeout=timeout)
+
+    page.wait_for_function(
+        f"() => document.querySelector('{locator}').innerText.includes('{substring}')",
+        timeout=timeout,
+    )
+
+    return page.locator(locator)

packages/pytest-simcore/src/pytest_simcore/helpers/webserver_login.py

@@ -11,8 +11,11 @@
 from simcore_service_webserver.db.models import UserRole, UserStatus
 from simcore_service_webserver.groups.api import auto_add_user_to_product_group
 from simcore_service_webserver.login._constants import MSG_LOGGED_IN
-from simcore_service_webserver.login._registration import create_invitation_token
-from simcore_service_webserver.login.storage import AsyncpgStorage, get_plugin_storage
+from simcore_service_webserver.login._invitations_service import create_invitation_token
+from simcore_service_webserver.login._login_repository_legacy import (
+    AsyncpgStorage,
+    get_plugin_storage,
+)
 from simcore_service_webserver.products.products_service import list_products
 from simcore_service_webserver.security.api import clean_auth_policy_cache
 from yarl import URL