diff --git a/api/specs/web-server/_nih_sparc.py b/api/specs/web-server/_nih_sparc.py index 7d457be55786..8c3fdb4a91b5 100644 --- a/api/specs/web-server/_nih_sparc.py +++ b/api/specs/web-server/_nih_sparc.py @@ -8,7 +8,7 @@ from fastapi import APIRouter from models_library.generics import Envelope from simcore_service_webserver._meta import API_VTAG -from simcore_service_webserver.studies_dispatcher._rest_handlers import ( +from simcore_service_webserver.studies_dispatcher._controller.rest.nih_schemas import ( ServiceGet, Viewer, ) diff --git a/api/specs/web-server/_nih_sparc_redirections.py b/api/specs/web-server/_nih_sparc_redirections.py index df1693e28773..f13a7b99ce5d 100644 --- a/api/specs/web-server/_nih_sparc_redirections.py +++ b/api/specs/web-server/_nih_sparc_redirections.py @@ -1,5 +1,4 @@ -""" Helper script to generate OAS automatically NIH-sparc portal API section -""" +"""Helper script to generate OAS automatically NIH-sparc portal API section""" # pylint: disable=protected-access # pylint: disable=redefined-outer-name @@ -11,7 +10,7 @@ from fastapi import APIRouter, status from fastapi.responses import RedirectResponse from models_library.projects import ProjectID -from models_library.services import ServiceKey, ServiceKeyVersion +from models_library.services_types import ServiceKey, ServiceVersion from pydantic import HttpUrl, PositiveInt router = APIRouter( @@ -31,7 +30,7 @@ async def get_redirection_to_viewer( file_type: str, viewer_key: ServiceKey, - viewer_version: ServiceKeyVersion, + viewer_version: ServiceVersion, file_size: PositiveInt, download_link: HttpUrl, file_name: str | None = "unknown", diff --git a/packages/pytest-simcore/src/pytest_simcore/__init__.py b/packages/pytest-simcore/src/pytest_simcore/__init__.py index 8716d997ef21..7a7da935aadc 100644 --- a/packages/pytest-simcore/src/pytest_simcore/__init__.py +++ b/packages/pytest-simcore/src/pytest_simcore/__init__.py @@ -25,7 +25,7 @@ def keep_docker_up(request: pytest.FixtureRequest) -> bool: return flag -@pytest.fixture +@pytest.fixture(scope="session") def is_pdb_enabled(request: pytest.FixtureRequest): """Returns true if tests are set to use interactive debugger, i.e. --pdb""" options = request.config.option diff --git a/packages/pytest-simcore/src/pytest_simcore/helpers/faker_factories.py b/packages/pytest-simcore/src/pytest_simcore/helpers/faker_factories.py index 7c50df70de5c..ef006f53105b 100644 --- a/packages/pytest-simcore/src/pytest_simcore/helpers/faker_factories.py +++ b/packages/pytest-simcore/src/pytest_simcore/helpers/faker_factories.py @@ -571,3 +571,32 @@ def random_itis_vip_available_download_item( data.update(**overrides) return data + + +def random_service_consume_filetype( + *, + service_key: str, + service_version: str, + fake: Faker = DEFAULT_FAKER, + **overrides, +) -> dict[str, Any]: + from simcore_postgres_database.models.services_consume_filetypes import ( + services_consume_filetypes, + ) + + data = { + "service_key": service_key, + "service_version": service_version, + "service_display_name": fake.company(), + "service_input_port": fake.word(), + "filetype": fake.random_element(["CSV", "VTK", "H5", "JSON", "TXT"]), + "preference_order": fake.pyint(min_value=0, max_value=10), + "is_guest_allowed": fake.pybool(), + } + + assert set(data.keys()).issubset( # nosec + {c.name for c in services_consume_filetypes.columns} + ) + + data.update(overrides) + return data diff --git a/packages/pytest-simcore/src/pytest_simcore/helpers/postgres_tools.py b/packages/pytest-simcore/src/pytest_simcore/helpers/postgres_tools.py index 1e854e8b6874..1086a61fc7ae 100644 --- a/packages/pytest-simcore/src/pytest_simcore/helpers/postgres_tools.py +++ b/packages/pytest-simcore/src/pytest_simcore/helpers/postgres_tools.py @@ -88,20 +88,54 @@ async def _async_insert_and_get_row( conn: AsyncConnection, table: sa.Table, values: dict[str, Any], - pk_col: sa.Column, + pk_col: sa.Column | None = None, pk_value: Any | None = None, + pk_cols: list[sa.Column] | None = None, + pk_values: list[Any] | None = None, ) -> sa.engine.Row: - result = await conn.execute(table.insert().values(**values).returning(pk_col)) + # Validate parameters + single_pk_provided = pk_col is not None + composite_pk_provided = pk_cols is not None + + if single_pk_provided == composite_pk_provided: + msg = "Must provide either pk_col or pk_cols, but not both" + raise ValueError(msg) + + if composite_pk_provided: + if pk_values is not None and len(pk_cols) != len(pk_values): + msg = "pk_cols and pk_values must have the same length" + raise ValueError(msg) + returning_cols = pk_cols + else: + returning_cols = [pk_col] + + result = await conn.execute( + table.insert().values(**values).returning(*returning_cols) + ) row = result.one() - # Get the pk_value from the row if not provided - if pk_value is None: - pk_value = getattr(row, pk_col.name) + if composite_pk_provided: + # Handle composite primary keys + if pk_values is None: + pk_values = [getattr(row, col.name) for col in pk_cols] + else: + for col, expected_value in zip(pk_cols, pk_values, strict=True): + assert getattr(row, col.name) == expected_value + + # Build WHERE clause for composite key + where_clause = sa.and_( + *[col == val for col, val in zip(pk_cols, pk_values, strict=True)] + ) else: - # NOTE: DO NO USE row[pk_col] since you will get a deprecation error (Background on SQLAlchemy 2.0 at: https://sqlalche.me/e/b8d9) - assert getattr(row, pk_col.name) == pk_value + # Handle single primary key (existing logic) + if pk_value is None: + pk_value = getattr(row, pk_col.name) + else: + assert getattr(row, pk_col.name) == pk_value + + where_clause = pk_col == pk_value - result = await conn.execute(sa.select(table).where(pk_col == pk_value)) + result = await conn.execute(sa.select(table).where(where_clause)) return result.one() @@ -109,20 +143,52 @@ def _sync_insert_and_get_row( conn: sa.engine.Connection, table: sa.Table, values: dict[str, Any], - pk_col: sa.Column, + pk_col: sa.Column | None = None, pk_value: Any | None = None, + pk_cols: list[sa.Column] | None = None, + pk_values: list[Any] | None = None, ) -> sa.engine.Row: - result = conn.execute(table.insert().values(**values).returning(pk_col)) + # Validate parameters + single_pk_provided = pk_col is not None + composite_pk_provided = pk_cols is not None + + if single_pk_provided == composite_pk_provided: + msg = "Must provide either pk_col or pk_cols, but not both" + raise ValueError(msg) + + if composite_pk_provided: + if pk_values is not None and len(pk_cols) != len(pk_values): + msg = "pk_cols and pk_values must have the same length" + raise ValueError(msg) + returning_cols = pk_cols + else: + returning_cols = [pk_col] + + result = conn.execute(table.insert().values(**values).returning(*returning_cols)) row = result.one() - # Get the pk_value from the row if not provided - if pk_value is None: - pk_value = getattr(row, pk_col.name) + if composite_pk_provided: + # Handle composite primary keys + if pk_values is None: + pk_values = [getattr(row, col.name) for col in pk_cols] + else: + for col, expected_value in zip(pk_cols, pk_values, strict=True): + assert getattr(row, col.name) == expected_value + + # Build WHERE clause for composite key + where_clause = sa.and_( + *[col == val for col, val in zip(pk_cols, pk_values, strict=True)] + ) else: - # NOTE: DO NO USE row[pk_col] since you will get a deprecation error (Background on SQLAlchemy 2.0 at: https://sqlalche.me/e/b8d9) - assert getattr(row, pk_col.name) == pk_value + # Handle single primary key (existing logic) + if pk_value is None: + pk_value = getattr(row, pk_col.name) + else: + assert getattr(row, pk_col.name) == pk_value + + where_clause = pk_col == pk_value - result = conn.execute(sa.select(table).where(pk_col == pk_value)) + result = conn.execute(sa.select(table).where(where_clause)) return result.one() @@ -132,17 +198,125 @@ async def insert_and_get_row_lifespan( *, table: sa.Table, values: dict[str, Any], - pk_col: sa.Column, + pk_col: sa.Column | None = None, pk_value: Any | None = None, + pk_cols: list[sa.Column] | None = None, + pk_values: list[Any] | None = None, ) -> AsyncIterator[dict[str, Any]]: + """ + Context manager that inserts a row into a table and automatically deletes it on exit. + + Args: + sqlalchemy_async_engine: Async SQLAlchemy engine + table: The table to insert into + values: Dictionary of column values to insert + pk_col: Primary key column for deletion (for single-column primary keys) + pk_value: Optional primary key value (if None, will be taken from inserted row) + pk_cols: List of primary key columns (for composite primary keys) + pk_values: Optional list of primary key values (if None, will be taken from inserted row) + + Yields: + dict: The inserted row as a dictionary + + Examples: + ## Single primary key usage: + + @pytest.fixture + async def user_in_db(asyncpg_engine: AsyncEngine) -> AsyncIterator[dict]: + user_data = random_user(name="test_user", email="test@example.com") + async with insert_and_get_row_lifespan( + asyncpg_engine, + table=users, + values=user_data, + pk_col=users.c.id, + ) as row: + yield row + + ##Composite primary key usage: + + @pytest.fixture + async def service_in_db(asyncpg_engine: AsyncEngine) -> AsyncIterator[dict]: + service_data = {"key": "simcore/services/comp/test", "version": "1.0.0", "name": "Test Service"} + async with insert_and_get_row_lifespan( + asyncpg_engine, + table=services, + values=service_data, + pk_cols=[services.c.key, services.c.version], + ) as row: + yield row + + ##Multiple rows with single primary keys using AsyncExitStack: + + @pytest.fixture + async def users_in_db(asyncpg_engine: AsyncEngine) -> AsyncIterator[list[dict]]: + users_data = [ + random_user(name="user1", email="user1@example.com"), + random_user(name="user2", email="user2@example.com"), + ] + + async with AsyncExitStack() as stack: + created_users = [] + for user_data in users_data: + row = await stack.enter_async_context( + insert_and_get_row_lifespan( + asyncpg_engine, + table=users, + values=user_data, + pk_col=users.c.id, + ) + ) + created_users.append(row) + + yield created_users + + ## Multiple rows with composite primary keys using AsyncExitStack: + + @pytest.fixture + async def services_in_db(asyncpg_engine: AsyncEngine) -> AsyncIterator[list[dict]]: + services_data = [ + {"key": "simcore/services/comp/service1", "version": "1.0.0", "name": "Service 1"}, + {"key": "simcore/services/comp/service2", "version": "2.0.0", "name": "Service 2"}, + {"key": "simcore/services/comp/service1", "version": "2.0.0", "name": "Service 1 v2"}, + ] + + async with AsyncExitStack() as stack: + created_services = [] + for service_data in services_data: + row = await stack.enter_async_context( + insert_and_get_row_lifespan( + asyncpg_engine, + table=services, + values=service_data, + pk_cols=[services.c.key, services.c.version], + ) + ) + created_services.append(row) + + yield created_services + """ # SETUP: insert & get async with sqlalchemy_async_engine.begin() as conn: row = await _async_insert_and_get_row( - conn, table=table, values=values, pk_col=pk_col, pk_value=pk_value + conn, + table=table, + values=values, + pk_col=pk_col, + pk_value=pk_value, + pk_cols=pk_cols, + pk_values=pk_values, ) - # If pk_value was None, get it from the row for deletion later - if pk_value is None: - pk_value = getattr(row, pk_col.name) + + # Get pk values for deletion + if pk_cols is not None: + if pk_values is None: + pk_values = [getattr(row, col.name) for col in pk_cols] + where_clause = sa.and_( + *[col == val for col, val in zip(pk_cols, pk_values, strict=True)] + ) + else: + if pk_value is None: + pk_value = getattr(row, pk_col.name) + where_clause = pk_col == pk_value assert row @@ -150,9 +324,9 @@ async def insert_and_get_row_lifespan( # pylint: disable=protected-access yield row._asdict() - # TEAD-DOWN: delete row + # TEARDOWN: delete row async with sqlalchemy_async_engine.begin() as conn: - await conn.execute(table.delete().where(pk_col == pk_value)) + await conn.execute(table.delete().where(where_clause)) @contextmanager @@ -161,23 +335,43 @@ def sync_insert_and_get_row_lifespan( *, table: sa.Table, values: dict[str, Any], - pk_col: sa.Column, + pk_col: sa.Column | None = None, pk_value: Any | None = None, + pk_cols: list[sa.Column] | None = None, + pk_values: list[Any] | None = None, ) -> Iterator[dict[str, Any]]: """sync version of insert_and_get_row_lifespan. TIP: more convenient for **module-scope fixtures** that setup the database tables before the app starts since it does not require an `event_loop` - fixture (which is funcition-scoped ) + fixture (which is function-scoped) + + Supports both single and composite primary keys using the same parameter patterns + as the async version. """ # SETUP: insert & get with sqlalchemy_sync_engine.begin() as conn: row = _sync_insert_and_get_row( - conn, table=table, values=values, pk_col=pk_col, pk_value=pk_value + conn, + table=table, + values=values, + pk_col=pk_col, + pk_value=pk_value, + pk_cols=pk_cols, + pk_values=pk_values, ) - # If pk_value was None, get it from the row for deletion later - if pk_value is None: - pk_value = getattr(row, pk_col.name) + + # Get pk values for deletion + if pk_cols is not None: + if pk_values is None: + pk_values = [getattr(row, col.name) for col in pk_cols] + where_clause = sa.and_( + *[col == val for col, val in zip(pk_cols, pk_values, strict=True)] + ) + else: + if pk_value is None: + pk_value = getattr(row, pk_col.name) + where_clause = pk_col == pk_value assert row @@ -187,4 +381,4 @@ def sync_insert_and_get_row_lifespan( # TEARDOWN: delete row with sqlalchemy_sync_engine.begin() as conn: - conn.execute(table.delete().where(pk_col == pk_value)) + conn.execute(table.delete().where(where_clause)) diff --git a/packages/pytest-simcore/src/pytest_simcore/helpers/webserver_fake_services_data.py b/packages/pytest-simcore/src/pytest_simcore/helpers/webserver_fake_services_data.py index 32d91d46c783..f3bb3c003f0b 100644 --- a/packages/pytest-simcore/src/pytest_simcore/helpers/webserver_fake_services_data.py +++ b/packages/pytest-simcore/src/pytest_simcore/helpers/webserver_fake_services_data.py @@ -1,6 +1,7 @@ """ - NOTE: avoid creating dependencies +NOTE: avoid creating dependencies """ + from typing import Any FAKE_FILE_CONSUMER_SERVICES = [ @@ -55,7 +56,7 @@ def list_fake_file_consumers() -> list[dict[str, Any]]: consumers = [] for service in FAKE_FILE_CONSUMER_SERVICES: for consumable in service["consumes"]: - filetype, port, *_ = consumable.split(":") + ["input_1"] + filetype, port, *_ = [*consumable.split(":"), "input_1"] consumer = { "key": service["key"], "version": service["version"], diff --git a/services/web/server/setup.cfg b/services/web/server/setup.cfg index 56ef26e3520a..5ee0c6c434ce 100644 --- a/services/web/server/setup.cfg +++ b/services/web/server/setup.cfg @@ -13,13 +13,13 @@ commit_args = --no-verify addopts = --strict-markers asyncio_mode = auto asyncio_default_fixture_loop_scope = function -markers = +markers = slow: marks tests as slow (deselect with '-m "not slow"') acceptance_test: "marks tests as 'acceptance tests' i.e. does the system do what the user expects? Typically those are workflows." testit: "marks test to run during development" heavy_load: "mark tests that require large amount of data" [mypy] -plugins = +plugins = pydantic.mypy sqlalchemy.ext.mypy.plugin diff --git a/services/web/server/src/simcore_service_webserver/api/v0/openapi.yaml b/services/web/server/src/simcore_service_webserver/api/v0/openapi.yaml index 1117ee5e120d..8557fa526c92 100644 --- a/services/web/server/src/simcore_service_webserver/api/v0/openapi.yaml +++ b/services/web/server/src/simcore_service_webserver/api/v0/openapi.yaml @@ -4528,6 +4528,13 @@ paths: type: string pattern: ^simcore/services/((comp|dynamic|frontend))/([a-z0-9][a-z0-9_.-]*/)*([a-z0-9-_]+[a-z0-9])$ title: Viewer Key + - name: viewer_version + in: query + required: true + schema: + type: string + pattern: ^(0|[1-9]\d*)(\.(0|[1-9]\d*)){2}(-(0|[1-9]\d*|\d*[-a-zA-Z][-\da-zA-Z]*)(\.(0|[1-9]\d*|\d*[-a-zA-Z][-\da-zA-Z]*))*)?(\+[-\da-zA-Z]+(\.[-\da-zA-Z-]+)*)?$ + title: Viewer Version - name: file_size in: query required: true @@ -4554,12 +4561,6 @@ paths: - type: 'null' default: unknown title: File Name - requestBody: - required: true - content: - application/json: - schema: - $ref: '#/components/schemas/ServiceKeyVersion' responses: '302': description: Opens osparc and starts viewer for selected data diff --git a/services/web/server/src/simcore_service_webserver/studies_dispatcher/_catalog.py b/services/web/server/src/simcore_service_webserver/studies_dispatcher/_catalog.py index 1a17651e13ea..30a9fbd35ecb 100644 --- a/services/web/server/src/simcore_service_webserver/studies_dispatcher/_catalog.py +++ b/services/web/server/src/simcore_service_webserver/studies_dispatcher/_catalog.py @@ -25,7 +25,7 @@ from simcore_postgres_database.utils_services import create_select_latest_services_query from ..db.plugin import get_database_engine_legacy -from ._errors import ServiceNotFound +from ._errors import ServiceNotFoundError from .settings import StudiesDispatcherSettings, get_plugin_settings LARGEST_PAGE_SIZE = 1000 @@ -156,7 +156,7 @@ async def validate_requested_service( row = await result.fetchone() if row is None: - raise ServiceNotFound( + raise ServiceNotFoundError( service_key=service_key, service_version=service_version ) diff --git a/services/web/server/src/simcore_service_webserver/studies_dispatcher/_constants.py b/services/web/server/src/simcore_service_webserver/studies_dispatcher/_constants.py index 3a2409cf4e9d..d92c75618eb1 100644 --- a/services/web/server/src/simcore_service_webserver/studies_dispatcher/_constants.py +++ b/services/web/server/src/simcore_service_webserver/studies_dispatcher/_constants.py @@ -24,11 +24,6 @@ _version=1, ) -MSG_GUESTS_NOT_ALLOWED: Final[str] = user_message( - "Access is restricted to registered users.

" - "If you don't have an account, please contact support to request one.

", - _version=1, -) MSG_TOO_MANY_GUESTS: Final[str] = user_message( "We have reached the maximum number of anonymous users allowed on the platform. " diff --git a/services/web/server/src/simcore_service_webserver/studies_dispatcher/_controller/__init__.py b/services/web/server/src/simcore_service_webserver/studies_dispatcher/_controller/__init__.py new file mode 100644 index 000000000000..a051fdcd7af5 --- /dev/null +++ b/services/web/server/src/simcore_service_webserver/studies_dispatcher/_controller/__init__.py @@ -0,0 +1,29 @@ +import logging + +from aiohttp import web + +from ...login.decorators import login_required +from .._controller.rest.redirects import get_redirection_to_viewer +from ..settings import StudiesDispatcherSettings +from .rest.nih import routes as nih_routes +from .rest.redirects import get_redirection_to_viewer + +_logger = logging.getLogger(__name__) + + +def setup_controller(app: web.Application, settings: StudiesDispatcherSettings): + # routes + redirect_handler = get_redirection_to_viewer + if settings.is_login_required(): + redirect_handler = login_required(get_redirection_to_viewer) + + _logger.info( + "'%s' config explicitly disables anonymous users from this feature", + __name__, + ) + + app.router.add_routes( + [web.get("/view", redirect_handler, name="get_redirection_to_viewer")] + ) + + app.router.add_routes(nih_routes) diff --git a/services/web/server/src/simcore_service_webserver/studies_dispatcher/_controller/rest/__init__.py b/services/web/server/src/simcore_service_webserver/studies_dispatcher/_controller/rest/__init__.py new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/services/web/server/src/simcore_service_webserver/studies_dispatcher/_controller/rest/nih.py b/services/web/server/src/simcore_service_webserver/studies_dispatcher/_controller/rest/nih.py new file mode 100644 index 000000000000..74174840fb38 --- /dev/null +++ b/services/web/server/src/simcore_service_webserver/studies_dispatcher/_controller/rest/nih.py @@ -0,0 +1,65 @@ +"""Handles requests to the Rest API""" + +import logging + +from aiohttp import web +from aiohttp.web import Request +from pydantic import ( + ValidationError, +) + +from ...._meta import API_VTAG +from ....products import products_web +from ....utils_aiohttp import envelope_json_response +from ... import _service +from ..._catalog import iter_latest_product_services +from .nih_schemas import ServiceGet, Viewer + +_logger = logging.getLogger(__name__) + + +routes = web.RouteTableDef() + + +@routes.get(f"/{API_VTAG}/services", name="list_latest_services") +async def list_latest_services(request: Request): + """Returns a list latest version of services""" + product_name = products_web.get_product_name(request) + + services = [] + async for service_data in iter_latest_product_services( + request.app, product_name=product_name + ): + try: + service = ServiceGet.create(service_data, request) + services.append(service) + except ValidationError as err: + _logger.debug("Invalid %s: %s", f"{service_data=}", err) + + return envelope_json_response(services) + + +@routes.get(f"/{API_VTAG}/viewers", name="list_viewers") +async def list_viewers(request: Request): + # filter: file_type=* + file_type: str | None = request.query.get("file_type", None) + + viewers = [ + Viewer.create(request, viewer).model_dump() + for viewer in await _service.list_viewers_info(request.app, file_type=file_type) + ] + return envelope_json_response(viewers) + + +@routes.get(f"/{API_VTAG}/viewers/default", name="list_default_viewers") +async def list_default_viewers(request: Request): + # filter: file_type=* + file_type: str | None = request.query.get("file_type", None) + + viewers = [ + Viewer.create(request, viewer).model_dump() + for viewer in await _service.list_viewers_info( + request.app, file_type=file_type, only_default=True + ) + ] + return envelope_json_response(viewers) diff --git a/services/web/server/src/simcore_service_webserver/studies_dispatcher/_rest_handlers.py b/services/web/server/src/simcore_service_webserver/studies_dispatcher/_controller/rest/nih_schemas.py similarity index 67% rename from services/web/server/src/simcore_service_webserver/studies_dispatcher/_rest_handlers.py rename to services/web/server/src/simcore_service_webserver/studies_dispatcher/_controller/rest/nih_schemas.py index 943893972fe6..b9579f0ff9c1 100644 --- a/services/web/server/src/simcore_service_webserver/studies_dispatcher/_rest_handlers.py +++ b/services/web/server/src/simcore_service_webserver/studies_dispatcher/_controller/rest/nih_schemas.py @@ -1,6 +1,3 @@ -""" Handles requests to the Rest API - -""" import logging from dataclasses import asdict @@ -13,18 +10,13 @@ ConfigDict, Field, TypeAdapter, - ValidationError, field_validator, ) from pydantic.networks import HttpUrl -from .._meta import API_VTAG -from ..products import products_web -from ..utils_aiohttp import envelope_json_response -from ._catalog import ServiceMetaData, iter_latest_product_services -from ._core import list_viewers_info -from ._models import ViewerInfo -from ._redirects_handlers import ViewerQueryParams +from ..._catalog import ServiceMetaData +from ..._models import ViewerInfo +from .redirects_schemas import ViewerQueryParams _logger = logging.getLogger(__name__) @@ -50,8 +42,8 @@ def _compose_service_only_dispatcher_prefix_url( request: web.Request, service_key: str, service_version: str ) -> HttpUrl: params = ViewerQueryParams( - viewer_key=ServiceKey(service_key), - viewer_version=ServiceVersion(service_version), + viewer_key=TypeAdapter(ServiceKey).validate_python(service_key), + viewer_version=TypeAdapter(ServiceVersion).validate_python(service_version), ).model_dump(exclude_none=True, exclude_unset=True) absolute_url = request.url.join( request.app.router["get_redirection_to_viewer"].url_for().with_query(**params) @@ -150,60 +142,3 @@ def remove_dot_prefix_from_extension(cls, v): } } ) - - -# -# API Handlers -# - - -routes = web.RouteTableDef() - - -@routes.get(f"/{API_VTAG}/services", name="list_latest_services") -async def list_latest_services(request: Request): - """Returns a list latest version of services""" - product_name = products_web.get_product_name(request) - - services = [] - async for service_data in iter_latest_product_services( - request.app, product_name=product_name - ): - try: - service = ServiceGet.create(service_data, request) - services.append(service) - except ValidationError as err: - _logger.debug("Invalid %s: %s", f"{service_data=}", err) - - return envelope_json_response(services) - - -@routes.get(f"/{API_VTAG}/viewers", name="list_viewers") -async def list_viewers(request: Request): - # filter: file_type=* - file_type: str | None = request.query.get("file_type", None) - - viewers = [ - Viewer.create(request, viewer).model_dump() - for viewer in await list_viewers_info(request.app, file_type=file_type) - ] - return envelope_json_response(viewers) - - -@routes.get(f"/{API_VTAG}/viewers/default", name="list_default_viewers") -async def list_default_viewers(request: Request): - # filter: file_type=* - file_type: str | None = request.query.get("file_type", None) - - viewers = [ - Viewer.create(request, viewer).model_dump() - for viewer in await list_viewers_info( - request.app, file_type=file_type, only_default=True - ) - ] - return envelope_json_response(viewers) - - -rest_handler_functions = { - fun.__name__: fun for fun in [list_default_viewers, list_viewers] -} diff --git a/services/web/server/src/simcore_service_webserver/studies_dispatcher/_redirects_handlers.py b/services/web/server/src/simcore_service_webserver/studies_dispatcher/_controller/rest/redirects.py similarity index 54% rename from services/web/server/src/simcore_service_webserver/studies_dispatcher/_redirects_handlers.py rename to services/web/server/src/simcore_service_webserver/studies_dispatcher/_controller/rest/redirects.py index e51303599154..332d628bd196 100644 --- a/services/web/server/src/simcore_service_webserver/studies_dispatcher/_redirects_handlers.py +++ b/services/web/server/src/simcore_service_webserver/studies_dispatcher/_controller/rest/redirects.py @@ -1,40 +1,39 @@ """Handles request to the viewers redirection entrypoints""" -import functools import logging -import urllib.parse -from typing import TypeAlias from aiohttp import web -from common_library.error_codes import create_error_code from models_library.projects import ProjectID from models_library.projects_nodes_io import NodeID -from models_library.services import ServiceKey, ServiceVersion -from pydantic import BaseModel, ConfigDict, ValidationError, field_validator -from servicelib.aiohttp import status from servicelib.aiohttp.requests_validation import parse_request_query_parameters_as -from servicelib.aiohttp.typing_extension import Handler -from servicelib.logging_errors import create_troubleshootting_log_kwargs - -from ..dynamic_scheduler import api as dynamic_scheduler_service -from ..products import products_web -from ..utils import compose_support_error_msg -from ..utils_aiohttp import create_redirect_to_page_response, get_api_base_url -from ._catalog import ValidService, validate_requested_service -from ._constants import MSG_UNEXPECTED_DISPATCH_ERROR -from ._core import validate_requested_file, validate_requested_viewer -from ._errors import InvalidRedirectionParams, StudyDispatcherError -from ._models import FileParams, ServiceInfo, ServiceParams, ViewerInfo -from ._projects import ( + +from ....dynamic_scheduler import api as dynamic_scheduler_service +from ....products import products_web +from ....utils_aiohttp import create_redirect_to_page_response, get_api_base_url +from ... import _service +from ..._catalog import ValidService, validate_requested_service +from ..._errors import ( + InvalidRedirectionParamsError, +) +from ..._models import ServiceInfo, ViewerInfo +from ..._projects import ( get_or_create_project_with_file, get_or_create_project_with_file_and_service, get_or_create_project_with_service, ) -from ._users import UserInfo, ensure_authentication, get_or_create_guest_user -from .settings import get_plugin_settings +from ..._users import UserInfo, ensure_authentication, get_or_create_guest_user +from ...settings import get_plugin_settings +from .redirects_exceptions import handle_errors_with_error_page +from .redirects_schemas import ( + FileQueryParams, + RedirectionQueryParams, + ServiceAndFileParams, + ServiceQueryParams, +) _logger = logging.getLogger(__name__) + # # HELPERS # @@ -58,18 +57,6 @@ def _create_redirect_response_to_view_page( ) -def _create_redirect_response_to_error_page( - app: web.Application, message: str, status_code: int -) -> web.HTTPFound: - # NOTE: these are 'error' page params and need to be interpreted by front-end correctly! - return create_redirect_to_page_response( - app, - page="error", - message=message, - status_code=status_code, - ) - - def _create_service_info_from(service: ValidService) -> ServiceInfo: values_map = { "key": service.key, @@ -82,133 +69,12 @@ def _create_service_info_from(service: ValidService) -> ServiceInfo: return ServiceInfo.model_construct(_fields_set=set(values_map.keys()), **values_map) -def _handle_errors_with_error_page(handler: Handler): - @functools.wraps(handler) - async def wrapper(request: web.Request) -> web.StreamResponse: - try: - return await handler(request) - - except (web.HTTPRedirection, web.HTTPSuccessful): - # NOTE: that response is a redirection that is reraised and not returned - raise - - except StudyDispatcherError as err: - raise _create_redirect_response_to_error_page( - request.app, - message=f"Sorry, we cannot dispatch your study: {err}", - status_code=status.HTTP_422_UNPROCESSABLE_ENTITY, # 422 - ) from err - - except web.HTTPUnauthorized as err: - raise _create_redirect_response_to_error_page( - request.app, - message=f"{err.text}. Please reload this page to login/register.", - status_code=err.status_code, - ) from err - - except web.HTTPUnprocessableEntity as err: - raise _create_redirect_response_to_error_page( - request.app, - message=f"Invalid parameters in link: {err.text}", - status_code=status.HTTP_422_UNPROCESSABLE_ENTITY, # 422 - ) from err - - except web.HTTPClientError as err: - _logger.exception("Client error with status code %d", err.status_code) - raise _create_redirect_response_to_error_page( - request.app, - message=err.reason, - status_code=err.status_code, - ) from err - - except (ValidationError, web.HTTPServerError, Exception) as err: - error_code = create_error_code(err) - - user_error_msg = compose_support_error_msg( - msg=MSG_UNEXPECTED_DISPATCH_ERROR, error_code=error_code - ) - _logger.exception( - **create_troubleshootting_log_kwargs( - user_error_msg, - error=err, - error_code=error_code, - error_context={"request": request}, - tip="Unexpected failure while dispatching study", - ) - ) - raise _create_redirect_response_to_error_page( - request.app, - message=user_error_msg, - status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, - ) from err - - return wrapper - - -# -# API Schemas -# - - -class ServiceQueryParams(ServiceParams): - model_config = ConfigDict(extra="forbid") - - -class FileQueryParams(FileParams): - model_config = ConfigDict(extra="forbid") - - @field_validator("file_type") - @classmethod - def ensure_extension_upper_and_dotless(cls, v): - # NOTE: see filetype constraint-check - if v and isinstance(v, str): - w = urllib.parse.unquote(v) - return w.upper().lstrip(".") - return v - - -class ServiceAndFileParams(FileQueryParams, ServiceParams): ... - - -class ViewerQueryParams(BaseModel): - file_type: str | None = None - viewer_key: ServiceKey - viewer_version: ServiceVersion - - @staticmethod - def from_viewer(viewer: ViewerInfo) -> "ViewerQueryParams": - # can safely construct w/o validation from a viewer - return ViewerQueryParams.model_construct( - file_type=viewer.filetype, - viewer_key=viewer.key, - viewer_version=viewer.version, - ) - - @field_validator("file_type") - @classmethod - def ensure_extension_upper_and_dotless(cls, v): - # NOTE: see filetype constraint-check - if v and isinstance(v, str): - w = urllib.parse.unquote(v) - return w.upper().lstrip(".") - return v - - -RedirectionQueryParams: TypeAlias = ( - # NOTE: Extra.forbid in FileQueryParams, ServiceQueryParams avoids bad casting when - # errors in ServiceAndFileParams - ServiceAndFileParams - | FileQueryParams - | ServiceQueryParams -) - - # -# API HANDLERS +# ROUTES # -@_handle_errors_with_error_page +@handle_errors_with_error_page async def get_redirection_to_viewer(request: web.Request): """ - validate request @@ -229,7 +95,7 @@ async def get_redirection_to_viewer(request: web.Request): file_params = service_params = query_params # NOTE: Cannot check file_size in from HEAD in a AWS download link so file_size is just infomative - viewer: ViewerInfo = await validate_requested_viewer( + viewer: ViewerInfo = await _service.validate_requested_viewer( request.app, file_type=file_params.file_type, file_size=file_params.file_size, @@ -298,7 +164,7 @@ async def get_redirection_to_viewer(request: web.Request): elif isinstance(query_params, FileQueryParams): file_params_ = query_params - validate_requested_file( + _service.validate_requested_file( app=request.app, file_type=file_params_.file_type, file_size=file_params_.file_size, @@ -336,7 +202,7 @@ async def get_redirection_to_viewer(request: web.Request): else: # NOTE: if query is done right, this should never happen - raise InvalidRedirectionParams + raise InvalidRedirectionParamsError(query_params=query_params) # Adds auth cookies (login) await ensure_authentication(user, request, response) diff --git a/services/web/server/src/simcore_service_webserver/studies_dispatcher/_controller/rest/redirects_exceptions.py b/services/web/server/src/simcore_service_webserver/studies_dispatcher/_controller/rest/redirects_exceptions.py new file mode 100644 index 000000000000..bdd7ee662a35 --- /dev/null +++ b/services/web/server/src/simcore_service_webserver/studies_dispatcher/_controller/rest/redirects_exceptions.py @@ -0,0 +1,167 @@ +import functools +import logging + +from aiohttp import web +from common_library.error_codes import create_error_code +from common_library.user_messages import user_message +from models_library.function_services_catalog._utils import ServiceNotFound +from servicelib.aiohttp import status +from servicelib.aiohttp.typing_extension import Handler +from servicelib.logging_errors import create_troubleshootting_log_kwargs + +from ....exception_handling import create_error_context_from_request +from ....utils import compose_support_error_msg +from ....utils_aiohttp import create_redirect_to_page_response +from ..._constants import MSG_UNEXPECTED_DISPATCH_ERROR +from ..._errors import ( + FileToLargeError, + GuestUserNotAllowedError, + GuestUsersLimitError, + IncompatibleServiceError, + InvalidRedirectionParamsError, + ProjectWorkbenchMismatchError, +) + +_logger = logging.getLogger(__name__) + +# +# HELPERS +# + + +def _create_redirect_response_to_error_page( + app: web.Application, message: str, status_code: int +) -> web.HTTPFound: + # NOTE: these are 'error' page params and need to be interpreted by front-end correctly! + return create_redirect_to_page_response( + app, + page="error", + message=message, + status_code=status_code, + ) + + +def _create_error_redirect_with_logging( + request: web.Request, + err: Exception, + *, + message: str, + status_code: int, + tip: str | None = None, +) -> web.HTTPFound: + """Helper to create error redirect with consistent logging""" + error_code = create_error_code(err) + user_error_msg = compose_support_error_msg(msg=message, error_code=error_code) + + _logger.exception( + **create_troubleshootting_log_kwargs( + user_error_msg, + error=err, + error_code=error_code, + error_context=create_error_context_from_request(request), + tip=tip, + ) + ) + + return _create_redirect_response_to_error_page( + request.app, + message=user_error_msg, + status_code=status_code, + ) + + +def _create_simple_error_redirect( + request: web.Request, + public_error: Exception, + *, + status_code: int, +) -> web.HTTPFound: + """Helper to create simple error redirect without logging + + WARNING: note that the `public_error` is exposed as-is in the user-message + """ + user_error_msg = user_message( + f"Unable to open your project: {public_error}", _version=1 + ) + return _create_redirect_response_to_error_page( + request.app, + message=user_error_msg, + status_code=status_code, + ) + + +def handle_errors_with_error_page(handler: Handler): + @functools.wraps(handler) + async def _wrapper(request: web.Request) -> web.StreamResponse: + try: + return await handler(request) + + except (web.HTTPRedirection, web.HTTPSuccessful): + # NOTE: that response is a redirection that is reraised and not returned + raise + + except GuestUserNotAllowedError as err: + raise _create_redirect_response_to_error_page( + request.app, + message=user_message( + "Access is restricted to registered users.

" + "If you don't have an account, please contact support to request one.

", + _version=2, + ), + status_code=status.HTTP_401_UNAUTHORIZED, + ) from err + + except ProjectWorkbenchMismatchError as err: + raise _create_error_redirect_with_logging( + request, + err, + message=MSG_UNEXPECTED_DISPATCH_ERROR, + status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, + tip="project might be corrupted", + ) from err + + except ( + ServiceNotFound, + FileToLargeError, + IncompatibleServiceError, + GuestUsersLimitError, + ) as err: + raise _create_simple_error_redirect( + request, + err, + status_code=status.HTTP_422_UNPROCESSABLE_ENTITY, + ) from err + + except (InvalidRedirectionParamsError, web.HTTPUnprocessableEntity) as err: + # Validation error in query parameters + raise _create_error_redirect_with_logging( + request, + err, + message=user_message( + "The link you provided is invalid because it doesn't contain valid information related to data or a service. " + "Please check the link and make sure it is correct.", + _version=1, + ), + status_code=status.HTTP_422_UNPROCESSABLE_ENTITY, + tip="The link might be corrupted", + ) from err + + except web.HTTPClientError as err: + raise _create_error_redirect_with_logging( + request, + err, + message="Fatal error while redirecting request", + status_code=err.status_code, + tip="The link might be corrupted", + ) from err + + except Exception as err: + raise _create_error_redirect_with_logging( + request, + err, + message=MSG_UNEXPECTED_DISPATCH_ERROR, + status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, + tip="Unexpected failure while dispatching study", + ) from err + + return _wrapper diff --git a/services/web/server/src/simcore_service_webserver/studies_dispatcher/_controller/rest/redirects_schemas.py b/services/web/server/src/simcore_service_webserver/studies_dispatcher/_controller/rest/redirects_schemas.py new file mode 100644 index 000000000000..3cbbd931d148 --- /dev/null +++ b/services/web/server/src/simcore_service_webserver/studies_dispatcher/_controller/rest/redirects_schemas.py @@ -0,0 +1,60 @@ +import urllib.parse +from typing import TypeAlias + +from models_library.services import ServiceKey, ServiceVersion +from pydantic import BaseModel, ConfigDict, field_validator + +from ..._models import FileParams, ServiceParams, ViewerInfo + + +class ServiceQueryParams(ServiceParams): + model_config = ConfigDict(extra="forbid") + + +class FileQueryParams(FileParams): + model_config = ConfigDict(extra="forbid") + + @field_validator("file_type") + @classmethod + def _ensure_extension_upper_and_dotless(cls, v): + # NOTE: see filetype constraint-check + if v and isinstance(v, str): + w = urllib.parse.unquote(v) + return w.upper().lstrip(".") + return v + + +class ServiceAndFileParams(FileQueryParams, ServiceParams): ... + + +class ViewerQueryParams(BaseModel): + file_type: str | None = None + viewer_key: ServiceKey + viewer_version: ServiceVersion + + @staticmethod + def from_viewer(viewer: ViewerInfo) -> "ViewerQueryParams": + # can safely construct w/o validation from a viewer + return ViewerQueryParams.model_construct( + file_type=viewer.filetype, + viewer_key=viewer.key, + viewer_version=viewer.version, + ) + + @field_validator("file_type") + @classmethod + def _ensure_extension_upper_and_dotless(cls, v): + # NOTE: see filetype constraint-check + if v and isinstance(v, str): + w = urllib.parse.unquote(v) + return w.upper().lstrip(".") + return v + + +RedirectionQueryParams: TypeAlias = ( + # NOTE: Extra.forbid in FileQueryParams, ServiceQueryParams avoids bad casting when + # errors in ServiceAndFileParams + ServiceAndFileParams + | FileQueryParams + | ServiceQueryParams +) diff --git a/services/web/server/src/simcore_service_webserver/studies_dispatcher/_core.py b/services/web/server/src/simcore_service_webserver/studies_dispatcher/_core.py deleted file mode 100644 index b5660f40241b..000000000000 --- a/services/web/server/src/simcore_service_webserver/studies_dispatcher/_core.py +++ /dev/null @@ -1,157 +0,0 @@ -import logging -import uuid -from collections import deque -from functools import lru_cache - -import sqlalchemy as sa -from aiohttp import web -from models_library.services import ServiceVersion -from models_library.utils.pydantic_tools_extension import parse_obj_or_none -from pydantic import ByteSize, TypeAdapter, ValidationError -from servicelib.logging_utils import log_decorator -from simcore_postgres_database.models.services_consume_filetypes import ( - services_consume_filetypes, -) -from sqlalchemy.dialects.postgresql import ARRAY, INTEGER - -from ..db.plugin import get_database_engine_legacy -from ._errors import FileToLarge, IncompatibleService -from ._models import ViewerInfo -from .settings import get_plugin_settings - -_BASE_UUID = uuid.UUID("ca2144da-eabb-4daf-a1df-a3682050e25f") - - -_logger = logging.getLogger(__name__) - - -@lru_cache -def compose_uuid_from(*values) -> uuid.UUID: - composition: str = "/".join(map(str, values)) - return uuid.uuid5(_BASE_UUID, composition) - - -async def list_viewers_info( - app: web.Application, file_type: str | None = None, *, only_default: bool = False -) -> list[ViewerInfo]: - # - # TODO: These services MUST be shared with EVERYBODY! Setup check on startup and fill - # with !? - # - consumers: deque = deque() - - async with get_database_engine_legacy(app).acquire() as conn: - # FIXME: ADD CONDITION: service MUST be shared with EVERYBODY! - query = services_consume_filetypes.select() - if file_type: - query = query.where(services_consume_filetypes.c.filetype == file_type) - - query = query.order_by("filetype", "preference_order") - - if file_type and only_default: - query = query.limit(1) - - _logger.debug("Listing viewers:\n%s", query) - - listed_filetype = set() - async for row in await conn.execute(query): - try: - # TODO: filter in database (see test_list_default_compatible_services ) - if only_default and row["filetype"] in listed_filetype: - continue - listed_filetype.add(row["filetype"]) - consumer = ViewerInfo.create_from_db(row) - consumers.append(consumer) - - except ValidationError as err: - _logger.warning("Review invalid service metadata %s: %s", row, err) - - return list(consumers) - - -async def get_default_viewer( - app: web.Application, - file_type: str, - file_size: int | None = None, -) -> ViewerInfo: - """ - - Raises: - IncompatibleService - FileToLarge - """ - try: - viewers = await list_viewers_info(app, file_type, only_default=True) - viewer = viewers[0] - except IndexError as err: - raise IncompatibleService(file_type=file_type) from err - - if current_size := parse_obj_or_none(ByteSize, file_size): - max_size: ByteSize = get_plugin_settings(app).STUDIES_MAX_FILE_SIZE_ALLOWED - if current_size > max_size: - raise FileToLarge(file_size_in_mb=current_size.to("MiB")) - - return viewer - - -@log_decorator(_logger, level=logging.DEBUG) -async def validate_requested_viewer( - app: web.Application, - file_type: str, - file_size: int | None = None, - service_key: str | None = None, - service_version: str | None = None, -) -> ViewerInfo: - """ - - Raises: - IncompatibleService: When there is no match - - """ - - def _version(column_or_value): - # converts version value string to array[integer] that can be compared - return sa.func.string_to_array(column_or_value, ".").cast(ARRAY(INTEGER)) - - if not service_key and not service_version: - return await get_default_viewer(app, file_type, file_size) - - if service_key and service_version: - async with get_database_engine_legacy(app).acquire() as conn: - query = ( - services_consume_filetypes.select() - .where( - (services_consume_filetypes.c.filetype == file_type) - & (services_consume_filetypes.c.service_key == service_key) - & ( - _version(services_consume_filetypes.c.service_version) - <= _version(service_version) - ) - ) - .order_by(_version(services_consume_filetypes.c.service_version).desc()) - .limit(1) - ) - - result = await conn.execute(query) - row = await result.first() - if row: - view = ViewerInfo.create_from_db(row) - view.version = TypeAdapter(ServiceVersion).validate_python( - service_version - ) - return view - - raise IncompatibleService(file_type=file_type) - - -@log_decorator(_logger, level=logging.DEBUG) -def validate_requested_file( - app: web.Application, file_type: str, file_size: int | None = None -): - # NOTE in the future we might want to prevent some types to be pulled - assert file_type # nosec - - if current_size := parse_obj_or_none(ByteSize, file_size): - max_size: ByteSize = get_plugin_settings(app).STUDIES_MAX_FILE_SIZE_ALLOWED - if current_size > max_size: - raise FileToLarge(file_size_in_mb=current_size.to("MiB")) diff --git a/services/web/server/src/simcore_service_webserver/studies_dispatcher/_errors.py b/services/web/server/src/simcore_service_webserver/studies_dispatcher/_errors.py index 4c7c0bbce731..f52b2c1f71af 100644 --- a/services/web/server/src/simcore_service_webserver/studies_dispatcher/_errors.py +++ b/services/web/server/src/simcore_service_webserver/studies_dispatcher/_errors.py @@ -1,28 +1,50 @@ +from common_library.user_messages import user_message + from ..errors import WebServerBaseError -class StudyDispatcherError(WebServerBaseError, ValueError): - ... +class StudyDispatcherError(WebServerBaseError, ValueError): ... -class IncompatibleService(StudyDispatcherError): - msg_template = "None of the registered services can handle '{file_type}'" +class IncompatibleServiceError(StudyDispatcherError): + msg_template = user_message( + "None of the registered services can handle '{file_type}' files.", _version=1 + ) -class FileToLarge(StudyDispatcherError): - msg_template = "File size {file_size_in_mb} MB is over allowed limit" +class FileToLargeError(StudyDispatcherError): + msg_template = user_message( + "File size {file_size_in_mb} MB exceeds the allowed limit.", _version=1 + ) -class ServiceNotFound(StudyDispatcherError): - msg_template = "Service {service_key}:{service_version} not found" +class ServiceNotFoundError(StudyDispatcherError): + msg_template = user_message( + "Service {service_key}:{service_version} could not be found.", _version=1 + ) -class InvalidRedirectionParams(StudyDispatcherError): - msg_template = ( - "The link you provided is invalid because it doesn't contain any information related to data or a service." - " Please check the link and make sure it is correct." +class InvalidRedirectionParamsError(StudyDispatcherError): + msg_template = user_message( + "The provided link is invalid or incomplete.", _version=1 ) class GuestUsersLimitError(StudyDispatcherError): - msg_template = "Maximum number of guests was reached. Please login with a registered user or try again later" + msg_template = user_message( + "Maximum number of guest users has been reached. Please log in with a registered account or try again later.", + _version=1, + ) + + +class GuestUserNotAllowedError(StudyDispatcherError): + msg_template = user_message( + "Guest users are not allowed to access this resource.", _version=1 + ) + + +class ProjectWorkbenchMismatchError(StudyDispatcherError): + msg_template = user_message( + "Project {project_uuid} appears to be corrupted and cannot be accessed properly.", + _version=1, + ) diff --git a/services/web/server/src/simcore_service_webserver/studies_dispatcher/_models.py b/services/web/server/src/simcore_service_webserver/studies_dispatcher/_models.py index a9a1cc23661f..a98a3eac3052 100644 --- a/services/web/server/src/simcore_service_webserver/studies_dispatcher/_models.py +++ b/services/web/server/src/simcore_service_webserver/studies_dispatcher/_models.py @@ -1,21 +1,16 @@ from typing import Annotated -from aiopg.sa.result import RowProxy from models_library.services import ServiceKey, ServiceVersion -from pydantic import BaseModel, Field, HttpUrl, PositiveInt, TypeAdapter +from pydantic import BaseModel, Field, HttpUrl, PositiveInt class ServiceInfo(BaseModel): key: ServiceKey version: ServiceVersion - label: Annotated[str, Field(..., description="Display name")] + label: Annotated[str, Field(description="Display name")] - thumbnail: HttpUrl = Field( - default=TypeAdapter(HttpUrl).validate_python( - "https://via.placeholder.com/170x120.png" - ) - ) + thumbnail: HttpUrl = HttpUrl("https://via.placeholder.com/170x120.png") is_guest_allowed: bool = True @@ -38,23 +33,12 @@ class ViewerInfo(ServiceInfo): to visualize a file of that type """ - filetype: str = Field(..., description="Filetype associated to this viewer") - - input_port_key: str = Field( - ..., - description="Name of the connection port, since it is service-dependent", - ) - - @classmethod - def create_from_db(cls, row: RowProxy) -> "ViewerInfo": - return cls( - key=row["service_key"], - version=row["service_version"], - filetype=row["filetype"], - label=row["service_display_name"] or row["service_key"].split("/")[-1], - input_port_key=row["service_input_port"], - is_guest_allowed=row["is_guest_allowed"], - ) + filetype: Annotated[str, Field(description="Filetype associated to this viewer")] + + input_port_key: Annotated[ + str, + Field(description="Name of the connection port, since it is service-dependent"), + ] class ServiceParams(BaseModel): diff --git a/services/web/server/src/simcore_service_webserver/studies_dispatcher/_projects.py b/services/web/server/src/simcore_service_webserver/studies_dispatcher/_projects.py index 168f60d69f75..669ae2233142 100644 --- a/services/web/server/src/simcore_service_webserver/studies_dispatcher/_projects.py +++ b/services/web/server/src/simcore_service_webserver/studies_dispatcher/_projects.py @@ -19,14 +19,14 @@ from models_library.projects_nodes_io import DownloadLink, NodeID, PortLink from models_library.services import ServiceKey, ServiceVersion from pydantic import AnyUrl, HttpUrl, TypeAdapter -from servicelib.logging_errors import create_troubleshootting_log_kwargs from servicelib.logging_utils import log_decorator from ..projects._projects_repository_legacy import ProjectDBAPI from ..projects._projects_service import get_project_for_user from ..projects.exceptions import ProjectInvalidRightsError, ProjectNotFoundError from ..utils import now_str -from ._core import compose_uuid_from +from . import _service +from ._errors import ProjectWorkbenchMismatchError from ._models import FileParams, ServiceInfo, ViewerInfo from ._users import UserInfo @@ -42,11 +42,11 @@ def _generate_nodeids(project_id: ProjectID) -> tuple[NodeID, NodeID]: - file_picker_id = compose_uuid_from( + file_picker_id = _service.compose_uuid_from( project_id, "4c69c0ce-00e4-4bd5-9cf0-59b67b3a9343", ) - viewer_id = compose_uuid_from( + viewer_id = _service.compose_uuid_from( project_id, "fc718e5a-bf07-4abe-b526-d9cafd34830c", ) @@ -265,7 +265,9 @@ async def get_or_create_project_with_file_and_service( # - if user requests several times, the same project is reused # - if user is not a guest, the project will be saved in it's account (desired?) # - project_uid: ProjectID = compose_uuid_from(user.id, viewer.footprint, download_link) + project_uid: ProjectID = _service.compose_uuid_from( + user.id, viewer.footprint, download_link + ) # Ids are linked to produce a footprint (see viewer_project_exists) file_picker_id, service_id = _generate_nodeids(project_uid) @@ -282,24 +284,14 @@ async def get_or_create_project_with_file_and_service( if is_valid: exists = True else: - http_500_error = web.HTTPInternalServerError() - _logger.error( - **create_troubleshootting_log_kwargs( - f"Project {project_uid} exists but does not seem to be a viewer generated by this module.", - error=http_500_error, - error_context={ - "user": user, - "viewer": viewer, - "download_link": download_link, - "project_db_workbench_keys": list( - project_db.get("workbench", {}).keys() - ), - "expected_keys": [file_picker_id, service_id], - }, - ) + raise ProjectWorkbenchMismatchError( + project_uuid=project_uid, + user=user, + viewer=viewer, + download_link=download_link, + project_db_workbench_keys=list(project_db.get("workbench", {}).keys()), + expected_keys=[file_picker_id, service_id], ) - # FIXME: CANNOT GUARANTEE!!, DELETE?? ERROR?? and cannot be viewed until verified? - raise http_500_error except (ProjectNotFoundError, ProjectInvalidRightsError): exists = False @@ -334,7 +326,7 @@ async def get_or_create_project_with_service( product_name: str, product_api_base_url: str, ) -> ProjectNodePair: - project_uid: ProjectID = compose_uuid_from(user.id, service_info.footprint) + project_uid: ProjectID = _service.compose_uuid_from(user.id, service_info.footprint) _, service_id = _generate_nodeids(project_uid) try: @@ -372,7 +364,7 @@ async def get_or_create_project_with_file( product_name: str, product_api_base_url: str, ) -> ProjectNodePair: - project_uid: ProjectID = compose_uuid_from(user.id, file_params.footprint) + project_uid: ProjectID = _service.compose_uuid_from(user.id, file_params.footprint) file_picker_id, _ = _generate_nodeids(project_uid) if not await _project_exists( diff --git a/services/web/server/src/simcore_service_webserver/studies_dispatcher/_repository.py b/services/web/server/src/simcore_service_webserver/studies_dispatcher/_repository.py index e69de29bb2d1..46ee8d110aef 100644 --- a/services/web/server/src/simcore_service_webserver/studies_dispatcher/_repository.py +++ b/services/web/server/src/simcore_service_webserver/studies_dispatcher/_repository.py @@ -0,0 +1,127 @@ +import logging +from collections.abc import AsyncIterator + +import sqlalchemy as sa +from models_library.services import ServiceVersion +from pydantic import TypeAdapter, ValidationError +from simcore_postgres_database.models.services_consume_filetypes import ( + services_consume_filetypes, +) +from simcore_postgres_database.utils_repos import pass_or_acquire_connection +from sqlalchemy.dialects.postgresql import ARRAY, INTEGER +from sqlalchemy.engine import Row +from sqlalchemy.ext.asyncio import AsyncConnection + +from ..db.base_repository import BaseRepository +from ._models import ViewerInfo + +_logger = logging.getLogger(__name__) + + +def _version(column_or_value): + """Converts version value string to array[integer] that can be compared.""" + return sa.func.string_to_array(column_or_value, ".").cast(ARRAY(INTEGER)) + + +def create_viewer_info_from_db(row: Row) -> ViewerInfo: + """Create ViewerInfo instance from database row.""" + return ViewerInfo( + key=row.service_key, + version=row.service_version, + filetype=row.filetype, + label=row.service_display_name or row.service_key.split("/")[-1], + input_port_key=row.service_input_port, + is_guest_allowed=row.is_guest_allowed, + ) + + +class StudiesDispatcherRepository(BaseRepository): + + async def list_viewers_info( + self, + connection: AsyncConnection | None = None, + *, + file_type: str | None = None, + only_default: bool = False, + ) -> list[ViewerInfo]: + """List viewer services that can consume the given file type.""" + + async def _iter_viewers() -> AsyncIterator[ViewerInfo]: + query = services_consume_filetypes.select() + if file_type: + query = query.where(services_consume_filetypes.c.filetype == file_type) + + query = query.order_by("filetype", "preference_order") + + if file_type and only_default: + query = query.limit(1) + + _logger.debug("Listing viewers:\n%s", query) + + async with pass_or_acquire_connection(self.engine, connection) as conn: + result = await conn.stream(query) + + listed_filetype = set() + async for row in result: + try: + # TODO: filter in database (see test_list_default_compatible_services ) + if only_default and row.filetype in listed_filetype: + continue + listed_filetype.add(row.filetype) + consumer = create_viewer_info_from_db(row) + yield consumer + + except ValidationError as err: + _logger.warning( + "Review invalid service metadata %s: %s", row, err + ) + + return [viewer async for viewer in _iter_viewers()] + + async def get_default_viewer_for_filetype( + self, + connection: AsyncConnection | None = None, + *, + file_type: str, + ) -> ViewerInfo | None: + """Get the default viewer for a specific file type.""" + viewers = await self.list_viewers_info( + connection=connection, file_type=file_type, only_default=True + ) + return viewers[0] if viewers else None + + async def find_compatible_viewer( + self, + connection: AsyncConnection | None = None, + *, + file_type: str, + service_key: str, + service_version: str, + ) -> ViewerInfo | None: + """Find a compatible viewer service for the given file type, service key, and version.""" + + query = ( + services_consume_filetypes.select() + .where( + (services_consume_filetypes.c.filetype == file_type) + & (services_consume_filetypes.c.service_key == service_key) + & ( + _version(services_consume_filetypes.c.service_version) + <= _version(service_version) + ) + ) + .order_by(_version(services_consume_filetypes.c.service_version).desc()) + .limit(1) + ) + + async with pass_or_acquire_connection(self.engine, connection) as conn: + result = await conn.execute(query) + row = result.one_or_none() + if row: + view = create_viewer_info_from_db(row) + view.version = TypeAdapter(ServiceVersion).validate_python( + service_version + ) + return view + + return None diff --git a/services/web/server/src/simcore_service_webserver/studies_dispatcher/_service.py b/services/web/server/src/simcore_service_webserver/studies_dispatcher/_service.py index e69de29bb2d1..beb83985b4d7 100644 --- a/services/web/server/src/simcore_service_webserver/studies_dispatcher/_service.py +++ b/services/web/server/src/simcore_service_webserver/studies_dispatcher/_service.py @@ -0,0 +1,99 @@ +import logging +import uuid +from functools import lru_cache + +from aiohttp import web +from models_library.utils.pydantic_tools_extension import parse_obj_or_none +from pydantic import ByteSize +from servicelib.logging_utils import log_decorator + +from ._errors import FileToLargeError, IncompatibleServiceError +from ._models import ViewerInfo +from ._repository import StudiesDispatcherRepository +from .settings import get_plugin_settings + +_BASE_UUID = uuid.UUID("ca2144da-eabb-4daf-a1df-a3682050e25f") + + +_logger = logging.getLogger(__name__) + + +@lru_cache +def compose_uuid_from(*values) -> uuid.UUID: + composition: str = "/".join(map(str, values)) + return uuid.uuid5(_BASE_UUID, composition) + + +async def list_viewers_info( + app: web.Application, file_type: str | None = None, *, only_default: bool = False +) -> list[ViewerInfo]: + repo = StudiesDispatcherRepository.create_from_app(app) + return await repo.list_viewers_info(file_type=file_type, only_default=only_default) + + +async def get_default_viewer( + app: web.Application, + file_type: str, + file_size: int | None = None, +) -> ViewerInfo: + """ + + Raises: + IncompatibleService + FileToLarge + """ + repo = StudiesDispatcherRepository.create_from_app(app) + viewer = await repo.get_default_viewer_for_filetype(file_type=file_type) + + if viewer is None: + raise IncompatibleServiceError(file_type=file_type) + + if current_size := parse_obj_or_none(ByteSize, file_size): + max_size: ByteSize = get_plugin_settings(app).STUDIES_MAX_FILE_SIZE_ALLOWED + if current_size > max_size: + raise FileToLargeError(file_size_in_mb=current_size.to("MiB")) + + return viewer + + +@log_decorator(_logger, level=logging.DEBUG) +async def validate_requested_viewer( + app: web.Application, + file_type: str, + file_size: int | None = None, + service_key: str | None = None, + service_version: str | None = None, +) -> ViewerInfo: + """ + + Raises: + IncompatibleService: When there is no match + + """ + if not service_key and not service_version: + return await get_default_viewer(app, file_type, file_size) + + if service_key and service_version: + repo = StudiesDispatcherRepository.create_from_app(app) + viewer = await repo.find_compatible_viewer( + file_type=file_type, + service_key=service_key, + service_version=service_version, + ) + if viewer: + return viewer + + raise IncompatibleServiceError(file_type=file_type) + + +@log_decorator(_logger, level=logging.DEBUG) +def validate_requested_file( + app: web.Application, file_type: str, file_size: int | None = None +): + # NOTE in the future we might want to prevent some types to be pulled + assert file_type # nosec + + if current_size := parse_obj_or_none(ByteSize, file_size): + max_size: ByteSize = get_plugin_settings(app).STUDIES_MAX_FILE_SIZE_ALLOWED + if current_size > max_size: + raise FileToLargeError(file_size_in_mb=current_size.to("MiB")) diff --git a/services/web/server/src/simcore_service_webserver/studies_dispatcher/_users.py b/services/web/server/src/simcore_service_webserver/studies_dispatcher/_users.py index 873274fe7125..b8479d3c2c35 100644 --- a/services/web/server/src/simcore_service_webserver/studies_dispatcher/_users.py +++ b/services/web/server/src/simcore_service_webserver/studies_dispatcher/_users.py @@ -37,8 +37,7 @@ from ..security import security_service, security_web from ..users import users_service from ..users.exceptions import UserNotFoundError -from ._constants import MSG_GUESTS_NOT_ALLOWED -from ._errors import GuestUsersLimitError +from ._errors import GuestUserNotAllowedError, GuestUsersLimitError from .settings import StudiesDispatcherSettings, get_plugin_settings _logger = logging.getLogger(__name__) @@ -187,7 +186,7 @@ async def get_or_create_guest_user( allow_anonymous_or_guest_users -- if True, it will create a temporary GUEST account Raises: - web.HTTPUnauthorized if ANONYMOUS users are not allowed (either w/o auth or as GUEST) + GuestUserNotAllowedError if ANONYMOUS users are not allowed (either w/o auth or as GUEST) """ user = None @@ -205,7 +204,11 @@ async def get_or_create_guest_user( if not allow_anonymous_or_guest_users and (not user or user.get("role") == GUEST): # NOTE: if allow_anonymous_users=False then GUEST users are NOT allowed! - raise web.HTTPUnauthorized(text=MSG_GUESTS_NOT_ALLOWED) + raise GuestUserNotAllowedError( + allow_anonymous_or_guest_users=allow_anonymous_or_guest_users, + user=user, + is_anonymous_user=is_anonymous_user, + ) assert isinstance(user, dict) # nosec diff --git a/services/web/server/src/simcore_service_webserver/studies_dispatcher/plugin.py b/services/web/server/src/simcore_service_webserver/studies_dispatcher/plugin.py index ef420205b734..2338fdd4c9b5 100644 --- a/services/web/server/src/simcore_service_webserver/studies_dispatcher/plugin.py +++ b/services/web/server/src/simcore_service_webserver/studies_dispatcher/plugin.py @@ -5,9 +5,8 @@ from ..login.decorators import login_required from ..products.plugin import setup_products -from . import _rest_handlers +from ._controller import setup_controller from ._projects_permalinks import setup_projects_permalinks -from ._redirects_handlers import get_redirection_to_viewer from ._studies_access import get_redirection_to_study_page from .settings import StudiesDispatcherSettings, get_plugin_settings @@ -48,20 +47,7 @@ def setup_studies_dispatcher(app: web.Application) -> bool: _setup_studies_access(app, settings) setup_projects_permalinks(app, settings) - # routes - redirect_handler = get_redirection_to_viewer - if settings.is_login_required(): - redirect_handler = login_required(get_redirection_to_viewer) - - _logger.info( - "'%s' config explicitly disables anonymous users from this feature", - __name__, - ) - - app.router.add_routes( - [web.get("/view", redirect_handler, name="get_redirection_to_viewer")] - ) - - app.router.add_routes(_rest_handlers.routes) + # rest controllers + setup_controller(app, settings) return True diff --git a/services/web/server/src/simcore_service_webserver/studies_dispatcher/settings.py b/services/web/server/src/simcore_service_webserver/studies_dispatcher/settings.py index 4d61119d0b72..fac7758d1a7a 100644 --- a/services/web/server/src/simcore_service_webserver/studies_dispatcher/settings.py +++ b/services/web/server/src/simcore_service_webserver/studies_dispatcher/settings.py @@ -1,60 +1,63 @@ from datetime import timedelta +from typing import Annotated, Final from aiohttp import web from common_library.pydantic_validators import validate_numeric_string_as_timedelta -from pydantic import ByteSize, HttpUrl, TypeAdapter, field_validator -from pydantic.fields import Field +from pydantic import ByteSize, Field, HttpUrl, TypeAdapter, field_validator from pydantic_settings import SettingsConfigDict from servicelib.aiohttp.application_keys import APP_SETTINGS_KEY from settings_library.base import BaseCustomSettings +_DEFAULT_THUMBNAIL: Final[HttpUrl] = TypeAdapter(HttpUrl).validate_python( + "https://via.placeholder.com/170x120.png" +) + class StudiesDispatcherSettings(BaseCustomSettings): - STUDIES_ACCESS_ANONYMOUS_ALLOWED: bool = Field( - default=False, - description="If enabled, the study links are accessible to anonymous users", - ) + STUDIES_ACCESS_ANONYMOUS_ALLOWED: Annotated[ + bool, + Field( + description="If enabled, the study links are accessible to anonymous users" + ), + ] = False - STUDIES_GUEST_ACCOUNT_LIFETIME: timedelta = Field( - default=timedelta(minutes=15), - description="Sets lifetime of a guest user until it is logged out " - " and removed by the GC", - ) + STUDIES_GUEST_ACCOUNT_LIFETIME: Annotated[ + timedelta, + Field( + description="Sets lifetime of a guest user until it is logged out and removed by the GC" + ), + ] = timedelta(minutes=15) - STUDIES_DEFAULT_SERVICE_THUMBNAIL: HttpUrl = Field( - default=TypeAdapter(HttpUrl).validate_python( - "https://via.placeholder.com/170x120.png" + STUDIES_DEFAULT_SERVICE_THUMBNAIL: Annotated[ + HttpUrl, + Field( + description="Default thumbnail for services or dispatch project with a service" ), - description="Default thumbnail for services or dispatch project with a service", - ) + ] = _DEFAULT_THUMBNAIL - STUDIES_DEFAULT_FILE_THUMBNAIL: HttpUrl = Field( - default=TypeAdapter(HttpUrl).validate_python( - "https://via.placeholder.com/170x120.png" + STUDIES_DEFAULT_FILE_THUMBNAIL: Annotated[ + HttpUrl, + Field( + description="Default thumbnail for dispatch projects with only data (i.e. file-picker)" ), - description="Default thumbnail for dispatch projects with only data (i.e. file-picker)", - ) + ] = _DEFAULT_THUMBNAIL - STUDIES_MAX_FILE_SIZE_ALLOWED: ByteSize = Field( - default=TypeAdapter(ByteSize).validate_python("50Mib"), - description="Limits the size of the files that can be dispatched" - "Note that the accuracy of the file size is not guaranteed and this limit might be surpassed", - ) + STUDIES_MAX_FILE_SIZE_ALLOWED: Annotated[ + ByteSize, + Field( + description="Limits the size of the files that can be dispatched. " + "Note that the accuracy of the file size is not guaranteed and this limit might be surpassed" + ), + ] = TypeAdapter(ByteSize).validate_python("50Mib") @field_validator("STUDIES_GUEST_ACCOUNT_LIFETIME") @classmethod def _is_positive_lifetime(cls, v): if v and isinstance(v, timedelta) and v.total_seconds() <= 0: - msg = f"Must be a positive number, got {v.total_seconds()=}" + msg = f"Must be a positive lifetime, got {v.total_seconds()=}" raise ValueError(msg) return v - def is_login_required(self): - """Used just to allow protecting the dispatcher redirect entrypoint programatically - Normally dispatcher entrypoints are openened - """ - return not self.STUDIES_ACCESS_ANONYMOUS_ALLOWED - _validate_studies_guest_account_lifetime = validate_numeric_string_as_timedelta( "STUDIES_GUEST_ACCOUNT_LIFETIME" ) @@ -68,6 +71,12 @@ def is_login_required(self): } ) + def is_login_required(self): + """Used just to allow protecting the dispatcher redirect entrypoint programatically + Normally dispatcher entrypoints are openened + """ + return not self.STUDIES_ACCESS_ANONYMOUS_ALLOWED + def get_plugin_settings(app: web.Application) -> StudiesDispatcherSettings: settings = app[APP_SETTINGS_KEY].WEBSERVER_STUDIES_DISPATCHER diff --git a/services/web/server/tests/unit/isolated/test_studies_dispatcher_models.py b/services/web/server/tests/unit/isolated/test_studies_dispatcher_models.py index 245709164634..d1de084671db 100644 --- a/services/web/server/tests/unit/isolated/test_studies_dispatcher_models.py +++ b/services/web/server/tests/unit/isolated/test_studies_dispatcher_models.py @@ -9,20 +9,38 @@ from urllib.parse import parse_qs import pytest +import simcore_service_webserver.studies_dispatcher from aiohttp.test_utils import make_mocked_request from models_library.utils.pydantic_tools_extension import parse_obj_or_none -from pydantic import ByteSize, TypeAdapter +from pydantic import BaseModel, ByteSize, TypeAdapter +from pytest_simcore.pydantic_models import ( + assert_validation_model, + walk_model_examples_in_package, +) from servicelib.aiohttp.requests_validation import parse_request_query_parameters_as +from simcore_service_webserver.studies_dispatcher._controller.rest.redirects_schemas import ( + FileQueryParams, + ServiceAndFileParams, +) from simcore_service_webserver.studies_dispatcher._models import ( FileParams, ServiceParams, ) -from simcore_service_webserver.studies_dispatcher._redirects_handlers import ( - FileQueryParams, - ServiceAndFileParams, -) from yarl import URL + +@pytest.mark.parametrize( + "model_cls, example_name, example_data", + walk_model_examples_in_package(simcore_service_webserver.studies_dispatcher), +) +def test_model_examples( + model_cls: type[BaseModel], example_name: str, example_data: Any +): + assert_validation_model( + model_cls, example_name=example_name, example_data=example_data + ) + + _SIZEBYTES = TypeAdapter(ByteSize).validate_python("3MiB") # SEE https://github.com/ITISFoundation/osparc-simcore/issues/3951#issuecomment-1489992645 @@ -79,9 +97,7 @@ def test_download_link_validators_2(file_and_service_params: dict[str, Any]): assert params.download_link assert params.download_link.host - assert params.download_link.host.endswith( - "s3.amazonaws.com" - ) + assert params.download_link.host.endswith("s3.amazonaws.com") query = parse_qs(params.download_link.query) assert {"AWSAccessKeyId", "Signature", "Expires", "x-amz-request-payer"} == set( diff --git a/services/web/server/tests/unit/with_dbs/04/studies_dispatcher/conftest.py b/services/web/server/tests/unit/with_dbs/04/studies_dispatcher/conftest.py index 93ec781cab84..47279c6f2dd2 100644 --- a/services/web/server/tests/unit/with_dbs/04/studies_dispatcher/conftest.py +++ b/services/web/server/tests/unit/with_dbs/04/studies_dispatcher/conftest.py @@ -1,12 +1,29 @@ +# pylint: disable=protected-access # pylint: disable=redefined-outer-name +# pylint: disable=too-many-arguments # pylint: disable=unused-argument # pylint: disable=unused-variable -# pylint: disable=too-many-arguments +from collections.abc import Iterator +from contextlib import ExitStack import pytest +import sqlalchemy as sa +from pytest_simcore.helpers.faker_factories import ( + random_service_access_rights, + random_service_consume_filetype, + random_service_meta_data, +) from pytest_simcore.helpers.monkeypatch_envs import setenvs_from_dict +from pytest_simcore.helpers.postgres_tools import sync_insert_and_get_row_lifespan from pytest_simcore.helpers.typing_env import EnvVarsDict +from simcore_postgres_database.models.services import ( + services_access_rights, + services_meta_data, +) +from simcore_postgres_database.models.services_consume_filetypes import ( + services_consume_filetypes, +) from simcore_service_webserver.studies_dispatcher.settings import ( StudiesDispatcherSettings, ) @@ -53,3 +70,217 @@ def app_environment( print(plugin_settings.model_dump_json(indent=1)) return {**app_environment, **envs_plugins, **envs_studies_dispatcher} + + +@pytest.fixture(scope="module") +def services_metadata_in_db( + postgres_db: sa.engine.Engine, +) -> Iterator[list[dict]]: + """Pre-populate services metadata table with test data maintaining original structure.""" + services_data = [ + random_service_meta_data( + key="simcore/services/dynamic/raw-graphs", + version="2.11.1", + name="2D plot", + description="2D plots powered by RAW Graphs", + thumbnail=None, + ), + random_service_meta_data( + key="simcore/services/dynamic/bio-formats-web", + version="1.0.1", + name="bio-formats", + description="Bio-Formats image viewer", + thumbnail="https://www.openmicroscopy.org/img/logos/bio-formats.svg", + ), + random_service_meta_data( + key="simcore/services/dynamic/jupyter-octave-python-math", + version="1.6.9", + name="JupyterLab Math", + description="JupyterLab Math with octave and python", + thumbnail=None, + ), + random_service_meta_data( + key="simcore/services/dynamic/s4l-ui-modeling", + version="3.2.300", + name="Hornet Flow", + description="Hornet Flow UI for Sim4Life", + thumbnail=None, + ), + ] + + with ExitStack() as stack: + created_services = [] + for service_data in services_data: + row = stack.enter_context( + sync_insert_and_get_row_lifespan( + postgres_db, + table=services_meta_data, + values=service_data, + pk_cols=[services_meta_data.c.key, services_meta_data.c.version], + ) + ) + created_services.append(row) + + yield created_services + + +@pytest.fixture(scope="module") +def services_consume_filetypes_in_db( + postgres_db: sa.engine.Engine, services_metadata_in_db: list[dict] +) -> Iterator[list[dict]]: + """Pre-populate services consume filetypes table with test data.""" + filetypes_data = [ + random_service_consume_filetype( + service_key="simcore/services/dynamic/bio-formats-web", + service_version="1.0.1", + service_display_name="bio-formats", + service_input_port="input_1", + filetype="PNG", + preference_order=0, + is_guest_allowed=True, + ), + random_service_consume_filetype( + service_key="simcore/services/dynamic/raw-graphs", + service_version="2.11.1", + service_display_name="RAWGraphs", + service_input_port="input_1", + filetype="CSV", + preference_order=0, + is_guest_allowed=True, + ), + random_service_consume_filetype( + service_key="simcore/services/dynamic/bio-formats-web", + service_version="1.0.1", + service_display_name="bio-formats", + service_input_port="input_1", + filetype="JPEG", + preference_order=0, + is_guest_allowed=True, + ), + random_service_consume_filetype( + service_key="simcore/services/dynamic/raw-graphs", + service_version="2.11.1", + service_display_name="RAWGraphs", + service_input_port="input_1", + filetype="TSV", + preference_order=0, + is_guest_allowed=True, + ), + random_service_consume_filetype( + service_key="simcore/services/dynamic/raw-graphs", + service_version="2.11.1", + service_display_name="RAWGraphs", + service_input_port="input_1", + filetype="XLSX", + preference_order=0, + is_guest_allowed=True, + ), + random_service_consume_filetype( + service_key="simcore/services/dynamic/raw-graphs", + service_version="2.11.1", + service_display_name="RAWGraphs", + service_input_port="input_1", + filetype="JSON", + preference_order=0, + is_guest_allowed=True, + ), + random_service_consume_filetype( + service_key="simcore/services/dynamic/jupyter-octave-python-math", + service_version="1.6.9", + service_display_name="JupyterLab Math", + service_input_port="input_1", + filetype="PY", + preference_order=0, + is_guest_allowed=False, + ), + random_service_consume_filetype( + service_key="simcore/services/dynamic/jupyter-octave-python-math", + service_version="1.6.9", + service_display_name="JupyterLab Math", + service_input_port="input_1", + filetype="IPYNB", + preference_order=0, + is_guest_allowed=False, + ), + random_service_consume_filetype( + service_key="simcore/services/dynamic/s4l-ui-modeling", + service_version="3.2.300", + service_display_name="Hornet Flow", + service_input_port="input_1", + filetype="HORNET_REPO", + preference_order=0, + is_guest_allowed=False, + ), + ] + + with ExitStack() as stack: + created_filetypes = [] + for filetype_data in filetypes_data: + row = stack.enter_context( + sync_insert_and_get_row_lifespan( + postgres_db, + table=services_consume_filetypes, + values=filetype_data, + pk_cols=[ + services_consume_filetypes.c.service_key, + services_consume_filetypes.c.service_version, + services_consume_filetypes.c.filetype, + ], + ) + ) + created_filetypes.append(row) + + yield created_filetypes + + +@pytest.fixture(scope="module") +def services_access_rights_in_db( + postgres_db: sa.engine.Engine, services_metadata_in_db: list[dict] +) -> Iterator[list[dict]]: + """Pre-populate services access rights table with test data.""" + access_rights_data = [ + random_service_access_rights( + key="simcore/services/dynamic/raw-graphs", + version="2.11.1", + gid=1, # everyone group + execute_access=True, + write_access=False, + product_name="osparc", + ), + random_service_access_rights( + key="simcore/services/dynamic/jupyter-octave-python-math", + version="1.6.9", + gid=1, # everyone group + execute_access=True, + write_access=False, + product_name="osparc", + ), + random_service_access_rights( + key="simcore/services/dynamic/s4l-ui-modeling", + version="3.2.300", + gid=1, # everyone group + execute_access=True, + write_access=False, + product_name="osparc", + ), + ] + + with ExitStack() as stack: + created_access_rights = [] + for access_data in access_rights_data: + row = stack.enter_context( + sync_insert_and_get_row_lifespan( + postgres_db, + table=services_access_rights, + values=access_data, + pk_cols=[ + services_access_rights.c.key, + services_access_rights.c.version, + services_access_rights.c.gid, + services_access_rights.c.product_name, + ], + ) + ) + created_access_rights.append(row) + + yield created_access_rights diff --git a/services/web/server/tests/unit/with_dbs/04/studies_dispatcher/test_studies_dispatcher_projects.py b/services/web/server/tests/unit/with_dbs/04/studies_dispatcher/test_studies_dispatcher_projects.py index a268b839f478..3f4c8e8e9c6a 100644 --- a/services/web/server/tests/unit/with_dbs/04/studies_dispatcher/test_studies_dispatcher_projects.py +++ b/services/web/server/tests/unit/with_dbs/04/studies_dispatcher/test_studies_dispatcher_projects.py @@ -120,8 +120,11 @@ async def test_add_new_project_from_model_instance( ): assert client.app - mock_directorv2_api = mocker.patch( - "simcore_service_webserver.director_v2.director_v2_service.create_or_update_pipeline", + import simcore_service_webserver.director_v2.director_v2_service + + mock_directorv2_api = mocker.patch.object( + simcore_service_webserver.director_v2.director_v2_service, + "create_or_update_pipeline", return_value=None, ) diff --git a/services/web/server/tests/unit/with_dbs/04/studies_dispatcher/test_studies_dispatcher_repository.py b/services/web/server/tests/unit/with_dbs/04/studies_dispatcher/test_studies_dispatcher_repository.py new file mode 100644 index 000000000000..8a508f360179 --- /dev/null +++ b/services/web/server/tests/unit/with_dbs/04/studies_dispatcher/test_studies_dispatcher_repository.py @@ -0,0 +1,203 @@ +# pylint: disable=protected-access +# pylint: disable=redefined-outer-name +# pylint: disable=too-many-arguments +# pylint: disable=unused-argument +# pylint: disable=unused-variable + +from collections.abc import AsyncIterator + +import pytest +from pytest_simcore.helpers.faker_factories import ( + random_service_consume_filetype, + random_service_meta_data, +) +from pytest_simcore.helpers.postgres_tools import insert_and_get_row_lifespan +from simcore_postgres_database.models.services import services_meta_data +from simcore_postgres_database.models.services_consume_filetypes import ( + services_consume_filetypes, +) +from simcore_service_webserver.studies_dispatcher._models import ViewerInfo +from simcore_service_webserver.studies_dispatcher._repository import ( + StudiesDispatcherRepository, +) +from sqlalchemy.ext.asyncio import AsyncEngine + + +@pytest.fixture +async def service_metadata_in_db(asyncpg_engine: AsyncEngine) -> AsyncIterator[dict]: + """Pre-populate services metadata table with test data.""" + service_data = random_service_meta_data( + key="simcore/services/dynamic/viewer", + version="1.0.0", + name="Test Viewer Service", + ) + # pylint: disable=contextmanager-generator-missing-cleanup + async with insert_and_get_row_lifespan( + asyncpg_engine, + table=services_meta_data, + values=service_data, + pk_col=services_meta_data.c.key, + pk_value=service_data["key"], + ) as row: + yield row + # cleanup happens automatically + + +@pytest.fixture +async def consume_filetypes_in_db( + asyncpg_engine: AsyncEngine, service_metadata_in_db: dict +): + """Pre-populate services consume filetypes table with test data.""" + consume_data = random_service_consume_filetype( + service_key=service_metadata_in_db["key"], + service_version=service_metadata_in_db["version"], + filetype="CSV", + service_display_name="CSV Viewer", + service_input_port="input_1", + preference_order=1, + is_guest_allowed=True, + ) + + # pylint: disable=contextmanager-generator-missing-cleanup + async with insert_and_get_row_lifespan( + asyncpg_engine, + table=services_consume_filetypes, + values=consume_data, + pk_col=services_consume_filetypes.c.service_key, + pk_value=consume_data["service_key"], + ) as row: + yield row + + +@pytest.fixture +def studies_dispatcher_repository( + asyncpg_engine: AsyncEngine, +) -> StudiesDispatcherRepository: + """Create StudiesDispatcherRepository instance.""" + return StudiesDispatcherRepository(engine=asyncpg_engine) + + +async def test_list_viewers_info_all( + studies_dispatcher_repository: StudiesDispatcherRepository, + consume_filetypes_in_db: dict, +): + """Test listing all viewer services.""" + # Act + viewers = await studies_dispatcher_repository.list_viewers_info() + + # Assert + assert len(viewers) == 1 + viewer = viewers[0] + assert isinstance(viewer, ViewerInfo) + assert viewer.key == consume_filetypes_in_db["service_key"] + assert viewer.version == consume_filetypes_in_db["service_version"] + assert viewer.filetype == consume_filetypes_in_db["filetype"] + assert viewer.label == consume_filetypes_in_db["service_display_name"] + assert viewer.input_port_key == consume_filetypes_in_db["service_input_port"] + assert viewer.is_guest_allowed == consume_filetypes_in_db["is_guest_allowed"] + + +async def test_list_viewers_info_filtered_by_filetype( + studies_dispatcher_repository: StudiesDispatcherRepository, + consume_filetypes_in_db: dict, +): + """Test listing viewer services filtered by file type.""" + # Act + viewers = await studies_dispatcher_repository.list_viewers_info(file_type="CSV") + + # Assert + assert len(viewers) == 1 + assert viewers[0].filetype == "CSV" + + # Test with non-existent filetype + viewers_empty = await studies_dispatcher_repository.list_viewers_info( + file_type="NONEXISTENT" + ) + assert len(viewers_empty) == 0 + + +async def test_list_viewers_info_only_default( + studies_dispatcher_repository: StudiesDispatcherRepository, + consume_filetypes_in_db: dict, +): + """Test listing only default viewer services.""" + # Act + viewers = await studies_dispatcher_repository.list_viewers_info( + file_type="CSV", only_default=True + ) + + # Assert + assert len(viewers) == 1 + assert viewers[0].filetype == "CSV" + + +async def test_get_default_viewer_for_filetype( + studies_dispatcher_repository: StudiesDispatcherRepository, + consume_filetypes_in_db: dict, +): + """Test getting the default viewer for a specific file type.""" + # Act + viewer = await studies_dispatcher_repository.get_default_viewer_for_filetype( + file_type="CSV" + ) + + # Assert + assert viewer is not None + assert isinstance(viewer, ViewerInfo) + assert viewer.key == consume_filetypes_in_db["service_key"] + assert viewer.version == consume_filetypes_in_db["service_version"] + assert viewer.filetype == "CSV" + assert viewer.label == consume_filetypes_in_db["service_display_name"] + + # Test with non-existent filetype + viewer_none = await studies_dispatcher_repository.get_default_viewer_for_filetype( + file_type="NONEXISTENT" + ) + assert viewer_none is None + + +async def test_find_compatible_viewer_found( + studies_dispatcher_repository: StudiesDispatcherRepository, + consume_filetypes_in_db: dict, +): + """Test finding a compatible viewer service that exists.""" + # Act + viewer = await studies_dispatcher_repository.find_compatible_viewer( + file_type="CSV", + service_key=consume_filetypes_in_db["service_key"], + service_version="1.0.0", + ) + + # Assert + assert viewer is not None + assert isinstance(viewer, ViewerInfo) + assert viewer.key == consume_filetypes_in_db["service_key"] + assert viewer.version == "1.0.0" # Should use the requested version + assert viewer.filetype == "CSV" + assert viewer.label == consume_filetypes_in_db["service_display_name"] + + +async def test_find_compatible_viewer_not_found( + studies_dispatcher_repository: StudiesDispatcherRepository, + consume_filetypes_in_db: dict, +): + """Test finding a compatible viewer service that doesn't exist.""" + # Act - test with non-existent service key + viewer = await studies_dispatcher_repository.find_compatible_viewer( + file_type="CSV", + service_key="simcore/services/dynamic/nonexistent", + service_version="1.0.0", + ) + + # Assert + assert viewer is None + + # Act - test with incompatible filetype + viewer_wrong_filetype = await studies_dispatcher_repository.find_compatible_viewer( + file_type="NONEXISTENT", + service_key=consume_filetypes_in_db["service_key"], + service_version="1.0.0", + ) + + # Assert + assert viewer_wrong_filetype is None diff --git a/services/web/server/tests/unit/with_dbs/04/studies_dispatcher/test_studies_dispatcher_rest_nih.py b/services/web/server/tests/unit/with_dbs/04/studies_dispatcher/test_studies_dispatcher_rest_nih.py new file mode 100644 index 000000000000..77b9e41f091f --- /dev/null +++ b/services/web/server/tests/unit/with_dbs/04/studies_dispatcher/test_studies_dispatcher_rest_nih.py @@ -0,0 +1,169 @@ +# pylint: disable=protected-access +# pylint: disable=redefined-outer-name +# pylint: disable=too-many-arguments +# pylint: disable=unused-argument +# pylint: disable=unused-variable + + +import pytest +from aiohttp.test_utils import TestClient, TestServer +from common_library.json_serialization import json_dumps +from common_library.serialization import model_dump_with_secrets +from pydantic import TypeAdapter +from pytest_simcore.helpers.assert_checks import assert_status +from pytest_simcore.helpers.monkeypatch_envs import setenvs_from_dict +from pytest_simcore.helpers.typing_env import EnvVarsDict +from servicelib.aiohttp import status +from settings_library.rabbit import RabbitSettings +from settings_library.redis import RedisSettings +from simcore_service_webserver.studies_dispatcher._controller.rest.nih_schemas import ( + ServiceGet, +) +from yarl import URL + +pytest_simcore_core_services_selection = [ + "rabbit", +] + + +@pytest.fixture +def app_environment( + app_environment: EnvVarsDict, + monkeypatch: pytest.MonkeyPatch, + rabbit_service: RabbitSettings, +) -> EnvVarsDict: + return setenvs_from_dict( + monkeypatch, + { + "WEBSERVER_RABBITMQ": json_dumps( + model_dump_with_secrets(rabbit_service, show_secrets=True) + ) + }, + ) + + +@pytest.fixture +def web_server( + redis_service: RedisSettings, + rabbit_service: RabbitSettings, + web_server: TestServer, + # Add dependencies to ensure database is populated before app starts + services_metadata_in_db: list[dict], + services_consume_filetypes_in_db: list[dict], + services_access_rights_in_db: list[dict], +) -> TestServer: + # + # Extends web_server to start redis_service and ensure DB is populated + # + print( + "Redis service started with settings: ", redis_service.model_dump_json(indent=1) + ) + return web_server + + +def _get_base_url(client: TestClient) -> str: + s = client.server + assert isinstance(s.scheme, str) + url = URL.build(scheme=s.scheme, host=s.host, port=s.port) + return f"{url}" + + +async def test_api_get_viewer_for_file(client: TestClient): + resp = await client.get("/v0/viewers/default?file_type=JPEG") + data, _ = await assert_status(resp, status.HTTP_200_OK) + + base_url = _get_base_url(client) + assert data == [ + { + "file_type": "JPEG", + "title": "Bio-formats v1.0.1", + "view_url": f"{base_url}/view?file_type=JPEG&viewer_key=simcore/services/dynamic/bio-formats-web&viewer_version=1.0.1", + }, + ] + + +async def test_api_get_viewer_for_unsupported_type(client: TestClient): + resp = await client.get("/v0/viewers/default?file_type=UNSUPPORTED_TYPE") + data, error = await assert_status(resp, status.HTTP_200_OK) + assert data == [] + assert error is None + + +async def test_api_list_supported_filetypes(client: TestClient): + resp = await client.get("/v0/viewers/default") + data, _ = await assert_status(resp, status.HTTP_200_OK) + + base_url = _get_base_url(client) + assert data == [ + { + "title": "Rawgraphs v2.11.1", + "file_type": "CSV", + "view_url": f"{base_url}/view?file_type=CSV&viewer_key=simcore/services/dynamic/raw-graphs&viewer_version=2.11.1", + }, + { + "file_type": "HORNET_REPO", + "title": "Hornet flow v3.2.300", + "view_url": f"{base_url}/view?file_type=HORNET_REPO&viewer_key=simcore/services/dynamic/s4l-ui-modeling&viewer_version=3.2.300", + }, + { + "title": "Jupyterlab math v1.6.9", + "file_type": "IPYNB", + "view_url": f"{base_url}/view?file_type=IPYNB&viewer_key=simcore/services/dynamic/jupyter-octave-python-math&viewer_version=1.6.9", + }, + { + "title": "Bio-formats v1.0.1", + "file_type": "JPEG", + "view_url": f"{base_url}/view?file_type=JPEG&viewer_key=simcore/services/dynamic/bio-formats-web&viewer_version=1.0.1", + }, + { + "title": "Rawgraphs v2.11.1", + "file_type": "JSON", + "view_url": f"{base_url}/view?file_type=JSON&viewer_key=simcore/services/dynamic/raw-graphs&viewer_version=2.11.1", + }, + { + "title": "Bio-formats v1.0.1", + "file_type": "PNG", + "view_url": f"{base_url}/view?file_type=PNG&viewer_key=simcore/services/dynamic/bio-formats-web&viewer_version=1.0.1", + }, + { + "title": "Jupyterlab math v1.6.9", + "file_type": "PY", + "view_url": f"{base_url}/view?file_type=PY&viewer_key=simcore/services/dynamic/jupyter-octave-python-math&viewer_version=1.6.9", + }, + { + "title": "Rawgraphs v2.11.1", + "file_type": "TSV", + "view_url": f"{base_url}/view?file_type=TSV&viewer_key=simcore/services/dynamic/raw-graphs&viewer_version=2.11.1", + }, + { + "title": "Rawgraphs v2.11.1", + "file_type": "XLSX", + "view_url": f"{base_url}/view?file_type=XLSX&viewer_key=simcore/services/dynamic/raw-graphs&viewer_version=2.11.1", + }, + ] + + +async def test_api_list_services(client: TestClient): + assert client.app + + url = client.app.router["list_latest_services"].url_for() + response = await client.get(f"{url}") + + data, error = await assert_status(response, status.HTTP_200_OK) + + services = TypeAdapter(list[ServiceGet]).validate_python(data) + assert services + + # latest versions of services with everyone + ospar-product (see services_access_rights_in_db) + assert services[0].key == "simcore/services/dynamic/raw-graphs" + assert services[0].file_extensions == ["CSV", "JSON", "TSV", "XLSX"] + + assert services[0].view_url.query + assert "2.11.1" in services[0].view_url.query + + assert services[2].key == "simcore/services/dynamic/jupyter-octave-python-math" + assert services[2].file_extensions == ["IPYNB", "PY"] + assert services[2].view_url.query + assert "1.6.9" in services[2].view_url.query + + assert error is None diff --git a/services/web/server/tests/unit/with_dbs/04/studies_dispatcher/test_studies_dispatcher_handlers.py b/services/web/server/tests/unit/with_dbs/04/studies_dispatcher/test_studies_dispatcher_rest_redirects.py similarity index 64% rename from services/web/server/tests/unit/with_dbs/04/studies_dispatcher/test_studies_dispatcher_handlers.py rename to services/web/server/tests/unit/with_dbs/04/studies_dispatcher/test_studies_dispatcher_rest_redirects.py index e48f716bd369..de3cbeec3fe1 100644 --- a/services/web/server/tests/unit/with_dbs/04/studies_dispatcher/test_studies_dispatcher_handlers.py +++ b/services/web/server/tests/unit/with_dbs/04/studies_dispatcher/test_studies_dispatcher_rest_redirects.py @@ -11,8 +11,6 @@ from unittest import mock import pytest -import simcore_service_webserver.studies_dispatcher -import sqlalchemy as sa from aiohttp import ClientResponse, ClientSession from aiohttp.test_utils import TestClient, TestServer from aioresponses import aioresponses @@ -20,70 +18,23 @@ from common_library.serialization import model_dump_with_secrets from common_library.users_enums import UserRole from models_library.projects_state import ProjectShareState, ProjectStatus -from pydantic import BaseModel, ByteSize, TypeAdapter +from pydantic import ByteSize, TypeAdapter from pytest_mock import MockerFixture from pytest_simcore.helpers.assert_checks import assert_status from pytest_simcore.helpers.monkeypatch_envs import setenvs_from_dict from pytest_simcore.helpers.typing_env import EnvVarsDict from pytest_simcore.helpers.webserver_users import UserInfoDict -from pytest_simcore.pydantic_models import ( - assert_validation_model, - walk_model_examples_in_package, -) from servicelib.aiohttp import status from settings_library.rabbit import RabbitSettings from settings_library.redis import RedisSettings from settings_library.utils_session import DEFAULT_SESSION_COOKIE_NAME -from simcore_service_webserver.studies_dispatcher._core import ViewerInfo -from simcore_service_webserver.studies_dispatcher._rest_handlers import ServiceGet -from sqlalchemy.sql import text +from simcore_service_webserver.studies_dispatcher._models import ViewerInfo from yarl import URL pytest_simcore_core_services_selection = [ "rabbit", ] -# -# FIXTURES OVERRIDES -# - - -@pytest.fixture(scope="module") -def postgres_db(postgres_db: sa.engine.Engine) -> sa.engine.Engine: - # - # Extends postgres_db fixture (called with web_server) to inject tables and start redis - # - stmt_create_services = text( - 'INSERT INTO "services_meta_data" ("key", "version", "owner", "name", "description", "thumbnail", "classifiers", "created", "modified", "quality") VALUES' - "('simcore/services/dynamic/raw-graphs', '2.11.1', NULL, '2D plot', '2D plots powered by RAW Graphs', NULL, '{}', '2021-03-02 16:08:28.655207', '2021-03-02 16:08:28.655207', '{}')," - "('simcore/services/dynamic/bio-formats-web', '1.0.1', NULL, 'bio-formats', 'Bio-Formats image viewer', 'https://www.openmicroscopy.org/img/logos/bio-formats.svg', '{}', '2021-03-02 16:08:28.420722', '2021-03-02 16:08:28.420722', '{}')," - "('simcore/services/dynamic/jupyter-octave-python-math', '1.6.9', NULL, 'JupyterLab Math', 'JupyterLab Math with octave and python', NULL, '{}', '2021-03-02 16:08:28.420722', '2021-03-02 16:08:28.420722', '{}');" - ) - stmt_create_services_consume_filetypes = text( - 'INSERT INTO "services_consume_filetypes" ("service_key", "service_version", "service_display_name", "service_input_port", "filetype", "preference_order", "is_guest_allowed") VALUES' - "('simcore/services/dynamic/bio-formats-web', '1.0.1', 'bio-formats', 'input_1', 'PNG', 0, '1')," - "('simcore/services/dynamic/raw-graphs', '2.11.1', 'RAWGraphs', 'input_1', 'CSV', 0, '1')," - "('simcore/services/dynamic/bio-formats-web', '1.0.1', 'bio-formats', 'input_1', 'JPEG', 0, '1')," - "('simcore/services/dynamic/raw-graphs', '2.11.1', 'RAWGraphs', 'input_1', 'TSV', 0, '1')," - "('simcore/services/dynamic/raw-graphs', '2.11.1', 'RAWGraphs', 'input_1', 'XLSX', 0, '1')," - "('simcore/services/dynamic/raw-graphs', '2.11.1', 'RAWGraphs', 'input_1', 'JSON', 0, '1')," - "('simcore/services/dynamic/jupyter-octave-python-math', '1.6.9', 'JupyterLab Math', 'input_1', 'PY', 0, '0')," - "('simcore/services/dynamic/jupyter-octave-python-math', '1.6.9', 'JupyterLab Math', 'input_1', 'IPYNB',0, '0');" - ) - - # NOTE: users default osparc project and everyone group (which should be by default already in tables) - stmt_create_services_access_rights = text( - ' INSERT INTO "services_access_rights" ("key", "version", "gid", "execute_access", "write_access", "created", "modified", "product_name") VALUES' - "('simcore/services/dynamic/raw-graphs', '2.11.1', 1, 't', 'f', '2022-05-23 08:44:45.418376', '2022-05-23 08:44:45.418376', 'osparc')," - "('simcore/services/dynamic/jupyter-octave-python-math', '1.6.9', 1, 't', 'f', '2022-05-23 08:44:45.418376', '2022-05-23 08:44:45.418376', 'osparc');" - ) - with postgres_db.connect() as conn: - conn.execute(stmt_create_services) - conn.execute(stmt_create_services_consume_filetypes) - conn.execute(stmt_create_services_access_rights) - - return postgres_db - @pytest.fixture def app_environment( @@ -103,10 +54,16 @@ def app_environment( @pytest.fixture def web_server( - redis_service: RedisSettings, rabbit_service: RabbitSettings, web_server: TestServer + redis_service: RedisSettings, + rabbit_service: RabbitSettings, + web_server: TestServer, + # Add dependencies to ensure database is populated before app starts + services_metadata_in_db: list[dict], + services_consume_filetypes_in_db: list[dict], + services_access_rights_in_db: list[dict], ) -> TestServer: # - # Extends web_server to start redis_service + # Extends web_server to start redis_service and ensure DB is populated # print( "Redis service started with settings: ", redis_service.model_dump_json(indent=1) @@ -181,123 +138,6 @@ async def director_v2_automock( ] -# REST-API -# Samples taken from trials on http://127.0.0.1:9081/dev/doc#/viewer/get_viewer_for_file -# - - -def _get_base_url(client: TestClient) -> str: - s = client.server - assert isinstance(s.scheme, str) - url = URL.build(scheme=s.scheme, host=s.host, port=s.port) - return f"{url}" - - -async def test_api_get_viewer_for_file(client: TestClient): - resp = await client.get("/v0/viewers/default?file_type=JPEG") - data, _ = await assert_status(resp, status.HTTP_200_OK) - - base_url = _get_base_url(client) - assert data == [ - { - "file_type": "JPEG", - "title": "Bio-formats v1.0.1", - "view_url": f"{base_url}/view?file_type=JPEG&viewer_key=simcore/services/dynamic/bio-formats-web&viewer_version=1.0.1", - }, - ] - - -async def test_api_get_viewer_for_unsupported_type(client: TestClient): - resp = await client.get("/v0/viewers/default?file_type=UNSUPPORTED_TYPE") - data, error = await assert_status(resp, status.HTTP_200_OK) - assert data == [] - assert error is None - - -async def test_api_list_supported_filetypes(client: TestClient): - resp = await client.get("/v0/viewers/default") - data, _ = await assert_status(resp, status.HTTP_200_OK) - - base_url = _get_base_url(client) - assert data == [ - { - "title": "Rawgraphs v2.11.1", - "file_type": "CSV", - "view_url": f"{base_url}/view?file_type=CSV&viewer_key=simcore/services/dynamic/raw-graphs&viewer_version=2.11.1", - }, - { - "title": "Jupyterlab math v1.6.9", - "file_type": "IPYNB", - "view_url": f"{base_url}/view?file_type=IPYNB&viewer_key=simcore/services/dynamic/jupyter-octave-python-math&viewer_version=1.6.9", - }, - { - "title": "Bio-formats v1.0.1", - "file_type": "JPEG", - "view_url": f"{base_url}/view?file_type=JPEG&viewer_key=simcore/services/dynamic/bio-formats-web&viewer_version=1.0.1", - }, - { - "title": "Rawgraphs v2.11.1", - "file_type": "JSON", - "view_url": f"{base_url}/view?file_type=JSON&viewer_key=simcore/services/dynamic/raw-graphs&viewer_version=2.11.1", - }, - { - "title": "Bio-formats v1.0.1", - "file_type": "PNG", - "view_url": f"{base_url}/view?file_type=PNG&viewer_key=simcore/services/dynamic/bio-formats-web&viewer_version=1.0.1", - }, - { - "title": "Jupyterlab math v1.6.9", - "file_type": "PY", - "view_url": f"{base_url}/view?file_type=PY&viewer_key=simcore/services/dynamic/jupyter-octave-python-math&viewer_version=1.6.9", - }, - { - "title": "Rawgraphs v2.11.1", - "file_type": "TSV", - "view_url": f"{base_url}/view?file_type=TSV&viewer_key=simcore/services/dynamic/raw-graphs&viewer_version=2.11.1", - }, - { - "title": "Rawgraphs v2.11.1", - "file_type": "XLSX", - "view_url": f"{base_url}/view?file_type=XLSX&viewer_key=simcore/services/dynamic/raw-graphs&viewer_version=2.11.1", - }, - ] - - -@pytest.mark.parametrize( - "model_cls, example_name, example_data", - walk_model_examples_in_package(simcore_service_webserver.studies_dispatcher), -) -def test_model_examples( - model_cls: type[BaseModel], example_name: str, example_data: Any -): - assert_validation_model( - model_cls, example_name=example_name, example_data=example_data - ) - - -async def test_api_list_services(client: TestClient): - assert client.app - - url = client.app.router["list_latest_services"].url_for() - response = await client.get(f"{url}") - - data, error = await assert_status(response, status.HTTP_200_OK) - - services = TypeAdapter(list[ServiceGet]).validate_python(data) - assert services - - # latest versions of services with everyone + ospar-product (see stmt_create_services_access_rights) - assert services[0].key == "simcore/services/dynamic/raw-graphs" - assert services[0].file_extensions == ["CSV", "JSON", "TSV", "XLSX"] - assert "2.11.1" in services[0].view_url.query - - assert services[1].key == "simcore/services/dynamic/jupyter-octave-python-math" - assert services[1].file_extensions == ["IPYNB", "PY"] - assert "1.6.9" in services[1].view_url.query - - assert error is None - - # REDIRECT ROUTES -------------------------------------------------------------------------------- @@ -430,7 +270,7 @@ async def test_dispatch_study_anonymously( return_value=None, ) mock_dynamic_scheduler_update_project_networks = mocker.patch( - "simcore_service_webserver.studies_dispatcher._redirects_handlers.dynamic_scheduler_service.update_projects_networks", + "simcore_service_webserver.studies_dispatcher._controller.rest.redirects.dynamic_scheduler_service.update_projects_networks", return_value=None, ) @@ -496,7 +336,7 @@ async def test_dispatch_logged_in_user( return_value=None, ) mock_dynamic_scheduler_update_project_networks = mocker.patch( - "simcore_service_webserver.studies_dispatcher._redirects_handlers.dynamic_scheduler_service.update_projects_networks", + "simcore_service_webserver.studies_dispatcher._controller.rest.redirects.dynamic_scheduler_service.update_projects_networks", return_value=None, ) @@ -580,7 +420,7 @@ async def test_viewer_redirect_with_file_type_errors(client: TestClient): message, status_code = assert_error_in_fragment(resp) assert status_code == status.HTTP_422_UNPROCESSABLE_ENTITY - assert "type" in message.lower() + assert "link" in message.lower() async def test_viewer_redirect_with_client_errors(client: TestClient): diff --git a/services/web/server/tests/unit/with_dbs/04/studies_dispatcher/test_studies_dispatcher_studies_access.py b/services/web/server/tests/unit/with_dbs/04/studies_dispatcher/test_studies_dispatcher_studies_access.py index 5321874a1686..056a64254ad7 100644 --- a/services/web/server/tests/unit/with_dbs/04/studies_dispatcher/test_studies_dispatcher_studies_access.py +++ b/services/web/server/tests/unit/with_dbs/04/studies_dispatcher/test_studies_dispatcher_studies_access.py @@ -12,6 +12,7 @@ from copy import deepcopy from pathlib import Path from pprint import pformat +from typing import Any from unittest import mock import pytest @@ -74,7 +75,7 @@ async def _get_user_projects(client) -> list[ProjectDict]: return projects -def _assert_same_projects(got: dict, expected: dict): +def _assert_same_projects(got: dict[str, Any], expected: dict[str, Any]): exclude = { "accessRights", "creationDate", @@ -89,9 +90,10 @@ def _assert_same_projects(got: dict, expected: dict): "type", "templateType", } - for key in expected: - if key not in exclude: - assert got[key] == expected[key], f"Failed in {key}" + expected_values = {k: v for k, v in expected.items() if k not in exclude} + got_values = {k: got[k] for k in expected if k not in exclude} + + assert got_values == expected_values def _is_user_authenticated(session: ClientSession) -> bool: @@ -172,8 +174,11 @@ def mocks_on_projects_api(mocker: MockerFixture) -> None: """ All projects in this module are UNLOCKED """ - mocker.patch( - "simcore_service_webserver.projects._projects_service._get_project_share_state", + import simcore_service_webserver.projects._projects_service + + mocker.patch.object( + simcore_service_webserver.projects._projects_service, + "_get_project_share_state", return_value=ProjectShareState( locked=False, status=ProjectStatus.CLOSED, current_user_groupids=[] ), @@ -306,7 +311,8 @@ async def _assert_redirected_to_study( async def test_access_to_invalid_study(client: TestClient, faker: Faker): - response = await client.get(f"/study/{faker.uuid4()}") + invalid_project_id = faker.uuid4() + response = await client.get(f"/study/{invalid_project_id}") _assert_redirected_to_error_page( response,