From d61e95eacd27cb0585a6e1676a1d913e28eba1d1 Mon Sep 17 00:00:00 2001 From: Pedro Crespo-Valero <32402063+pcrespov@users.noreply.github.com> Date: Fri, 26 Sep 2025 18:30:12 +0200 Subject: [PATCH 01/20] =?UTF-8?q?=E2=9C=A8=20[Service]=20Introduce=20Group?= =?UTF-8?q?ClassifierRepository=20and=20refactor=20related=20services=20an?= =?UTF-8?q?d=20tests?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../groups/_classifiers_repository.py | 44 +++++++++ .../groups/_classifiers_rest.py | 39 ++++---- .../groups/_classifiers_service.py | 94 +++++++++---------- .../with_dbs/01/test_groups_classifiers.py | 2 +- 4 files changed, 109 insertions(+), 70 deletions(-) create mode 100644 services/web/server/src/simcore_service_webserver/groups/_classifiers_repository.py diff --git a/services/web/server/src/simcore_service_webserver/groups/_classifiers_repository.py b/services/web/server/src/simcore_service_webserver/groups/_classifiers_repository.py new file mode 100644 index 000000000000..461378de3cbe --- /dev/null +++ b/services/web/server/src/simcore_service_webserver/groups/_classifiers_repository.py @@ -0,0 +1,44 @@ +import logging +from typing import Any + +import sqlalchemy as sa +from simcore_postgres_database.models.classifiers import group_classifiers +from simcore_postgres_database.utils_repos import pass_or_acquire_connection +from sqlalchemy.engine import Row +from sqlalchemy.ext.asyncio import AsyncConnection + +from ..db.base_repository import BaseRepository + +_logger = logging.getLogger(__name__) + + +class GroupClassifierRepository(BaseRepository): + + async def _get_bundle( + self, gid: int, connection: AsyncConnection | None = None + ) -> Row | None: + async with pass_or_acquire_connection(self.engine, connection) as conn: + result = await conn.execute( + sa.select(group_classifiers.c.bundle).where( + group_classifiers.c.gid == gid + ) + ) + return result.one_or_none() + + async def get_classifiers_from_bundle(self, gid: int) -> dict[str, Any] | None: + bundle_row = await self._get_bundle(gid) + if bundle_row: + return bundle_row.bundle + return None + + async def group_uses_scicrunch( + self, gid: int, connection: AsyncConnection | None = None + ) -> bool: + async with pass_or_acquire_connection(self.engine, connection) as conn: + result = await conn.execute( + sa.select(group_classifiers.c.uses_scicrunch).where( + group_classifiers.c.gid == gid + ) + ) + row = result.one_or_none() + return bool(row.uses_scicrunch if row else False) diff --git a/services/web/server/src/simcore_service_webserver/groups/_classifiers_rest.py b/services/web/server/src/simcore_service_webserver/groups/_classifiers_rest.py index e9113e5b6666..9c6ecdc65dbc 100644 --- a/services/web/server/src/simcore_service_webserver/groups/_classifiers_rest.py +++ b/services/web/server/src/simcore_service_webserver/groups/_classifiers_rest.py @@ -9,12 +9,11 @@ from .._meta import API_VTAG from ..login.decorators import login_required from ..scicrunch.db import ResearchResourceRepository -from ..scicrunch.errors import ScicrunchError from ..scicrunch.models import ResearchResource, ResourceHit from ..scicrunch.service_client import SciCrunch from ..security.decorators import permission_required from ..utils_aiohttp import envelope_json_response -from ._classifiers_service import GroupClassifierRepository, build_rrids_tree_view +from ._classifiers_service import GroupClassifiersService from ._common.exceptions_handlers import handle_plugin_requests_exceptions from ._common.schemas import GroupsClassifiersQuery, GroupsPathParams @@ -29,23 +28,15 @@ @permission_required("groups.*") @handle_plugin_requests_exceptions async def get_group_classifiers(request: web.Request): - try: - path_params = parse_request_path_parameters_as(GroupsPathParams, request) - query_params: GroupsClassifiersQuery = parse_request_query_parameters_as( - GroupsClassifiersQuery, request - ) - - repo = GroupClassifierRepository(request.app) - if not await repo.group_uses_scicrunch(path_params.gid): - bundle = await repo.get_classifiers_from_bundle(path_params.gid) - return envelope_json_response(bundle) - - # otherwise, build dynamic tree with RRIDs - view = await build_rrids_tree_view( - request.app, tree_view_mode=query_params.tree_view - ) - except ScicrunchError: - view = {} + path_params = parse_request_path_parameters_as(GroupsPathParams, request) + query_params: GroupsClassifiersQuery = parse_request_query_parameters_as( + GroupsClassifiersQuery, request + ) + + service = GroupClassifiersService(request.app) + view = await service.get_group_classifiers( + path_params.gid, tree_view_mode=query_params.tree_view + ) return envelope_json_response(view) @@ -110,3 +101,13 @@ async def search_scicrunch_resources(request: web.Request): hits: list[ResourceHit] = await scicrunch.search_resource(guess_name) return envelope_json_response([hit.model_dump() for hit in hits]) + + +@handle_plugin_requests_exceptions +async def search_scicrunch_resources(request: web.Request): + guess_name = str(request.query["guess_name"]).strip() + + scicrunch = SciCrunch.get_instance(request.app) + hits: list[ResourceHit] = await scicrunch.search_resource(guess_name) + + return envelope_json_response([hit.model_dump() for hit in hits]) diff --git a/services/web/server/src/simcore_service_webserver/groups/_classifiers_service.py b/services/web/server/src/simcore_service_webserver/groups/_classifiers_service.py index 1e1155613e25..02ec763f6bb6 100644 --- a/services/web/server/src/simcore_service_webserver/groups/_classifiers_service.py +++ b/services/web/server/src/simcore_service_webserver/groups/_classifiers_service.py @@ -11,9 +11,7 @@ import logging from typing import Annotated, Any, Final, Literal, TypeAlias -import sqlalchemy as sa from aiohttp import web -from aiopg.sa.result import RowProxy from common_library.logging.logging_errors import create_troubleshooting_log_kwargs from pydantic import ( BaseModel, @@ -24,11 +22,11 @@ ValidationError, field_validator, ) -from simcore_postgres_database.models.classifiers import group_classifiers -from ..db.plugin import get_database_engine_legacy from ..scicrunch.db import ResearchResourceRepository +from ..scicrunch.errors import ScicrunchError from ..scicrunch.service_client import SciCrunch +from ._classifiers_repository import GroupClassifierRepository _logger = logging.getLogger(__name__) MAX_SIZE_SHORT_MSG: Final[int] = 100 @@ -50,11 +48,13 @@ class ClassifierItem(BaseModel): ) display_name: str short_description: str | None - url: HttpUrl | None = Field( - None, - description="Link to more information", - examples=["https://scicrunch.org/resources/Any/search?q=osparc&l=osparc"], - ) + url: Annotated[ + HttpUrl | None, + Field( + description="Link to more information", + examples=["https://scicrunch.org/resources/Any/search?q=osparc&l=osparc"], + ), + ] = None @field_validator("short_description", mode="before") @classmethod @@ -73,51 +73,45 @@ class Classifiers(BaseModel): classifiers: dict[TreePath, ClassifierItem] -# DATABASE -------- +# SERVICE LAYER -------- -class GroupClassifierRepository: +class GroupClassifiersService: def __init__(self, app: web.Application): - self.engine = get_database_engine_legacy(app) - - async def _get_bundle(self, gid: int) -> RowProxy | None: - async with self.engine.acquire() as conn: - bundle: RowProxy | None = await conn.scalar( - sa.select(group_classifiers.c.bundle).where( - group_classifiers.c.gid == gid - ) - ) - return bundle - - async def get_classifiers_from_bundle(self, gid: int) -> dict[str, Any]: - bundle = await self._get_bundle(gid) - if bundle: - try: - # truncate bundle to what is needed and drop the rest - return Classifiers(**bundle).model_dump( - exclude_unset=True, exclude_none=True - ) - except ValidationError as err: - _logger.exception( - **create_troubleshooting_log_kwargs( - f"DB corrupt data in 'groups_classifiers' table. Invalid classifier for gid={gid}. Returning empty bundle.", - error=err, - error_context={ - "gid": gid, - "bundle": bundle, - }, + self.app = app + self._repo = GroupClassifierRepository.create_from_app(app) + + async def get_group_classifiers( + self, gid: int, tree_view_mode: Literal["std"] = "std" + ) -> dict[str, Any]: + """Get classifiers for a group, either from bundle or dynamic tree view.""" + if not await self._repo.group_uses_scicrunch(gid): + bundle = await self._repo.get_classifiers_from_bundle(gid) + if bundle: + try: + # truncate bundle to what is needed and drop the rest + return Classifiers(**bundle).model_dump( + exclude_unset=True, exclude_none=True ) - ) - return {} - - async def group_uses_scicrunch(self, gid: int) -> bool: - async with self.engine.acquire() as conn: - value: RowProxy | None = await conn.scalar( - sa.select(group_classifiers.c.uses_scicrunch).where( - group_classifiers.c.gid == gid - ) - ) - return bool(value) + except ValidationError as err: + _logger.exception( + **create_troubleshooting_log_kwargs( + f"DB corrupt data in 'groups_classifiers' table. Invalid classifier for gid={gid}. Returning empty bundle.", + error=err, + error_context={ + "gid": gid, + "bundle": bundle, + }, + ) + ) + return {} + + # otherwise, build dynamic tree with RRIDs + try: + return await build_rrids_tree_view(self.app, tree_view_mode=tree_view_mode) + except ScicrunchError: + # Return empty view on any error (including ScicrunchError) + return {} # HELPERS FOR API HANDLERS -------------- diff --git a/services/web/server/tests/unit/with_dbs/01/test_groups_classifiers.py b/services/web/server/tests/unit/with_dbs/01/test_groups_classifiers.py index c77f1335015a..9c15e6c4fccd 100644 --- a/services/web/server/tests/unit/with_dbs/01/test_groups_classifiers.py +++ b/services/web/server/tests/unit/with_dbs/01/test_groups_classifiers.py @@ -42,7 +42,7 @@ async def app(postgres_dsn: dict, inject_tables): async def test_classfiers_from_bundle(app): - repo = GroupClassifierRepository(app) + repo = GroupClassifierRepository.create_from_app(app) assert not await repo.group_uses_scicrunch(gid=1) From 35c6278908151f586dd403e637692a410edffaac Mon Sep 17 00:00:00 2001 From: Pedro Crespo-Valero <32402063+pcrespov@users.noreply.github.com> Date: Fri, 26 Sep 2025 18:31:46 +0200 Subject: [PATCH 02/20] rename --- ...roups_classifiers.py => test_groups_classifiers_repository.py} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename services/web/server/tests/unit/with_dbs/01/{test_groups_classifiers.py => test_groups_classifiers_repository.py} (100%) diff --git a/services/web/server/tests/unit/with_dbs/01/test_groups_classifiers.py b/services/web/server/tests/unit/with_dbs/01/test_groups_classifiers_repository.py similarity index 100% rename from services/web/server/tests/unit/with_dbs/01/test_groups_classifiers.py rename to services/web/server/tests/unit/with_dbs/01/test_groups_classifiers_repository.py From 99c7bb6f66729010ad24dfd10cfdd98d39aee36b Mon Sep 17 00:00:00 2001 From: Pedro Crespo-Valero <32402063+pcrespov@users.noreply.github.com> Date: Fri, 26 Sep 2025 18:35:22 +0200 Subject: [PATCH 03/20] =?UTF-8?q?=E2=9C=A8=20[Tests]=20Add=20random=5Fgrou?= =?UTF-8?q?p=5Fclassifier=20factory=20and=20related=20tests=20for=20GroupC?= =?UTF-8?q?lassifierRepository?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../pytest_simcore/helpers/faker_factories.py | 62 +++++++++++ .../01/test_groups_classifiers_repository.py | 105 ++++++++++++------ 2 files changed, 133 insertions(+), 34 deletions(-) diff --git a/packages/pytest-simcore/src/pytest_simcore/helpers/faker_factories.py b/packages/pytest-simcore/src/pytest_simcore/helpers/faker_factories.py index b03d29c67005..56174c1e4388 100644 --- a/packages/pytest-simcore/src/pytest_simcore/helpers/faker_factories.py +++ b/packages/pytest-simcore/src/pytest_simcore/helpers/faker_factories.py @@ -629,3 +629,65 @@ def random_service_consume_filetype( data.update(overrides) return data + + +def random_group_classifier( + *, + gid: int, + fake: Faker = DEFAULT_FAKER, + **overrides, +) -> dict[str, Any]: + from simcore_postgres_database.models.classifiers import group_classifiers + + data = { + "gid": gid, + "bundle": { + "vcs_ref": "asdfasdf", + "vcs_url": "https://foo.classifiers.git", + "build_date": "2021-01-20T15:19:30Z", + "classifiers": { + "project::dak": { + "url": None, + "logo": None, + "aliases": [], + "related": [], + "markdown": "", + "released": None, + "classifier": "project::dak", + "created_by": "Nicolas Chavannes", + "github_url": None, + "display_name": "DAK", + "wikipedia_url": None, + "short_description": None, + }, + "organization::zmt": { + "url": "https://zmt.swiss/", + "logo": None, + "aliases": ["Zurich MedTech AG"], + "related": [], + "markdown": "Zurich MedTech AG (ZMT) offers tools and best practices for targeted life sciences applications to simulate, analyze, and predict complex and dynamic biological processes and interactions. ZMT is a member of Zurich43", + "released": None, + "classifier": "organization::zmt", + "created_by": "crespo", + "github_url": None, + "display_name": "ZMT", + "wikipedia_url": None, + "short_description": "ZMT is a member of Zurich43", + }, + }, + "collections": { + "jupyterlab-math": { + "items": ["crespo/osparc-demo"], + "markdown": "Curated collection of repositories with examples of notebooks to run in jupyter-python-octave-math service", + "created_by": "crespo", + "display_name": "jupyterlab-math", + } + }, + }, + "uses_scicrunch": False, + } + + assert set(data.keys()).issubset({c.name for c in group_classifiers.columns}) + + data.update(overrides) + return data diff --git a/services/web/server/tests/unit/with_dbs/01/test_groups_classifiers_repository.py b/services/web/server/tests/unit/with_dbs/01/test_groups_classifiers_repository.py index 9c15e6c4fccd..0a546ac6cec1 100644 --- a/services/web/server/tests/unit/with_dbs/01/test_groups_classifiers_repository.py +++ b/services/web/server/tests/unit/with_dbs/01/test_groups_classifiers_repository.py @@ -3,54 +3,91 @@ # pylint: disable=unused-variable # pylint: disable=too-many-arguments - import pytest -import sqlalchemy as sa -from servicelib.common_aiopg_utils import DataSourceName, create_pg_engine -from simcore_service_webserver.constants import APP_AIOPG_ENGINE_KEY -from simcore_service_webserver.groups._classifiers_service import ( +from pytest_simcore.helpers.faker_factories import random_group_classifier +from pytest_simcore.helpers.postgres_tools import insert_and_get_row_lifespan +from simcore_postgres_database.models.classifiers import group_classifiers +from simcore_service_webserver.groups._classifiers_repository import ( GroupClassifierRepository, ) -from sqlalchemy.sql import text +from sqlalchemy.ext.asyncio import AsyncEngine @pytest.fixture -def inject_tables(postgres_db: sa.engine.Engine): - stmt = text( - """\ - INSERT INTO "group_classifiers" ("id", "bundle", "created", "modified", "gid", "uses_scicrunch") VALUES - (2, '{"vcs_ref": "asdfasdf", "vcs_url": "https://foo.classifiers.git", "build_date": "2021-01-20T15:19:30Z", "classifiers": {"project::dak": {"url": null, "logo": null, "aliases": [], "related": [], "markdown": "", "released": null, "classifier": "project::dak", "created_by": "Nicolas Chavannes", "github_url": null, "display_name": "DAK", "wikipedia_url": null, "short_description": null}, "organization::zmt": {"url": "https://zmt.swiss/", "logo": null, "aliases": ["Zurich MedTech AG"], "related": [], "markdown": "Zurich MedTech AG (ZMT) offers tools and best practices for targeted life sciences applications to simulate, analyze, and predict complex and dynamic biological processes and interactions. ZMT is a member of Zurich43", "released": null, "classifier": "organization::zmt", "created_by": "crespo", "github_url": null, "display_name": "ZMT", "wikipedia_url": null, "short_description": "ZMT is a member of Zurich43"}}, "collections": {"jupyterlab-math": {"items": ["crespo/osparc-demo"], "markdown": "Curated collection of repositories with examples of notebooks to run in jupyter-python-octave-math service", "created_by": "crespo", "display_name": "jupyterlab-math"}}}', '2021-03-04 23:17:43.373258', '2021-03-04 23:17:43.373258', 1, '0'); - """ +async def group_classifier_in_db(asyncpg_engine: AsyncEngine): + """Pre-populate group_classifiers table with test data.""" + data = random_group_classifier( + gid=1, ) - with postgres_db.connect() as conn: - conn.execute(stmt) + + async with insert_and_get_row_lifespan( + asyncpg_engine, + table=group_classifiers, + values=data, + pk_col=group_classifiers.c.id, + pk_value=data.get("id"), + ) as row: + yield row @pytest.fixture -async def app(postgres_dsn: dict, inject_tables): - dsn = DataSourceName( - user=postgres_dsn["user"], - password=postgres_dsn["password"], - database=postgres_dsn["database"], - host=postgres_dsn["host"], - port=postgres_dsn["port"], +def group_classifier_repository( + asyncpg_engine: AsyncEngine, +) -> GroupClassifierRepository: + """Create GroupClassifierRepository instance.""" + return GroupClassifierRepository(engine=asyncpg_engine) + + +async def test_get_classifiers_from_bundle_returns_bundle( + group_classifier_repository: GroupClassifierRepository, + group_classifier_in_db: dict, +): + """Test get_classifiers_from_bundle returns the stored bundle.""" + # Act + bundle = await group_classifier_repository.get_classifiers_from_bundle( + gid=group_classifier_in_db["gid"] ) - async with create_pg_engine(dsn) as engine: - fake_app = {APP_AIOPG_ENGINE_KEY: engine} - yield fake_app + # Assert + assert bundle is not None + assert bundle["vcs_ref"] == "asdfasdf" + assert bundle["vcs_url"] == "https://foo.classifiers.git" + assert "classifiers" in bundle + assert "project::dak" in bundle["classifiers"] + assert bundle["classifiers"]["project::dak"]["display_name"] == "DAK" + + +async def test_group_uses_scicrunch_returns_false( + group_classifier_repository: GroupClassifierRepository, + group_classifier_in_db: dict, +): + """Test group_uses_scicrunch returns False for non-scicrunch group.""" + # Act + uses_scicrunch = await group_classifier_repository.group_uses_scicrunch( + gid=group_classifier_in_db["gid"] + ) + + # Assert + assert uses_scicrunch is False + +async def test_get_classifiers_from_bundle_returns_none_for_missing_gid( + group_classifier_repository: GroupClassifierRepository, +): + """Test get_classifiers_from_bundle returns None for non-existent gid.""" + # Act + bundle = await group_classifier_repository.get_classifiers_from_bundle(gid=999999) -async def test_classfiers_from_bundle(app): - repo = GroupClassifierRepository.create_from_app(app) + # Assert + assert bundle is None - assert not await repo.group_uses_scicrunch(gid=1) - bundle = await repo.get_classifiers_from_bundle(gid=1) - assert bundle +async def test_group_uses_scicrunch_returns_false_for_missing_gid( + group_classifier_repository: GroupClassifierRepository, +): + """Test group_uses_scicrunch returns False for non-existent gid.""" + # Act + uses_scicrunch = await group_classifier_repository.group_uses_scicrunch(gid=999999) - # Prunes extras and excludes unset and nones - assert bundle["classifiers"]["project::dak"] == { - "classifier": "project::dak", - "display_name": "DAK", - } + # Assert + assert uses_scicrunch is False From 772bed7d35bc2368ee0b45fdddb7202d5d573521 Mon Sep 17 00:00:00 2001 From: Pedro Crespo-Valero <32402063+pcrespov@users.noreply.github.com> Date: Fri, 26 Sep 2025 18:41:02 +0200 Subject: [PATCH 04/20] wrong exceptions --- .../simcore_service_webserver/groups/_groups_repository.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/services/web/server/src/simcore_service_webserver/groups/_groups_repository.py b/services/web/server/src/simcore_service_webserver/groups/_groups_repository.py index ede591b76589..5abfdc024f74 100644 --- a/services/web/server/src/simcore_service_webserver/groups/_groups_repository.py +++ b/services/web/server/src/simcore_service_webserver/groups/_groups_repository.py @@ -17,7 +17,6 @@ StandardGroupUpdate, ) from models_library.users import UserID, UserNameID -from simcore_postgres_database.aiopg_errors import UniqueViolation from simcore_postgres_database.models.users import users from simcore_postgres_database.utils_products import get_or_create_product_group from simcore_postgres_database.utils_repos import ( @@ -28,6 +27,7 @@ from sqlalchemy import and_ from sqlalchemy.dialects.postgresql import insert from sqlalchemy.engine.row import Row +from sqlalchemy.exc import UniqueViolationError from sqlalchemy.ext.asyncio import AsyncConnection from ..db.models import groups, user_to_groups, users @@ -789,7 +789,7 @@ async def add_new_user_in_group( uid=new_user_id, gid=group_id, access_rights=user_access_rights ) ) - except UniqueViolation as exc: + except UniqueViolationError as exc: raise UserAlreadyInGroupError( uid=new_user_id, gid=group_id, From a117544ff41b1a69390457c66a2109cc9a7d1a7d Mon Sep 17 00:00:00 2001 From: Pedro Crespo-Valero <32402063+pcrespov@users.noreply.github.com> Date: Fri, 26 Sep 2025 18:53:58 +0200 Subject: [PATCH 05/20] rename --- .../src/simcore_service_webserver/exporter/_formatter/_sds.py | 2 +- .../src/simcore_service_webserver/groups/_classifiers_rest.py | 2 +- .../simcore_service_webserver/groups/_classifiers_service.py | 2 +- .../scicrunch/{db.py => repository.py} | 0 4 files changed, 3 insertions(+), 3 deletions(-) rename services/web/server/src/simcore_service_webserver/scicrunch/{db.py => repository.py} (100%) diff --git a/services/web/server/src/simcore_service_webserver/exporter/_formatter/_sds.py b/services/web/server/src/simcore_service_webserver/exporter/_formatter/_sds.py index 806e33b5df59..36432311a70d 100644 --- a/services/web/server/src/simcore_service_webserver/exporter/_formatter/_sds.py +++ b/services/web/server/src/simcore_service_webserver/exporter/_formatter/_sds.py @@ -11,7 +11,7 @@ from ...projects._projects_service import get_project_for_user from ...projects.exceptions import BaseProjectError from ...projects.models import ProjectDict -from ...scicrunch.db import ResearchResourceRepository +from ...scicrunch.repository import ResearchResourceRepository from ..exceptions import SDSException from .template_json import write_template_json from .xlsx.code_description import ( diff --git a/services/web/server/src/simcore_service_webserver/groups/_classifiers_rest.py b/services/web/server/src/simcore_service_webserver/groups/_classifiers_rest.py index 9c6ecdc65dbc..ff5729e56867 100644 --- a/services/web/server/src/simcore_service_webserver/groups/_classifiers_rest.py +++ b/services/web/server/src/simcore_service_webserver/groups/_classifiers_rest.py @@ -8,8 +8,8 @@ from .._meta import API_VTAG from ..login.decorators import login_required -from ..scicrunch.db import ResearchResourceRepository from ..scicrunch.models import ResearchResource, ResourceHit +from ..scicrunch.repository import ResearchResourceRepository from ..scicrunch.service_client import SciCrunch from ..security.decorators import permission_required from ..utils_aiohttp import envelope_json_response diff --git a/services/web/server/src/simcore_service_webserver/groups/_classifiers_service.py b/services/web/server/src/simcore_service_webserver/groups/_classifiers_service.py index 02ec763f6bb6..2696cd88e39a 100644 --- a/services/web/server/src/simcore_service_webserver/groups/_classifiers_service.py +++ b/services/web/server/src/simcore_service_webserver/groups/_classifiers_service.py @@ -23,8 +23,8 @@ field_validator, ) -from ..scicrunch.db import ResearchResourceRepository from ..scicrunch.errors import ScicrunchError +from ..scicrunch.repository import ResearchResourceRepository from ..scicrunch.service_client import SciCrunch from ._classifiers_repository import GroupClassifierRepository diff --git a/services/web/server/src/simcore_service_webserver/scicrunch/db.py b/services/web/server/src/simcore_service_webserver/scicrunch/repository.py similarity index 100% rename from services/web/server/src/simcore_service_webserver/scicrunch/db.py rename to services/web/server/src/simcore_service_webserver/scicrunch/repository.py From fb166e42e86a5dc76046b1d381abb1184cd2e559 Mon Sep 17 00:00:00 2001 From: Pedro Crespo-Valero <32402063+pcrespov@users.noreply.github.com> Date: Fri, 26 Sep 2025 19:05:00 +0200 Subject: [PATCH 06/20] exposes interface --- .../exporter/_formatter/_sds.py | 6 +- .../groups/_classifiers_rest.py | 12 +-- .../groups/_classifiers_service.py | 6 +- .../scicrunch/_repository.py | 65 ++++++++++++++ .../scicrunch/_service.py | 90 +++++++++++++++++++ .../scicrunch/repository.py | 75 ---------------- .../scicrunch/scicrunch_service.py | 4 + 7 files changed, 171 insertions(+), 87 deletions(-) create mode 100644 services/web/server/src/simcore_service_webserver/scicrunch/_repository.py create mode 100644 services/web/server/src/simcore_service_webserver/scicrunch/_service.py delete mode 100644 services/web/server/src/simcore_service_webserver/scicrunch/repository.py create mode 100644 services/web/server/src/simcore_service_webserver/scicrunch/scicrunch_service.py diff --git a/services/web/server/src/simcore_service_webserver/exporter/_formatter/_sds.py b/services/web/server/src/simcore_service_webserver/exporter/_formatter/_sds.py index 36432311a70d..a0275acfdbc5 100644 --- a/services/web/server/src/simcore_service_webserver/exporter/_formatter/_sds.py +++ b/services/web/server/src/simcore_service_webserver/exporter/_formatter/_sds.py @@ -11,7 +11,7 @@ from ...projects._projects_service import get_project_for_user from ...projects.exceptions import BaseProjectError from ...projects.models import ProjectDict -from ...scicrunch.repository import ResearchResourceRepository +from ...scicrunch.scicrunch_service import ScicrunchResourcesService from ..exceptions import SDSException from .template_json import write_template_json from .xlsx.code_description import ( @@ -70,10 +70,10 @@ async def _add_rrid_entries( ) -> None: rrid_entires: deque[RRIDEntry] = deque() - repo = ResearchResourceRepository(app) + service = ScicrunchResourcesService(app) classifiers = project_data["classifiers"] for classifier in classifiers: - scicrunch_resource = await repo.get(rrid=classifier) + scicrunch_resource = await service.get_resource_atdb(rrid=classifier) if scicrunch_resource is None: continue diff --git a/services/web/server/src/simcore_service_webserver/groups/_classifiers_rest.py b/services/web/server/src/simcore_service_webserver/groups/_classifiers_rest.py index ff5729e56867..ab109c0a167b 100644 --- a/services/web/server/src/simcore_service_webserver/groups/_classifiers_rest.py +++ b/services/web/server/src/simcore_service_webserver/groups/_classifiers_rest.py @@ -9,7 +9,7 @@ from .._meta import API_VTAG from ..login.decorators import login_required from ..scicrunch.models import ResearchResource, ResourceHit -from ..scicrunch.repository import ResearchResourceRepository +from ..scicrunch.scicrunch_service import ScicrunchResourcesService from ..scicrunch.service_client import SciCrunch from ..security.decorators import permission_required from ..utils_aiohttp import envelope_json_response @@ -53,8 +53,8 @@ async def get_scicrunch_resource(request: web.Request): rrid = SciCrunch.validate_identifier(rrid) # check if in database first - repo = ResearchResourceRepository(request.app) - resource: ResearchResource | None = await repo.get_resource(rrid) + service = ScicrunchResourcesService(request.app) + resource: ResearchResource | None = await service.get_resource(rrid) if not resource: # otherwise, request to scicrunch service scicrunch = SciCrunch.get_instance(request.app) @@ -74,15 +74,15 @@ async def add_scicrunch_resource(request: web.Request): rrid = request.match_info["rrid"] # check if exists - repo = ResearchResourceRepository(request.app) - resource: ResearchResource | None = await repo.get_resource(rrid) + service = ScicrunchResourcesService(request.app) + resource: ResearchResource | None = await service.get_resource(rrid) if not resource: # then request scicrunch service scicrunch = SciCrunch.get_instance(request.app) resource = await scicrunch.get_resource_fields(rrid) # insert new or if exists, then update - await repo.upsert(resource) + await service.upsert_resource(resource) return envelope_json_response(resource.model_dump()) diff --git a/services/web/server/src/simcore_service_webserver/groups/_classifiers_service.py b/services/web/server/src/simcore_service_webserver/groups/_classifiers_service.py index 2696cd88e39a..5b0871bf4780 100644 --- a/services/web/server/src/simcore_service_webserver/groups/_classifiers_service.py +++ b/services/web/server/src/simcore_service_webserver/groups/_classifiers_service.py @@ -24,7 +24,7 @@ ) from ..scicrunch.errors import ScicrunchError -from ..scicrunch.repository import ResearchResourceRepository +from ..scicrunch.scicrunch_service import ScicrunchResourcesService from ..scicrunch.service_client import SciCrunch from ._classifiers_repository import GroupClassifierRepository @@ -126,10 +126,10 @@ async def build_rrids_tree_view( ) scicrunch = SciCrunch.get_instance(app) - repo = ResearchResourceRepository(app) + service = ScicrunchResourcesService(app) flat_tree_view: dict[TreePath, ClassifierItem] = {} - for resource in await repo.list_resources(): + for resource in await service.list_resources(): try: validated_item = ClassifierItem( classifier=resource.rrid, diff --git a/services/web/server/src/simcore_service_webserver/scicrunch/_repository.py b/services/web/server/src/simcore_service_webserver/scicrunch/_repository.py new file mode 100644 index 000000000000..74fdc41ab7d7 --- /dev/null +++ b/services/web/server/src/simcore_service_webserver/scicrunch/_repository.py @@ -0,0 +1,65 @@ +""" +Repository for scicrunch_resources table operations using asyncpg +""" + +import logging +from typing import Any + +import sqlalchemy as sa +from simcore_postgres_database.models.scicrunch_resources import scicrunch_resources +from simcore_postgres_database.utils_repos import ( + pass_or_acquire_connection, + transaction_context, +) +from sqlalchemy.dialects.postgresql import insert as sa_pg_insert +from sqlalchemy.engine import Row +from sqlalchemy.ext.asyncio import AsyncConnection + +from ..db.base_repository import BaseRepository + +_logger = logging.getLogger(__name__) + + +class ScicrunchResourcesRepository(BaseRepository): + """Repository for managing scicrunch_resources operations.""" + + async def list_all_resources( + self, connection: AsyncConnection | None = None + ) -> list[Row]: + """List all research resources with basic fields.""" + async with pass_or_acquire_connection(self.engine, connection) as conn: + stmt = sa.select( + scicrunch_resources.c.rrid, + scicrunch_resources.c.name, + scicrunch_resources.c.description, + ) + result = await conn.execute(stmt) + return result.fetchall() + + async def get_resource_by_rrid( + self, rrid: str, connection: AsyncConnection | None = None + ) -> Row | None: + """Get a research resource by RRID.""" + async with pass_or_acquire_connection(self.engine, connection) as conn: + stmt = sa.select(scicrunch_resources).where( + scicrunch_resources.c.rrid == rrid + ) + result = await conn.execute(stmt) + return result.one_or_none() + + async def upsert_resource( + self, resource_data: dict[str, Any], connection: AsyncConnection | None = None + ) -> Row: + """Insert or update a research resource.""" + async with transaction_context(self.engine, connection) as conn: + stmt = ( + sa_pg_insert(scicrunch_resources) + .values(resource_data) + .on_conflict_do_update( + index_elements=[scicrunch_resources.c.rrid], + set_=resource_data, + ) + .returning(*scicrunch_resources.c) + ) + result = await conn.execute(stmt) + return result.one() diff --git a/services/web/server/src/simcore_service_webserver/scicrunch/_service.py b/services/web/server/src/simcore_service_webserver/scicrunch/_service.py new file mode 100644 index 000000000000..71fc2dc1e1ca --- /dev/null +++ b/services/web/server/src/simcore_service_webserver/scicrunch/_service.py @@ -0,0 +1,90 @@ +""" +Service layer for scicrunch research resources operations +""" + +import logging + +from aiohttp import web +from common_library.logging.logging_errors import create_troubleshooting_log_kwargs +from pydantic import ValidationError + +from ._repository import ScicrunchResourcesRepository +from .models import ResearchResource, ResearchResourceAtdB + +_logger = logging.getLogger(__name__) + + +class ScicrunchResourcesService: + """Service layer handling business logic for scicrunch resources.""" + + def __init__(self, app: web.Application): + self.app = app + self._repo = ScicrunchResourcesRepository.create_from_app(app) + + async def list_resources(self) -> list[ResearchResource]: + """List all research resources as domain models.""" + rows = await self._repo.list_all_resources() + if not rows: + return [] + + resources = [] + for row in rows: + try: + resource = ResearchResource.model_validate(dict(row)) + resources.append(resource) + except ValidationError as err: + _logger.warning( + **create_troubleshooting_log_kwargs( + f"Invalid data for resource {row.rrid}", + error=err, + error_context={"row_data": dict(row)}, + ) + ) + continue + + return resources + + async def get_resource_atdb(self, rrid: str) -> ResearchResourceAtdB | None: + """Get resource with all database fields.""" + row = await self._repo.get_resource_by_rrid(rrid) + if not row: + return None + + try: + return ResearchResourceAtdB.model_validate(dict(row)) + except ValidationError as err: + _logger.exception( + **create_troubleshooting_log_kwargs( + f"Invalid data for resource {rrid}", + error=err, + error_context={"rrid": rrid, "row_data": dict(row)}, + ) + ) + return None + + async def get_resource(self, rrid: str) -> ResearchResource | None: + """Get resource as domain model.""" + resource_atdb = await self.get_resource_atdb(rrid) + if not resource_atdb: + return None + + try: + return ResearchResource.model_validate(resource_atdb.model_dump()) + except ValidationError as err: + _logger.exception( + **create_troubleshooting_log_kwargs( + f"Failed to convert resource {rrid} to domain model", + error=err, + error_context={ + "rrid": rrid, + "resource_data": resource_atdb.model_dump(), + }, + ) + ) + return None + + async def upsert_resource(self, resource: ResearchResource) -> ResearchResource: + """Create or update a research resource.""" + values = resource.model_dump(exclude_unset=True) + row = await self._repo.upsert_resource(values) + return ResearchResource.model_validate(dict(row)) diff --git a/services/web/server/src/simcore_service_webserver/scicrunch/repository.py b/services/web/server/src/simcore_service_webserver/scicrunch/repository.py deleted file mode 100644 index c99c82f2aed7..000000000000 --- a/services/web/server/src/simcore_service_webserver/scicrunch/repository.py +++ /dev/null @@ -1,75 +0,0 @@ -""" -Access to postgres database scicrunch_resources table where USED rrids get stored -""" - -import logging - -import sqlalchemy as sa -from aiohttp import web -from aiopg.sa.result import ResultProxy, RowProxy -from simcore_postgres_database.models.scicrunch_resources import scicrunch_resources -from sqlalchemy.dialects.postgresql import insert as sa_pg_insert - -from ..db.plugin import get_database_engine_legacy -from .models import ResearchResource, ResearchResourceAtdB - -logger = logging.getLogger(__name__) - - -class ResearchResourceRepository: - """Hides interaction with scicrunch_resources pg tables - - acquires & releases connection **per call** - - uses aiopg[sa] - - implements CRUD on rrids - """ - - # WARNING: interfaces to both ResarchResource and ResearchResourceAtDB - - def __init__(self, app: web.Application): - self._engine = get_database_engine_legacy(app) - - async def list_resources(self) -> list[ResearchResource]: - async with self._engine.acquire() as conn: - stmt = sa.select( - [ - scicrunch_resources.c.rrid, - scicrunch_resources.c.name, - scicrunch_resources.c.description, - ] - ) - res: ResultProxy = await conn.execute(stmt) - rows: list[RowProxy] = await res.fetchall() - return ( - [ResearchResource.model_validate(row) for row in rows] if rows else [] - ) - - async def get(self, rrid: str) -> ResearchResourceAtdB | None: - async with self._engine.acquire() as conn: - stmt = sa.select(scicrunch_resources).where( - scicrunch_resources.c.rrid == rrid - ) - rows = await conn.execute(stmt) - row = await rows.fetchone() - return ResearchResourceAtdB(**row) if row else None - - async def get_resource(self, rrid: str) -> ResearchResource | None: - resource: ResearchResourceAtdB | None = await self.get(rrid) - if resource: - return ResearchResource(**resource.model_dump()) - return resource - - async def upsert(self, resource: ResearchResource): - async with self._engine.acquire() as conn: - values = resource.model_dump(exclude_unset=True) - - stmt = ( - sa_pg_insert(scicrunch_resources) - .values(values) - .on_conflict_do_update( - index_elements=[ - scicrunch_resources.c.rrid, - ], - set_=values, - ) - ) - await conn.execute(stmt) diff --git a/services/web/server/src/simcore_service_webserver/scicrunch/scicrunch_service.py b/services/web/server/src/simcore_service_webserver/scicrunch/scicrunch_service.py new file mode 100644 index 000000000000..10039e86aa73 --- /dev/null +++ b/services/web/server/src/simcore_service_webserver/scicrunch/scicrunch_service.py @@ -0,0 +1,4 @@ +from ._service import ScicrunchResourcesService + +__all__: tuple[str, ...] = ("ScicrunchResourcesService",) +# nopycln: file From ec78ff2aa049e2c547cdb93e6a2e5395ded257f2 Mon Sep 17 00:00:00 2001 From: Pedro Crespo-Valero <32402063+pcrespov@users.noreply.github.com> Date: Fri, 26 Sep 2025 19:19:40 +0200 Subject: [PATCH 07/20] moving scicrungh unders the service layer --- .../groups/_classifiers_rest.py | 35 ++------- .../groups/_classifiers_service.py | 72 +++++++++---------- .../scicrunch/_service.py | 50 ++++++++++++- .../scicrunch/models.py | 8 ++- 4 files changed, 90 insertions(+), 75 deletions(-) diff --git a/services/web/server/src/simcore_service_webserver/groups/_classifiers_rest.py b/services/web/server/src/simcore_service_webserver/groups/_classifiers_rest.py index ab109c0a167b..5e2d57b48c58 100644 --- a/services/web/server/src/simcore_service_webserver/groups/_classifiers_rest.py +++ b/services/web/server/src/simcore_service_webserver/groups/_classifiers_rest.py @@ -8,9 +8,8 @@ from .._meta import API_VTAG from ..login.decorators import login_required -from ..scicrunch.models import ResearchResource, ResourceHit +from ..scicrunch.models import ResourceHit from ..scicrunch.scicrunch_service import ScicrunchResourcesService -from ..scicrunch.service_client import SciCrunch from ..security.decorators import permission_required from ..utils_aiohttp import envelope_json_response from ._classifiers_service import GroupClassifiersService @@ -50,15 +49,9 @@ async def get_group_classifiers(request: web.Request): @handle_plugin_requests_exceptions async def get_scicrunch_resource(request: web.Request): rrid = request.match_info["rrid"] - rrid = SciCrunch.validate_identifier(rrid) - # check if in database first service = ScicrunchResourcesService(request.app) - resource: ResearchResource | None = await service.get_resource(rrid) - if not resource: - # otherwise, request to scicrunch service - scicrunch = SciCrunch.get_instance(request.app) - resource = await scicrunch.get_resource_fields(rrid) + resource = await service.get_or_fetch_resource(rrid) return envelope_json_response(resource.model_dump()) @@ -73,16 +66,8 @@ async def get_scicrunch_resource(request: web.Request): async def add_scicrunch_resource(request: web.Request): rrid = request.match_info["rrid"] - # check if exists service = ScicrunchResourcesService(request.app) - resource: ResearchResource | None = await service.get_resource(rrid) - if not resource: - # then request scicrunch service - scicrunch = SciCrunch.get_instance(request.app) - resource = await scicrunch.get_resource_fields(rrid) - - # insert new or if exists, then update - await service.upsert_resource(resource) + resource = await service.add_resource(rrid) return envelope_json_response(resource.model_dump()) @@ -97,17 +82,7 @@ async def add_scicrunch_resource(request: web.Request): async def search_scicrunch_resources(request: web.Request): guess_name = str(request.query["guess_name"]).strip() - scicrunch = SciCrunch.get_instance(request.app) - hits: list[ResourceHit] = await scicrunch.search_resource(guess_name) - - return envelope_json_response([hit.model_dump() for hit in hits]) - - -@handle_plugin_requests_exceptions -async def search_scicrunch_resources(request: web.Request): - guess_name = str(request.query["guess_name"]).strip() - - scicrunch = SciCrunch.get_instance(request.app) - hits: list[ResourceHit] = await scicrunch.search_resource(guess_name) + service = ScicrunchResourcesService(request.app) + hits: list[ResourceHit] = await service.search_resources(guess_name) return envelope_json_response([hit.model_dump() for hit in hits]) diff --git a/services/web/server/src/simcore_service_webserver/groups/_classifiers_service.py b/services/web/server/src/simcore_service_webserver/groups/_classifiers_service.py index 5b0871bf4780..a0a86d652d5c 100644 --- a/services/web/server/src/simcore_service_webserver/groups/_classifiers_service.py +++ b/services/web/server/src/simcore_service_webserver/groups/_classifiers_service.py @@ -25,7 +25,6 @@ from ..scicrunch.errors import ScicrunchError from ..scicrunch.scicrunch_service import ScicrunchResourcesService -from ..scicrunch.service_client import SciCrunch from ._classifiers_repository import GroupClassifierRepository _logger = logging.getLogger(__name__) @@ -108,46 +107,41 @@ async def get_group_classifiers( # otherwise, build dynamic tree with RRIDs try: - return await build_rrids_tree_view(self.app, tree_view_mode=tree_view_mode) + return await self._build_rrids_tree_view(tree_view_mode=tree_view_mode) except ScicrunchError: # Return empty view on any error (including ScicrunchError) return {} - -# HELPERS FOR API HANDLERS -------------- - - -async def build_rrids_tree_view( - app: web.Application, tree_view_mode: Literal["std"] = "std" -) -> dict[str, Any]: - if tree_view_mode != "std": - raise web.HTTPNotImplemented( - text="Currently only 'std' option for the classifiers tree view is implemented" + async def _build_rrids_tree_view( + self, tree_view_mode: Literal["std"] = "std" + ) -> dict[str, Any]: + if tree_view_mode != "std": + msg = "Currently only 'std' option for the classifiers tree view is implemented" + raise NotImplementedError(msg) + + service = ScicrunchResourcesService(self.app) + + flat_tree_view: dict[TreePath, ClassifierItem] = {} + for resource in await service.list_resources(include_url=True): + try: + validated_item = ClassifierItem( + classifier=resource.rrid, + display_name=resource.name.title(), + short_description=resource.description, + url=resource.url, + ) + + node = TypeAdapter(TreePath).validate_python( + validated_item.display_name.replace(":", " ") + ) + flat_tree_view[node] = validated_item + + except ValidationError as err: + _logger.warning( + "Cannot convert RRID into a classifier item. Skipping. Details: %s", + err, + ) + + return Classifiers.model_construct(classifiers=flat_tree_view).model_dump( + exclude_unset=True ) - - scicrunch = SciCrunch.get_instance(app) - service = ScicrunchResourcesService(app) - - flat_tree_view: dict[TreePath, ClassifierItem] = {} - for resource in await service.list_resources(): - try: - validated_item = ClassifierItem( - classifier=resource.rrid, - display_name=resource.name.title(), - short_description=resource.description, - url=scicrunch.get_resolver_web_url(resource.rrid), - ) - - node = TypeAdapter(TreePath).validate_python( - validated_item.display_name.replace(":", " ") - ) - flat_tree_view[node] = validated_item - - except ValidationError as err: - _logger.warning( - "Cannot convert RRID into a classifier item. Skipping. Details: %s", err - ) - - return Classifiers.model_construct(classifiers=flat_tree_view).model_dump( - exclude_unset=True - ) diff --git a/services/web/server/src/simcore_service_webserver/scicrunch/_service.py b/services/web/server/src/simcore_service_webserver/scicrunch/_service.py index 71fc2dc1e1ca..c1f314d29951 100644 --- a/services/web/server/src/simcore_service_webserver/scicrunch/_service.py +++ b/services/web/server/src/simcore_service_webserver/scicrunch/_service.py @@ -9,7 +9,8 @@ from pydantic import ValidationError from ._repository import ScicrunchResourcesRepository -from .models import ResearchResource, ResearchResourceAtdB +from .models import ResearchResource, ResearchResourceAtdB, ResourceHit +from .service_client import SciCrunch _logger = logging.getLogger(__name__) @@ -20,8 +21,9 @@ class ScicrunchResourcesService: def __init__(self, app: web.Application): self.app = app self._repo = ScicrunchResourcesRepository.create_from_app(app) + self._scicrunch = SciCrunch.get_instance(self.app) - async def list_resources(self) -> list[ResearchResource]: + async def list_resources(self, include_url: bool = False) -> list[ResearchResource]: """List all research resources as domain models.""" rows = await self._repo.list_all_resources() if not rows: @@ -30,7 +32,15 @@ async def list_resources(self) -> list[ResearchResource]: resources = [] for row in rows: try: - resource = ResearchResource.model_validate(dict(row)) + resource_data = dict(row) + + # Add resolver URL if requested + if include_url: + resource_data["url"] = self._scicrunch.get_resolver_web_url( + row.rrid + ) + + resource = ResearchResource.model_validate(resource_data) resources.append(resource) except ValidationError as err: _logger.warning( @@ -88,3 +98,37 @@ async def upsert_resource(self, resource: ResearchResource) -> ResearchResource: values = resource.model_dump(exclude_unset=True) row = await self._repo.upsert_resource(values) return ResearchResource.model_validate(dict(row)) + + async def search_resources(self, guess_name: str) -> list[ResourceHit]: + """Search for research resources using SciCrunch API.""" + guess_name = guess_name.strip() + if not guess_name: + return [] + + return await self._scicrunch.search_resource(guess_name) + + async def add_resource(self, rrid: str) -> ResearchResource: + """Add a research resource by RRID, fetching from SciCrunch if not in database.""" + # Check if exists in database first + resource = await self.get_resource(rrid) + if resource: + return resource + + # If not found, request from scicrunch service + resource = await self._scicrunch.get_resource_fields(rrid) + + # Insert new or update if exists + return await self.upsert_resource(resource) + + async def get_or_fetch_resource(self, rrid: str) -> ResearchResource: + """Get resource from database first, fetch from SciCrunch API if not found.""" + # Validate the RRID format first + validated_rrid = SciCrunch.validate_identifier(rrid) + + # Check if in database first + resource = await self.get_resource(validated_rrid) + if resource: + return resource + + # Otherwise, request from scicrunch service + return await self._scicrunch.get_resource_fields(validated_rrid) diff --git a/services/web/server/src/simcore_service_webserver/scicrunch/models.py b/services/web/server/src/simcore_service_webserver/scicrunch/models.py index 2140f88ea335..4418c96d27ab 100644 --- a/services/web/server/src/simcore_service_webserver/scicrunch/models.py +++ b/services/web/server/src/simcore_service_webserver/scicrunch/models.py @@ -1,12 +1,12 @@ """ - Domain models at every interface: scicrunch API, pg database and webserver API +Domain models at every interface: scicrunch API, pg database and webserver API """ import logging import re from datetime import datetime -from pydantic import field_validator, ConfigDict, BaseModel, Field +from pydantic import BaseModel, ConfigDict, Field, HttpUrl, field_validator logger = logging.getLogger(__name__) @@ -62,11 +62,13 @@ class ResearchResource(BaseModel): ) name: str description: str + url: HttpUrl | None = None @field_validator("rrid", mode="before") @classmethod - def format_rrid(cls, v): + def _format_rrid(cls, v): return normalize_rrid_tags(v, with_prefix=True) + model_config = ConfigDict(from_attributes=True, str_strip_whitespace=True) From 7281b1da2a41ec5a648daa7c65c30cb777a0e835 Mon Sep 17 00:00:00 2001 From: Pedro Crespo-Valero <32402063+pcrespov@users.noreply.github.com> Date: Fri, 26 Sep 2025 19:35:51 +0200 Subject: [PATCH 08/20] fixes error --- .../simcore_service_webserver/groups/_groups_repository.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/services/web/server/src/simcore_service_webserver/groups/_groups_repository.py b/services/web/server/src/simcore_service_webserver/groups/_groups_repository.py index 5abfdc024f74..765935c85e5c 100644 --- a/services/web/server/src/simcore_service_webserver/groups/_groups_repository.py +++ b/services/web/server/src/simcore_service_webserver/groups/_groups_repository.py @@ -27,7 +27,7 @@ from sqlalchemy import and_ from sqlalchemy.dialects.postgresql import insert from sqlalchemy.engine.row import Row -from sqlalchemy.exc import UniqueViolationError +from sqlalchemy.exc import IntegrityError from sqlalchemy.ext.asyncio import AsyncConnection from ..db.models import groups, user_to_groups, users @@ -789,7 +789,7 @@ async def add_new_user_in_group( uid=new_user_id, gid=group_id, access_rights=user_access_rights ) ) - except UniqueViolationError as exc: + except IntegrityError as exc: raise UserAlreadyInGroupError( uid=new_user_id, gid=group_id, From aec68f4611cda9e9f52e925b6d8d86d419c1e86c Mon Sep 17 00:00:00 2001 From: Pedro Crespo-Valero <32402063+pcrespov@users.noreply.github.com> Date: Fri, 26 Sep 2025 19:40:39 +0200 Subject: [PATCH 09/20] fix --- .../groups/_classifiers_service.py | 4 ++-- .../scicrunch/_service.py | 15 ++++++--------- .../simcore_service_webserver/scicrunch/models.py | 3 +-- 3 files changed, 9 insertions(+), 13 deletions(-) diff --git a/services/web/server/src/simcore_service_webserver/groups/_classifiers_service.py b/services/web/server/src/simcore_service_webserver/groups/_classifiers_service.py index a0a86d652d5c..3813fde62adb 100644 --- a/services/web/server/src/simcore_service_webserver/groups/_classifiers_service.py +++ b/services/web/server/src/simcore_service_webserver/groups/_classifiers_service.py @@ -122,13 +122,13 @@ async def _build_rrids_tree_view( service = ScicrunchResourcesService(self.app) flat_tree_view: dict[TreePath, ClassifierItem] = {} - for resource in await service.list_resources(include_url=True): + for resource in await service.list_resources(): try: validated_item = ClassifierItem( classifier=resource.rrid, display_name=resource.name.title(), short_description=resource.description, - url=resource.url, + url=service.get_resolver_web_url(resource.rrid), ) node = TypeAdapter(TreePath).validate_python( diff --git a/services/web/server/src/simcore_service_webserver/scicrunch/_service.py b/services/web/server/src/simcore_service_webserver/scicrunch/_service.py index c1f314d29951..d2cf31cdf340 100644 --- a/services/web/server/src/simcore_service_webserver/scicrunch/_service.py +++ b/services/web/server/src/simcore_service_webserver/scicrunch/_service.py @@ -6,7 +6,7 @@ from aiohttp import web from common_library.logging.logging_errors import create_troubleshooting_log_kwargs -from pydantic import ValidationError +from pydantic import HttpUrl, ValidationError from ._repository import ScicrunchResourcesRepository from .models import ResearchResource, ResearchResourceAtdB, ResourceHit @@ -23,7 +23,7 @@ def __init__(self, app: web.Application): self._repo = ScicrunchResourcesRepository.create_from_app(app) self._scicrunch = SciCrunch.get_instance(self.app) - async def list_resources(self, include_url: bool = False) -> list[ResearchResource]: + async def list_resources(self) -> list[ResearchResource]: """List all research resources as domain models.""" rows = await self._repo.list_all_resources() if not rows: @@ -33,13 +33,6 @@ async def list_resources(self, include_url: bool = False) -> list[ResearchResour for row in rows: try: resource_data = dict(row) - - # Add resolver URL if requested - if include_url: - resource_data["url"] = self._scicrunch.get_resolver_web_url( - row.rrid - ) - resource = ResearchResource.model_validate(resource_data) resources.append(resource) except ValidationError as err: @@ -54,6 +47,10 @@ async def list_resources(self, include_url: bool = False) -> list[ResearchResour return resources + def get_resolver_web_url(self, rrid: str) -> HttpUrl: + """Get the resolver web URL for a given RRID.""" + return self._scicrunch.get_resolver_web_url(rrid) + async def get_resource_atdb(self, rrid: str) -> ResearchResourceAtdB | None: """Get resource with all database fields.""" row = await self._repo.get_resource_by_rrid(rrid) diff --git a/services/web/server/src/simcore_service_webserver/scicrunch/models.py b/services/web/server/src/simcore_service_webserver/scicrunch/models.py index 4418c96d27ab..0d87a7f33afd 100644 --- a/services/web/server/src/simcore_service_webserver/scicrunch/models.py +++ b/services/web/server/src/simcore_service_webserver/scicrunch/models.py @@ -6,7 +6,7 @@ import re from datetime import datetime -from pydantic import BaseModel, ConfigDict, Field, HttpUrl, field_validator +from pydantic import BaseModel, ConfigDict, Field, field_validator logger = logging.getLogger(__name__) @@ -62,7 +62,6 @@ class ResearchResource(BaseModel): ) name: str description: str - url: HttpUrl | None = None @field_validator("rrid", mode="before") @classmethod From 251633dfc6061dd2553463451d20012d17cb5d2a Mon Sep 17 00:00:00 2001 From: Pedro Crespo-Valero <32402063+pcrespov@users.noreply.github.com> Date: Mon, 29 Sep 2025 09:48:20 +0200 Subject: [PATCH 10/20] mypy fix --- .../groups/_classifiers_repository.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/services/web/server/src/simcore_service_webserver/groups/_classifiers_repository.py b/services/web/server/src/simcore_service_webserver/groups/_classifiers_repository.py index 461378de3cbe..4898082ef00f 100644 --- a/services/web/server/src/simcore_service_webserver/groups/_classifiers_repository.py +++ b/services/web/server/src/simcore_service_webserver/groups/_classifiers_repository.py @@ -28,7 +28,8 @@ async def _get_bundle( async def get_classifiers_from_bundle(self, gid: int) -> dict[str, Any] | None: bundle_row = await self._get_bundle(gid) if bundle_row: - return bundle_row.bundle + # pylint: disable=protected-access + return dict(bundle_row.bundle._mapping) # noqa: SLF001 return None async def group_uses_scicrunch( From 555b07dad996a2a8480ed6727f19d5b009d4c7b6 Mon Sep 17 00:00:00 2001 From: Pedro Crespo-Valero <32402063+pcrespov@users.noreply.github.com> Date: Mon, 29 Sep 2025 10:15:46 +0200 Subject: [PATCH 11/20] cleanup --- .../groups/_classifiers_rest.py | 6 +- .../groups/_classifiers_service.py | 2 +- .../scicrunch/_service.py | 100 ++++++++++-------- .../01/test_groups_classifiers_repository.py | 7 +- 4 files changed, 62 insertions(+), 53 deletions(-) diff --git a/services/web/server/src/simcore_service_webserver/groups/_classifiers_rest.py b/services/web/server/src/simcore_service_webserver/groups/_classifiers_rest.py index 5e2d57b48c58..678174c1a4f5 100644 --- a/services/web/server/src/simcore_service_webserver/groups/_classifiers_rest.py +++ b/services/web/server/src/simcore_service_webserver/groups/_classifiers_rest.py @@ -51,7 +51,7 @@ async def get_scicrunch_resource(request: web.Request): rrid = request.match_info["rrid"] service = ScicrunchResourcesService(request.app) - resource = await service.get_or_fetch_resource(rrid) + resource = await service.get_or_fetch_reseach_resource(rrid) return envelope_json_response(resource.model_dump()) @@ -67,7 +67,7 @@ async def add_scicrunch_resource(request: web.Request): rrid = request.match_info["rrid"] service = ScicrunchResourcesService(request.app) - resource = await service.add_resource(rrid) + resource = await service.create_research_resource(rrid) return envelope_json_response(resource.model_dump()) @@ -83,6 +83,6 @@ async def search_scicrunch_resources(request: web.Request): guess_name = str(request.query["guess_name"]).strip() service = ScicrunchResourcesService(request.app) - hits: list[ResourceHit] = await service.search_resources(guess_name) + hits: list[ResourceHit] = await service.search_research_resources(guess_name) return envelope_json_response([hit.model_dump() for hit in hits]) diff --git a/services/web/server/src/simcore_service_webserver/groups/_classifiers_service.py b/services/web/server/src/simcore_service_webserver/groups/_classifiers_service.py index 3813fde62adb..e892dc7ae473 100644 --- a/services/web/server/src/simcore_service_webserver/groups/_classifiers_service.py +++ b/services/web/server/src/simcore_service_webserver/groups/_classifiers_service.py @@ -122,7 +122,7 @@ async def _build_rrids_tree_view( service = ScicrunchResourcesService(self.app) flat_tree_view: dict[TreePath, ClassifierItem] = {} - for resource in await service.list_resources(): + for resource in await service.list_research_resources(): try: validated_item = ClassifierItem( classifier=resource.rrid, diff --git a/services/web/server/src/simcore_service_webserver/scicrunch/_service.py b/services/web/server/src/simcore_service_webserver/scicrunch/_service.py index d2cf31cdf340..b9f42198bb74 100644 --- a/services/web/server/src/simcore_service_webserver/scicrunch/_service.py +++ b/services/web/server/src/simcore_service_webserver/scicrunch/_service.py @@ -16,14 +16,18 @@ class ScicrunchResourcesService: - """Service layer handling business logic for scicrunch resources.""" + """Service layer handling business logic for scicrunch resources. + + - Research Resources operations (RRID = Research Resource ID) + """ def __init__(self, app: web.Application): self.app = app self._repo = ScicrunchResourcesRepository.create_from_app(app) - self._scicrunch = SciCrunch.get_instance(self.app) + # client to interact with scicrunch.org service + self._client = SciCrunch.get_instance(self.app) - async def list_resources(self) -> list[ResearchResource]: + async def list_research_resources(self) -> list[ResearchResource]: """List all research resources as domain models.""" rows = await self._repo.list_all_resources() if not rows: @@ -47,85 +51,89 @@ async def list_resources(self) -> list[ResearchResource]: return resources - def get_resolver_web_url(self, rrid: str) -> HttpUrl: - """Get the resolver web URL for a given RRID.""" - return self._scicrunch.get_resolver_web_url(rrid) - - async def get_resource_atdb(self, rrid: str) -> ResearchResourceAtdB | None: - """Get resource with all database fields.""" - row = await self._repo.get_resource_by_rrid(rrid) - if not row: + async def get_research_resource(self, rrid: str) -> ResearchResource | None: + """Get resource as domain model.""" + resource_atdb = await self.get_resource_atdb(rrid) + if not resource_atdb: return None try: - return ResearchResourceAtdB.model_validate(dict(row)) + return ResearchResource.model_validate(resource_atdb.model_dump()) except ValidationError as err: _logger.exception( **create_troubleshooting_log_kwargs( - f"Invalid data for resource {rrid}", + f"Failed to convert resource {rrid} to domain model", error=err, - error_context={"rrid": rrid, "row_data": dict(row)}, + error_context={ + "rrid": rrid, + "resource_data": resource_atdb.model_dump(), + }, ) ) return None - async def get_resource(self, rrid: str) -> ResearchResource | None: - """Get resource as domain model.""" - resource_atdb = await self.get_resource_atdb(rrid) - if not resource_atdb: + async def get_or_fetch_reseach_resource(self, rrid: str) -> ResearchResource: + """Get resource from database first, fetch from SciCrunch API if not found.""" + # Validate the RRID format first + validated_rrid = SciCrunch.validate_identifier(rrid) + + # Check if in database first + resource = await self.get_research_resource(validated_rrid) + if resource: + return resource + + # Otherwise, request from scicrunch service + return await self._client.get_resource_fields(validated_rrid) + + async def get_resource_atdb(self, rrid: str) -> ResearchResourceAtdB | None: + """Get resource with all database fields.""" + row = await self._repo.get_resource_by_rrid(rrid) + if not row: return None try: - return ResearchResource.model_validate(resource_atdb.model_dump()) + return ResearchResourceAtdB.model_validate(dict(row)) except ValidationError as err: _logger.exception( **create_troubleshooting_log_kwargs( - f"Failed to convert resource {rrid} to domain model", + f"Invalid data for resource {rrid}", error=err, - error_context={ - "rrid": rrid, - "resource_data": resource_atdb.model_dump(), - }, + error_context={"rrid": rrid, "row_data": dict(row)}, ) ) return None - async def upsert_resource(self, resource: ResearchResource) -> ResearchResource: - """Create or update a research resource.""" - values = resource.model_dump(exclude_unset=True) - row = await self._repo.upsert_resource(values) - return ResearchResource.model_validate(dict(row)) - - async def search_resources(self, guess_name: str) -> list[ResourceHit]: + async def search_research_resources(self, guess_name: str) -> list[ResourceHit]: """Search for research resources using SciCrunch API.""" guess_name = guess_name.strip() if not guess_name: return [] - return await self._scicrunch.search_resource(guess_name) + return await self._client.search_resource(guess_name) - async def add_resource(self, rrid: str) -> ResearchResource: + async def create_research_resource(self, rrid: str) -> ResearchResource: """Add a research resource by RRID, fetching from SciCrunch if not in database.""" # Check if exists in database first - resource = await self.get_resource(rrid) + resource = await self.get_research_resource(rrid) if resource: return resource # If not found, request from scicrunch service - resource = await self._scicrunch.get_resource_fields(rrid) + resource = await self._client.get_resource_fields(rrid) # Insert new or update if exists - return await self.upsert_resource(resource) + return await self.upsert_research_resource(resource) - async def get_or_fetch_resource(self, rrid: str) -> ResearchResource: - """Get resource from database first, fetch from SciCrunch API if not found.""" - # Validate the RRID format first - validated_rrid = SciCrunch.validate_identifier(rrid) + async def upsert_research_resource( + self, resource: ResearchResource + ) -> ResearchResource: + """Create or update a research resource.""" + values = resource.model_dump(exclude_unset=True) + row = await self._repo.upsert_resource(values) + return ResearchResource.model_validate(dict(row)) - # Check if in database first - resource = await self.get_resource(validated_rrid) - if resource: - return resource + # HELPERS -- - # Otherwise, request from scicrunch service - return await self._scicrunch.get_resource_fields(validated_rrid) + def get_resolver_web_url(self, rrid: str) -> HttpUrl: + """Get the resolver web URL for a given RRID.""" + return self._client.get_resolver_web_url(rrid) diff --git a/services/web/server/tests/unit/with_dbs/01/test_groups_classifiers_repository.py b/services/web/server/tests/unit/with_dbs/01/test_groups_classifiers_repository.py index 0a546ac6cec1..78717cae7841 100644 --- a/services/web/server/tests/unit/with_dbs/01/test_groups_classifiers_repository.py +++ b/services/web/server/tests/unit/with_dbs/01/test_groups_classifiers_repository.py @@ -4,6 +4,7 @@ # pylint: disable=too-many-arguments import pytest +from models_library.groups import EVERYONE_GROUP_ID from pytest_simcore.helpers.faker_factories import random_group_classifier from pytest_simcore.helpers.postgres_tools import insert_and_get_row_lifespan from simcore_postgres_database.models.classifiers import group_classifiers @@ -16,10 +17,10 @@ @pytest.fixture async def group_classifier_in_db(asyncpg_engine: AsyncEngine): """Pre-populate group_classifiers table with test data.""" - data = random_group_classifier( - gid=1, - ) + data = random_group_classifier(gid=EVERYONE_GROUP_ID) + # pylint: disable=contextmanager-generator-missing-cleanup + # NOTE: this code is safe since `@asynccontextmanager` takes care of the cleanup async with insert_and_get_row_lifespan( asyncpg_engine, table=group_classifiers, From eca30ba005b8a654a6c3fff1dafb9dcff32c12fc Mon Sep 17 00:00:00 2001 From: Pedro Crespo-Valero <32402063+pcrespov@users.noreply.github.com> Date: Mon, 29 Sep 2025 10:23:39 +0200 Subject: [PATCH 12/20] cleanup models --- .../scicrunch/models.py | 31 +++++++++++-------- 1 file changed, 18 insertions(+), 13 deletions(-) diff --git a/services/web/server/src/simcore_service_webserver/scicrunch/models.py b/services/web/server/src/simcore_service_webserver/scicrunch/models.py index 0d87a7f33afd..ab5bc79485c8 100644 --- a/services/web/server/src/simcore_service_webserver/scicrunch/models.py +++ b/services/web/server/src/simcore_service_webserver/scicrunch/models.py @@ -5,8 +5,10 @@ import logging import re from datetime import datetime +from functools import partial +from typing import Annotated -from pydantic import BaseModel, ConfigDict, Field, field_validator +from pydantic import BaseModel, BeforeValidator, ConfigDict, Field logger = logging.getLogger(__name__) @@ -49,26 +51,29 @@ def normalize_rrid_tags(rrid_tag: str, *, with_prefix: bool = True) -> str: class ResourceHit(BaseModel): - rrid: str = Field(..., alias="rid") + rrid: Annotated[str, Field(alias="rid")] name: str # webserver API models ----------------------------------------- class ResearchResource(BaseModel): - rrid: str = Field( - ..., - description="Unique identifier used as classifier, i.e. to tag studies and services", - pattern=STRICT_RRID_PATTERN, - ) + rrid: Annotated[ + str, + BeforeValidator( + partial(normalize_rrid_tags, with_prefix=True), + ), + Field( + description="Unique identifier used as classifier, i.e. to tag studies and services", + pattern=STRICT_RRID_PATTERN, + ), + ] name: str description: str - @field_validator("rrid", mode="before") - @classmethod - def _format_rrid(cls, v): - return normalize_rrid_tags(v, with_prefix=True) - - model_config = ConfigDict(from_attributes=True, str_strip_whitespace=True) + model_config = ConfigDict( + from_attributes=True, + str_strip_whitespace=True, + ) # postgres_database.scicrunch_resources ORM -------------------- From bfd8972ed493926868e311ca633e899c48599de9 Mon Sep 17 00:00:00 2001 From: Pedro Crespo-Valero <32402063+pcrespov@users.noreply.github.com> Date: Mon, 29 Sep 2025 10:56:12 +0200 Subject: [PATCH 13/20] rename --- .../src/simcore_service_webserver/groups/_classifiers_rest.py | 2 +- .../server/src/simcore_service_webserver/scicrunch/_service.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/services/web/server/src/simcore_service_webserver/groups/_classifiers_rest.py b/services/web/server/src/simcore_service_webserver/groups/_classifiers_rest.py index 678174c1a4f5..f8228ba883e6 100644 --- a/services/web/server/src/simcore_service_webserver/groups/_classifiers_rest.py +++ b/services/web/server/src/simcore_service_webserver/groups/_classifiers_rest.py @@ -51,7 +51,7 @@ async def get_scicrunch_resource(request: web.Request): rrid = request.match_info["rrid"] service = ScicrunchResourcesService(request.app) - resource = await service.get_or_fetch_reseach_resource(rrid) + resource = await service.get_or_fetch_research_resource(rrid) return envelope_json_response(resource.model_dump()) diff --git a/services/web/server/src/simcore_service_webserver/scicrunch/_service.py b/services/web/server/src/simcore_service_webserver/scicrunch/_service.py index b9f42198bb74..3b44c01b3f20 100644 --- a/services/web/server/src/simcore_service_webserver/scicrunch/_service.py +++ b/services/web/server/src/simcore_service_webserver/scicrunch/_service.py @@ -72,7 +72,7 @@ async def get_research_resource(self, rrid: str) -> ResearchResource | None: ) return None - async def get_or_fetch_reseach_resource(self, rrid: str) -> ResearchResource: + async def get_or_fetch_research_resource(self, rrid: str) -> ResearchResource: """Get resource from database first, fetch from SciCrunch API if not found.""" # Validate the RRID format first validated_rrid = SciCrunch.validate_identifier(rrid) From b75387a3671df5c101665e0e811b686464e88689 Mon Sep 17 00:00:00 2001 From: Pedro Crespo-Valero <32402063+pcrespov@users.noreply.github.com> Date: Mon, 29 Sep 2025 10:59:07 +0200 Subject: [PATCH 14/20] fake --- .../src/pytest_simcore/helpers/faker_factories.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/packages/pytest-simcore/src/pytest_simcore/helpers/faker_factories.py b/packages/pytest-simcore/src/pytest_simcore/helpers/faker_factories.py index 56174c1e4388..fe43da30f158 100644 --- a/packages/pytest-simcore/src/pytest_simcore/helpers/faker_factories.py +++ b/packages/pytest-simcore/src/pytest_simcore/helpers/faker_factories.py @@ -642,8 +642,8 @@ def random_group_classifier( data = { "gid": gid, "bundle": { - "vcs_ref": "asdfasdf", - "vcs_url": "https://foo.classifiers.git", + "vcs_ref": fake.lexify(text="???????"), + "vcs_url": "https://organization.classifiers.git", "build_date": "2021-01-20T15:19:30Z", "classifiers": { "project::dak": { @@ -654,7 +654,7 @@ def random_group_classifier( "markdown": "", "released": None, "classifier": "project::dak", - "created_by": "Nicolas Chavannes", + "created_by": fake.user_name(), "github_url": None, "display_name": "DAK", "wikipedia_url": None, @@ -665,10 +665,10 @@ def random_group_classifier( "logo": None, "aliases": ["Zurich MedTech AG"], "related": [], - "markdown": "Zurich MedTech AG (ZMT) offers tools and best practices for targeted life sciences applications to simulate, analyze, and predict complex and dynamic biological processes and interactions. ZMT is a member of Zurich43", + "markdown": fake.text(), "released": None, "classifier": "organization::zmt", - "created_by": "crespo", + "created_by": fake.user_name(), "github_url": None, "display_name": "ZMT", "wikipedia_url": None, @@ -678,8 +678,8 @@ def random_group_classifier( "collections": { "jupyterlab-math": { "items": ["crespo/osparc-demo"], - "markdown": "Curated collection of repositories with examples of notebooks to run in jupyter-python-octave-math service", - "created_by": "crespo", + "markdown": fake.text(), + "created_by": fake.user_name(), "display_name": "jupyterlab-math", } }, From 64cf9115d6e12ec1be2659c32b0748a9e2eb7cfb Mon Sep 17 00:00:00 2001 From: Pedro Crespo-Valero <32402063+pcrespov@users.noreply.github.com> Date: Mon, 29 Sep 2025 11:04:56 +0200 Subject: [PATCH 15/20] fixes --- .../groups/_classifiers_repository.py | 5 ++--- .../unit/with_dbs/01/test_groups_classifiers_repository.py | 3 +-- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/services/web/server/src/simcore_service_webserver/groups/_classifiers_repository.py b/services/web/server/src/simcore_service_webserver/groups/_classifiers_repository.py index 4898082ef00f..54a2a04c1291 100644 --- a/services/web/server/src/simcore_service_webserver/groups/_classifiers_repository.py +++ b/services/web/server/src/simcore_service_webserver/groups/_classifiers_repository.py @@ -1,5 +1,5 @@ import logging -from typing import Any +from typing import Any, cast import sqlalchemy as sa from simcore_postgres_database.models.classifiers import group_classifiers @@ -28,8 +28,7 @@ async def _get_bundle( async def get_classifiers_from_bundle(self, gid: int) -> dict[str, Any] | None: bundle_row = await self._get_bundle(gid) if bundle_row: - # pylint: disable=protected-access - return dict(bundle_row.bundle._mapping) # noqa: SLF001 + return cast(dict[str, Any], bundle_row.bundle) return None async def group_uses_scicrunch( diff --git a/services/web/server/tests/unit/with_dbs/01/test_groups_classifiers_repository.py b/services/web/server/tests/unit/with_dbs/01/test_groups_classifiers_repository.py index 78717cae7841..3eb480dd2e6a 100644 --- a/services/web/server/tests/unit/with_dbs/01/test_groups_classifiers_repository.py +++ b/services/web/server/tests/unit/with_dbs/01/test_groups_classifiers_repository.py @@ -51,8 +51,7 @@ async def test_get_classifiers_from_bundle_returns_bundle( # Assert assert bundle is not None - assert bundle["vcs_ref"] == "asdfasdf" - assert bundle["vcs_url"] == "https://foo.classifiers.git" + assert bundle["vcs_url"] == "https://organization.classifiers.git" assert "classifiers" in bundle assert "project::dak" in bundle["classifiers"] assert bundle["classifiers"]["project::dak"]["display_name"] == "DAK" From 17ba3afa6937d4a47586480eadf5dfee1314d7e1 Mon Sep 17 00:00:00 2001 From: Pedro Crespo-Valero <32402063+pcrespov@users.noreply.github.com> Date: Mon, 29 Sep 2025 14:20:30 +0200 Subject: [PATCH 16/20] refactor: update scicrunch service integration and improve dependency management --- .../groups/_classifiers_rest.py | 8 ++++---- .../simcore_service_webserver/groups/plugin.py | 2 ++ .../scicrunch/_service.py | 8 +++----- .../scicrunch/plugin.py | 16 ++++++++++++++-- .../scicrunch/scicrunch_service.py | 11 ++++++++++- .../scicrunch/service_client.py | 8 ++++---- .../scicrunch/test_scicrunch_service_client.py | 4 +++- 7 files changed, 40 insertions(+), 17 deletions(-) diff --git a/services/web/server/src/simcore_service_webserver/groups/_classifiers_rest.py b/services/web/server/src/simcore_service_webserver/groups/_classifiers_rest.py index f8228ba883e6..d96531098b79 100644 --- a/services/web/server/src/simcore_service_webserver/groups/_classifiers_rest.py +++ b/services/web/server/src/simcore_service_webserver/groups/_classifiers_rest.py @@ -9,7 +9,7 @@ from .._meta import API_VTAG from ..login.decorators import login_required from ..scicrunch.models import ResourceHit -from ..scicrunch.scicrunch_service import ScicrunchResourcesService +from ..scicrunch.scicrunch_service import SCICRUNCH_SERVICE_APPKEY from ..security.decorators import permission_required from ..utils_aiohttp import envelope_json_response from ._classifiers_service import GroupClassifiersService @@ -50,7 +50,7 @@ async def get_group_classifiers(request: web.Request): async def get_scicrunch_resource(request: web.Request): rrid = request.match_info["rrid"] - service = ScicrunchResourcesService(request.app) + service = request.app[SCICRUNCH_SERVICE_APPKEY] resource = await service.get_or_fetch_research_resource(rrid) return envelope_json_response(resource.model_dump()) @@ -66,7 +66,7 @@ async def get_scicrunch_resource(request: web.Request): async def add_scicrunch_resource(request: web.Request): rrid = request.match_info["rrid"] - service = ScicrunchResourcesService(request.app) + service = request.app[SCICRUNCH_SERVICE_APPKEY] resource = await service.create_research_resource(rrid) return envelope_json_response(resource.model_dump()) @@ -82,7 +82,7 @@ async def add_scicrunch_resource(request: web.Request): async def search_scicrunch_resources(request: web.Request): guess_name = str(request.query["guess_name"]).strip() - service = ScicrunchResourcesService(request.app) + service = request.app[SCICRUNCH_SERVICE_APPKEY] hits: list[ResourceHit] = await service.search_research_resources(guess_name) return envelope_json_response([hit.model_dump() for hit in hits]) diff --git a/services/web/server/src/simcore_service_webserver/groups/plugin.py b/services/web/server/src/simcore_service_webserver/groups/plugin.py index 1ac79222337a..5654cb0c39a8 100644 --- a/services/web/server/src/simcore_service_webserver/groups/plugin.py +++ b/services/web/server/src/simcore_service_webserver/groups/plugin.py @@ -1,6 +1,7 @@ import logging from aiohttp import web +from simcore_service_webserver.scicrunch.plugin import setup_scicrunch from ..application_keys import APP_SETTINGS_APPKEY from ..application_setup import ModuleCategory, app_setup_func @@ -22,6 +23,7 @@ def setup_groups(app: web.Application): # plugin dependencies setup_products(app) + setup_scicrunch(app) app.router.add_routes(_groups_rest.routes) app.router.add_routes(_classifiers_rest.routes) diff --git a/services/web/server/src/simcore_service_webserver/scicrunch/_service.py b/services/web/server/src/simcore_service_webserver/scicrunch/_service.py index 3b44c01b3f20..f6880e841fbd 100644 --- a/services/web/server/src/simcore_service_webserver/scicrunch/_service.py +++ b/services/web/server/src/simcore_service_webserver/scicrunch/_service.py @@ -4,7 +4,6 @@ import logging -from aiohttp import web from common_library.logging.logging_errors import create_troubleshooting_log_kwargs from pydantic import HttpUrl, ValidationError @@ -21,11 +20,10 @@ class ScicrunchResourcesService: - Research Resources operations (RRID = Research Resource ID) """ - def __init__(self, app: web.Application): - self.app = app - self._repo = ScicrunchResourcesRepository.create_from_app(app) + def __init__(self, repo: ScicrunchResourcesRepository, client: SciCrunch): + self._repo = repo # client to interact with scicrunch.org service - self._client = SciCrunch.get_instance(self.app) + self._client = client async def list_research_resources(self) -> list[ResearchResource]: """List all research resources as domain models.""" diff --git a/services/web/server/src/simcore_service_webserver/scicrunch/plugin.py b/services/web/server/src/simcore_service_webserver/scicrunch/plugin.py index ef82e673c30d..01a96e50362f 100644 --- a/services/web/server/src/simcore_service_webserver/scicrunch/plugin.py +++ b/services/web/server/src/simcore_service_webserver/scicrunch/plugin.py @@ -5,8 +5,12 @@ import logging from aiohttp import web +from conftest import app +from simcore_service_webserver.scicrunch._repository import ScicrunchResourcesRepository from ..application_setup import ModuleCategory, app_setup_func +from ._service import ScicrunchResourcesService +from .scicrunch_service import SCICRUNCH_SERVICE_APPKEY from .service_client import SciCrunch from .settings import get_plugin_settings @@ -15,8 +19,16 @@ async def _on_startup(app: web.Application): settings = get_plugin_settings(app) - api = SciCrunch.acquire_instance(app, settings) - assert api == SciCrunch.get_instance(app) # nosec + + client = SciCrunch.acquire_instance(app, settings) + assert client == SciCrunch.get_instance(app) # nosec + + service = ScicrunchResourcesService( + repo=ScicrunchResourcesRepository.create_from_app(app), + client=SciCrunch.get_instance(app), + ) + + app[SCICRUNCH_SERVICE_APPKEY] = service @app_setup_func( diff --git a/services/web/server/src/simcore_service_webserver/scicrunch/scicrunch_service.py b/services/web/server/src/simcore_service_webserver/scicrunch/scicrunch_service.py index 10039e86aa73..c22f6fa16da5 100644 --- a/services/web/server/src/simcore_service_webserver/scicrunch/scicrunch_service.py +++ b/services/web/server/src/simcore_service_webserver/scicrunch/scicrunch_service.py @@ -1,4 +1,13 @@ +from typing import Final + +from aiohttp import web + from ._service import ScicrunchResourcesService -__all__: tuple[str, ...] = ("ScicrunchResourcesService",) +SCICRUNCH_SERVICE_APPKEY: Final = web.AppKey( + ScicrunchResourcesService.__name__, ScicrunchResourcesService +) + + +__all__: tuple[str, ...] = ("SCICRUNCH_SERVICE_APPKEY",) # nopycln: file diff --git a/services/web/server/src/simcore_service_webserver/scicrunch/service_client.py b/services/web/server/src/simcore_service_webserver/scicrunch/service_client.py index 3f9f4fe2653b..6354194a1f2a 100644 --- a/services/web/server/src/simcore_service_webserver/scicrunch/service_client.py +++ b/services/web/server/src/simcore_service_webserver/scicrunch/service_client.py @@ -61,15 +61,15 @@ def acquire_instance( cls, app: web.Application, settings: SciCrunchSettings ) -> "SciCrunch": """Returns single instance for the application and stores it""" - obj: SciCrunch | None = app.get(_SCICRUNCH_APPKEY) + obj: SciCrunch | None = app.get(SCICRUNCH_CLIENT_APPKEY) if obj is None: session = get_client_session(app) - app[_SCICRUNCH_APPKEY] = obj = cls(session, settings) + app[SCICRUNCH_CLIENT_APPKEY] = obj = cls(session, settings) return obj @classmethod def get_instance(cls, app: web.Application) -> "SciCrunch": - obj: SciCrunch | None = app.get(_SCICRUNCH_APPKEY) + obj: SciCrunch | None = app.get(SCICRUNCH_CLIENT_APPKEY) if obj is None: raise ScicrunchConfigError( details="Services on scicrunch.org are currently disabled" @@ -174,4 +174,4 @@ async def search_resource(self, name_as: str) -> list[ResourceHit]: return hits.root -_SCICRUNCH_APPKEY: Final = web.AppKey(SciCrunch.__name__, SciCrunch) +SCICRUNCH_CLIENT_APPKEY: Final = web.AppKey(SciCrunch.__name__, SciCrunch) diff --git a/services/web/server/tests/unit/isolated/scicrunch/test_scicrunch_service_client.py b/services/web/server/tests/unit/isolated/scicrunch/test_scicrunch_service_client.py index d32013c33885..26cccbb62f37 100644 --- a/services/web/server/tests/unit/isolated/scicrunch/test_scicrunch_service_client.py +++ b/services/web/server/tests/unit/isolated/scicrunch/test_scicrunch_service_client.py @@ -19,7 +19,9 @@ from servicelib.aiohttp.application import create_safe_application from servicelib.aiohttp.client_session import get_client_session from simcore_service_webserver.application_settings import setup_settings -from simcore_service_webserver.scicrunch.plugin import setup_scicrunch +from simcore_service_webserver.scicrunch.plugin import ( + setup_scicrunch, +) from simcore_service_webserver.scicrunch.service_client import ( ResearchResource, SciCrunch, From 9e90443562d73d80daac87225251d221f37e7669 Mon Sep 17 00:00:00 2001 From: Pedro Crespo-Valero <32402063+pcrespov@users.noreply.github.com> Date: Mon, 29 Sep 2025 14:26:29 +0200 Subject: [PATCH 17/20] refactor: replace ScicrunchResourcesService with SCICRUNCH_SERVICE_APPKEY for improved service integration --- .../exporter/_formatter/_sds.py | 4 ++-- .../exporter/plugin.py | 3 +++ .../groups/_classifiers_service.py | 4 ++-- .../{service_client.py => _client.py} | 0 .../scicrunch/_service.py | 2 +- .../scicrunch/plugin.py | 21 ++++++++++++------- .../test_scicrunch_service_client.py | 8 +++---- 7 files changed, 25 insertions(+), 17 deletions(-) rename services/web/server/src/simcore_service_webserver/scicrunch/{service_client.py => _client.py} (100%) diff --git a/services/web/server/src/simcore_service_webserver/exporter/_formatter/_sds.py b/services/web/server/src/simcore_service_webserver/exporter/_formatter/_sds.py index a0275acfdbc5..081ab82549b2 100644 --- a/services/web/server/src/simcore_service_webserver/exporter/_formatter/_sds.py +++ b/services/web/server/src/simcore_service_webserver/exporter/_formatter/_sds.py @@ -11,7 +11,7 @@ from ...projects._projects_service import get_project_for_user from ...projects.exceptions import BaseProjectError from ...projects.models import ProjectDict -from ...scicrunch.scicrunch_service import ScicrunchResourcesService +from ...scicrunch.scicrunch_service import SCICRUNCH_SERVICE_APPKEY from ..exceptions import SDSException from .template_json import write_template_json from .xlsx.code_description import ( @@ -70,7 +70,7 @@ async def _add_rrid_entries( ) -> None: rrid_entires: deque[RRIDEntry] = deque() - service = ScicrunchResourcesService(app) + service = app[SCICRUNCH_SERVICE_APPKEY] classifiers = project_data["classifiers"] for classifier in classifiers: scicrunch_resource = await service.get_resource_atdb(rrid=classifier) diff --git a/services/web/server/src/simcore_service_webserver/exporter/plugin.py b/services/web/server/src/simcore_service_webserver/exporter/plugin.py index 30f9ec6407f3..7f6ca183b6d1 100644 --- a/services/web/server/src/simcore_service_webserver/exporter/plugin.py +++ b/services/web/server/src/simcore_service_webserver/exporter/plugin.py @@ -1,6 +1,7 @@ import logging from aiohttp import web +from simcore_service_webserver.scicrunch.plugin import setup_scicrunch from ..application_setup import ModuleCategory, app_setup_func from . import _handlers @@ -19,4 +20,6 @@ def setup_exporter(app: web.Application) -> bool: # Rest-API routes: maps handlers with routes tags with "viewer" based on OAS operation_id app.router.add_routes(_handlers.routes) + setup_scicrunch(app) + return True diff --git a/services/web/server/src/simcore_service_webserver/groups/_classifiers_service.py b/services/web/server/src/simcore_service_webserver/groups/_classifiers_service.py index e892dc7ae473..2f2eca7ea3d6 100644 --- a/services/web/server/src/simcore_service_webserver/groups/_classifiers_service.py +++ b/services/web/server/src/simcore_service_webserver/groups/_classifiers_service.py @@ -24,7 +24,7 @@ ) from ..scicrunch.errors import ScicrunchError -from ..scicrunch.scicrunch_service import ScicrunchResourcesService +from ..scicrunch.scicrunch_service import SCICRUNCH_SERVICE_APPKEY from ._classifiers_repository import GroupClassifierRepository _logger = logging.getLogger(__name__) @@ -119,7 +119,7 @@ async def _build_rrids_tree_view( msg = "Currently only 'std' option for the classifiers tree view is implemented" raise NotImplementedError(msg) - service = ScicrunchResourcesService(self.app) + service = self.app[SCICRUNCH_SERVICE_APPKEY] flat_tree_view: dict[TreePath, ClassifierItem] = {} for resource in await service.list_research_resources(): diff --git a/services/web/server/src/simcore_service_webserver/scicrunch/service_client.py b/services/web/server/src/simcore_service_webserver/scicrunch/_client.py similarity index 100% rename from services/web/server/src/simcore_service_webserver/scicrunch/service_client.py rename to services/web/server/src/simcore_service_webserver/scicrunch/_client.py diff --git a/services/web/server/src/simcore_service_webserver/scicrunch/_service.py b/services/web/server/src/simcore_service_webserver/scicrunch/_service.py index f6880e841fbd..8f75767509b8 100644 --- a/services/web/server/src/simcore_service_webserver/scicrunch/_service.py +++ b/services/web/server/src/simcore_service_webserver/scicrunch/_service.py @@ -7,9 +7,9 @@ from common_library.logging.logging_errors import create_troubleshooting_log_kwargs from pydantic import HttpUrl, ValidationError +from ._client import SciCrunch from ._repository import ScicrunchResourcesRepository from .models import ResearchResource, ResearchResourceAtdB, ResourceHit -from .service_client import SciCrunch _logger = logging.getLogger(__name__) diff --git a/services/web/server/src/simcore_service_webserver/scicrunch/plugin.py b/services/web/server/src/simcore_service_webserver/scicrunch/plugin.py index 01a96e50362f..78bdf8f44252 100644 --- a/services/web/server/src/simcore_service_webserver/scicrunch/plugin.py +++ b/services/web/server/src/simcore_service_webserver/scicrunch/plugin.py @@ -1,17 +1,13 @@ -""" -Notice that this is used as a submodule of groups'a app module -""" - import logging from aiohttp import web -from conftest import app +from simcore_service_webserver.db.plugin import setup_db from simcore_service_webserver.scicrunch._repository import ScicrunchResourcesRepository from ..application_setup import ModuleCategory, app_setup_func +from ._client import SciCrunch from ._service import ScicrunchResourcesService from .scicrunch_service import SCICRUNCH_SERVICE_APPKEY -from .service_client import SciCrunch from .settings import get_plugin_settings _logger = logging.getLogger(__name__) @@ -20,14 +16,21 @@ async def _on_startup(app: web.Application): settings = get_plugin_settings(app) + # 1. scicrunch http client client = SciCrunch.acquire_instance(app, settings) assert client == SciCrunch.get_instance(app) # nosec + # 2. scicrunch repository (uses app[ENGINE_DB_CLIENT_APPKEY]) + repo = ScicrunchResourcesRepository.create_from_app(app) + assert repo # nosec + + # 3. scicrunch resources service service = ScicrunchResourcesService( - repo=ScicrunchResourcesRepository.create_from_app(app), - client=SciCrunch.get_instance(app), + repo=repo, + client=client, ) + # store service in app app[SCICRUNCH_SERVICE_APPKEY] = service @@ -40,4 +43,6 @@ async def _on_startup(app: web.Application): def setup_scicrunch(app: web.Application): assert get_plugin_settings(app) # nosec + setup_db(app) # needs engine + app.on_startup.append(_on_startup) diff --git a/services/web/server/tests/unit/isolated/scicrunch/test_scicrunch_service_client.py b/services/web/server/tests/unit/isolated/scicrunch/test_scicrunch_service_client.py index 26cccbb62f37..bc249ec89958 100644 --- a/services/web/server/tests/unit/isolated/scicrunch/test_scicrunch_service_client.py +++ b/services/web/server/tests/unit/isolated/scicrunch/test_scicrunch_service_client.py @@ -19,13 +19,13 @@ from servicelib.aiohttp.application import create_safe_application from servicelib.aiohttp.client_session import get_client_session from simcore_service_webserver.application_settings import setup_settings -from simcore_service_webserver.scicrunch.plugin import ( - setup_scicrunch, -) -from simcore_service_webserver.scicrunch.service_client import ( +from simcore_service_webserver.scicrunch._client import ( ResearchResource, SciCrunch, ) +from simcore_service_webserver.scicrunch.plugin import ( + setup_scicrunch, +) @pytest.fixture From 4ef00006c303993192f94f513a011d2b59917f2c Mon Sep 17 00:00:00 2001 From: Pedro Crespo-Valero <32402063+pcrespov@users.noreply.github.com> Date: Mon, 29 Sep 2025 14:27:17 +0200 Subject: [PATCH 18/20] @GitHK review: fix #5160 --- .../helpers/scrunch_citations.py | 14 ++++++----- .../01/scicrunch/test_scicrunch__resolver.py | 24 ++++++++++--------- .../test_scicrunch_service_client.py | 17 ++++++++++++- 3 files changed, 37 insertions(+), 18 deletions(-) diff --git a/packages/pytest-simcore/src/pytest_simcore/helpers/scrunch_citations.py b/packages/pytest-simcore/src/pytest_simcore/helpers/scrunch_citations.py index ea6e0287ccce..80edcce57c40 100644 --- a/packages/pytest-simcore/src/pytest_simcore/helpers/scrunch_citations.py +++ b/packages/pytest-simcore/src/pytest_simcore/helpers/scrunch_citations.py @@ -1,10 +1,10 @@ # Citations according to https://scicrunch.org/resources """ - NOTES: +NOTES: - - scicrunch API ONLY recognizes RRIDs from SciCrunch registry of tools (i.e. with prefix "SCR") - - scicrunch web search handles ALL RRIDs (see below example of citations from other) - - scicrunch API does NOT uses 'RRID:' prefix in rrid request parameters +- scicrunch API ONLY recognizes RRIDs from SciCrunch registry of tools (i.e. with prefix "SCR") +- scicrunch web search handles ALL RRIDs (see below example of citations from other) +- scicrunch API does NOT uses 'RRID:' prefix in rrid request parameters """ @@ -15,14 +15,16 @@ def split_citations(citations: list[str]) -> list[tuple[str, str]]: def _split(citation: str) -> tuple[str, str]: if "," not in citation: citation = citation.replace("(", "(,") - name, rrid = re.match(r"^\((.*),\s*RRID:(.+)\)$", citation).groups() + name, rrid = re.match( + r"^[\(]{0,1}(.*),\s*RRID:(.+)[\)]{0,1}$", citation + ).groups() return name, rrid return list(map(_split, citations)) # http://antibodyregistry.org/AB_90755 -ANTIBODY_CITATIONS = split_citations(["(Millipore Cat# AB1542, RRID:AB_90755)"]) +ANTIBODY_CITATIONS = split_citations(["Millipore Cat# AB1542, RRID:AB_90755)"]) # https://www.addgene.org/44362/ PLAMID_CITATIONS = split_citations(["(RRID:Addgene_44362)"]) diff --git a/services/web/server/tests/integration/01/scicrunch/test_scicrunch__resolver.py b/services/web/server/tests/integration/01/scicrunch/test_scicrunch__resolver.py index 99683e08cff8..97680acfc454 100644 --- a/services/web/server/tests/integration/01/scicrunch/test_scicrunch__resolver.py +++ b/services/web/server/tests/integration/01/scicrunch/test_scicrunch__resolver.py @@ -16,9 +16,6 @@ from simcore_service_webserver.scicrunch.settings import SciCrunchSettings -@pytest.mark.skip( - "requires a fix see case https://github.com/ITISFoundation/osparc-simcore/issues/5160" -) @pytest.mark.parametrize( "name,rrid", TOOL_CITATIONS + ANTIBODY_CITATIONS + PLAMID_CITATIONS + ORGANISM_CITATIONS, @@ -56,6 +53,7 @@ async def test_scicrunch_resolves_all_valid_rrids( ) else: # proper_citation includes both 'name' and 'rrid' but in different formats! + # AND has been changing in time. BELOW are some of the formats found! # # NOTE: why CELL_LINE_CITATIONS are removed from test parametrization ? @@ -64,13 +62,17 @@ async def test_scicrunch_resolves_all_valid_rrids( # sometimes (BCRJ Cat# 0226, RRID:CVCL_0033) appears as first hit instead # of the reference in CELL_LINE_CITATIONS # + valid_formats = ( + f"({name}, RRID:{rrid})", + f"{name}, RRID:{rrid}", # without parentheses + f"({name},RRID:{rrid})", + f"{name} (RRID:{rrid})", + ) + resolved_citations = [ + resolved.proper_citation for resolved in resolved_items + ] assert any( - resolved.proper_citation - in ( - f"({name}, RRID:{rrid})", - f"({name},RRID:{rrid})", - f"{name} (RRID:{rrid})", - ) - for resolved in resolved_items - ) + proper_citation in valid_formats + for proper_citation in resolved_citations + ), f"No proper_citation found with both name and rrid: {resolved_citations=} not in {valid_formats=}" diff --git a/services/web/server/tests/unit/isolated/scicrunch/test_scicrunch_service_client.py b/services/web/server/tests/unit/isolated/scicrunch/test_scicrunch_service_client.py index bc249ec89958..6d6884f7ff35 100644 --- a/services/web/server/tests/unit/isolated/scicrunch/test_scicrunch_service_client.py +++ b/services/web/server/tests/unit/isolated/scicrunch/test_scicrunch_service_client.py @@ -14,7 +14,8 @@ import pytest from aiohttp import web -from aioresponses import aioresponses as AioResponsesMock # noqa: N812 +from aioresponses import aioresponses as AioResponsesMock +from pytest_mock import MockerFixture # noqa: N812 from pytest_simcore.helpers.typing_env import EnvVarsDict from servicelib.aiohttp.application import create_safe_application from servicelib.aiohttp.client_session import get_client_session @@ -105,14 +106,28 @@ async def mock_scicrunch_service_resolver( async def app( mock_env_devel_environment: EnvVarsDict, aiohttp_server: Callable, + mocker: MockerFixture, ) -> web.Application: app_ = create_safe_application() + # mock access to db in this test-suite + mock_setup_db = mocker.patch( + "simcore_service_webserver.scicrunch.plugin.setup_db", + return_value=True, + ) + + get_async_engine = mocker.patch( + "simcore_service_webserver.db.base_repository._asyncpg.get_async_engine", + ) + setup_settings(app_) setup_scicrunch(app_) + assert mock_setup_db.called + assert not get_async_engine.called server = await aiohttp_server(app_) assert server.app == app_ + assert get_async_engine.called return server.app From 6982f4257a6334a13d36ff91a56b3a61016df2d4 Mon Sep 17 00:00:00 2001 From: Pedro Crespo-Valero <32402063+pcrespov@users.noreply.github.com> Date: Mon, 29 Sep 2025 21:48:29 +0200 Subject: [PATCH 19/20] fixes fomratting --- .../src/pytest_simcore/helpers/scrunch_citations.py | 10 ++++++---- .../01/scicrunch/test_scicrunch__resolver.py | 2 +- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/packages/pytest-simcore/src/pytest_simcore/helpers/scrunch_citations.py b/packages/pytest-simcore/src/pytest_simcore/helpers/scrunch_citations.py index 80edcce57c40..1b2ff9e55c08 100644 --- a/packages/pytest-simcore/src/pytest_simcore/helpers/scrunch_citations.py +++ b/packages/pytest-simcore/src/pytest_simcore/helpers/scrunch_citations.py @@ -13,18 +13,20 @@ def split_citations(citations: list[str]) -> list[tuple[str, str]]: def _split(citation: str) -> tuple[str, str]: + assert citation.startswith("("), f"Got {citation=}" + assert citation.endswith(")"), f"Got {citation=}" + # make sure there is a comma before RRID: so we can split if "," not in citation: citation = citation.replace("(", "(,") - name, rrid = re.match( - r"^[\(]{0,1}(.*),\s*RRID:(.+)[\)]{0,1}$", citation - ).groups() + + name, rrid = re.match(r"^\((.*),\s*RRID:(.+)\)$", citation).groups() return name, rrid return list(map(_split, citations)) # http://antibodyregistry.org/AB_90755 -ANTIBODY_CITATIONS = split_citations(["Millipore Cat# AB1542, RRID:AB_90755)"]) +ANTIBODY_CITATIONS = split_citations(["(Millipore Cat# AB1542, RRID:AB_90755)"]) # https://www.addgene.org/44362/ PLAMID_CITATIONS = split_citations(["(RRID:Addgene_44362)"]) diff --git a/services/web/server/tests/integration/01/scicrunch/test_scicrunch__resolver.py b/services/web/server/tests/integration/01/scicrunch/test_scicrunch__resolver.py index 97680acfc454..47845898e2d4 100644 --- a/services/web/server/tests/integration/01/scicrunch/test_scicrunch__resolver.py +++ b/services/web/server/tests/integration/01/scicrunch/test_scicrunch__resolver.py @@ -75,4 +75,4 @@ async def test_scicrunch_resolves_all_valid_rrids( assert any( proper_citation in valid_formats for proper_citation in resolved_citations - ), f"No proper_citation found with both name and rrid: {resolved_citations=} not in {valid_formats=}" + ), f"No proper_citation found with both {name=} and {rrid=}: {resolved_citations=} not in {valid_formats=}" From af4d430efd0462f95a374733211b20288eb832d9 Mon Sep 17 00:00:00 2001 From: Pedro Crespo-Valero <32402063+pcrespov@users.noreply.github.com> Date: Mon, 29 Sep 2025 22:07:50 +0200 Subject: [PATCH 20/20] fixes tests --- .../exporter/plugin.py | 3 +-- .../scicrunch/settings.py | 18 ++++++++---------- .../01/test_exporter_requests_handlers.py | 13 ++++++++++++- 3 files changed, 21 insertions(+), 13 deletions(-) diff --git a/services/web/server/src/simcore_service_webserver/exporter/plugin.py b/services/web/server/src/simcore_service_webserver/exporter/plugin.py index 7f6ca183b6d1..99977626f00f 100644 --- a/services/web/server/src/simcore_service_webserver/exporter/plugin.py +++ b/services/web/server/src/simcore_service_webserver/exporter/plugin.py @@ -16,10 +16,9 @@ logger=_logger, ) def setup_exporter(app: web.Application) -> bool: + setup_scicrunch(app) # Rest-API routes: maps handlers with routes tags with "viewer" based on OAS operation_id app.router.add_routes(_handlers.routes) - setup_scicrunch(app) - return True diff --git a/services/web/server/src/simcore_service_webserver/scicrunch/settings.py b/services/web/server/src/simcore_service_webserver/scicrunch/settings.py index 4a88fd3d1dc6..6b04f70be780 100644 --- a/services/web/server/src/simcore_service_webserver/scicrunch/settings.py +++ b/services/web/server/src/simcore_service_webserver/scicrunch/settings.py @@ -1,3 +1,5 @@ +from typing import Annotated + from aiohttp import web from pydantic import Field, HttpUrl, SecretStr, TypeAdapter from settings_library.base import BaseCustomSettings @@ -10,21 +12,17 @@ class SciCrunchSettings(BaseCustomSettings): - SCICRUNCH_API_BASE_URL: HttpUrl = Field( - default=TypeAdapter(HttpUrl).validate_python(f"{SCICRUNCH_DEFAULT_URL}/api/1"), - description="Base url to scicrunch API's entrypoint", - ) + SCICRUNCH_API_BASE_URL: Annotated[ + HttpUrl, Field(description="Base url to scicrunch API's entrypoint") + ] = TypeAdapter(HttpUrl).validate_python(f"{SCICRUNCH_DEFAULT_URL}/api/1") # NOTE: Login in https://scicrunch.org and get API Key under My Account -> API Keys # WARNING: this needs to be setup in osparc-ops before deploying SCICRUNCH_API_KEY: SecretStr - SCICRUNCH_RESOLVER_BASE_URL: HttpUrl = Field( - default=TypeAdapter(HttpUrl).validate_python( - f"{SCICRUNCH_DEFAULT_URL}/resolver" - ), - description="Base url to scicrunch resolver entrypoint", - ) + SCICRUNCH_RESOLVER_BASE_URL: Annotated[ + HttpUrl, Field(description="Base url to scicrunch resolver entrypoint") + ] = TypeAdapter(HttpUrl).validate_python(f"{SCICRUNCH_DEFAULT_URL}/resolver") def get_plugin_settings(app: web.Application) -> SciCrunchSettings: diff --git a/services/web/server/tests/integration/01/test_exporter_requests_handlers.py b/services/web/server/tests/integration/01/test_exporter_requests_handlers.py index b471f6883c85..161b81ccebfd 100644 --- a/services/web/server/tests/integration/01/test_exporter_requests_handlers.py +++ b/services/web/server/tests/integration/01/test_exporter_requests_handlers.py @@ -14,6 +14,7 @@ import pytest import redis.asyncio as aioredis from aiohttp.test_utils import TestClient +from pytest_simcore.helpers.monkeypatch_envs import setenvs_from_dict from pytest_simcore.helpers.webserver_login import LoggedUser, UserInfoDict from pytest_simcore.helpers.webserver_projects import ( create_project, @@ -152,6 +153,7 @@ async def client( redis_client: aioredis.Redis, rabbit_service: RabbitSettings, simcore_services_ready: None, + monkeypatch: pytest.MonkeyPatch, ) -> Iterable[TestClient]: # test config & env vars ---------------------- cfg = deepcopy(app_config) @@ -163,11 +165,20 @@ async def client( monkeypatch_setenv_from_app_config(cfg) + monkeypatch.delenv("WEBSERVER_SCICRUNCH", raising=False) + setenvs_from_dict( + monkeypatch, {"SCICRUNCH_API_KEY": "REPLACE_ME_with_valid_api_key"} + ) + # app setup ---------------------------------- app = create_safe_application(cfg) # activates only security+restAPI sub-modules - assert setup_settings(app) + app_settings = setup_settings(app) + + assert app_settings.WEBSERVER_SCICRUNCH is not None + assert app_settings.WEBSERVER_RABBITMQ is not None + assert app_settings.WEBSERVER_EXPORTER is not None assert ( exporter_settings.get_plugin_settings(app) is not None ), "Should capture defaults"