♻️ Refactor groups/classifiers and scicrunch to use asyncpg with service/repository separation

packages/pytest-simcore/src/pytest_simcore/helpers/faker_factories.py

@@ -629,3 +629,65 @@ def random_service_consume_filetype(
 
     data.update(overrides)
     return data
+
+
+def random_group_classifier(
+    *,
+    gid: int,
+    fake: Faker = DEFAULT_FAKER,
+    **overrides,
+) -> dict[str, Any]:
+    from simcore_postgres_database.models.classifiers import group_classifiers
+
+    data = {
+        "gid": gid,
+        "bundle": {
+            "vcs_ref": fake.lexify(text="???????"),
+            "vcs_url": "https://organization.classifiers.git",
+            "build_date": "2021-01-20T15:19:30Z",
+            "classifiers": {
+                "project::dak": {
+                    "url": None,
+                    "logo": None,
+                    "aliases": [],
+                    "related": [],
+                    "markdown": "",
+                    "released": None,
+                    "classifier": "project::dak",
+                    "created_by": fake.user_name(),
+                    "github_url": None,
+                    "display_name": "DAK",
+                    "wikipedia_url": None,
+                    "short_description": None,
+                },
+                "organization::zmt": {
+                    "url": "https://zmt.swiss/",
+                    "logo": None,
+                    "aliases": ["Zurich MedTech AG"],
+                    "related": [],
+                    "markdown": fake.text(),
+                    "released": None,
+                    "classifier": "organization::zmt",
+                    "created_by": fake.user_name(),
+                    "github_url": None,
+                    "display_name": "ZMT",
+                    "wikipedia_url": None,
+                    "short_description": "ZMT is a member of Zurich43",
+                },
+            },
+            "collections": {
+                "jupyterlab-math": {
+                    "items": ["crespo/osparc-demo"],
+                    "markdown": fake.text(),
+                    "created_by": fake.user_name(),
+                    "display_name": "jupyterlab-math",
+                }
+            },
+        },
+        "uses_scicrunch": False,
+    }
+
+    assert set(data.keys()).issubset({c.name for c in group_classifiers.columns})
+
+    data.update(overrides)
+    return data

packages/pytest-simcore/src/pytest_simcore/helpers/scrunch_citations.py

@@ -1,10 +1,10 @@
 # Citations according to https://scicrunch.org/resources
 """
-    NOTES:
+NOTES:
 
-    - scicrunch API ONLY recognizes RRIDs from SciCrunch registry of tools (i.e. with prefix "SCR")
-    - scicrunch web search handles ALL RRIDs (see below example of citations from other)
-    - scicrunch API does NOT uses 'RRID:' prefix in rrid request parameters
+- scicrunch API ONLY recognizes RRIDs from SciCrunch registry of tools (i.e. with prefix "SCR")
+- scicrunch web search handles ALL RRIDs (see below example of citations from other)
+- scicrunch API does NOT uses 'RRID:' prefix in rrid request parameters
 
 """
 
@@ -13,8 +13,12 @@
 
 def split_citations(citations: list[str]) -> list[tuple[str, str]]:
     def _split(citation: str) -> tuple[str, str]:
+        assert citation.startswith("("), f"Got {citation=}"
+        assert citation.endswith(")"), f"Got {citation=}"
+        # make sure there is a comma before RRID: so we can split
         if "," not in citation:
             citation = citation.replace("(", "(,")
+
         name, rrid = re.match(r"^\((.*),\s*RRID:(.+)\)$", citation).groups()
         return name, rrid
 

services/web/server/src/simcore_service_webserver/exporter/_formatter/_sds.py

@@ -11,7 +11,7 @@
 from ...projects._projects_service import get_project_for_user
 from ...projects.exceptions import BaseProjectError
 from ...projects.models import ProjectDict
-from ...scicrunch.db import ResearchResourceRepository
+from ...scicrunch.scicrunch_service import SCICRUNCH_SERVICE_APPKEY
 from ..exceptions import SDSException
 from .template_json import write_template_json
 from .xlsx.code_description import (
@@ -70,10 +70,10 @@ async def _add_rrid_entries(
 ) -> None:
     rrid_entires: deque[RRIDEntry] = deque()
 
-    repo = ResearchResourceRepository(app)
+    service = app[SCICRUNCH_SERVICE_APPKEY]
     classifiers = project_data["classifiers"]
     for classifier in classifiers:
-        scicrunch_resource = await repo.get(rrid=classifier)
+        scicrunch_resource = await service.get_resource_atdb(rrid=classifier)
         if scicrunch_resource is None:
             continue
 

services/web/server/src/simcore_service_webserver/exporter/plugin.py

@@ -1,6 +1,7 @@
 import logging
 
 from aiohttp import web
+from simcore_service_webserver.scicrunch.plugin import setup_scicrunch
 
 from ..application_setup import ModuleCategory, app_setup_func
 from . import _handlers
@@ -15,6 +16,7 @@
     logger=_logger,
 )
 def setup_exporter(app: web.Application) -> bool:
+    setup_scicrunch(app)
 
     # Rest-API routes: maps handlers with routes tags with "viewer" based on OAS operation_id
     app.router.add_routes(_handlers.routes)

services/web/server/src/simcore_service_webserver/groups/_classifiers_repository.py

@@ -0,0 +1,44 @@
+import logging
+from typing import Any, cast
+
+import sqlalchemy as sa
+from simcore_postgres_database.models.classifiers import group_classifiers
+from simcore_postgres_database.utils_repos import pass_or_acquire_connection
+from sqlalchemy.engine import Row
+from sqlalchemy.ext.asyncio import AsyncConnection
+
+from ..db.base_repository import BaseRepository
+
+_logger = logging.getLogger(__name__)
+
+
+class GroupClassifierRepository(BaseRepository):
+
+    async def _get_bundle(
+        self, gid: int, connection: AsyncConnection | None = None
+    ) -> Row | None:
+        async with pass_or_acquire_connection(self.engine, connection) as conn:
+            result = await conn.execute(
+                sa.select(group_classifiers.c.bundle).where(
+                    group_classifiers.c.gid == gid
+                )
+            )
+            return result.one_or_none()
+
+    async def get_classifiers_from_bundle(self, gid: int) -> dict[str, Any] | None:
+        bundle_row = await self._get_bundle(gid)
+        if bundle_row:
+            return cast(dict[str, Any], bundle_row.bundle)
+        return None
+
+    async def group_uses_scicrunch(
+        self, gid: int, connection: AsyncConnection | None = None
+    ) -> bool:
+        async with pass_or_acquire_connection(self.engine, connection) as conn:
+            result = await conn.execute(
+                sa.select(group_classifiers.c.uses_scicrunch).where(
+                    group_classifiers.c.gid == gid
+                )
+            )
+            row = result.one_or_none()
+            return bool(row.uses_scicrunch if row else False)

services/web/server/src/simcore_service_webserver/groups/_classifiers_rest.py

@@ -8,13 +8,11 @@
 
 from .._meta import API_VTAG
 from ..login.decorators import login_required
-from ..scicrunch.db import ResearchResourceRepository
-from ..scicrunch.errors import ScicrunchError
-from ..scicrunch.models import ResearchResource, ResourceHit
-from ..scicrunch.service_client import SciCrunch
+from ..scicrunch.models import ResourceHit
+from ..scicrunch.scicrunch_service import SCICRUNCH_SERVICE_APPKEY
 from ..security.decorators import permission_required
 from ..utils_aiohttp import envelope_json_response
-from ._classifiers_service import GroupClassifierRepository, build_rrids_tree_view
+from ._classifiers_service import GroupClassifiersService
 from ._common.exceptions_handlers import handle_plugin_requests_exceptions
 from ._common.schemas import GroupsClassifiersQuery, GroupsPathParams
 
@@ -29,23 +27,15 @@
 @permission_required("groups.*")
 @handle_plugin_requests_exceptions
 async def get_group_classifiers(request: web.Request):
-    try:
-        path_params = parse_request_path_parameters_as(GroupsPathParams, request)
-        query_params: GroupsClassifiersQuery = parse_request_query_parameters_as(
-            GroupsClassifiersQuery, request
-        )
-
-        repo = GroupClassifierRepository(request.app)
-        if not await repo.group_uses_scicrunch(path_params.gid):
-            bundle = await repo.get_classifiers_from_bundle(path_params.gid)
-            return envelope_json_response(bundle)
-
-        # otherwise, build dynamic tree with RRIDs
-        view = await build_rrids_tree_view(
-            request.app, tree_view_mode=query_params.tree_view
-        )
-    except ScicrunchError:
-        view = {}
+    path_params = parse_request_path_parameters_as(GroupsPathParams, request)
+    query_params: GroupsClassifiersQuery = parse_request_query_parameters_as(
+        GroupsClassifiersQuery, request
+    )
+
+    service = GroupClassifiersService(request.app)
+    view = await service.get_group_classifiers(
+        path_params.gid, tree_view_mode=query_params.tree_view
+    )
 
     return envelope_json_response(view)
 
@@ -59,15 +49,9 @@ async def get_group_classifiers(request: web.Request):
 @handle_plugin_requests_exceptions
 async def get_scicrunch_resource(request: web.Request):
     rrid = request.match_info["rrid"]
-    rrid = SciCrunch.validate_identifier(rrid)
 
-    # check if in database first
-    repo = ResearchResourceRepository(request.app)
-    resource: ResearchResource | None = await repo.get_resource(rrid)
-    if not resource:
-        # otherwise, request to scicrunch service
-        scicrunch = SciCrunch.get_instance(request.app)
-        resource = await scicrunch.get_resource_fields(rrid)
+    service = request.app[SCICRUNCH_SERVICE_APPKEY]
+    resource = await service.get_or_fetch_research_resource(rrid)
 
     return envelope_json_response(resource.model_dump())
 
@@ -82,16 +66,8 @@ async def get_scicrunch_resource(request: web.Request):
 async def add_scicrunch_resource(request: web.Request):
     rrid = request.match_info["rrid"]
 
-    # check if exists
-    repo = ResearchResourceRepository(request.app)
-    resource: ResearchResource | None = await repo.get_resource(rrid)
-    if not resource:
-        # then request scicrunch service
-        scicrunch = SciCrunch.get_instance(request.app)
-        resource = await scicrunch.get_resource_fields(rrid)
-
-        # insert new or if exists, then update
-        await repo.upsert(resource)
+    service = request.app[SCICRUNCH_SERVICE_APPKEY]
+    resource = await service.create_research_resource(rrid)
 
     return envelope_json_response(resource.model_dump())
 
@@ -106,7 +82,7 @@ async def add_scicrunch_resource(request: web.Request):
 async def search_scicrunch_resources(request: web.Request):
     guess_name = str(request.query["guess_name"]).strip()
 
-    scicrunch = SciCrunch.get_instance(request.app)
-    hits: list[ResourceHit] = await scicrunch.search_resource(guess_name)
+    service = request.app[SCICRUNCH_SERVICE_APPKEY]
+    hits: list[ResourceHit] = await service.search_research_resources(guess_name)
 
     return envelope_json_response([hit.model_dump() for hit in hits])