fixes scicrunch API due to change in OAS (ITISFoundation#3668)

Author: pcrespov

services/web/server/src/simcore_service_webserver/scicrunch/_resolver.py

@@ -6,10 +6,10 @@
 
 import logging
 from datetime import datetime
-from typing import Any, Dict, List, Optional
+from typing import Any
 
 from aiohttp import ClientSession
-from pydantic import Field
+from pydantic import Field, ValidationError
 from pydantic.main import BaseModel
 from pydantic.types import NonNegativeInt
 
@@ -39,7 +39,7 @@ class HitSource(BaseModel):
     item: ItemInfo
     rrid: RRIDInfo
 
-    def flatten_dict(self) -> Dict[str, Any]:
+    def flatten_dict(self) -> dict[str, Any]:
         """Used as an output"""
         return {**self.item.dict(), **self.rrid.dict()}
 
@@ -50,7 +50,7 @@ class HitDetail(BaseModel):
 
 class Hits(BaseModel):
     total: NonNegativeInt
-    hits: List[HitDetail]
+    hits: list[HitDetail]
 
 
 class ResolverInfo(BaseModel):
@@ -77,10 +77,12 @@ class ResolvedItem(BaseModel):
 
 
 async def resolve_rrid(
-    identifier: str, client: ClientSession, settings: SciCrunchSettings
-) -> Optional[ResolvedItem]:
+    identifier: str,
+    client: ClientSession,
+    settings: SciCrunchSettings,
+) -> list[ResolvedItem]:
     """
-    Provides a API to access to results as provided by this web https://scicrunch.org/resolver
+    API to access to results of https://scicrunch.org/resolver
 
     """
     # Example https://scicrunch.org/resolver/RRID:AB_90755.json
@@ -95,25 +97,24 @@ async def resolve_rrid(
     if resolved.hits.total == 0:
         return None
 
-    #  FIXME: Not sure why the same RRID can have multiple hits.
+    #  WARNING: Not sure why the same RRID can have multiple hits.
     #  We have experience that the order of hits is not preserve and
     #  therefore selecting the first hit is not the right way to go ...
     #
-    #  WARNING: Since Sep.2021, hits returned by resolver does not guarantee order.
+    #  WARNING: scicrunch API has been changing:
+    #  - Since Sep.2021, hits returned by resolver DOES NOT guarantee order.
     #  For instance, https://scicrunch.org/resolver/RRID:CVCL_0033.json changes
     #  the order every call and the first hit flips between
     #  '(BCRJ Cat# 0226, RRID:CVCL_0033)' and '(ATCC Cat# HTB-30, RRID:CVCL_0033)'
+    #  - Since Dec.2022 hits returned by https://scicrunch.org/resolver/RRID:AB_90755.json
+    #  also changed order '(Sigma-Aldrich Cat# AB1542, RRID:AB_90755)' and
+    #  '(Millipore Cat# AB1542, RRID:AB_90755)'. Decided to return all hits.
     #
-    hit = resolved.hits.hits[0].source
-
-    if resolved.hits.total > 1:
-        logger.warning(
-            "Multiple hits (%d) for '%s'. Returning first",
-            resolved.hits.total,
-            identifier,
-        )
-    else:
-        assert resolved.hits.total == 1  # nosec
-
-    output = ResolvedItem.parse_obj(hit.flatten_dict())
-    return output
+    items = []
+    for hit in resolved.hits.hits:
+        try:
+            items.append(ResolvedItem.parse_obj(hit.source.flatten_dict()))
+        except ValidationError as err:
+            logger.warning("Skipping unexpected response %s: %s", url, err)
+
+    return items

services/web/server/src/simcore_service_webserver/scicrunch/service_client.py

@@ -6,7 +6,7 @@
 
 import asyncio
 import logging
-from typing import Any, List, MutableMapping, Optional
+from typing import Any, MutableMapping
 
 from aiohttp import ClientSession, client_exceptions
 from pydantic import ValidationError
@@ -125,12 +125,17 @@ async def get_resource_fields(self, rrid: str) -> ResearchResource:
             # NOTE: replaces former call to API.
             # Resolver entrypoint does NOT require authentication
             # and has an associated website
-            resolved: Optional[ResolvedItem] = await resolve_rrid(
+            resolved_items: list[ResolvedItem] = await resolve_rrid(
                 rrid, self.client, self.settings
             )
-            if not resolved:
+            if not resolved_items:
                 raise InvalidRRID(f".Could not resolve {rrid}")
 
+            # WARNING: currently we only take the first, but it might
+            # have multiple hits. Nonetheless, test_scicrunch_resolves_all_valid_rrids
+            # checks them all
+            resolved = resolved_items[0]
+
             return ResearchResource(
                 rrid=rrid,
                 name=resolved.name,
@@ -154,7 +159,7 @@ async def get_resource_fields(self, rrid: str) -> ResearchResource:
             # https://docs.aiohttp.org/en/stable/client_reference.html#hierarchy-of-exceptions
             raise ScicrunchServiceError("Failed to connect scicrunch service") from err
 
-    async def search_resource(self, name_as: str) -> List[ResourceHit]:
+    async def search_resource(self, name_as: str) -> list[ResourceHit]:
         # Safe: returns empty string if fails!
         # Might be slow and timeout!
         # Might be good to know that scicrunch.org is not reachable and cannot perform search now?

services/web/server/tests/integration/02/scicrunch/test_scicrunch__resolver.py

@@ -6,9 +6,7 @@
 import pytest
 from aiohttp import ClientSession
 from aiohttp.client import ClientTimeout
-
-# FIXME: PC check the CELL_LINE_CITATIONS test please
-from pytest_simcore.helpers.utils_scrunch_citations import (  # CELL_LINE_CITATIONS,
+from pytest_simcore.helpers.utils_scrunch_citations import (
     ANTIBODY_CITATIONS,
     ORGANISM_CITATIONS,
     PLAMID_CITATIONS,
@@ -20,8 +18,7 @@
 
 @pytest.mark.parametrize(
     "name,rrid",
-    TOOL_CITATIONS + ANTIBODY_CITATIONS + PLAMID_CITATIONS + ORGANISM_CITATIONS
-    # + CELL_LINE_CITATIONS, PC: this one fails
+    TOOL_CITATIONS + ANTIBODY_CITATIONS + PLAMID_CITATIONS + ORGANISM_CITATIONS,
 )
 async def test_scicrunch_resolves_all_valid_rrids(
     name: str, rrid: str, settings: SciCrunchSettings
@@ -32,21 +29,27 @@ async def test_scicrunch_resolves_all_valid_rrids(
     # This tests checks some of the structure "deduced" from the responses so far.
 
     async with ClientSession(timeout=ClientTimeout(total=30)) as client:
-        resolved = await resolve_rrid(identifier=rrid, client=client, settings=settings)
+        resolved_items: list[ResolvedItem] = await resolve_rrid(
+            identifier=rrid, client=client, settings=settings
+        )
 
-        assert resolved
-        assert isinstance(resolved, ResolvedItem)
+        for resolved in resolved_items:
+            assert resolved
+            assert isinstance(resolved, ResolvedItem)
 
-        if resolved.is_unique and name:
-            assert name in resolved.proper_citation
+            if resolved.is_unique and name:
+                assert name in resolved.proper_citation
 
-        assert rrid in resolved.proper_citation
+            assert rrid in resolved.proper_citation
 
         # NOTE: proper_citation does not seem to have a standard format.
         # So far I found four different formats!! :-o
         if not name:
             # only rrid with a prefix
-            assert resolved.proper_citation == f"RRID:{rrid}"
+            assert any(
+                resolved.proper_citation == f"RRID:{rrid}"
+                for resolved in resolved_items
+            )
         else:
             # proper_citation includes both 'name' and 'rrid' but in different formats!
 
@@ -58,8 +61,12 @@ async def test_scicrunch_resolves_all_valid_rrids(
             #   of the reference in CELL_LINE_CITATIONS
             #
 
-            assert resolved.proper_citation in (
-                f"({name}, RRID:{rrid})",
-                f"({name},RRID:{rrid})",
-                f"{name} (RRID:{rrid})",
+            assert any(
+                resolved.proper_citation
+                in (
+                    f"({name}, RRID:{rrid})",
+                    f"({name},RRID:{rrid})",
+                    f"{name} (RRID:{rrid})",
+                )
+                for resolved in resolved_items
             )