Rework ClinGen LDH Access Endpoint Environment Variable

Author: bencap

settings/.env.template

@@ -60,3 +60,12 @@ HGVS_SEQREPO_DIR=/usr/local/share/seqrepo/2024-12-20
 MAVEDB_BASE_URL=http://app:8000
 MAVEDB_API_KEY=secret
 DCD_MAPPING_URL=http://dcd-mapping:8000
+
+####################################################################################################
+# Environment variables for ClinGen
+####################################################################################################
+
+GENBOREE_ACCOUNT_NAME=
+GENBOREE_ACCOUNT_PASSWORD=
+CLIN_GEN_TENANT=
+CLIN_GEN_SUBMISSION_ENABLED=

src/mavedb/lib/clingen/constants.py

@@ -1,19 +1,19 @@
 import os
 
-CLIN_GEN_SUBMISSION_ENABLED = os.getenv("SUBMISSION_ENABLED", "false").lower() == "true"
+CLIN_GEN_SUBMISSION_ENABLED = os.getenv("CLIN_GEN_SUBMISSION_ENABLED", "false").lower() == "true"
 
 GENBOREE_ACCOUNT_NAME = os.getenv("GENBOREE_ACCOUNT_NAME")
 GENBOREE_ACCOUNT_PASSWORD = os.getenv("GENBOREE_ACCOUNT_PASSWORD")
 
 CLIN_GEN_TENANT = os.getenv("CLIN_GEN_TENANT")
-LDH_TENANT = os.getenv("LDH_TENANT")
 
 LDH_SUBMISSION_TYPE = "cg-ldh-ld-submission"
 LDH_ENTITY_NAME = "MaveDBMapping"
 LDH_ENTITY_ENDPOINT = "maveDb"  # for some reason, not the same :/
 
 DEFAULT_LDH_SUBMISSION_BATCH_SIZE = 100
-LDH_SUBMISSION_URL = f"https://genboree.org/mq/brdg/pulsar/{CLIN_GEN_TENANT}/ldh/submissions/{LDH_ENTITY_ENDPOINT}"
-LDH_LINKED_DATA_URL = f"https://genboree.org/{LDH_TENANT}/{LDH_ENTITY_NAME}/id"
+LDH_SUBMISSION_ENDPOINT = f"https://genboree.org/mq/brdg/pulsar/{CLIN_GEN_TENANT}/ldh/submissions/{LDH_ENTITY_ENDPOINT}"
+LDH_ACCESS_ENDPOINT = os.getenv("LDH_ACCESS_ENDPOINT", "https://genboree.org/ldh")
+LDH_MAVE_ACCESS_ENDPOINT = f"{LDH_ACCESS_ENDPOINT}/{LDH_ENTITY_NAME}/id"
 
 LINKED_DATA_RETRY_THRESHOLD = 0.95

src/mavedb/lib/clingen/linked_data_hub.py

@@ -9,7 +9,7 @@
 from jose import jwt
 
 from mavedb.lib.logging.context import logging_context, save_to_logging_context, format_raised_exception_info_as_dict
-from mavedb.lib.clingen.constants import GENBOREE_ACCOUNT_NAME, GENBOREE_ACCOUNT_PASSWORD, LDH_LINKED_DATA_URL
+from mavedb.lib.clingen.constants import GENBOREE_ACCOUNT_NAME, GENBOREE_ACCOUNT_PASSWORD, LDH_MAVE_ACCESS_ENDPOINT
 
 from mavedb.lib.types.clingen import LdhSubmission
 from mavedb.lib.utils import batched
@@ -212,7 +212,7 @@ def get_clingen_variation(urn: str) -> Optional[dict]:
                         or None if the request fails.
     """
     response = requests.get(
-        f"{LDH_LINKED_DATA_URL}/{parse.quote_plus(urn)}",
+        f"{LDH_MAVE_ACCESS_ENDPOINT}/{parse.quote_plus(urn)}",
         headers={"Accept": "application/json"},
     )
 

src/mavedb/scripts/clingen_ldh_submission.py

@@ -10,15 +10,15 @@
 from mavedb.models.mapped_variant import MappedVariant
 from mavedb.scripts.environment import with_database_session
 from mavedb.lib.clingen.linked_data_hub import ClinGenLdhService
-from mavedb.lib.clingen.constants import DEFAULT_LDH_SUBMISSION_BATCH_SIZE, LDH_SUBMISSION_URL
+from mavedb.lib.clingen.constants import DEFAULT_LDH_SUBMISSION_BATCH_SIZE, LDH_SUBMISSION_ENDPOINT
 from mavedb.lib.clingen.content_constructors import construct_ldh_submission
 from mavedb.lib.variants import hgvs_from_mapped_variant
 
 logger = logging.getLogger(__name__)
 
 
 def submit_urns_to_clingen(db: Session, urns: Sequence[str], debug: bool) -> list[str]:
-    ldh_service = ClinGenLdhService(url=LDH_SUBMISSION_URL)
+    ldh_service = ClinGenLdhService(url=LDH_SUBMISSION_ENDPOINT)
     ldh_service.authenticate()
 
     submitted_entities = []

src/mavedb/worker/jobs.py

@@ -17,7 +17,7 @@
 from mavedb.db.view import refresh_all_mat_views
 from mavedb.lib.clingen.constants import (
     DEFAULT_LDH_SUBMISSION_BATCH_SIZE,
-    LDH_SUBMISSION_URL,
+    LDH_SUBMISSION_ENDPOINT,
     LINKED_DATA_RETRY_THRESHOLD,
     CLIN_GEN_SUBMISSION_ENABLED,
 )
@@ -792,7 +792,7 @@ async def submit_score_set_mappings_to_ldh(ctx: dict, correlation_id: str, score
         return {"success": False, "retried": False, "enqueued_job": None}
 
     try:
-        ldh_service = ClinGenLdhService(url=LDH_SUBMISSION_URL)
+        ldh_service = ClinGenLdhService(url=LDH_SUBMISSION_ENDPOINT)
         ldh_service.authenticate()
     except Exception as e:
         send_slack_error(e)

tests/lib/clingen/test_linked_data_hub.py

@@ -11,7 +11,7 @@
 cdot = pytest.importorskip("cdot")
 fastapi = pytest.importorskip("fastapi")
 
-from mavedb.lib.clingen.constants import LDH_LINKED_DATA_URL, GENBOREE_ACCOUNT_NAME, GENBOREE_ACCOUNT_PASSWORD
+from mavedb.lib.clingen.constants import LDH_MAVE_ACCESS_ENDPOINT, GENBOREE_ACCOUNT_NAME, GENBOREE_ACCOUNT_PASSWORD
 from mavedb.lib.utils import batched
 from mavedb.lib.clingen.linked_data_hub import (
     ClinGenLdhService,
@@ -225,7 +225,7 @@ def test_get_clingen_variation_success(mock_get):
 
     assert result == mocked_response_json
     mock_get.assert_called_once_with(
-        f"{LDH_LINKED_DATA_URL}/{parse.quote_plus(urn)}",
+        f"{LDH_MAVE_ACCESS_ENDPOINT}/{parse.quote_plus(urn)}",
         headers={"Accept": "application/json"},
     )
 
@@ -242,7 +242,7 @@ def test_get_clingen_variation_failure(mock_get):
 
     assert result is None
     mock_get.assert_called_once_with(
-        f"{LDH_LINKED_DATA_URL}/{parse.quote_plus(urn)}",
+        f"{LDH_MAVE_ACCESS_ENDPOINT}/{parse.quote_plus(urn)}",
         headers={"Accept": "application/json"},
     )
 

tests/worker/test_jobs.py

@@ -2027,11 +2027,26 @@ async def test_link_score_set_mappings_to_ldh_objects_exception_while_parsing_li
         standalone_worker_context,
     )
 
+    async def dummy_linking_job():
+        return [
+            (variant_urn, TEST_CLINGEN_LDH_LINKING_RESPONSE)
+            for variant_urn in session.scalars(
+                select(Variant.urn).join(ScoreSetDbModel).where(ScoreSetDbModel.urn == score_set.urn)
+            ).all()
+        ]
+
     # We are unable to mock requests via requests_mock that occur inside another event loop. Instead, patch the return
     # value of the EventLoop itself, which would have made the request.
-    with patch(
-        "mavedb.lib.clingen.linked_data_hub.clingen_allele_id_from_ldh_variation",
-        side_effect=Exception(),
+    with (
+        patch.object(
+            _UnixSelectorEventLoop,
+            "run_in_executor",
+            return_value=dummy_linking_job(),
+        ),
+        patch(
+            "mavedb.worker.jobs.clingen_allele_id_from_ldh_variation",
+            side_effect=Exception(),
+        ),
     ):
         result = await link_clingen_variants(standalone_worker_context, uuid4().hex, score_set.id, 1)