Merge pull request #300 from VariantEffect/release-2024.4.0

Release 2024.4.0

Author: bencap

alembic/versions/1d4933b4b6f7_merge_76e1e55bc5c1_and_d7e6f8c3b9dc.py

@@ -0,0 +1,24 @@
+"""merge 76e1e55bc5c1 and d7e6f8c3b9dc
+
+Revision ID: 1d4933b4b6f7
+Revises: 76e1e55bc5c1, d7e6f8c3b9dc
+Create Date: 2024-09-04 16:17:20.875937
+
+"""
+from alembic import op
+import sqlalchemy as sa
+
+
+# revision identifiers, used by Alembic.
+revision = '1d4933b4b6f7'
+down_revision = ('76e1e55bc5c1', 'd7e6f8c3b9dc')
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    pass
+
+
+def downgrade():
+    pass

alembic/versions/d7e6f8c3b9dc_scoreset_mapping_columns.py

@@ -0,0 +1,48 @@
+"""scoreset_mapping_columns
+
+Revision ID: d7e6f8c3b9dc
+Revises: f36cf612e029
+Create Date: 2024-08-28 09:54:08.249077
+
+"""
+
+from alembic import op
+from sqlalchemy.dialects import postgresql
+import sqlalchemy as sa
+
+
+# revision identifiers, used by Alembic.
+revision = "d7e6f8c3b9dc"
+down_revision = "f36cf612e029"
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    op.add_column(
+        "scoresets",
+        sa.Column(
+            "mapping_state",
+            sa.Enum(
+                "incomplete",
+                "processing",
+                "failed",
+                "complete",
+                "pending_variant_processing",
+                "not_attempted",
+                "queued",
+                name="mappingstate",
+                native_enum=False,
+                create_constraint=True,
+                length=32
+            ),
+            nullable=True,
+        ),
+    )
+    op.add_column("scoresets", sa.Column("mapping_errors", postgresql.JSONB, nullable=True))
+
+
+def downgrade():
+    op.drop_constraint("mappingstate", table_name="scoresets")
+    op.drop_column("scoresets", "mapping_state")
+    op.drop_column("scoresets", "mapping_errors")

alembic/versions/f36cf612e029_additional_mapping_columns.py

@@ -0,0 +1,102 @@
+"""Additional mapping columns
+
+Revision ID: f36cf612e029
+Revises: ec5d2787bec9
+Create Date: 2024-08-21 16:06:06.793541
+
+"""
+
+from alembic import op
+import sqlalchemy as sa
+from sqlalchemy.dialects import postgresql
+
+from datetime import datetime
+
+# revision identifiers, used by Alembic.
+revision = "f36cf612e029"
+down_revision = "ec5d2787bec9"
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.add_column(
+        "mapped_variants",
+        sa.Column("vrs_version", sa.String(), nullable=True, server_default="1.3"),
+    )
+    op.add_column("mapped_variants", sa.Column("error_message", sa.String(), nullable=True))
+    op.add_column(
+        "mapped_variants",
+        sa.Column(
+            "modification_date",
+            sa.Date(),
+            nullable=False,
+            server_default = sa.func.current_date(),
+        ),
+    )
+    op.add_column(
+        "mapped_variants",
+        sa.Column(
+            "mapped_date",
+            sa.Date(),
+            nullable=False,
+            server_default=datetime.fromisocalendar(2024, 1, 1).date().strftime("%Y-%m-%d"),
+        ),
+    )
+    op.add_column(
+        "mapped_variants",
+        sa.Column("mapping_api_version", sa.String(), nullable=False, server_default="0.0.0"),
+    )
+    op.add_column(
+        "mapped_variants",
+        sa.Column("current", sa.Boolean(), nullable=False, server_default=sa.false()),
+        )
+    op.alter_column(
+        "mapped_variants",
+        "pre_mapped",
+        existing_type=postgresql.JSONB(astext_type=sa.Text()),
+        nullable=True,
+    )
+    op.alter_column(
+        "mapped_variants",
+        "post_mapped",
+        existing_type=postgresql.JSONB(astext_type=sa.Text()),
+        nullable=True,
+    )
+
+    op.add_column(
+        "target_genes",
+        sa.Column("pre_mapped_metadata", postgresql.JSONB, nullable=True),
+    )
+    op.add_column(
+        "target_genes",
+        sa.Column("post_mapped_metadata", postgresql.JSONB, nullable=True),
+    )
+    # ### end Alembic commands ###
+
+
+def downgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.alter_column(
+        "mapped_variants",
+        "post_mapped",
+        existing_type=postgresql.JSONB(astext_type=sa.Text()),
+        nullable=False,
+    )
+    op.alter_column(
+        "mapped_variants",
+        "pre_mapped",
+        existing_type=postgresql.JSONB(astext_type=sa.Text()),
+        nullable=False,
+    )
+    op.drop_column("mapped_variants", "current")
+    op.drop_column("mapped_variants", "mapping_api_version")
+    op.drop_column("mapped_variants", "mapped_date")
+    op.drop_column("mapped_variants", "modification_date")
+    op.drop_column("mapped_variants", "error_message")
+    op.drop_column("mapped_variants", "vrs_version")
+
+    op.drop_column("target_genes", "pre_mapped_metadata")
+    op.drop_column("target_genes", "post_mapped_metadata")
+    # ### end Alembic commands ###

docker-compose-dev.yml

@@ -63,6 +63,24 @@ services:
     volumes:
       - mavedb-redis-dev:/data
 
+  dcd-mapping:
+    image: dcd-mapping:dev
+    command: bash -c "uvicorn api.server_main:app --host 0.0.0.0 --port 8000 --reload"
+    depends_on:
+      - db
+      - seqrepo
+    env_file:
+      - settings/.env.dev
+    ports:
+      - "8004:8000"
+    volumes:
+      - mavedb-seqrepo-dev:/usr/local/share/seqrepo
+
+  seqrepo:
+    image: biocommons/seqrepo:2021-01-29
+    volumes:
+      - mavedb-seqrepo-dev:/usr/local/share/seqrepo
+
 #  rabbitmq:
 #    image: rabbitmq:3.8.3
 #    ports:
@@ -71,3 +89,4 @@ services:
 volumes:
   mavedb-data-dev:
   mavedb-redis-dev:
+  mavedb-seqrepo-dev:

pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "poetry.core.masonry.api"
 
 [tool.poetry]
 name = "mavedb"
-version = "2024.3.0"
+version = "2024.4.0"
 description = "API for MaveDB, the database of Multiplexed Assays of Variant Effect."
 license = "AGPL-3.0-only"
 readme = "README.md"

settings/.env.template

@@ -32,3 +32,30 @@ SLACK_WEBHOOK_URL=
 ORCID_CLIENT_ID=client-id
 ORCID_CLIENT_SECRET=secret
 ORCID_JWT_SIGNING_PUBLIC_KEY="-----BEGIN PUBLIC KEY-----\npublic key\n-----END PUBLIC KEY-----"
+
+####################################################################################################
+# Environment variables for vrs-mapping
+####################################################################################################
+
+GENE_NORM_DB_URL=postgres://postgres:postgres@db:5432/gene_normalizer
+MAVEDB_STORAGE_DIR=/root/.local/share/dcd-mapping
+
+####################################################################################################
+# Environment variables for UTA connection via CoolSeqTool
+####################################################################################################
+
+UTA_DB_URL=postgresql://anonymous:[email protected]:5432/uta/uta_20180821
+
+####################################################################################################
+# Environment variables for seqrepo
+####################################################################################################
+
+SEQREPO_ROOT_DIR=/usr/local/share/seqrepo/2021-01-29
+
+####################################################################################################
+# Environment variables for mapping MaveDB connection
+####################################################################################################
+
+MAVEDB_BASE_URL=http://app:8000
+MAVEDB_API_KEY=secret
+DCD_MAPPING_URL=http://dcd-mapping:8000

src/mavedb/__init__.py

@@ -6,6 +6,6 @@
 logger = module_logging.getLogger(__name__)
 
 __project__ = "mavedb-api"
-__version__ = "2024.3.0"
+__version__ = "2024.4.0"
 
 logger.info(f"MaveDB {__version__}")

src/mavedb/data_providers/services.py

@@ -1,14 +1,49 @@
+import os
+import requests
+from datetime import date
+from typing import Optional, TypedDict
+
 from cdot.hgvs.dataproviders import ChainedSeqFetcher, FastaSeqFetcher, RESTDataProvider
 
 GENOMIC_FASTA_FILES = [
     "/data/GCF_000001405.39_GRCh38.p13_genomic.fna.gz",
     "/data/GCF_000001405.25_GRCh37.p13_genomic.fna.gz",
 ]
 
+DCD_MAP_URL = os.environ.get("DCD_MAPPING_URL", "http://dcd-mapping:8000")
+
 
 def seqfetcher() -> ChainedSeqFetcher:
     return ChainedSeqFetcher(*[FastaSeqFetcher(file) for file in GENOMIC_FASTA_FILES])
 
 
 def cdot_rest() -> RESTDataProvider:
     return RESTDataProvider(seqfetcher=seqfetcher())
+
+
+class VRSMap:
+    url: str
+
+    class ScoreSetMappingResults(TypedDict):
+        metadata: Optional[dict[str, str]]
+        dcd_mapping_version: str
+        mapped_date_utc: date
+        computed_genomic_reference_sequence: Optional[dict[str, str]]
+        mapped_genomic_reference_sequence: Optional[dict[str, str]]
+        computed_protein_reference_sequence: Optional[dict[str, str]]
+        mapped_protein_reference_sequence: Optional[dict[str, str]]
+        mapped_scores: Optional[list[dict]]
+        error_message: Optional[str]
+
+    def __init__(self, url: str) -> None:
+        self.url = url
+
+    def map_score_set(self, score_set_urn: str) -> ScoreSetMappingResults:
+        uri = f"{self.url}/api/v1/map/{score_set_urn}"
+        response = requests.post(uri)
+        response.raise_for_status()
+        return response.json()
+
+
+def vrs_mapper(url: Optional[str] = None) -> VRSMap:
+    return VRSMap(DCD_MAP_URL) if not url else VRSMap(url)

src/mavedb/lib/exceptions.py

@@ -160,3 +160,19 @@ class NonexistentOrcidUserError(ValueError):
     """Raised when a user tries to create a contributor with a non-existent ORCID ID"""
 
     pass
+
+
+class NonexistentMappingResultsError(ValueError):
+    """Raised when score set mapping results do not contain mapping results"""
+
+    pass
+
+
+class NonexistentMappingReferenceError(ValueError):
+    """Raised when score set mapping results do not contain a valid reference sequence"""
+
+    pass
+
+
+class MappingEnqueueError(ValueError):
+    """Raised when a mapping job fails to be enqueued despite appearing as if it should have been"""

src/mavedb/lib/logging/canonical.py

@@ -27,7 +27,6 @@ async def log_job(ctx: dict) -> None:
 
     if not result:
         logger.warning(msg=f"Job finished, but could not retrieve a job result for job {job_id}.", extra=log_context)
-        log_context.pop("message")
     else:
         log_context = {
             **log_context,
@@ -55,10 +54,8 @@ async def log_job(ctx: dict) -> None:
 
     if result is None:
         logger.error(msg="Job result could not be found.", extra=log_context)
-    elif result.result == "success":
-        logger.info(msg="Job completed successfully.", extra=log_context)
-    elif result.result != "success":
-        logger.warning(msg="Job completed with handled exception.", extra=log_context)
+    elif result.result is not None:
+        logger.info(msg="Job completed with result.", extra=log_context)
     else:
         logger.error(msg="Job completed with unhandled exception.", extra=log_context)