Merge branch 'release-2025.0.1' into estelle/debugShowTmpSupersedingScoreSet

Author: EstelleDa

alembic/versions/aa73d39b3705_score_set_level_score_thresholds.py

@@ -0,0 +1,29 @@
+"""score set level score thresholds
+
+Revision ID: aa73d39b3705
+Revises: 68a0ec57694e
+Create Date: 2024-11-13 11:23:57.917725
+
+"""
+
+from alembic import op
+import sqlalchemy as sa
+from sqlalchemy.dialects import postgresql
+
+# revision identifiers, used by Alembic.
+revision = "aa73d39b3705"
+down_revision = "68a0ec57694e"
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.add_column("scoresets", sa.Column("score_calibrations", postgresql.JSONB(astext_type=sa.Text()), nullable=True))
+    # ### end Alembic commands ###
+
+
+def downgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.drop_column("scoresets", "score_calibrations")
+    # ### end Alembic commands ###

pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "poetry.core.masonry.api"
 
 [tool.poetry]
 name = "mavedb"
-version = "2024.4.2"
+version = "2025.0.0"
 description = "API for MaveDB, the database of Multiplexed Assays of Variant Effect."
 license = "AGPL-3.0-only"
 readme = "README.md"

src/mavedb/__init__.py

@@ -6,6 +6,6 @@
 logger = module_logging.getLogger(__name__)
 
 __project__ = "mavedb-api"
-__version__ = "2024.4.2"
+__version__ = "2025.0.0"
 
 logger.info(f"MaveDB {__version__}")

src/mavedb/lib/experiments.py

@@ -1,7 +1,7 @@
 import logging
 from typing import Optional
 
-from sqlalchemy import func, or_
+from sqlalchemy import func, or_, not_
 from sqlalchemy.orm import Session
 
 from mavedb.lib.logging.context import logging_context, save_to_logging_context
@@ -99,6 +99,19 @@ def search_experiments(
             )
         )
 
+    if search.meta_analysis is not None:
+        if not search.meta_analysis:
+            query = query.filter(
+                or_(
+                    # Keep experiments without any score sets
+                    not_(Experiment.score_sets.any()),
+                    # Keep experiments where score sets exist but have no meta_analyzes_score_sets
+                    Experiment.score_sets.any(not_(ScoreSet.meta_analyzes_score_sets.any()))
+                )
+            )
+        else:
+            query = query.filter(Experiment.score_sets.any(ScoreSet.meta_analyzes_score_sets.any()))
+
     items: list[Experiment] = query.order_by(Experiment.urn, Experiment.title).all()
     if not items:
         items = []

src/mavedb/lib/score_sets.py

@@ -22,6 +22,7 @@
 )
 from mavedb.lib.mave.utils import is_csv_null
 from mavedb.lib.validation.constants.general import null_values_list
+from mavedb.lib.validation.utilities import is_null as validate_is_null
 from mavedb.models.contributor import Contributor
 from mavedb.models.controlled_keyword import ControlledKeyword
 from mavedb.models.doi_identifier import DoiIdentifier
@@ -402,6 +403,7 @@ def get_score_set_counts_as_csv(
     score_set: ScoreSet,
     start: Optional[int] = None,
     limit: Optional[int] = None,
+    drop_na_columns: Optional[bool] = None,
 ) -> str:
     assert type(score_set.dataset_columns) is dict
     count_columns = [str(x) for x in list(score_set.dataset_columns.get("count_columns", []))]
@@ -420,6 +422,9 @@ def get_score_set_counts_as_csv(
     variants = db.scalars(variants_query).all()
 
     rows_data = variants_to_csv_rows(variants, columns=columns, dtype=type_column)
+    if drop_na_columns:
+        rows_data, columns = drop_na_columns_from_csv_file_rows(rows_data, columns)
+
     stream = io.StringIO()
     writer = csv.DictWriter(stream, fieldnames=columns, quoting=csv.QUOTE_MINIMAL)
     writer.writeheader()
@@ -432,6 +437,7 @@ def get_score_set_scores_as_csv(
     score_set: ScoreSet,
     start: Optional[int] = None,
     limit: Optional[int] = None,
+    drop_na_columns: Optional[bool] = None,
 ) -> str:
     assert type(score_set.dataset_columns) is dict
     score_columns = [str(x) for x in list(score_set.dataset_columns.get("score_columns", []))]
@@ -450,13 +456,38 @@ def get_score_set_scores_as_csv(
     variants = db.scalars(variants_query).all()
 
     rows_data = variants_to_csv_rows(variants, columns=columns, dtype=type_column)
+    if drop_na_columns:
+        rows_data, columns = drop_na_columns_from_csv_file_rows(rows_data, columns)
+
     stream = io.StringIO()
     writer = csv.DictWriter(stream, fieldnames=columns, quoting=csv.QUOTE_MINIMAL)
     writer.writeheader()
     writer.writerows(rows_data)
     return stream.getvalue()
 
 
+def drop_na_columns_from_csv_file_rows(
+    rows_data: Iterable[dict[str, Any]],
+    columns: list[str]
+) -> tuple[list[dict[str, Any]], list[str]]:
+    """Process rows_data for downloadable CSV by removing empty columns."""
+    # Convert map to list.
+    rows_data = list(rows_data)
+    columns_to_check = ["hgvs_nt", "hgvs_splice", "hgvs_pro"]
+    columns_to_remove = []
+
+    # Check if all values in a column are None or "NA"
+    for col in columns_to_check:
+        if all(validate_is_null(row[col]) for row in rows_data):
+            columns_to_remove.append(col)
+            for row in rows_data:
+                row.pop(col, None)  # Remove column from each row
+
+    # Remove these columns from the header list
+    columns = [col for col in columns if col not in columns_to_remove]
+    return rows_data, columns
+
+
 null_values_re = re.compile(r"\s+|none|nan|na|undefined|n/a|null|nil", flags=re.IGNORECASE)
 
 

src/mavedb/models/score_set.py

@@ -157,6 +157,7 @@ class ScoreSet(Base):
 
     target_genes: Mapped[List["TargetGene"]] = relationship(back_populates="score_set", cascade="all, delete-orphan")
     score_ranges = Column(JSONB, nullable=True)
+    score_calibrations = Column(JSONB, nullable=True)
 
     # Unfortunately, we can't use association_proxy here, because in spite of what the documentation seems to imply, it
     # doesn't check for a pre-existing keyword with the same text.

src/mavedb/routers/experiments.py

@@ -7,6 +7,7 @@
 from fastapi import APIRouter, Depends, HTTPException
 from fastapi.encoders import jsonable_encoder
 from sqlalchemy.orm import Session
+from sqlalchemy import or_
 
 from mavedb import deps
 from mavedb.lib.authentication import UserData, get_current_user
@@ -44,7 +45,7 @@
 )
 
 
-# TODO: Rewrite this function.
+# None of any part calls this function. Feel free to modify it if we need it in the future.
 @router.get(
     "/experiments/",
     status_code=200,
@@ -54,30 +55,26 @@
 def list_experiments(
     *,
     editable: Optional[bool] = None,
-    q: Optional[str] = None,
     db: Session = Depends(deps.get_db),
     user_data: Optional[UserData] = Depends(get_current_user),
 ) -> list[Experiment]:
     """
     List experiments.
     """
     query = db.query(Experiment)
-    if q is not None:
-        save_to_logging_context({"query_string": q})
 
+    if editable:
         if user_data is None or user_data.user is None:
             logger.debug(msg="User is anonymous; Cannot list their experiments.", extra=logging_context())
             return []
 
-        if len(q) > 0:
-            logger.debug(msg="Listing experiments for the current user.", extra=logging_context())
-            query = query.filter(
-                Experiment.created_by_id == user_data.user.id
-            )  # .filter(Experiment.published_date is None)
-        # else:
-        #     query = query.filter(Experiment.created_by_id == user.id).filter(Experiment.published_date is None)
-    else:
-        logger.debug(msg="No query string was provided; Listing all experiments.", extra=logging_context())
+        logger.debug(msg="Listing experiments for the current user.", extra=logging_context())
+        query = query.filter(
+            or_(
+                Experiment.created_by_id == user_data.user.id,
+                Experiment.contributors.any(Contributor.orcid_id == user_data.user.username)
+            )
+        )
 
     items = query.order_by(Experiment.urn).all()
     return items

src/mavedb/routers/hgvs.py

@@ -11,7 +11,7 @@
 
 router = APIRouter(
     prefix="/api/v1/hgvs",
-    tags=["hgvs"],
+    tags=["transcripts"],
     responses={404: {"description": "Not found"}},
 )
 
@@ -85,7 +85,7 @@ def gene_info(gene: str, hdp: RESTDataProvider = Depends(hgvs_data_provider)) ->
     return gene_info
 
 
-@router.get("/transcripts/gene/{gene}", status_code=200, response_model=list[str])
+@router.get("/gene/{gene}", status_code=200, response_model=list[str])
 def list_transcripts_for_gene(gene: str, hdp: RESTDataProvider = Depends(hgvs_data_provider)) -> list[str]:
     """
     List transcripts associated with a particular gene
@@ -98,7 +98,7 @@ def list_transcripts_for_gene(gene: str, hdp: RESTDataProvider = Depends(hgvs_da
     return list(transcripts)
 
 
-@router.get("/transcripts/{transcript}", status_code=200, response_model=dict[str, Any])
+@router.get("/{transcript}", status_code=200, response_model=dict[str, Any])
 def transcript_info(transcript: str, hdp: RESTDataProvider = Depends(hgvs_data_provider)) -> dict[str, Any]:
     """
     List transcript information for a particular transcript
@@ -111,7 +111,7 @@ def transcript_info(transcript: str, hdp: RESTDataProvider = Depends(hgvs_data_p
     return transcript_info
 
 
-@router.get("/transcripts/protein/{transcript}", status_code=200, response_model=str)
+@router.get("/protein/{transcript}", status_code=200, response_model=str)
 def convert_to_protein(transcript: str, hdp: RESTDataProvider = Depends(hgvs_data_provider)) -> str:
     """
     Convert a provided transcript from it's nucleotide accession identifier to its protein accession identifier

src/mavedb/routers/score_sets.py

@@ -9,13 +9,18 @@
 from fastapi.encoders import jsonable_encoder
 from fastapi.exceptions import HTTPException
 from fastapi.responses import StreamingResponse
-from sqlalchemy import or_
-from sqlalchemy.exc import MultipleResultsFound
+from sqlalchemy import or_, select
+from sqlalchemy.exc import MultipleResultsFound, NoResultFound
 from sqlalchemy.orm import Session
 
 from mavedb import deps
 from mavedb.lib.authentication import UserData
-from mavedb.lib.authorization import get_current_user, require_current_user, require_current_user_with_email
+from mavedb.lib.authorization import (
+    get_current_user,
+    require_current_user,
+    require_current_user_with_email,
+    RoleRequirer,
+)
 from mavedb.lib.contributors import find_or_create_contributor
 from mavedb.lib.exceptions import MixedTargetError, NonexistentOrcidUserError, ValidationError
 from mavedb.lib.identifiers import (
@@ -50,6 +55,7 @@
 )
 from mavedb.models.contributor import Contributor
 from mavedb.models.enums.processing_state import ProcessingState
+from mavedb.models.enums.user_role import UserRole
 from mavedb.models.experiment import Experiment
 from mavedb.models.license import License
 from mavedb.models.mapped_variant import MappedVariant
@@ -58,7 +64,7 @@
 from mavedb.models.target_gene import TargetGene
 from mavedb.models.target_sequence import TargetSequence
 from mavedb.models.variant import Variant
-from mavedb.view_models import mapped_variant, score_set
+from mavedb.view_models import mapped_variant, score_set, calibration
 from mavedb.view_models.search import ScoreSetsSearch
 
 logger = logging.getLogger(__name__)
@@ -185,6 +191,7 @@ def get_score_set_scores_csv(
     urn: str,
     start: int = Query(default=None, description="Start index for pagination"),
     limit: int = Query(default=None, description="Number of variants to return"),
+    drop_na_columns: Optional[bool] = None,
     db: Session = Depends(deps.get_db),
     user_data: Optional[UserData] = Depends(get_current_user),
 ) -> Any:
@@ -219,7 +226,7 @@ def get_score_set_scores_csv(
 
     assert_permission(user_data, score_set, Action.READ)
 
-    csv_str = get_score_set_scores_as_csv(db, score_set, start, limit)
+    csv_str = get_score_set_scores_as_csv(db, score_set, start, limit, drop_na_columns)
     return StreamingResponse(iter([csv_str]), media_type="text/csv")
 
 
@@ -239,6 +246,7 @@ async def get_score_set_counts_csv(
     urn: str,
     start: int = Query(default=None, description="Start index for pagination"),
     limit: int = Query(default=None, description="Number of variants to return"),
+    drop_na_columns: Optional[bool] = None,
     db: Session = Depends(deps.get_db),
     user_data: Optional[UserData] = Depends(get_current_user),
 ) -> Any:
@@ -273,7 +281,7 @@ async def get_score_set_counts_csv(
 
     assert_permission(user_data, score_set, Action.READ)
 
-    csv_str = get_score_set_counts_as_csv(db, score_set, start, limit)
+    csv_str = get_score_set_counts_as_csv(db, score_set, start, limit, drop_na_columns)
     return StreamingResponse(iter([csv_str]), media_type="text/csv")
 
 
@@ -347,8 +355,10 @@ async def create_score_set(
             raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail="Unknown experiment")
         # Not allow add score set in meta-analysis experiments.
         if any(s.meta_analyzes_score_sets for s in experiment.score_sets):
-            raise HTTPException(status_code=status.HTTP_403_FORBIDDEN,
-                                detail="Score sets may not be added to a meta-analysis experiment.")
+            raise HTTPException(
+                status_code=status.HTTP_403_FORBIDDEN,
+                detail="Score sets may not be added to a meta-analysis experiment.",
+            )
 
         save_to_logging_context({"experiment": experiment.urn})
         assert_permission(user_data, experiment, Action.ADD_SCORE_SET)
@@ -668,6 +678,43 @@ async def upload_score_set_variant_data(
     return item
 
 
+@router.post(
+    "/score-sets/{urn}/calibration/data",
+    response_model=score_set.ScoreSet,
+    responses={422: {}},
+    response_model_exclude_none=True,
+)
+async def update_score_set_calibration_data(
+    *,
+    urn: str,
+    calibration_update: dict[str, calibration.Calibration],
+    db: Session = Depends(deps.get_db),
+    user_data: UserData = Depends(RoleRequirer([UserRole.admin])),
+):
+    """
+    Update thresholds / score calibrations for a score set.
+    """
+    save_to_logging_context({"requested_resource": urn, "resource_property": "score_thresholds"})
+
+    try:
+        item = db.scalars(select(ScoreSet).where(ScoreSet.urn == urn)).one()
+    except NoResultFound:
+        logger.info(
+            msg="Failed to add score thresholds; The requested score set does not exist.", extra=logging_context()
+        )
+        raise HTTPException(status_code=404, detail=f"score set with URN '{urn}' not found")
+
+    assert_permission(user_data, item, Action.UPDATE)
+
+    item.score_calibrations = {k: v.dict() for k, v in calibration_update.items()}
+    db.add(item)
+    db.commit()
+    db.refresh(item)
+
+    save_to_logging_context({"updated_resource": item.urn})
+    return item
+
+
 @router.put(
     "/score-sets/{urn}", response_model=score_set.ScoreSet, responses={422: {}}, response_model_exclude_none=True
 )

src/mavedb/scripts/populate_mapped_variants.py

@@ -66,7 +66,11 @@ def populate_mapped_variant_data(db: Session, urns: Sequence[Optional[str]], all
 
         try:
             existing_mapped_variants = (
-                db.query(MappedVariant).join(Variant).join(ScoreSet).filter(MappedVariant.current.is_(True)).all()
+                db.query(MappedVariant)
+                .join(Variant)
+                .join(ScoreSet)
+                .filter(ScoreSet.id == ss_id, MappedVariant.current.is_(True))
+                .all()
             )
 
             for variant in existing_mapped_variants: