Merge pull request #385 from VariantEffect/estelle/dropDownloadNullHgvsColumns

Remove NA columns from dowloading scores and counts files.

Author: EstelleDa

alembic/versions/aa73d39b3705_score_set_level_score_thresholds.py

@@ -0,0 +1,29 @@
+"""score set level score thresholds
+
+Revision ID: aa73d39b3705
+Revises: 68a0ec57694e
+Create Date: 2024-11-13 11:23:57.917725
+
+"""
+
+from alembic import op
+import sqlalchemy as sa
+from sqlalchemy.dialects import postgresql
+
+# revision identifiers, used by Alembic.
+revision = "aa73d39b3705"
+down_revision = "68a0ec57694e"
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.add_column("scoresets", sa.Column("score_calibrations", postgresql.JSONB(astext_type=sa.Text()), nullable=True))
+    # ### end Alembic commands ###
+
+
+def downgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.drop_column("scoresets", "score_calibrations")
+    # ### end Alembic commands ###

src/mavedb/lib/score_sets.py

@@ -21,6 +21,7 @@
 )
 from mavedb.lib.mave.utils import is_csv_null
 from mavedb.lib.validation.constants.general import null_values_list
+from mavedb.lib.validation.utilities import is_null as validate_is_null
 from mavedb.models.contributor import Contributor
 from mavedb.models.controlled_keyword import ControlledKeyword
 from mavedb.models.doi_identifier import DoiIdentifier
@@ -311,6 +312,7 @@ def get_score_set_counts_as_csv(
     score_set: ScoreSet,
     start: Optional[int] = None,
     limit: Optional[int] = None,
+    drop_na_columns: Optional[bool] = None,
 ) -> str:
     assert type(score_set.dataset_columns) is dict
     count_columns = [str(x) for x in list(score_set.dataset_columns.get("count_columns", []))]
@@ -329,6 +331,9 @@ def get_score_set_counts_as_csv(
     variants = db.scalars(variants_query).all()
 
     rows_data = variants_to_csv_rows(variants, columns=columns, dtype=type_column)
+    if drop_na_columns:
+        rows_data, columns = drop_na_columns_from_csv_file_rows(rows_data, columns)
+
     stream = io.StringIO()
     writer = csv.DictWriter(stream, fieldnames=columns, quoting=csv.QUOTE_MINIMAL)
     writer.writeheader()
@@ -341,6 +346,7 @@ def get_score_set_scores_as_csv(
     score_set: ScoreSet,
     start: Optional[int] = None,
     limit: Optional[int] = None,
+    drop_na_columns: Optional[bool] = None,
 ) -> str:
     assert type(score_set.dataset_columns) is dict
     score_columns = [str(x) for x in list(score_set.dataset_columns.get("score_columns", []))]
@@ -359,13 +365,38 @@ def get_score_set_scores_as_csv(
     variants = db.scalars(variants_query).all()
 
     rows_data = variants_to_csv_rows(variants, columns=columns, dtype=type_column)
+    if drop_na_columns:
+        rows_data, columns = drop_na_columns_from_csv_file_rows(rows_data, columns)
+
     stream = io.StringIO()
     writer = csv.DictWriter(stream, fieldnames=columns, quoting=csv.QUOTE_MINIMAL)
     writer.writeheader()
     writer.writerows(rows_data)
     return stream.getvalue()
 
 
+def drop_na_columns_from_csv_file_rows(
+    rows_data: Iterable[dict[str, Any]],
+    columns: list[str]
+) -> tuple[list[dict[str, Any]], list[str]]:
+    """Process rows_data for downloadable CSV by removing empty columns."""
+    # Convert map to list.
+    rows_data = list(rows_data)
+    columns_to_check = ["hgvs_nt", "hgvs_splice", "hgvs_pro"]
+    columns_to_remove = []
+
+    # Check if all values in a column are None or "NA"
+    for col in columns_to_check:
+        if all(validate_is_null(row[col]) for row in rows_data):
+            columns_to_remove.append(col)
+            for row in rows_data:
+                row.pop(col, None)  # Remove column from each row
+
+    # Remove these columns from the header list
+    columns = [col for col in columns if col not in columns_to_remove]
+    return rows_data, columns
+
+
 null_values_re = re.compile(r"\s+|none|nan|na|undefined|n/a|null|nil", flags=re.IGNORECASE)
 
 

src/mavedb/models/score_set.py

@@ -157,6 +157,7 @@ class ScoreSet(Base):
 
     target_genes: Mapped[List["TargetGene"]] = relationship(back_populates="score_set", cascade="all, delete-orphan")
     score_ranges = Column(JSONB, nullable=True)
+    score_calibrations = Column(JSONB, nullable=True)
 
     # Unfortunately, we can't use association_proxy here, because in spite of what the documentation seems to imply, it
     # doesn't check for a pre-existing keyword with the same text.

src/mavedb/routers/score_sets.py

@@ -9,13 +9,18 @@
 from fastapi.encoders import jsonable_encoder
 from fastapi.exceptions import HTTPException
 from fastapi.responses import StreamingResponse
-from sqlalchemy import or_
-from sqlalchemy.exc import MultipleResultsFound
+from sqlalchemy import or_, select
+from sqlalchemy.exc import MultipleResultsFound, NoResultFound
 from sqlalchemy.orm import Session
 
 from mavedb import deps
 from mavedb.lib.authentication import UserData
-from mavedb.lib.authorization import get_current_user, require_current_user, require_current_user_with_email
+from mavedb.lib.authorization import (
+    get_current_user,
+    require_current_user,
+    require_current_user_with_email,
+    RoleRequirer,
+)
 from mavedb.lib.contributors import find_or_create_contributor
 from mavedb.lib.exceptions import MixedTargetError, NonexistentOrcidUserError, ValidationError
 from mavedb.lib.identifiers import (
@@ -49,6 +54,7 @@
 )
 from mavedb.models.contributor import Contributor
 from mavedb.models.enums.processing_state import ProcessingState
+from mavedb.models.enums.user_role import UserRole
 from mavedb.models.experiment import Experiment
 from mavedb.models.license import License
 from mavedb.models.mapped_variant import MappedVariant
@@ -57,7 +63,7 @@
 from mavedb.models.target_gene import TargetGene
 from mavedb.models.target_sequence import TargetSequence
 from mavedb.models.variant import Variant
-from mavedb.view_models import mapped_variant, score_set
+from mavedb.view_models import mapped_variant, score_set, calibration
 from mavedb.view_models.search import ScoreSetsSearch
 
 logger = logging.getLogger(__name__)
@@ -174,6 +180,7 @@ def get_score_set_scores_csv(
     urn: str,
     start: int = Query(default=None, description="Start index for pagination"),
     limit: int = Query(default=None, description="Number of variants to return"),
+    drop_na_columns: Optional[bool] = None,
     db: Session = Depends(deps.get_db),
     user_data: Optional[UserData] = Depends(get_current_user),
 ) -> Any:
@@ -208,7 +215,7 @@ def get_score_set_scores_csv(
 
     assert_permission(user_data, score_set, Action.READ)
 
-    csv_str = get_score_set_scores_as_csv(db, score_set, start, limit)
+    csv_str = get_score_set_scores_as_csv(db, score_set, start, limit, drop_na_columns)
     return StreamingResponse(iter([csv_str]), media_type="text/csv")
 
 
@@ -228,6 +235,7 @@ async def get_score_set_counts_csv(
     urn: str,
     start: int = Query(default=None, description="Start index for pagination"),
     limit: int = Query(default=None, description="Number of variants to return"),
+    drop_na_columns: Optional[bool] = None,
     db: Session = Depends(deps.get_db),
     user_data: Optional[UserData] = Depends(get_current_user),
 ) -> Any:
@@ -262,7 +270,7 @@ async def get_score_set_counts_csv(
 
     assert_permission(user_data, score_set, Action.READ)
 
-    csv_str = get_score_set_counts_as_csv(db, score_set, start, limit)
+    csv_str = get_score_set_counts_as_csv(db, score_set, start, limit, drop_na_columns)
     return StreamingResponse(iter([csv_str]), media_type="text/csv")
 
 
@@ -336,8 +344,10 @@ async def create_score_set(
             raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail="Unknown experiment")
         # Not allow add score set in meta-analysis experiments.
         if any(s.meta_analyzes_score_sets for s in experiment.score_sets):
-            raise HTTPException(status_code=status.HTTP_403_FORBIDDEN,
-                                detail="Score sets may not be added to a meta-analysis experiment.")
+            raise HTTPException(
+                status_code=status.HTTP_403_FORBIDDEN,
+                detail="Score sets may not be added to a meta-analysis experiment.",
+            )
 
         save_to_logging_context({"experiment": experiment.urn})
         assert_permission(user_data, experiment, Action.ADD_SCORE_SET)
@@ -656,6 +666,43 @@ async def upload_score_set_variant_data(
     return item
 
 
+@router.post(
+    "/score-sets/{urn}/calibration/data",
+    response_model=score_set.ScoreSet,
+    responses={422: {}},
+    response_model_exclude_none=True,
+)
+async def update_score_set_calibration_data(
+    *,
+    urn: str,
+    calibration_update: dict[str, calibration.Calibration],
+    db: Session = Depends(deps.get_db),
+    user_data: UserData = Depends(RoleRequirer([UserRole.admin])),
+):
+    """
+    Update thresholds / score calibrations for a score set.
+    """
+    save_to_logging_context({"requested_resource": urn, "resource_property": "score_thresholds"})
+
+    try:
+        item = db.scalars(select(ScoreSet).where(ScoreSet.urn == urn)).one()
+    except NoResultFound:
+        logger.info(
+            msg="Failed to add score thresholds; The requested score set does not exist.", extra=logging_context()
+        )
+        raise HTTPException(status_code=404, detail=f"score set with URN '{urn}' not found")
+
+    assert_permission(user_data, item, Action.UPDATE)
+
+    item.score_calibrations = {k: v.dict() for k, v in calibration_update.items()}
+    db.add(item)
+    db.commit()
+    db.refresh(item)
+
+    save_to_logging_context({"updated_resource": item.urn})
+    return item
+
+
 @router.put(
     "/score-sets/{urn}", response_model=score_set.ScoreSet, responses={422: {}}, response_model_exclude_none=True
 )

src/mavedb/view_models/calibration.py

@@ -0,0 +1,43 @@
+from typing import Union
+
+from pydantic import root_validator
+
+from mavedb.lib.validation.exceptions import ValidationError
+from mavedb.view_models.base.base import BaseModel
+
+
+class PillarProjectParameters(BaseModel):
+    skew: float
+    location: float
+    scale: float
+
+
+class PillarProjectParameterSet(BaseModel):
+    functionally_altering: PillarProjectParameters
+    functionally_normal: PillarProjectParameters
+    fraction_functionally_altering: float
+
+
+class PillarProjectCalibration(BaseModel):
+    parameter_sets: list[PillarProjectParameterSet]
+    evidence_strengths: list[int]
+    thresholds: list[float]
+    positive_likelihood_ratios: list[float]
+    prior_probability_pathogenicity: float
+
+    @root_validator
+    def validate_all_calibrations_have_a_pairwise_companion(cls, values):
+        num_es = len(values.get("evidence_strengths"))
+        num_st = len(values.get("thresholds"))
+        num_plr = len(values.get("positive_likelihood_ratios"))
+
+        if len(set((num_es, num_st, num_plr))) != 1:
+            raise ValidationError(
+                "Calibration object must provide the same number of evidence strengths, score thresholds, and positive likelihood ratios. "
+                "One or more of these provided objects was not the same length as the others."
+            )
+
+        return values
+
+
+Calibration = Union[PillarProjectCalibration]

src/mavedb/view_models/score_set.py

@@ -15,6 +15,7 @@
 from mavedb.models.enums.processing_state import ProcessingState
 from mavedb.view_models import PublicationIdentifiersGetter, record_type_validator, set_record_type
 from mavedb.view_models.base.base import BaseModel, validator
+from mavedb.view_models.calibration import Calibration
 from mavedb.view_models.contributor import Contributor, ContributorCreate
 from mavedb.view_models.doi_identifier import (
     DoiIdentifier,
@@ -387,6 +388,7 @@ class SavedScoreSet(ScoreSetBase):
     external_links: Dict[str, ExternalLink]
     contributors: list[Contributor]
     score_ranges: Optional[ScoreRanges]
+    score_calibrations: Optional[dict[str, Calibration]]
 
     _record_type_factory = record_type_validator()(set_record_type)
 

tests/helpers/constants.py

@@ -658,10 +658,41 @@
     ],
 }
 
+
 TEST_SAVED_SCORESET_RANGE = {
     "wtScore": 1.0,
     "ranges": [
         {"label": "test1", "classification": "normal", "range": [0.0, 2.0]},
         {"label": "test2", "classification": "abnormal", "range": [-2.0, 0.0]},
     ],
 }
+
+
+TEST_SCORE_CALIBRATION = {
+    "parameter_sets": [
+        {
+            "functionally_altering": {"skew": 1.15, "location": -2.20, "scale": 1.20},
+            "functionally_normal": {"skew": -1.5, "location": 2.25, "scale": 0.8},
+            "fraction_functionally_altering": 0.20,
+        },
+    ],
+    "evidence_strengths": [3, 2, 1, -1],
+    "thresholds": [1.25, 2.5, 3, 5.5],
+    "positive_likelihood_ratios": [100, 10, 1, 0.1],
+    "prior_probability_pathogenicity": 0.20,
+}
+
+
+TEST_SAVED_SCORE_CALIBRATION = {
+    "parameterSets": [
+        {
+            "functionallyAltering": {"skew": 1.15, "location": -2.20, "scale": 1.20},
+            "functionallyNormal": {"skew": -1.5, "location": 2.25, "scale": 0.8},
+            "fractionFunctionallyAltering": 0.20,
+        },
+    ],
+    "evidenceStrengths": [3, 2, 1, -1],
+    "thresholds": [1.25, 2.5, 3, 5.5],
+    "positiveLikelihoodRatios": [100, 10, 1, 0.1],
+    "priorProbabilityPathogenicity": 0.20,
+}