Add Threshold Calibrations to Score Set Models and Tests

Author: bencap

alembic/versions/aa73d39b3705_score_set_level_score_thresholds.py

@@ -19,11 +19,11 @@
 
 def upgrade():
     # ### commands auto generated by Alembic - please adjust! ###
-    op.add_column("scoresets", sa.Column("score_thresholds", postgresql.JSONB(astext_type=sa.Text()), nullable=True))
+    op.add_column("scoresets", sa.Column("score_calibrations", postgresql.JSONB(astext_type=sa.Text()), nullable=True))
     # ### end Alembic commands ###
 
 
 def downgrade():
     # ### commands auto generated by Alembic - please adjust! ###
-    op.drop_column("scoresets", "score_thresholds")
+    op.drop_column("scoresets", "score_calibrations")
     # ### end Alembic commands ###

src/mavedb/models/score_set.py

@@ -157,7 +157,7 @@ class ScoreSet(Base):
 
     target_genes: Mapped[List["TargetGene"]] = relationship(back_populates="score_set", cascade="all, delete-orphan")
     score_ranges = Column(JSONB, nullable=True)
-    score_thresholds = Column(JSONB, nullable=True)
+    score_calibrations = Column(JSONB, nullable=True)
 
     # Unfortunately, we can't use association_proxy here, because in spite of what the documentation seems to imply, it
     # doesn't check for a pre-existing keyword with the same text.

src/mavedb/routers/score_sets.py

@@ -9,13 +9,18 @@
 from fastapi.encoders import jsonable_encoder
 from fastapi.exceptions import HTTPException
 from fastapi.responses import StreamingResponse
-from sqlalchemy import or_
-from sqlalchemy.exc import MultipleResultsFound
+from sqlalchemy import or_, select
+from sqlalchemy.exc import MultipleResultsFound, NoResultFound
 from sqlalchemy.orm import Session
 
 from mavedb import deps
 from mavedb.lib.authentication import UserData
-from mavedb.lib.authorization import get_current_user, require_current_user, require_current_user_with_email
+from mavedb.lib.authorization import (
+    get_current_user,
+    require_current_user,
+    require_current_user_with_email,
+    RoleRequirer,
+)
 from mavedb.lib.contributors import find_or_create_contributor
 from mavedb.lib.exceptions import MixedTargetError, NonexistentOrcidUserError, ValidationError
 from mavedb.lib.identifiers import (
@@ -49,6 +54,7 @@
 )
 from mavedb.models.contributor import Contributor
 from mavedb.models.enums.processing_state import ProcessingState
+from mavedb.models.enums.user_role import UserRole
 from mavedb.models.experiment import Experiment
 from mavedb.models.license import License
 from mavedb.models.mapped_variant import MappedVariant
@@ -57,7 +63,7 @@
 from mavedb.models.target_gene import TargetGene
 from mavedb.models.target_sequence import TargetSequence
 from mavedb.models.variant import Variant
-from mavedb.view_models import mapped_variant, score_set
+from mavedb.view_models import mapped_variant, score_set, calibration
 from mavedb.view_models.search import ScoreSetsSearch
 
 logger = logging.getLogger(__name__)
@@ -336,8 +342,10 @@ async def create_score_set(
             raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail="Unknown experiment")
         # Not allow add score set in meta-analysis experiments.
         if any(s.meta_analyzes_score_sets for s in experiment.score_sets):
-            raise HTTPException(status_code=status.HTTP_403_FORBIDDEN,
-                                detail="Score sets may not be added to a meta-analysis experiment.")
+            raise HTTPException(
+                status_code=status.HTTP_403_FORBIDDEN,
+                detail="Score sets may not be added to a meta-analysis experiment.",
+            )
 
         save_to_logging_context({"experiment": experiment.urn})
         assert_permission(user_data, experiment, Action.ADD_SCORE_SET)
@@ -656,6 +664,43 @@ async def upload_score_set_variant_data(
     return item
 
 
+@router.post(
+    "/score-sets/{urn}/calibration/data",
+    response_model=score_set.ScoreSet,
+    responses={422: {}},
+    response_model_exclude_none=True,
+)
+async def update_score_set_calibration_data(
+    *,
+    urn: str,
+    calibration_update: dict[str, calibration.Calibration],
+    db: Session = Depends(deps.get_db),
+    user_data: UserData = Depends(RoleRequirer([UserRole.admin])),
+):
+    """
+    Update thresholds / score calibrations for a score set.
+    """
+    save_to_logging_context({"requested_resource": urn, "resource_property": "score_thresholds"})
+
+    try:
+        item = db.scalars(select(ScoreSet).where(ScoreSet.urn == urn)).one()
+    except NoResultFound:
+        logger.info(
+            msg="Failed to add score thresholds; The requested score set does not exist.", extra=logging_context()
+        )
+        raise HTTPException(status_code=404, detail=f"score set with URN '{urn}' not found")
+
+    assert_permission(user_data, item, Action.UPDATE)
+
+    item.score_calibrations = {k: v.dict() for k, v in calibration_update.items()}
+    db.add(item)
+    db.commit()
+    db.refresh(item)
+
+    save_to_logging_context({"updated_resource": item.urn})
+    return item
+
+
 @router.put(
     "/score-sets/{urn}", response_model=score_set.ScoreSet, responses={422: {}}, response_model_exclude_none=True
 )

src/mavedb/view_models/calibration.py

@@ -0,0 +1,43 @@
+from typing import Union
+
+from pydantic import root_validator
+
+from mavedb.lib.validation.exceptions import ValidationError
+from mavedb.view_models.base.base import BaseModel
+
+
+class PillarProjectParameters(BaseModel):
+    skew: float
+    location: float
+    scale: float
+
+
+class PillarProjectParameterSet(BaseModel):
+    functionally_altering: PillarProjectParameters
+    functionally_normal: PillarProjectParameters
+    fraction_functionally_altering: float
+
+
+class PillarProjectCalibration(BaseModel):
+    parameter_sets: list[PillarProjectParameterSet]
+    evidence_strengths: list[int]
+    thresholds: list[float]
+    positive_likelihood_ratios: list[float]
+    prior_probability_pathogenicity: float
+
+    @root_validator
+    def validate_all_calibrations_have_a_pairwise_companion(cls, values):
+        num_es = len(values.get("evidence_strengths"))
+        num_st = len(values.get("thresholds"))
+        num_plr = len(values.get("positive_likelihood_ratios"))
+
+        if len(set((num_es, num_st, num_plr))) != 1:
+            raise ValidationError(
+                "Calibration object must provide the same number of evidence strenghts, score thresholds, and positive likelihood ratios. "
+                "One or more of these provided objects was not the same length as the others."
+            )
+
+        return values
+
+
+Calibration = Union[PillarProjectCalibration]

src/mavedb/view_models/score_set.py

@@ -15,6 +15,7 @@
 from mavedb.models.enums.processing_state import ProcessingState
 from mavedb.view_models import PublicationIdentifiersGetter, record_type_validator, set_record_type
 from mavedb.view_models.base.base import BaseModel, validator
+from mavedb.view_models.calibration import Calibration
 from mavedb.view_models.contributor import Contributor, ContributorCreate
 from mavedb.view_models.doi_identifier import (
     DoiIdentifier,
@@ -387,6 +388,7 @@ class SavedScoreSet(ScoreSetBase):
     external_links: Dict[str, ExternalLink]
     contributors: list[Contributor]
     score_ranges: Optional[ScoreRanges]
+    score_calibrations: Optional[dict[str, Calibration]]
 
     _record_type_factory = record_type_validator()(set_record_type)
 

tests/helpers/constants.py

@@ -658,10 +658,41 @@
     ],
 }
 
+
 TEST_SAVED_SCORESET_RANGE = {
     "wtScore": 1.0,
     "ranges": [
         {"label": "test1", "classification": "normal", "range": [0.0, 2.0]},
         {"label": "test2", "classification": "abnormal", "range": [-2.0, 0.0]},
     ],
 }
+
+
+TEST_SCORE_CALIBRATION = {
+    "parameter_sets": [
+        {
+            "functionally_altering": {"skew": 1.15, "location": -2.20, "scale": 1.20},
+            "functionally_normal": {"skew": -1.5, "location": 2.25, "scale": 0.8},
+            "fraction_functionally_altering": 0.20,
+        },
+    ],
+    "evidence_strengths": [3, 2, 1, -1],
+    "thresholds": [1.25, 2.5, 3, 5.5],
+    "positive_likelihood_ratios": [100, 10, 1, 0.1],
+    "prior_probability_pathogenicity": 0.20,
+}
+
+
+TEST_SAVED_SCORE_CALIBRATION = {
+    "parameterSets": [
+        {
+            "functionallyAltering": {"skew": 1.15, "location": -2.20, "scale": 1.20},
+            "functionallyNormal": {"skew": -1.5, "location": 2.25, "scale": 0.8},
+            "fractionFunctionallyAltering": 0.20,
+        },
+    ],
+    "evidenceStrengths": [3, 2, 1, -1],
+    "thresholds": [1.25, 2.5, 3, 5.5],
+    "positiveLikelihoodRatios": [100, 10, 1, 0.1],
+    "priorProbabilityPathogenicity": 0.20,
+}

tests/routers/test_score_set.py

@@ -34,6 +34,8 @@
     SAVED_EXTRA_CONTRIBUTOR,
     SAVED_PUBMED_PUBLICATION,
     SAVED_SHORT_EXTRA_LICENSE,
+    TEST_SCORE_CALIBRATION,
+    TEST_SAVED_SCORE_CALIBRATION,
 )
 from tests.helpers.dependency_overrider import DependencyOverrider
 from tests.helpers.util import (
@@ -1557,3 +1559,71 @@ def test_can_modify_metadata_for_score_set_with_inactive_license(session, client
     assert response.status_code == 200
     response_data = response.json()
     assert ("title", response_data["title"]) == ("title", "Update title")
+
+
+def test_anonymous_user_cannot_add_score_calibrations_to_score_set(client, setup_router_db, anonymous_app_overrides):
+    experiment = create_experiment(client)
+    score_set = create_seq_score_set(client, experiment["urn"])
+    calibration_payload = deepcopy(TEST_SCORE_CALIBRATION)
+
+    with DependencyOverrider(anonymous_app_overrides):
+        response = client.post(
+            f"/api/v1/score-sets/{score_set['urn']}/calibration/data", json={"test_calibrations": calibration_payload}
+        )
+        response_data = response.json()
+
+    assert response.status_code == 401
+    assert "score_calibrations" not in response_data
+
+
+def test_user_cannot_add_score_calibrations_to_own_score_set(client, setup_router_db, anonymous_app_overrides):
+    experiment = create_experiment(client)
+    score_set = create_seq_score_set(client, experiment["urn"])
+    calibration_payload = deepcopy(TEST_SCORE_CALIBRATION)
+
+    response = client.post(
+        f"/api/v1/score-sets/{score_set['urn']}/calibration/data", json={"test_calibrations": calibration_payload}
+    )
+    response_data = response.json()
+
+    assert response.status_code == 401
+    assert "score_calibrations" not in response_data
+
+
+def test_admin_can_add_score_calibrations_to_score_set(client, setup_router_db, admin_app_overrides):
+    experiment = create_experiment(client)
+    score_set = create_seq_score_set(client, experiment["urn"])
+    calibration_payload = deepcopy(TEST_SCORE_CALIBRATION)
+
+    with DependencyOverrider(admin_app_overrides):
+        response = client.post(
+            f"/api/v1/score-sets/{score_set['urn']}/calibration/data", json={"test_calibrations": calibration_payload}
+        )
+        response_data = response.json()
+
+    expected_response = update_expected_response_for_created_resources(
+        deepcopy(TEST_MINIMAL_SEQ_SCORESET_RESPONSE), experiment, score_set
+    )
+    expected_response["scoreCalibrations"] = {"test_calibrations": deepcopy(TEST_SAVED_SCORE_CALIBRATION)}
+
+    assert response.status_code == 200
+    for key in expected_response:
+        assert (key, expected_response[key]) == (key, response_data[key])
+
+
+def test_score_set_not_found_for_non_existent_score_set_when_adding_score_calibrations(
+    client, setup_router_db, admin_app_overrides
+):
+    experiment = create_experiment(client)
+    score_set = create_seq_score_set(client, experiment["urn"])
+    calibration_payload = deepcopy(TEST_SCORE_CALIBRATION)
+
+    with DependencyOverrider(admin_app_overrides):
+        response = client.post(
+            f"/api/v1/score-sets/{score_set['urn']+'xxx'}/calibration/data",
+            json={"test_calibrations": calibration_payload},
+        )
+        response_data = response.json()
+
+    assert response.status_code == 404
+    assert "score_calibrations" not in response_data