Merge branch 'release-2024.4.2' into bugfix/bencap/329/editable-dataset-metadata

Author: bencap

alembic/manual_migrations/refresh_published_tmp_urns.py

@@ -0,0 +1,33 @@
+import sqlalchemy as sa
+from sqlalchemy.orm import Session, configure_mappers
+
+from mavedb.models import *
+
+from mavedb.lib.score_sets import refresh_variant_urns
+
+from mavedb.models.score_set import ScoreSet
+from mavedb.models.variant import Variant
+
+from mavedb.db.session import SessionLocal
+
+configure_mappers()
+
+
+def do_migration(db: Session):
+    published_score_sets_with_associated_tmp_variants: sa.ScalarResult[str]
+    published_score_sets_with_associated_tmp_variants = db.execute(
+        sa.select(sa.distinct(ScoreSet.urn)).join(Variant).where(ScoreSet.published_date.is_not(None), Variant.urn.like("%tmp:%"))
+    ).scalars()
+
+    for score_set_urn in published_score_sets_with_associated_tmp_variants:
+        refresh_variant_urns(db, db.execute(sa.select(ScoreSet).where(ScoreSet.urn == score_set_urn)).scalar_one())
+
+
+if __name__ == "__main__":
+    db = SessionLocal()
+    db.current_user = None  # type: ignore
+
+    do_migration(db)
+
+    db.commit()
+    db.close()

poetry.lock

@@ -26,7 +26,7 @@ python = "^3.9"
 
 fqfa = "~1.3.0"
 pyhumps = "~3.8.0"
-pyyaml = "~5.1"
+pyyaml = "~6.0.1"
 IDUtils = "~1.2.0"
 mavehgvs = "~0.6.0"
 eutils = "~0.6.0"

pyproject.toml

@@ -99,7 +99,7 @@ def search_experiments(
             )
         )
 
-    items: list[Experiment] = query.order_by(Experiment.title).all()
+    items: list[Experiment] = query.order_by(Experiment.urn, Experiment.title).all()
     if not items:
         items = []
 

src/mavedb/lib/experiments.py

@@ -617,6 +617,21 @@ def create_variants(db, score_set: ScoreSet, variants_data: list[VariantData], b
     return len(score_set.variants)
 
 
+def refresh_variant_urns(db: Session, score_set: ScoreSet):
+    variants = db.execute(select(Variant).where(Variant.score_set_id == score_set.id)).scalars()
+
+    for variant in variants:
+        if not variant.urn:
+            raise ValueError("All variants should have an associated URN.")
+
+        variant_number = variant.urn.split("#")[1]
+        refreshed_urn = f"{score_set.urn}#{variant_number}"
+        variant.urn = refreshed_urn
+        db.add(variant)
+
+    db.commit()
+
+
 def bulk_create_urns(n, score_set, reset_counter=False) -> list[str]:
     start_value = 0 if reset_counter else score_set.num_variants
     parent_urn = score_set.urn

src/mavedb/lib/score_sets.py

@@ -39,6 +39,7 @@
 )
 from mavedb.lib.score_sets import (
     search_score_sets as _search_score_sets,
+    refresh_variant_urns,
 )
 from mavedb.lib.taxonomies import find_or_create_taxonomy
 from mavedb.lib.urns import (
@@ -333,6 +334,10 @@ async def create_score_set(
                 msg="Failed to create score set; The requested experiment does not exist.", extra=logging_context()
             )
             raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail="Unknown experiment")
+        # Not allow add score set in meta-analysis experiments.
+        if any(s.meta_analyzes_score_sets for s in experiment.score_sets):
+            raise HTTPException(status_code=status.HTTP_403_FORBIDDEN,
+                                detail="Score sets may not be added to a meta-analysis experiment.")
 
         save_to_logging_context({"experiment": experiment.urn})
         assert_permission(user_data, experiment, Action.ADD_SCORE_SET)
@@ -385,7 +390,7 @@ async def create_score_set(
             )
 
     if len(meta_analyzes_score_sets) > 0:
-        # If any existing score set is a meta-analysis for score sets in the same collection of exepriment sets, use its
+        # If any existing score set is a meta-analysis for score sets in the same collection of experiment sets, use its
         # experiment as the parent of our new meta-analysis. Otherwise, create a new experiment.
         meta_analyzes_experiment_sets = list(
             set(
@@ -1006,6 +1011,7 @@ def publish_score_set(
     item.urn = generate_score_set_urn(db, item.experiment)
     item.private = False
     item.published_date = published_date
+    refresh_variant_urns(db, item)
 
     save_to_logging_context({"score_set": item.urn})
 

src/mavedb/routers/score_sets.py

@@ -0,0 +1,4 @@
+hgvs_nt,hgvs_pro,c_0,c_1
+c.1A>T,p.Thr1Ser,10,20
+c.2C>T,p.Thr1Met,8,8
+c.6T>A,p.Phe2Leu,90,2

tests/routers/data/counts_utf8_encoded.csv

@@ -0,0 +1,4 @@
+hgvs_nt,hgvs_pro,score
+c.1A>T,p.Thr1Ser,0.3
+c.2C>T,p.Thr1Met,0
+c.6T>A,p.Phe2Leu,-1.65

tests/routers/data/scores_utf8_encoded.csv

@@ -7,11 +7,13 @@
 import pytest
 from arq import ArqRedis
 from humps import camelize
+from sqlalchemy import select
 
 from mavedb.lib.validation.urn_re import MAVEDB_TMP_URN_RE, MAVEDB_SCORE_SET_URN_RE, MAVEDB_EXPERIMENT_URN_RE
 from mavedb.models.enums.processing_state import ProcessingState
 from mavedb.models.experiment import Experiment as ExperimentDbModel
 from mavedb.models.score_set import ScoreSet as ScoreSetDbModel
+from mavedb.models.variant import Variant as VariantDbModel
 from mavedb.view_models.orcid import OrcidUser
 from mavedb.view_models.score_set import ScoreSet, ScoreSetCreate
 from tests.helpers.constants import (
@@ -531,6 +533,59 @@ def test_add_score_set_variants_scores_and_counts_endpoint(session, client, setu
     assert score_set == response_data
 
 
+def test_add_score_set_variants_scores_only_endpoint_utf8_encoded(client, setup_router_db, data_files):
+    experiment = create_experiment(client)
+    score_set = create_seq_score_set(client, experiment["urn"])
+    scores_csv_path = data_files / "scores_utf8_encoded.csv"
+    with (
+        open(scores_csv_path, "rb") as scores_file,
+        patch.object(ArqRedis, "enqueue_job", return_value=None) as queue,
+    ):
+        response = client.post(
+            f"/api/v1/score-sets/{score_set['urn']}/variants/data",
+            files={"scores_file": (scores_csv_path.name, scores_file, "text/csv")},
+        )
+        queue.assert_called_once()
+
+    assert response.status_code == 200
+    response_data = response.json()
+    jsonschema.validate(instance=response_data, schema=ScoreSet.schema())
+
+    # We test the worker process that actually adds the variant data separately. Here, we take it as
+    # fact that it would have succeeded.
+    score_set.update({"processingState": "processing"})
+    assert score_set == response_data
+
+
+def test_add_score_set_variants_scores_and_counts_endpoint_utf8_encoded(session, client, setup_router_db, data_files):
+    experiment = create_experiment(client)
+    score_set = create_seq_score_set(client, experiment["urn"])
+    scores_csv_path = data_files / "scores_utf8_encoded.csv"
+    counts_csv_path = data_files / "counts_utf8_encoded.csv"
+    with (
+        open(scores_csv_path, "rb") as scores_file,
+        open(counts_csv_path, "rb") as counts_file,
+        patch.object(ArqRedis, "enqueue_job", return_value=None) as queue,
+    ):
+        response = client.post(
+            f"/api/v1/score-sets/{score_set['urn']}/variants/data",
+            files={
+                "scores_file": (scores_csv_path.name, scores_file, "text/csv"),
+                "counts_file": (counts_csv_path.name, counts_file, "text/csv"),
+            },
+        )
+        queue.assert_called_once()
+
+    assert response.status_code == 200
+    response_data = response.json()
+    jsonschema.validate(instance=response_data, schema=ScoreSet.schema())
+
+    # We test the worker process that actually adds the variant data separately. Here, we take it as
+    # fact that it would have succeeded.
+    score_set.update({"processingState": "processing"})
+    assert score_set == response_data
+
+
 def test_cannot_add_scores_to_score_set_without_email(session, client, setup_router_db, data_files):
     experiment = create_experiment(client)
     score_set = create_seq_score_set(client, experiment["urn"])
@@ -788,6 +843,11 @@ def test_publish_score_set(session, data_provider, client, setup_router_db, data
     for key in expected_response:
         assert (key, expected_response[key]) == (key, score_set[key])
 
+    score_set_variants = session.execute(
+        select(VariantDbModel).join(ScoreSetDbModel).where(ScoreSetDbModel.urn == score_set["urn"])
+    ).scalars()
+    assert all([variant.urn.startswith("urn:mavedb:") for variant in score_set_variants])
+
 
 def test_publish_multiple_score_sets(session, data_provider, client, setup_router_db, data_files):
     experiment = create_experiment(client)
@@ -820,6 +880,19 @@ def test_publish_multiple_score_sets(session, data_provider, client, setup_route
     assert pub_score_set_3_data["title"] == score_set_3["title"]
     assert pub_score_set_3_data["experiment"]["urn"] == "urn:mavedb:00000001-a"
 
+    score_set_1_variants = session.execute(
+        select(VariantDbModel).join(ScoreSetDbModel).where(ScoreSetDbModel.urn == score_set_1["urn"])
+    ).scalars()
+    assert all([variant.urn.startswith("urn:mavedb:") for variant in score_set_1_variants])
+    score_set_2_variants = session.execute(
+        select(VariantDbModel).join(ScoreSetDbModel).where(ScoreSetDbModel.urn == score_set_2["urn"])
+    ).scalars()
+    assert all([variant.urn.startswith("urn:mavedb:") for variant in score_set_2_variants])
+    score_set_3_variants = session.execute(
+        select(VariantDbModel).join(ScoreSetDbModel).where(ScoreSetDbModel.urn == score_set_3["urn"])
+    ).scalars()
+    assert all([variant.urn.startswith("urn:mavedb:") for variant in score_set_3_variants])
+
 
 def test_cannot_publish_score_set_without_variants(client, setup_router_db):
     experiment = create_experiment(client)
@@ -917,6 +990,11 @@ def test_contributor_can_publish_other_users_score_set(session, data_provider, c
     for key in expected_response:
         assert (key, expected_response[key]) == (key, score_set[key])
 
+    score_set_variants = session.execute(
+        select(VariantDbModel).join(ScoreSetDbModel).where(ScoreSetDbModel.urn == score_set["urn"])
+    ).scalars()
+    assert all([variant.urn.startswith("urn:mavedb:") for variant in score_set_variants])
+
 
 def test_admin_cannot_publish_other_user_private_score_set(
     session, data_provider, client, admin_app_overrides, setup_router_db, data_files