Use published variants materialized view for statistics dashboard

Author: bencap

src/mavedb/routers/statistics.py

@@ -19,8 +19,8 @@
 )
 from mavedb.models.experiment_controlled_keyword import ExperimentControlledKeywordAssociation
 from mavedb.models.experiment_publication_identifier import ExperimentPublicationIdentifierAssociation
-from mavedb.models.mapped_variant import MappedVariant
 from mavedb.models.publication_identifier import PublicationIdentifier
+from mavedb.models.published_variant import PublishedVariantsMV
 from mavedb.models.raw_read_identifier import RawReadIdentifier
 from mavedb.models.refseq_identifier import RefseqIdentifier
 from mavedb.models.refseq_offset import RefseqOffset
@@ -37,7 +37,6 @@
 from mavedb.models.uniprot_identifier import UniprotIdentifier
 from mavedb.models.uniprot_offset import UniprotOffset
 from mavedb.models.user import User
-from mavedb.models.variant import Variant
 
 router = APIRouter(
     prefix="/api/v1/statistics",
@@ -484,9 +483,12 @@ def variant_counts(group: Optional[GroupBy] = None, db: Session = Depends(get_db
     Returns a dictionary of counts for the number of published and distinct variants in the database.
     Optionally, group the counts by the day on which the score set (and by extension, the variant) was published.
     """
-    query = _join_model_and_filter_unpublished(select(ScoreSet.published_date, func.count(Variant.id)), ScoreSet)
+    variants = db.execute(
+        select(PublishedVariantsMV.published_date, func.count(PublishedVariantsMV.variant_id))
+        .group_by(PublishedVariantsMV.published_date)
+        .order_by(PublishedVariantsMV.published_date)
+    ).all()
 
-    variants = db.execute(query.group_by(ScoreSet.published_date).order_by(ScoreSet.published_date)).all()
     if group == GroupBy.month:
         grouped = {k: sum(c for _, c in g) for k, g in itertools.groupby(variants, lambda t: t[0].strftime("%Y-%m"))}
     elif group == GroupBy.year:
@@ -506,17 +508,15 @@ def mapped_variant_counts(
     Optionally, group the counts by the day on which the score set (and by extension, the variant) was published.
     Optionally, return the count of all mapped variants, not just the current/most up to date ones.
     """
-    query = _join_model_and_filter_unpublished(
-        select(ScoreSet.published_date, func.count(MappedVariant.id)).join(
-            Variant, Variant.id == MappedVariant.variant_id
-        ),
-        ScoreSet,
-    )
+    query = select(PublishedVariantsMV.published_date, func.count(PublishedVariantsMV.mapped_variant_id))
 
     if onlyCurrent:
-        query = query.where(MappedVariant.current.is_(True))
+        query = query.where(PublishedVariantsMV.current_mapped_variant.is_(True))
+
+    variants = db.execute(
+        query.group_by(PublishedVariantsMV.published_date).order_by(PublishedVariantsMV.published_date)
+    ).all()
 
-    variants = db.execute(query.group_by(ScoreSet.published_date).order_by(ScoreSet.published_date)).all()
     if group == GroupBy.month:
         grouped = {k: sum(c for _, c in g) for k, g in itertools.groupby(variants, lambda t: t[0].strftime("%Y-%m"))}
     elif group == GroupBy.year:

tests/conftest.py

@@ -1,5 +1,5 @@
 import os
-import logging
+import logging  # noqa: F401
 import sys
 from concurrent import futures
 from inspect import getsourcefile
@@ -44,7 +44,7 @@
 @pytest.fixture()
 def session(postgresql):
     # Un-comment this line to log all database queries:
-    logging.getLogger("sqlalchemy.engine").setLevel(logging.INFO)
+    # logging.getLogger("sqlalchemy.engine").setLevel(logging.INFO)
 
     connection = (
         f"postgresql+psycopg2://{postgresql.info.user}:"

tests/routers/conftest.py

@@ -8,6 +8,7 @@
 from mavedb.models.controlled_keyword import ControlledKeyword
 from mavedb.models.contributor import Contributor
 from mavedb.models.enums.user_role import UserRole
+from mavedb.models.published_variant import PublishedVariantsMV
 from mavedb.models.license import License
 from mavedb.models.role import Role
 from mavedb.models.taxonomy import Taxonomy
@@ -80,6 +81,10 @@ def setup_seq_scoreset(setup_router_db, session, data_provider, client, data_fil
     create_mapped_variants_for_score_set(session, score_set["urn"])
     publish_score_set(client, score_set["urn"])
 
+    # Note that we have not created indexes for this view when it is generated via metadata. This differs
+    # from the database created via alembic, which does create indexes.
+    PublishedVariantsMV.refresh(session, False)
+
 
 @pytest.fixture
 def mock_publication_fetch(request, requests_mock):