Merge pull request #178 from VariantEffect/2.1.0-release

2.1.0 release

Author: afrubin

Dockerfile

@@ -19,7 +19,7 @@ ENV APP_USER=mavedb
 RUN useradd -m ${APP_USER}
 RUN apt-get update && apt-get -y upgrade
 RUN apt-get install -y build-essential
-RUN apt-get install -y git wget curl pandoc
+RUN apt-get install -y git wget curl pandoc graphviz
 
 # Local directory with project source
 ENV HOST_SRC=.
@@ -73,12 +73,28 @@ WORKDIR ${APP_SOURCE}
 
 COPY ${HOST_SRC} .
 
-# Build the Sphinx documentation
+# Build the Sphinx documentation for MaveDB
 WORKDIR ${APP_SOURCE}/docs
 RUN make html
 
-#WORKDIR ${APP_SOURCE}/docs/mavehgvs
-#RUN make html; exit 0
+# Build the Sphinx documentation for mavehgvs
+WORKDIR /tmp
+# get the correct mavehgvs version by tag
+RUN RAW_MAVE_HGVS_VERSION=`pip3 show mavehgvs | grep Version`; \
+    MAVE_HGVS_VERSION="v${RAW_MAVE_HGVS_VERSION##* }"; \
+    git clone --depth 1 --branch ${MAVE_HGVS_VERSION} https://github.com/VariantEffect/mavehgvs.git
+WORKDIR /tmp/mavehgvs/docs/
+RUN make html
+RUN mv _build ${APP_SOURCE}/docs/build/docs/mavehgvs
+RUN rm -rf /tmp/mavehgvs
+
+# Build the Sphinx documentation for MaveTools
+WORKDIR /tmp
+RUN git clone --depth 1 https://github.com/VariantEffect/mavetools.git
+WORKDIR /tmp/mavetools/docs/
+RUN make html
+RUN mv _build ${APP_SOURCE}/docs/build/docs/mavetools
+RUN rm -rf /tmp/mavetools
 
 WORKDIR ${APP_SOURCE}
 

accounts/templates/accounts/profile_settings.html

@@ -33,7 +33,7 @@ <h2> Profile settings
 
   <div class="api-access-wrapper">
     <div class="h1-api-access">
-      <h3>API Access
+      <h3>API access
         <a><i class="icon fas fa-key" style="font-size: 36px;"></i></a>
       </h3>
     </div>

api/constants.py

@@ -0,0 +1,3 @@
+# We should set up a periodic celery job to clear out files here
+# by oldest modified date or something later.
+BASE_RESULTS_DIR = '/tmp/results'

api/tasks.py

@@ -0,0 +1,44 @@
+import json
+import os
+from pathlib import Path
+import zipfile
+
+from django.db import transaction
+from django.contrib.auth import get_user_model
+
+from .constants import BASE_RESULTS_DIR
+from .utilities import format_variant_get_response
+from celery.utils.log import get_task_logger
+from core.tasks import BaseTask
+from mavedb import celery_app
+from variant.models import Variant
+
+User = get_user_model()
+logger = get_task_logger("api.tasks")
+
+@celery_app.task(ignore_result=False, base=BaseTask)
+def format_variant_large_get_response(results_uuid, variant_urn, offset, limit):
+    '''
+    For large responses, asynchronously call format_variant_get_response()
+    and save the output to a file for the user to access/download later.
+    '''
+    results_dir = BASE_RESULTS_DIR
+    Path(results_dir).mkdir(parents=True, exist_ok=True)
+
+    results_identifier = results_uuid
+    results_json_filepath = f'{results_dir}/{results_identifier}.json'
+    results = format_variant_get_response(variant_urn, offset, limit)
+    with open(results_json_filepath, 'w') as f:
+        json.dump(results, f)
+
+    # If we need more files like metadata or something, we can add them here.
+    files_to_zip = []
+    files_to_zip.append(results_json_filepath)
+
+    zip_filename = f'{BASE_RESULTS_DIR}/{results_identifier}.zip'
+    zip_file = zipfile.ZipFile(zip_filename, 'w')
+
+    for filepath in files_to_zip:
+        fdir, fname = os.path.split(filepath)
+        zip_file.write(filepath, fname)
+    zip_file.close()

api/tests/test_views.py

@@ -684,3 +684,183 @@ def test_can_download_metadata(self):
             ).content.decode()
         )
         self.assertEqual(response, scs.extra_metadata)
+
+class TestVariantAPIViews(TestCase):
+    factory = VariantFactory
+    url = "variants"
+    variants_base_url = "/api/variants"
+
+    def setUp(self):
+        Variant.objects.all().delete()
+
+    def tearDown(self):
+        Variant.objects.all().delete()
+
+    def test_raises_400_error_missing_params(self):
+        response = self.client.get(f"{self.variants_base_url}/NOT_A_REAL_URN")
+        self.assertEqual(response.status_code, 400)
+        variant = self.factory()
+        variant.save()
+        response = self.client.get(f"{self.variants_base_url}/{variant.urn}")
+        self.assertEqual(response.status_code, 400)
+
+    def test_raises_404_error_missing_variant(self):
+        variant_id = 1
+        response = self.client.get(
+            f"{self.variants_base_url}/NOT_A_REAL_URN?variant_id={variant_id}"
+        )
+        self.assertEqual(response.status_code, 404)
+
+    def test_can_get_response_for_variant(self):
+        variant = self.factory()
+        variant_urn = variant.urn
+        variant_id = 1
+        variant.urn = variant_urn + f'#{variant_id}'
+        variant.save()
+        response = self.client.get(
+            f"{self.variants_base_url}/{variant_urn}?variant_id={variant_id}"
+        )
+        self.assertEqual(response.status_code, 200)
+        response_json = json.loads(response.content.decode())
+        self.assertEqual(
+            len(response_json['experiment']['scoreset']['variants']), 1
+        )
+
+    def test_can_use_offset(self):
+        variant_fake_base_urn = 'tmp:abcdef123456'
+        variants_count = 5
+        offset = 3
+        vs = [self.factory() for _ in range(variants_count)]
+        for i in range(len(vs)):
+            # Overwrite the variant's urn so they all share a urn_prefix
+            v = vs[i]
+            v.urn = variant_fake_base_urn + f'#{i}'
+            v.save()
+        v = Variant.objects.get(urn=f'{variant_fake_base_urn}#0')
+        response = self.client.get(
+            f"{self.variants_base_url}/{variant_fake_base_urn}?variant_id=0" \
+            f"&offset={offset}"
+        )
+        response_json = json.loads(response.content.decode())
+        self.assertEqual(
+            len(response_json['experiment']['scoreset']['variants']),
+            variants_count-offset
+        )
+
+    def test_can_use_limit(self):
+        variant_fake_base_urn = 'tmp:abcdef123456'
+        variants_count = 5
+        limit = 3
+        vs = [self.factory() for _ in range(variants_count)]
+        for i in range(len(vs)):
+            v = vs[i]
+            v.urn = variant_fake_base_urn + f'#{i}'
+            v.save()
+
+        response = self.client.get(
+            f"{self.variants_base_url}/{variant_fake_base_urn}?variant_id=0" \
+            f"&limit={limit}"
+        )
+        response_json = json.loads(response.content.decode())
+        self.assertEqual(
+            len(response_json['experiment']['scoreset']['variants']), limit
+        )
+
+    def test_requested_variant_is_always_first(self):
+        """
+        The desired behavior is that the requested variant is always
+        returned, and is always returned first.
+        """
+        variant_fake_base_urn = 'tmp:abcdef123456'
+        variants_count = 5
+        vs = [self.factory() for _ in range(variants_count)]
+        for i in range(len(vs)):
+            v = vs[i]
+            v.urn = variant_fake_base_urn + f'#{i}'
+            v.save()
+
+        requested_variant_id = 1
+        # Make sure by default (no limit, no offset), the requested variant
+        # is returned first.
+        response = self.client.get(
+            f"{self.variants_base_url}/{variant_fake_base_urn}" \
+            f"?variant_id={requested_variant_id}"
+        )
+        response_json = json.loads(response.content.decode())
+        self.assertEqual(
+            response_json['experiment']['scoreset']['variants'][0]['urn'],
+            f'{variant_fake_base_urn}#{requested_variant_id}'
+        )
+
+        # For some representative combinations of limits and offsets, make sure
+        # the requested variant is still the first returned.
+        limits_and_offsets = [(0, 0), (0, 3), (2, 0), (5, 0), (5, 3)]
+        for limit_and_offset in limits_and_offsets:
+            limit = limit_and_offset[0]
+            offset = limit_and_offset[1]
+            response = self.client.get(
+                f"{self.variants_base_url}/{variant_fake_base_urn}" \
+                f"?variant_id={requested_variant_id}" \
+                f"&limit={limit}&offset={offset}"
+            )
+            response_json = json.loads(response.content.decode())
+            self.assertEqual(
+                response_json['experiment']['scoreset']['variants'][0]['urn'],
+                f'{variant_fake_base_urn}#{requested_variant_id}'
+            )
+
+    def test_can_get_large_response_for_variant(self):
+        variant = self.factory()
+        variant_urn = variant.urn
+        variant_id = 1
+        variant.urn = variant_urn + f'#{variant_id}'
+        variant.save()
+        response = self.client.get(
+            f"{self.variants_base_url}/{variant_urn}?variant_id={variant_id}" \
+            "&limit=51"
+        )
+        self.assertEqual(response.status_code, 202)
+        response_json = json.loads(response.content.decode())
+        self.assertTrue('results_uuid' in response_json)
+
+        response = self.client.get(
+            f"{self.variants_base_url}/{variant_urn}?variant_id={variant_id}" \
+            "&limit=-1"
+        )
+        self.assertEqual(response.status_code, 202)
+        response_json = json.loads(response.content.decode())
+        self.assertTrue('results_uuid' in response_json)
+
+class TestResultsAPIViews(TestCase):
+    factory = VariantFactory
+    url = "results"
+    variants_base_url = "/api/variants"
+    results_base_url = "/api/results"
+
+    def setUp(self):
+        Variant.objects.all().delete()
+
+    def tearDown(self):
+        Variant.objects.all().delete()
+
+    def test_raises_404_missing_results_uuid(self):
+        response = self.client.get(
+            f"{self.results_base_url}/NOT_A_REAL_UUID"
+        )
+        self.assertEqual(response.status_code, 404)
+
+    def test_can_get_zipfile(self):
+        variant = self.factory()
+        variant_urn = variant.urn
+        variant_id = 1
+        variant.urn = variant_urn + f'#{variant_id}'
+        variant.save()
+        response = self.client.get(
+            f"{self.variants_base_url}/{variant_urn}?variant_id={variant_id}" \
+            "&limit=51"
+        )
+        response_json = json.loads(response.content.decode())
+        results_uuid = response_json['results_uuid']
+
+        response = self.client.get(f"{self.results_base_url}/{results_uuid}")
+        self.assertEqual(response.status_code, 200)

api/urls.py

@@ -22,7 +22,12 @@
 router.register("target", views.TargetGeneViewSet)
 router.register("reference", views.ReferenceGenomeViewSet)
 
-scoreset_urls = [
+urlpatterns = [
+    url(
+        r"^results/(?P<results_uuid>.*)$",
+        views.ResultsView.as_view(),
+        name="results",
+    ),
     url(
         r"^scoresets/(?P<urn>{})/scores/$".format(scoreset_url_pattern),
         views.scoreset_score_data,
@@ -38,7 +43,11 @@
         views.scoreset_metadata,
         name="api_download_metadata",
     ),
+    url(
+        r"^variants/(?P<urn>.*)$",
+        views.VariantView.as_view(),
+        name="variant",
+    ),
 ]
 
-
-urlpatterns = router.urls + scoreset_urls
+urlpatterns += router.urls