Merge pull request #1486 from ImagingDataCommons/release-48-sp

#1409

Author: s-paquette

idc/demo_views.py

@@ -54,7 +54,7 @@ def cohort_detail(request, cohort_id):
     if debug: logger.debug('Called {}'.format(sys._getframe().f_code.co_name))
 
     try:
-        req = request.GET if request.GET else request.POST
+        req = request.GET if request.method == 'GET' else request.POST
         is_dicofdic = (req.get('is_dicofdic', "False").lower() == "true")
         source = req.get('data_source_type', DataSource.SOLR)
         fields = json.loads(req.get('fields', '[]'))

idc/urls.py

@@ -60,11 +60,11 @@
     re_path(r'^cart/$', views.cart_page, name='cart'),
     re_path(r'^explore/cart/$', views.cart_page, name='get_explore_cart'),
     re_path(r'^cart_data/$', views.cart_data, name='get_cart_data'),
-    re_path(r'^cart_data_stats/$', views.cart_data_stats, name='get_cart_data_stats'),
     re_path(r'^series_ids/(?P<patient_id>[A-Za-z0-9\.\-_]+)/$', views.get_series, name='get_series_by_case'),
     re_path(r'^series_ids/(?P<patient_id>[A-Za-z0-9\.\-_]+)/(?P<study_uid>[0-9\.]+)/$', views.get_series, name='get_series'),
     re_path(r'^collaborators/$', views.collaborators, name='collaborators'),
     re_path(r'^collections/', include('idc_collections.urls')),
+    re_path(r'^citations/', views.get_citations, name='get_citations'),
     # re_path(r'^share/', include('sharing.urls')),
 ]
 

idc/views.py

@@ -32,11 +32,12 @@
 from django.urls import reverse
 from django.contrib import messages
 from django.utils.html import escape
+from django.utils.encoding import uri_to_iri
 
 from google_helpers.stackdriver import StackDriverLogger
 from cohorts.models import Cohort, Cohort_Perms
 
-from idc_collections.models import Program, DataSource, Collection, ImagingDataCommonsVersion, Attribute, Attribute_Tooltips, DataSetType
+from idc_collections.models import Program, DataSource, Collection, ImagingDataCommonsVersion, Attribute, Attribute_Tooltips, DataSetType, Citation
 from idc_collections.collex_metadata_utils import build_explorer_context, get_collex_metadata, create_file_manifest, get_cart_data_serieslvl, get_cart_data_studylvl, get_table_data_with_cart_data
 from allauth.socialaccount.models import SocialAccount
 from django.core.exceptions import ObjectDoesNotExist
@@ -366,6 +367,27 @@ def populate_tables(request):
     return JsonResponse(response, status=status)
 
 
+def get_citations(request):
+    resp = { 'message': 'error', 'citations': None}
+    try:
+        req = request.GET if request.method == 'GET' else request.POST
+        if request.method == 'GET':
+            dois = [uri_to_iri(x) for x in req.getlist("doi", [])]
+        else:
+            body_unicode = request.body.decode('utf-8')
+            body = json.loads(body_unicode)
+            dois = body.get("doi", [])
+        cites = Citation.objects.filter(doi__in=dois)
+        resp['citations'] = {x.doi: x.cite for x in cites}
+        code = 200
+    except Exception as e:
+        logger.error("[ERROR] While fetching citations: ")
+        logger.exception(e)
+        resp['message'] = "There was an error while attempting to fetch these citations."
+        code = 500
+    return JsonResponse(resp, status=code)
+
+
 # Data exploration and cohort creation page
 def explore_data_page(request, filter_path=False, path_filters=None):
     context = {'request': request}
@@ -377,8 +399,7 @@ def explore_data_page(request, filter_path=False, path_filters=None):
         request.session.create()
 
     try:
-        req = request.GET or request.POST
-
+        req = request.GET if request.method == 'GET' else request.POST
         is_dicofdic = (req.get('is_dicofdic', "False").lower() == "true")
         source = req.get('data_source_type', DataSource.SOLR)
         versions = json.loads(req.get('versions', '[]'))
@@ -428,6 +449,7 @@ def explore_data_page(request, filter_path=False, path_filters=None):
             collapse_on, is_json, uniques=uniques, totals=totals, with_stats=with_stats, disk_size=disk_size
         )
 
+        print(context.keys())
         if not('totals' in context):
           context['totals']={}
         if not('PatientID' in context['totals']):
@@ -614,39 +636,10 @@ def cart_page(request):
     return render(request, 'collections/cart_list.html', context)
 
 
-def cart_data_stats(request):
-    status = 200
-    response = {}
-    field_list = ['collection_id', 'PatientID', 'StudyInstanceUID', 'SeriesInstanceUID', 'aws_bucket']
-    try:
-
-        req = request.GET if request.GET else request.POST
-        current_filters = json.loads(req.get('filters', '{}'))
-        filtergrp_list = json.loads(req.get('filtergrp_list', '{}'))
-        aggregate_level = req.get('aggregate_level', 'StudyInstanceUID')
-        results_level = req.get('results_level', 'StudyInstanceUID')
-
-        partitions = json.loads(req.get('partitions', '{}'))
-
-        limit = int(req.get('limit', 1000))
-        offset = int(req.get('offset', 0))
-        length = int(req.get('length', 100))
-        mxseries = int(req.get('mxseries',1000))
-
-        response = get_cart_and_filterset_stats(current_filters,filtergrp_list, partitions, limit, offset, length, mxseries, results_lvl=results_level)
-
-    except Exception as e:
-        logger.error("[ERROR] While loading cart:")
-        logger.exception(e)
-        status = 400
-
-    return JsonResponse(response, status=status)
-
-
 def cart_data(request):
     status = 200
     response = {}
-    field_list = ['collection_id', 'PatientID', 'StudyInstanceUID', 'SeriesInstanceUID', 'aws_bucket']
+    field_list = ['collection_id', 'PatientID', 'StudyInstanceUID', 'SeriesInstanceUID', 'aws_bucket', "source_DOI"]
     try:
         req = request.GET if request.GET else request.POST
         filtergrp_list = json.loads(req.get('filtergrp_list', '{}'))

static/js/citations_modal.js

@@ -66,7 +66,7 @@ require([
 
     A11y.Core();
 
-    var downloadToken = new Date().getTime();
+    let csrftoken = $.getCookie('csrftoken')
 
     $('#citations-modal').on('show.bs.modal', async function(event) {
         let button = $(event.relatedTarget);
@@ -76,36 +76,40 @@ require([
         cites_list.html(`
             Formatting citation(s)... <i class="fa fa-compass fa-spin"></i>
         `);
-        let citations = [];
-        await Promise.all(dois.map(async function(cite){
-            if(!DOI_CACHE[cite]) {
-                let response = await fetch(`https://doi.org/${cite}`, {
-                    headers: {
-                        "Accept": "text/x-bibliography; style=elsevier-vancouver-no-et-al"
-                    }
+        let dois_to_get = dois.filter((d) => (DOI_CACHE[d] === null || DOI_CACHE[d] === undefined));
+        if(dois_to_get.length > 0) {
+            let resp = null;
+            if(dois_to_get.join(",").length > 2048) {
+                resp = await fetch(`${BASE_URL}/citations/`, {
+                    method: "POST",
+                    body: JSON.stringify({
+                        'doi': dois_to_get
+                    }),
+                    headers: {"X-CSRFToken": csrftoken},
+                    "content-type": "application/json"
                 });
-                if (!response.ok) {
-                    citations.push(`Encountered an error requesting DOI ${cite}`);
-                } else {
-                    DOI_CACHE[cite] = await response.text();
-                }
+            } else {
+                let encoded_dois = dois_to_get.map(d => `doi=${encodeURIComponent(d)}`);
+                resp = await fetch(`${BASE_URL}/citations/?${encoded_dois.join("&")}`);
             }
-            citations.push(DOI_CACHE[cite]);
-        }));
+            if(!resp.ok) {
+                cites_list.html("Failed to retrieve citations!");
+                throw new Error("Failed to retrieve citations!");
+            }
+            let new_cites = await resp.json();
+            DOI_CACHE = {
+                ...new_cites['citations'],
+                ...DOI_CACHE
+            };
+        }
+        let citations = dois.map(d => DOI_CACHE[d]);
         cites_list.html(citations.join("\n\n"));
         copy_cites.attr('content', citations.join("\n\n"));
     });
 
     $('#citations-modal').on('hide.bs.modal', function() {
-
-    });
-
-    $('#export-manifest-modal').on('hidden.bs.modal', function() {
         $(".citations-list").empty();
-    });
-
-    $('.copy-this').on('click', function(e) {
-        // copy the entire set
+        $("#citations-modal .copy-this").attr('content', "");
     });
 
     tippy.delegate('#citations-modal', {

static/js/filterutils.js

@@ -73,7 +73,8 @@ define(['jquery', 'base'], function($, base) {
                 }
             })
             $('.bq-string, .citations-list').html("");
-            $('.bq-string-copy, .citations-button').attr('filter-params', "");
+            $('.bq-string-copy').attr('filter-params', "");
+            $('.citations-button').attr('data-dois', "");
             $('.hide-filter-uri').triggerHandler('click');
             $('.url-too-long').hide();
             $('#export-manifest-form').attr(
@@ -92,8 +93,7 @@ define(['jquery', 'base'], function($, base) {
                     $(this).removeAttr("disabled");
                 }
             }));
-            $('.bq-string-display').attr('filter-params', JSON.stringify(filters));
-            $('.bq-string-copy, .citations-button').attr('filter-params', JSON.stringify(filters));
+            $('.bq-string-display, .bq-string-copy').attr('filter-params', JSON.stringify(filters));
             $('#export-manifest-form input[name="filters"]').val(JSON.stringify(filters));
             $('#export-manifest').removeAttr("data-no-filters");
             let url = BASE_URL+"/explore/filters/?";

static/js/image_search.js

@@ -176,6 +176,11 @@ require([
                     $('input[name="async_download"]').val(
                         async_download ? "True" : "False"
                     );
+                    if('dois' in data) {
+                        $('.citations-button').attr("data-dois", Object.keys(data['dois']).join("||"));
+                    } else {
+                        $('.citations-button').attr("data-dois", "");
+                    }
                     if (('filtered_counts' in data) && ('origin_set' in data['filtered_counts']) &&
                         ('access' in data['filtered_counts']['origin_set']['All']['attributes']) &&
                         ('Limited' in data['filtered_counts']['origin_set']['All']['attributes']['access']) &&