Merge pull request #1487 from ImagingDataCommons/idc-prod-sp

v22 / r48

Author: s-paquette

README.md

@@ -21,7 +21,9 @@ From there perform the following steps...
 
 ## Configuring PyCharm
 
-PyCharm Pro can be used to run your Web Application as a native Django application.
+PyCharm Pro can be used to run your Web Application as a native Django application. 
+
+**NOTE:** PyCharm 2025 versions no longer include support for remote Python interpreters. Use 2024 or older!
 
 ### Setup
 
@@ -61,6 +63,7 @@ To run your server in PyCharm:
   * Kernel header update: `sudo apt-get -y install dkms build-essential linux-headers-$(uname -r)`
     * NOTE: you may get a 'package not found' error here; if so, you'll need to look up the current header package for this install and use that instead.
   * Guest Additions ISO mounting and installation: https://docs.bitnami.com/virtual-machine/faq/configuration/install-virtualbox-guest-additions/
+  * NOTE: in newer versions of VirtualBox the CD/DVD drives are added under Basic>Storage
  3. Next, set the `shell/python-su.sh` script to executable in the vagrant machine's command line with the command `chmod +x /home/vagrant/www/shell/python-su.sh`
  4. You can now click on the Run or Debug icons in the toolbar (upper-right corner of the PyCharm GUI)
   * Your server will start and the PyCharm console should show all the logs and output from the system. 

etl/etl.py

@@ -42,7 +42,7 @@
 
 from idc_collections.models import Program, Collection, Attribute, Attribute_Ranges, \
     Attribute_Display_Values, DataSource, DataSourceJoin, DataVersion, DataSetType, \
-    Attribute_Set_Type, Attribute_Display_Category, ImagingDataCommonsVersion, Attribute_Tooltips
+    Attribute_Set_Type, Attribute_Display_Category, ImagingDataCommonsVersion, Attribute_Tooltips, Citation
 from google_helpers.bigquery.bq_support import BigQuerySupport
 
 from django.contrib.auth.models import User
@@ -319,6 +319,35 @@ def add_source_joins(froms, from_col, tos=None, to_col=None):
         DataSourceJoin.objects.bulk_create(src_joins)
 
 
+def load_citations(filename):
+    try:
+        cites_file = open(filename,"r")
+        current_cites = [x.doi for x in Citation.objects.all()]
+        new_cites = []
+        updated_cites = {}
+        for line in csv_reader(cites_file):
+            if "doi, citation" in line:
+                print("[STATUS] Saw header line during citation load - skipping!")
+                continue
+            if line[0] in current_cites:
+                updated_cites[line[0]] = line[1]
+            else:
+                new_cites.append(Citation(doi=line[0], cite=line[1]))
+        if len(new_cites):
+            Citation.objects.bulk_create(new_cites)
+            print("[STATUS] The following {} DOI citations were added: {}".format(len(new_cites), "  ".join([x.doi for x in new_cites])))
+        if len(updated_cites):
+            to_update = Citation.objects.filter(doi__in=updated_cites.keys())
+            for upd in to_update:
+                upd.cite = updated_cites[upd.doi]
+            Citation.objects.bulk_update(to_update, ["cite"])
+            print("[STATUS] {} DOI citations were updated.".format(len(updated_cites)))
+    except Exception as e:
+        ERRORS_SEEN.append("Error seen while loading citations, check the logs!")
+        logger.error("[ERROR] While trying to load citations: ")
+        logger.exception(e)
+
+
 def load_collections(filename, data_version="8.0"):
     try:
         collection_file = open(filename, "r")
@@ -428,12 +457,12 @@ def create_solr_params(schema_src, solr_src):
     solr_index_strings = []
     field_types = ''
     add_copy_field = ''
-    SCHEMA_BASE = '{{field_types}add-field": {fields}{add_copy_field}'
+    SCHEMA_BASE = '{{{field_types}"add-field": {fields}{add_copy_field}}}'
     if len(TOKENIZED_FIELDS):
         field_types = '"add-field-type": { "name":"tokenizedText", "class":"solr.TextField", "analyzer" : { "tokenizer": { "name":"nGram" }}}, '
-        copy_fields = ",".join(['{"source":"{field}","dest":"{field{}_tokenized"}'.format(field) for field in TOKENIZED_FIELDS])
-        add_copy_field = ', "add-copy-field": [{copy_fields}]'.format(copy_fields)
-    CORE_CREATE_STRING = "sudo -u solr /opt/bitnami/solr/bin/solr create -c {solr_src} -s 2 -rf 2"
+        copy_fields = ",".join(['{{"source":"{field}","dest":"{field}_tokenized"}}'.format(field=field) for field in TOKENIZED_FIELDS])
+        add_copy_field = ', "add-copy-field": [{copy_fields}]'.format(copy_fields=copy_fields)
+    CORE_CREATE_STRING = "sudo -u solr /opt/bitnami/solr/bin/solr create -c {solr_src}"
     SCHEMA_STRING = "curl -u {solr_user}:{solr_pwd} -X POST -H 'Content-type:application/json' --data-binary '{schema}' https://localhost:8983/solr/{solr_src}/schema --cacert solr-ssl.pem"
     INDEX_STRING = "curl -u {solr_user}:{solr_pwd} -X POST 'https://localhost:8983/solr/{solr_src}/update?commit=yes{params}' --data-binary @{file_name}.csv -H 'Content-type:application/csv' --cacert solr-ssl.pem"
     for field in schema:
@@ -463,7 +492,8 @@ def create_solr_params(schema_src, solr_src):
     with open("{}_solr_cmds.txt".format(solr_src.name), "w") as cmd_outfile:
         schema_array = SCHEMA_BASE.format(
             field_types=field_types,
-            add_copy_field=add_copy_field
+            add_copy_field=add_copy_field,
+            fields=solr_schema
         )
         params = "&{}".format("&".join(solr_index_strings))
         cmd_outfile.write(CORE_CREATE_STRING.format(solr_src=solr_src.name))
@@ -780,7 +810,8 @@ def parse_args():
 
     parser = ArgumentParser()
     parser.add_argument('-j', '--config-file', type=str, default='', help='JSON file of version data to update')
-    parser.add_argument('-c', '--collex-file', type=str, default='', help='CSV data of collections to update/create')
+    parser.add_argument('-c', '--collex-file', type=str, default='', help='CSV data of citations to update/create')
+    parser.add_argument('-i', '--cites-file', type=str, default='', help='CSV data of collections to update/create')
     parser.add_argument('-d', '--display-vals', type=str, default='', help='CSV data of display values to add/update')
     parser.add_argument('-p', '--programs-file', type=str, default='', help='CSV data of programs to add/update')
     parser.add_argument('-a', '--attributes-file', type=str, default='', help='CSV data of attributes to add/update')
@@ -820,6 +851,8 @@ def main():
         len(args.programs_file) and load_programs(args.programs_file)
         # Add/update collections - any new programs must be added first
         len(args.collex_file) and load_collections(args.collex_file)
+        # Add/update any citations
+        len(args.cites_file) and load_citations(args.cites_file)
         # Add/update display values for attributes
         if len(args.display_vals):
             dvals = load_display_vals(args.display_vals)

idc/demo_views.py

@@ -54,7 +54,7 @@ def cohort_detail(request, cohort_id):
     if debug: logger.debug('Called {}'.format(sys._getframe().f_code.co_name))
 
     try:
-        req = request.GET if request.GET else request.POST
+        req = request.GET if request.method == 'GET' else request.POST
         is_dicofdic = (req.get('is_dicofdic', "False").lower() == "true")
         source = req.get('data_source_type', DataSource.SOLR)
         fields = json.loads(req.get('fields', '[]'))

idc/urls.py

@@ -30,13 +30,13 @@
 urlpatterns = [
 
     re_path(r'^$', views.landing_page, name='landing_page'),
-    re_path(r'^quota/', views.quota_page, name='quota_page'),
+    re_path(r'^quota/$', views.quota_page, name='quota_page'),
     re_path(r'^users/(?P<user_id>\d+)/$', views.user_detail, name='user_detail'),
     re_path(r'^users/api/', views_api.user_detail, name='user_detail_api'),
 
     re_path(r'^cohort_detail/(?P<cohort_id>\d+)/$', demo_views.cohort_detail, name='cohort_detail'),
-    re_path(r'robots.txt', TemplateView.as_view(template_name="robots.txt", content_type="text/plain"), name='robots'),
-    re_path(r'sitemap.xml', TemplateView.as_view(template_name="sitemap.xml", content_type="text/xml"), name='sitemap'),
+    re_path(r'robots.txt$', TemplateView.as_view(template_name="robots.txt", content_type="text/plain"), name='robots'),
+    re_path(r'sitemap.xml$', TemplateView.as_view(template_name="sitemap.xml", content_type="text/xml"), name='sitemap'),
 
     re_path(r'^cohorts/', include('cohorts.urls')),
     path('admin/', admin.site.urls),
@@ -52,19 +52,19 @@
     re_path(r'^tables/', views.populate_tables, name='populate_tables'),
     re_path(r'^studymp/', views.studymp, name='studymp'),
     re_path(r'^warning/', views.warn_page, name='warn'),
-    re_path(r'^about/', views.about_page, name='about_page'),
+    re_path(r'^about/$', views.about_page, name='about_page'),
     re_path(r'^dashboard/', views.dashboard_page, name='dashboard'),
     re_path(r'^extended_login/$', views.extended_login_view, name='extended_login'),
-    re_path(r'^privacy/', views.privacy_policy, name='privacy'),
-    re_path(r'^news/', views.news_page, name='news'),
+    re_path(r'^privacy/$', views.privacy_policy, name='privacy'),
+    re_path(r'^news/$', views.news_page, name='news'),
     re_path(r'^cart/$', views.cart_page, name='cart'),
     re_path(r'^explore/cart/$', views.cart_page, name='get_explore_cart'),
     re_path(r'^cart_data/$', views.cart_data, name='get_cart_data'),
-    re_path(r'^cart_data_stats/$', views.cart_data_stats, name='get_cart_data_stats'),
     re_path(r'^series_ids/(?P<patient_id>[A-Za-z0-9\.\-_]+)/$', views.get_series, name='get_series_by_case'),
     re_path(r'^series_ids/(?P<patient_id>[A-Za-z0-9\.\-_]+)/(?P<study_uid>[0-9\.]+)/$', views.get_series, name='get_series'),
-    re_path(r'^collaborators/', views.collaborators, name='collaborators'),
+    re_path(r'^collaborators/$', views.collaborators, name='collaborators'),
     re_path(r'^collections/', include('idc_collections.urls')),
+    re_path(r'^citations/', views.get_citations, name='get_citations'),
     # re_path(r'^share/', include('sharing.urls')),
 ]
 

idc/views.py

@@ -32,11 +32,12 @@
 from django.urls import reverse
 from django.contrib import messages
 from django.utils.html import escape
+from django.utils.encoding import uri_to_iri
 
 from google_helpers.stackdriver import StackDriverLogger
 from cohorts.models import Cohort, Cohort_Perms
 
-from idc_collections.models import Program, DataSource, Collection, ImagingDataCommonsVersion, Attribute, Attribute_Tooltips, DataSetType
+from idc_collections.models import Program, DataSource, Collection, ImagingDataCommonsVersion, Attribute, Attribute_Tooltips, DataSetType, Citation
 from idc_collections.collex_metadata_utils import build_explorer_context, get_collex_metadata, create_file_manifest, get_cart_data_serieslvl, get_cart_data_studylvl, get_table_data_with_cart_data
 from allauth.socialaccount.models import SocialAccount
 from django.core.exceptions import ObjectDoesNotExist
@@ -76,7 +77,8 @@ def landing_page(request):
         "Testicles": "Testis",
         "Adrenal Glands": "Adrenal Gland",
         "Adrenal": "Adrenal Gland",
-        "Lymph Node": "Lymph Nodes"
+        "Lymph Node": "Lymph Nodes",
+        "Bone Marrow": "Blood"
     }
 
     skip = [
@@ -91,7 +93,9 @@ def landing_page(request):
         "Chest-Abdomen-Pelvis, Leg, TSpine",
         "Abdomen, Arm, Bladder, Chest, Head-Neck, Kidney, Leg, Retroperitoneum, Stomach, Uterus",
         "Blood, Bone",
-        "Esophagus, Lung, Pancreas, Thymus"
+        "Esophagus, Lung, Pancreas, Thymus",
+        "Esophagus, Head-Neck, Lung, Pancreas, Rectum, Thymus",
+        "Arm, Bladder, Buttock, Colon, Liver, Myometrium, Pancreas, Rectum, Shoulder, Scapula"
     ]
 
     for collection in collex:
@@ -363,6 +367,27 @@ def populate_tables(request):
     return JsonResponse(response, status=status)
 
 
+def get_citations(request):
+    resp = { 'message': 'error', 'citations': None}
+    try:
+        req = request.GET if request.method == 'GET' else request.POST
+        if request.method == 'GET':
+            dois = [uri_to_iri(x) for x in req.getlist("doi", [])]
+        else:
+            body_unicode = request.body.decode('utf-8')
+            body = json.loads(body_unicode)
+            dois = body.get("doi", [])
+        cites = Citation.objects.filter(doi__in=dois)
+        resp['citations'] = {x.doi: x.cite for x in cites}
+        code = 200
+    except Exception as e:
+        logger.error("[ERROR] While fetching citations: ")
+        logger.exception(e)
+        resp['message'] = "There was an error while attempting to fetch these citations."
+        code = 500
+    return JsonResponse(resp, status=code)
+
+
 # Data exploration and cohort creation page
 def explore_data_page(request, filter_path=False, path_filters=None):
     context = {'request': request}
@@ -374,8 +399,7 @@ def explore_data_page(request, filter_path=False, path_filters=None):
         request.session.create()
 
     try:
-        req = request.GET or request.POST
-
+        req = request.GET if request.method == 'GET' else request.POST
         is_dicofdic = (req.get('is_dicofdic', "False").lower() == "true")
         source = req.get('data_source_type', DataSource.SOLR)
         versions = json.loads(req.get('versions', '[]'))
@@ -425,6 +449,7 @@ def explore_data_page(request, filter_path=False, path_filters=None):
             collapse_on, is_json, uniques=uniques, totals=totals, with_stats=with_stats, disk_size=disk_size
         )
 
+        print(context.keys())
         if not('totals' in context):
           context['totals']={}
         if not('PatientID' in context['totals']):
@@ -611,39 +636,10 @@ def cart_page(request):
     return render(request, 'collections/cart_list.html', context)
 
 
-def cart_data_stats(request):
-    status = 200
-    response = {}
-    field_list = ['collection_id', 'PatientID', 'StudyInstanceUID', 'SeriesInstanceUID', 'aws_bucket']
-    try:
-
-        req = request.GET if request.GET else request.POST
-        current_filters = json.loads(req.get('filters', '{}'))
-        filtergrp_list = json.loads(req.get('filtergrp_list', '{}'))
-        aggregate_level = req.get('aggregate_level', 'StudyInstanceUID')
-        results_level = req.get('results_level', 'StudyInstanceUID')
-
-        partitions = json.loads(req.get('partitions', '{}'))
-
-        limit = int(req.get('limit', 1000))
-        offset = int(req.get('offset', 0))
-        length = int(req.get('length', 100))
-        mxseries = int(req.get('mxseries',1000))
-
-        response = get_cart_and_filterset_stats(current_filters,filtergrp_list, partitions, limit, offset, length, mxseries, results_lvl=results_level)
-
-    except Exception as e:
-        logger.error("[ERROR] While loading cart:")
-        logger.exception(e)
-        status = 400
-
-    return JsonResponse(response, status=status)
-
-
 def cart_data(request):
     status = 200
     response = {}
-    field_list = ['collection_id', 'PatientID', 'StudyInstanceUID', 'SeriesInstanceUID', 'aws_bucket']
+    field_list = ['collection_id', 'PatientID', 'StudyInstanceUID', 'SeriesInstanceUID', 'aws_bucket', "source_DOI"]
     try:
         req = request.GET if request.GET else request.POST
         filtergrp_list = json.loads(req.get('filtergrp_list', '{}'))

requirements.txt

@@ -1,5 +1,5 @@
 cryptography==44.0.2
-django==4.2.20
+django==4.2.24
 django-allauth==0.63.1
 django-anymail[mailgun]
 django-axes==5.40.0
@@ -23,7 +23,7 @@ pyjwt==2.4.0
 pyopenssl==25.0.0
 pytz==2025.2
 PyYAML==6.0.2
-requests==2.32.3
+requests==2.32.4
 requests-oauthlib==0.7.0
 rsa==4.7
 simplejson==3.20.1

shell/vagrant-set-env.sh

@@ -4,3 +4,6 @@ echo 'export SECURE_LOCAL_PATH=../parentDir/secure_files/idc/' | tee -a /home/va
 echo 'export DJANGO_SETTINGS_MODULE=idc.settings' | tee -a /home/vagrant/.bash_profile
 source /home/vagrant/.bash_profile
 chmod +x /home/vagrant/www/shell/python-su.sh
+if [[ ! -d /vagrant ]]; then
+  sudo ln -s /home/vagrant/www /vagrant
+fi

static/css/style.css

@@ -2555,6 +2555,7 @@ pre.bq-string {
 pre.citations-list {
   white-space: pre-wrap;
   word-break: break-word;
+  max-height: 600px;
 }
 
 .bq-string-display img {

static/js/citations_modal.js

@@ -66,7 +66,7 @@ require([
 
     A11y.Core();
 
-    var downloadToken = new Date().getTime();
+    let csrftoken = $.getCookie('csrftoken')
 
     $('#citations-modal').on('show.bs.modal', async function(event) {
         let button = $(event.relatedTarget);
@@ -76,36 +76,40 @@ require([
         cites_list.html(`
             Formatting citation(s)... <i class="fa fa-compass fa-spin"></i>
         `);
-        let citations = [];
-        await Promise.all(dois.map(async function(cite){
-            if(!DOI_CACHE[cite]) {
-                let response = await fetch(`https://doi.org/${cite}`, {
-                    headers: {
-                        "Accept": "text/x-bibliography; style=elsevier-vancouver-no-et-al"
-                    }
+        let dois_to_get = dois.filter((d) => (DOI_CACHE[d] === null || DOI_CACHE[d] === undefined));
+        if(dois_to_get.length > 0) {
+            let resp = null;
+            if(dois_to_get.join(",").length > 2048) {
+                resp = await fetch(`${BASE_URL}/citations/`, {
+                    method: "POST",
+                    body: JSON.stringify({
+                        'doi': dois_to_get
+                    }),
+                    headers: {"X-CSRFToken": csrftoken},
+                    "content-type": "application/json"
                 });
-                if (!response.ok) {
-                    citations.push(`Encountered an error requesting DOI ${cite}`);
-                } else {
-                    DOI_CACHE[cite] = await response.text();
-                }
+            } else {
+                let encoded_dois = dois_to_get.map(d => `doi=${encodeURIComponent(d)}`);
+                resp = await fetch(`${BASE_URL}/citations/?${encoded_dois.join("&")}`);
             }
-            citations.push(DOI_CACHE[cite]);
-        }));
+            if(!resp.ok) {
+                cites_list.html("Failed to retrieve citations!");
+                throw new Error("Failed to retrieve citations!");
+            }
+            let new_cites = await resp.json();
+            DOI_CACHE = {
+                ...new_cites['citations'],
+                ...DOI_CACHE
+            };
+        }
+        let citations = dois.map(d => DOI_CACHE[d]);
         cites_list.html(citations.join("\n\n"));
         copy_cites.attr('content', citations.join("\n\n"));
     });
 
     $('#citations-modal').on('hide.bs.modal', function() {
-
-    });
-
-    $('#export-manifest-modal').on('hidden.bs.modal', function() {
         $(".citations-list").empty();
-    });
-
-    $('.copy-this').on('click', function(e) {
-        // copy the entire set
+        $("#citations-modal .copy-this").attr('content', "");
     });
 
     tippy.delegate('#citations-modal', {