Merge pull request #1485 from ImagingDataCommons/release-48-sp

s-paquette · web-flow · commit 9be804e73cb3 · 2025-09-11T14:11:15.000-07:00
Release 48
diff --git a/etl/etl.py b/etl/etl.py
@@ -42,7 +42,7 @@
 
 from idc_collections.models import Program, Collection, Attribute, Attribute_Ranges, \
     Attribute_Display_Values, DataSource, DataSourceJoin, DataVersion, DataSetType, \
-    Attribute_Set_Type, Attribute_Display_Category, ImagingDataCommonsVersion, Attribute_Tooltips
+    Attribute_Set_Type, Attribute_Display_Category, ImagingDataCommonsVersion, Attribute_Tooltips, Citation
 from google_helpers.bigquery.bq_support import BigQuerySupport
 
 from django.contrib.auth.models import User
@@ -319,6 +319,35 @@ def add_source_joins(froms, from_col, tos=None, to_col=None):
         DataSourceJoin.objects.bulk_create(src_joins)
 
 
+def load_citations(filename):
+    try:
+        cites_file = open(filename,"r")
+        current_cites = [x.doi for x in Citation.objects.all()]
+        new_cites = []
+        updated_cites = {}
+        for line in csv_reader(cites_file):
+            if "doi, citation" in line:
+                print("[STATUS] Saw header line during citation load - skipping!")
+                continue
+            if line[0] in current_cites:
+                updated_cites[line[0]] = line[1]
+            else:
+                new_cites.append(Citation(doi=line[0], cite=line[1]))
+        if len(new_cites):
+            Citation.objects.bulk_create(new_cites)
+            print("[STATUS] The following {} DOI citations were added: {}".format(len(new_cites), "  ".join([x.doi for x in new_cites])))
+        if len(updated_cites):
+            to_update = Citation.objects.filter(doi__in=updated_cites.keys())
+            for upd in to_update:
+                upd.cite = updated_cites[upd.doi]
+            Citation.objects.bulk_update(to_update, ["cite"])
+            print("[STATUS] {} DOI citations were updated.".format(len(updated_cites)))
+    except Exception as e:
+        ERRORS_SEEN.append("Error seen while loading citations, check the logs!")
+        logger.error("[ERROR] While trying to load citations: ")
+        logger.exception(e)
+
+
 def load_collections(filename, data_version="8.0"):
     try:
         collection_file = open(filename, "r")
@@ -781,7 +810,8 @@ def parse_args():
 
     parser = ArgumentParser()
     parser.add_argument('-j', '--config-file', type=str, default='', help='JSON file of version data to update')
-    parser.add_argument('-c', '--collex-file', type=str, default='', help='CSV data of collections to update/create')
+    parser.add_argument('-c', '--collex-file', type=str, default='', help='CSV data of citations to update/create')
+    parser.add_argument('-i', '--cites-file', type=str, default='', help='CSV data of collections to update/create')
     parser.add_argument('-d', '--display-vals', type=str, default='', help='CSV data of display values to add/update')
     parser.add_argument('-p', '--programs-file', type=str, default='', help='CSV data of programs to add/update')
     parser.add_argument('-a', '--attributes-file', type=str, default='', help='CSV data of attributes to add/update')
@@ -821,6 +851,8 @@ def main():
         len(args.programs_file) and load_programs(args.programs_file)
         # Add/update collections - any new programs must be added first
         len(args.collex_file) and load_collections(args.collex_file)
+        # Add/update any citations
+        len(args.cites_file) and load_citations(args.cites_file)
         # Add/update display values for attributes
         if len(args.display_vals):
             dvals = load_display_vals(args.display_vals)
diff --git a/idc/views.py b/idc/views.py
@@ -76,7 +76,8 @@ def landing_page(request):
         "Testicles": "Testis",
         "Adrenal Glands": "Adrenal Gland",
         "Adrenal": "Adrenal Gland",
-        "Lymph Node": "Lymph Nodes"
+        "Lymph Node": "Lymph Nodes",
+        "Bone Marrow": "Blood"
     }
 
     skip = [
diff --git a/static/js/filterutils.js b/static/js/filterutils.js
@@ -196,7 +196,6 @@ define(['jquery', 'base'], function($, base) {
         if (sliders.length > 0) {
             load_sliders(sliders, false);
         }
-
         //mkFiltText();
         //return updateFacetsData(true).promise();
         return handleFilterSelectionUpdate(null, true, true)
@@ -633,7 +632,6 @@ define(['jquery', 'base'], function($, base) {
         }
     };
 
-
     window.resetFilters = function(){
         $('input:checkbox').not('.hide-zeros').not('.tbl-sel').prop('checked',false);
         $('input:checkbox').not('.hide-zeros').not('.tbl-sel').prop('indeterminate',false);
@@ -664,34 +662,22 @@ define(['jquery', 'base'], function($, base) {
             } else {
                 first_filter_load = false;
             }
-        }
+          }
         }
 
         if (doUpdate){
             var mxstudies = 0;
             var mxseries = 0;
-
             var projArr=[];
             var serverdata = [updateFacetsData(true)];
             projArr = Object.keys(window.proj_in_cart);
-            /*
-            $('#projects_table').find('tr').each(function(){
-                if ($(this).hasClass('someInCart')){
-                   var projid = $(this).attr('data-projectid');
-                   projArr.push(projid)
-                }
-
-            });*/
-
-             if (projArr.length>0)
-            {
+            if (projArr.length>0) {
                 serverdata.push(getProjectCartStats(projArr));
             }
 
             $('.spinner').show();
             //$.when.apply(undefined, serverdata).then(function(ret)
-            promise = Promise.all(serverdata).then(function(ret)
-            {
+            promise = Promise.all(serverdata).then(function(ret) {
 
                 var collFilt = ret[0][0];
                 var collectionData = ret[0][1];
@@ -711,22 +697,18 @@ define(['jquery', 'base'], function($, base) {
                     cartStats={}
                 }
                 //var numStudiesRet = totals.StudyInstanceUID;
-
-
-
                 createPlots('search_orig_set');
-               createPlots('search_derived_set');
-               createPlots('tcga_clinical');
+                createPlots('search_derived_set');
+                createPlots('tcga_clinical');
 
                 if ($('.search-configuration').find('.hide-zeros')[0].checked) {
-                        addSliders('search_orig_set', false, true, '');
-                        addSliders('quantitative', false, true, 'quantitative.');
-                        addSliders('tcga_clinical', false, true, 'tcga_clinical.');
-                    }
+                    addSliders('search_orig_set', false, true, '');
+                    addSliders('quantitative', false, true, 'quantitative.');
+                    addSliders('tcga_clinical', false, true, 'tcga_clinical.');
+                }
 
                 updateTablesAfterFilter(collFilt, collectionData, collectionStats,cartStats);
                 $('.spinner').hide();
-                //updateTableCounts(1)
             });
 
 
diff --git a/static/js/image_search.js b/static/js/image_search.js
@@ -624,7 +624,7 @@ require([
         }
     }
 
-     $(document).ready(function () {
+     $(document).ready(async function () {
 
         tables.initializeTableCacheData();
         tables.initializeTableViewedItemsData();
@@ -694,7 +694,7 @@ require([
         window.cartHist.push(cartSel);
         window.proj_in_cart = new Object();
 
-        filterutils.load_preset_filters();
+        await filterutils.load_preset_filters();
         $('.hide-filter-uri').on('click',function() {
             $(this).hide();
             $('.get-filter-uri').show();
diff --git a/static/js/sapien.js b/static/js/sapien.js
@@ -73,14 +73,17 @@ require([
       Chest: 'rgb(0, 233, 255)',
     };
 
+    // Map the sapiens.js expected location names (typically uses a dash) to the TumorLocation
+    // value stored in DICOM (freeform text to an extent, not normalized)
+    //format: { <sapiens.js name>: [<TumorLocation>, ...], ...}
     var variantNames = {
         'Colorectal': ['Colon','Rectum'],
-        "Blood": ["Marrow, Blood", "Marrow and Blood", "Blood"],
+        "Blood": ["Marrow, Blood", "Marrow and Blood", "Blood", "Bone Marrow", "Blood, Bone"],
         "Bile-Duct": ["Bile Duct"],
         "Adrenal-Gland": ["Adrenal Glands", "Adrenal"],
         "Testis": ["Testicles"],
         "Head-and-Neck": ["Head", "Head-Neck"],
-        "Lymph-Nodes": ["Lymph Node", "Lymph Nodes"]
+        "Lymph-Nodes": ["Lymph Node", "Lymph Nodes"],
     };
 
     let data = case_counts.sort(function(a,b){
@@ -119,7 +122,7 @@ require([
 
   root.prepend(rawsvg.buildHumanBody(null,null,' '));
 
-  let initChartWidth = 400;
+  let initChartWidth = 500;
   let initChartHeight = 470;
   let top = 75;
   let labelSize ='12px';
@@ -130,7 +133,7 @@ require([
   let fileCountKey = 'fileCount';
 
   const plotHeight = initChartHeight - 30;
-  const barStartOffset = 130;
+  const barStartOffset = 200;
   const barWidth = initChartWidth - barStartOffset;
   const maxCases = Math.max(...data.map(d => d[caseCountKey]));
   const halfPixel = 0.5;
@@ -188,7 +191,7 @@ require([
     .attr('fill', 'rgb(10, 10, 10)')
     .attr('font-size', labelSize)
     .style('text-anchor', 'end')
-    .text(d => d[primarySiteKey])
+    .text(d => d[primarySiteKey] === "Blood" ? "Blood and Bone Marrow" : d[primarySiteKey])
     .on('mouseover', function (d, i) { // needs `this`
       const organSelector = toClassName(d[primarySiteKey]);
       const organ = document.getElementById(organSelector);
diff --git a/static/js/sapien/raw-svg.js b/static/js/sapien/raw-svg.js
diff --git a/static/js/tables.js b/static/js/tables.js
@@ -482,10 +482,8 @@ define(['cartutils','filterutils','tippy','jquery', 'base'], function(cartutils,
             }
         });
          return deferred.promise();
-
     }
 
-
     const caseTableColDefs = function(){
         return [
             {className: "ckbx studyview","targets": [0]},
@@ -927,6 +925,9 @@ define(['cartutils','filterutils','tippy','jquery', 'base'], function(cartutils,
                         }
                     }, {
                         "type": "text", "orderable": true, data: 'StudyDate', render: function (data) {
+                            if(data === "" || data === undefined || data === null) {
+                                return "";
+                            }
                             // fix when StudyData is an array of values
                             var dt = new Date(Date.parse(data));
                             var dtStr = (dt.getUTCMonth() + 1).toLocaleString('en-US', {minimumIntegerDigits: 2}) + "-" + dt.getUTCDate().toLocaleString('en-US', {minimumIntegerDigits: 2}) + "-" + dt.getUTCFullYear().toString();

Original file line number	Diff line number	Diff line change
`@@ -76,7 +76,8 @@ def landing_page(request):`
`76`	`76`	`"Testicles": "Testis",`
`77`	`77`	`"Adrenal Glands": "Adrenal Gland",`
`78`	`78`	`"Adrenal": "Adrenal Gland",`
`79`		`- "Lymph Node": "Lymph Nodes"`
	`79`	`+ "Lymph Node": "Lymph Nodes",`
	`80`	`+ "Bone Marrow": "Blood"`
`80`	`81`	`}`
`81`	`82`
`82`	`83`	`skip = [`
Original file line number	Diff line number	Diff line change
`@@ -624,7 +624,7 @@ require([`
`624`	`624`	`}`
`625`	`625`	`}`
`626`	`626`
`627`		`- $(document).ready(function () {`
	`627`	`+ $(document).ready(async function () {`
`628`	`628`
`629`	`629`	`tables.initializeTableCacheData();`
`630`	`630`	`tables.initializeTableViewedItemsData();`
`@@ -694,7 +694,7 @@ require([`
`694`	`694`	`window.cartHist.push(cartSel);`
`695`	`695`	`window.proj_in_cart = new Object();`
`696`	`696`
`697`		`- filterutils.load_preset_filters();`
	`697`	`+ await filterutils.load_preset_filters();`
`698`	`698`	`$('.hide-filter-uri').on('click',function() {`
`699`	`699`	`$(this).hide();`
`700`	`700`	`$('.get-filter-uri').show();`
Original file line number	Diff line number	Diff line change
`@@ -482,10 +482,8 @@ define(['cartutils','filterutils','tippy','jquery', 'base'], function(cartutils,`
`482`	`482`	`}`
`483`	`483`	`});`
`484`	`484`	`return deferred.promise();`
`485`		`-`
`486`	`485`	`}`
`487`	`486`
`488`		`-`
`489`	`487`	`const caseTableColDefs = function(){`
`490`	`488`	`return [`
`491`	`489`	`{className: "ckbx studyview","targets": [0]},`
`@@ -927,6 +925,9 @@ define(['cartutils','filterutils','tippy','jquery', 'base'], function(cartutils,`
`927`	`925`	`}`
`928`	`926`	`}, {`
`929`	`927`	`"type": "text", "orderable": true, data: 'StudyDate', render: function (data) {`
	`928`	`+ if(data === "" \|\| data === undefined \|\| data === null) {`
	`929`	`+ return "";`
	`930`	`+ }`
`930`	`931`	`// fix when StudyData is an array of values`
`931`	`932`	`var dt = new Date(Date.parse(data));`
`932`	`933`	`var dtStr = (dt.getUTCMonth() + 1).toLocaleString('en-US', {minimumIntegerDigits: 2}) + "-" + dt.getUTCDate().toLocaleString('en-US', {minimumIntegerDigits: 2}) + "-" + dt.getUTCFullYear().toString();`