-> For sapien, add Bone Marrow to Blood, rename 'Blood and Bone Marrow'

Author: s-paquette

etl/etl.py

@@ -42,7 +42,7 @@
 
 from idc_collections.models import Program, Collection, Attribute, Attribute_Ranges, \
     Attribute_Display_Values, DataSource, DataSourceJoin, DataVersion, DataSetType, \
-    Attribute_Set_Type, Attribute_Display_Category, ImagingDataCommonsVersion, Attribute_Tooltips
+    Attribute_Set_Type, Attribute_Display_Category, ImagingDataCommonsVersion, Attribute_Tooltips, Citation
 from google_helpers.bigquery.bq_support import BigQuerySupport
 
 from django.contrib.auth.models import User
@@ -319,6 +319,35 @@ def add_source_joins(froms, from_col, tos=None, to_col=None):
         DataSourceJoin.objects.bulk_create(src_joins)
 
 
+def load_citations(filename):
+    try:
+        cites_file = open(filename,"r")
+        current_cites = [x.doi for x in Citation.objects.all()]
+        new_cites = []
+        updated_cites = {}
+        for line in csv_reader(cites_file):
+            if "doi, citation" in line:
+                print("[STATUS] Saw header line during citation load - skipping!")
+                continue
+            if line[0] in current_cites:
+                updated_cites[line[0]] = line[1]
+            else:
+                new_cites.append(Citation(doi=line[0], cite=line[1]))
+        if len(new_cites):
+            Citation.objects.bulk_create(new_cites)
+            print("[STATUS] The following {} DOI citations were added: {}".format(len(new_cites), "  ".join([x.doi for x in new_cites])))
+        if len(updated_cites):
+            to_update = Citation.objects.filter(doi__in=updated_cites.keys())
+            for upd in to_update:
+                upd.cite = updated_cites[upd.doi]
+            Citation.objects.bulk_update(to_update, ["cite"])
+            print("[STATUS] {} DOI citations were updated.".format(len(updated_cites)))
+    except Exception as e:
+        ERRORS_SEEN.append("Error seen while loading citations, check the logs!")
+        logger.error("[ERROR] While trying to load citations: ")
+        logger.exception(e)
+
+
 def load_collections(filename, data_version="8.0"):
     try:
         collection_file = open(filename, "r")
@@ -781,7 +810,8 @@ def parse_args():
 
     parser = ArgumentParser()
     parser.add_argument('-j', '--config-file', type=str, default='', help='JSON file of version data to update')
-    parser.add_argument('-c', '--collex-file', type=str, default='', help='CSV data of collections to update/create')
+    parser.add_argument('-c', '--collex-file', type=str, default='', help='CSV data of citations to update/create')
+    parser.add_argument('-i', '--cites-file', type=str, default='', help='CSV data of collections to update/create')
     parser.add_argument('-d', '--display-vals', type=str, default='', help='CSV data of display values to add/update')
     parser.add_argument('-p', '--programs-file', type=str, default='', help='CSV data of programs to add/update')
     parser.add_argument('-a', '--attributes-file', type=str, default='', help='CSV data of attributes to add/update')
@@ -821,6 +851,8 @@ def main():
         len(args.programs_file) and load_programs(args.programs_file)
         # Add/update collections - any new programs must be added first
         len(args.collex_file) and load_collections(args.collex_file)
+        # Add/update any citations
+        len(args.cites_file) and load_citations(args.cites_file)
         # Add/update display values for attributes
         if len(args.display_vals):
             dvals = load_display_vals(args.display_vals)

idc/views.py

@@ -76,7 +76,8 @@ def landing_page(request):
         "Testicles": "Testis",
         "Adrenal Glands": "Adrenal Gland",
         "Adrenal": "Adrenal Gland",
-        "Lymph Node": "Lymph Nodes"
+        "Lymph Node": "Lymph Nodes",
+        "Bone Marrow": "Blood"
     }
 
     skip = [

static/js/sapien.js

@@ -73,14 +73,17 @@ require([
       Chest: 'rgb(0, 233, 255)',
     };
 
+    // Map the sapiens.js expected location names (typically uses a dash) to the TumorLocation
+    // value stored in DICOM (freeform text to an extent, not normalized)
+    //format: { <sapiens.js name>: [<TumorLocation>, ...], ...}
     var variantNames = {
         'Colorectal': ['Colon','Rectum'],
-        "Blood": ["Marrow, Blood", "Marrow and Blood", "Blood"],
+        "Blood": ["Marrow, Blood", "Marrow and Blood", "Blood", "Bone Marrow"],
         "Bile-Duct": ["Bile Duct"],
         "Adrenal-Gland": ["Adrenal Glands", "Adrenal"],
         "Testis": ["Testicles"],
         "Head-and-Neck": ["Head", "Head-Neck"],
-        "Lymph-Nodes": ["Lymph Node", "Lymph Nodes"]
+        "Lymph-Nodes": ["Lymph Node", "Lymph Nodes"],
     };
 
     let data = case_counts.sort(function(a,b){
@@ -119,7 +122,7 @@ require([
 
   root.prepend(rawsvg.buildHumanBody(null,null,' '));
 
-  let initChartWidth = 400;
+  let initChartWidth = 500;
   let initChartHeight = 470;
   let top = 75;
   let labelSize ='12px';
@@ -130,7 +133,7 @@ require([
   let fileCountKey = 'fileCount';
 
   const plotHeight = initChartHeight - 30;
-  const barStartOffset = 130;
+  const barStartOffset = 200;
   const barWidth = initChartWidth - barStartOffset;
   const maxCases = Math.max(...data.map(d => d[caseCountKey]));
   const halfPixel = 0.5;
@@ -188,7 +191,7 @@ require([
     .attr('fill', 'rgb(10, 10, 10)')
     .attr('font-size', labelSize)
     .style('text-anchor', 'end')
-    .text(d => d[primarySiteKey])
+    .text(d => d[primarySiteKey] === "Blood" ? "Blood and Bone Marrow" : d[primarySiteKey])
     .on('mouseover', function (d, i) { // needs `this`
       const organSelector = toClassName(d[primarySiteKey]);
       const organ = document.getElementById(organSelector);