-> #1446 (and related)

Author: s-paquette

etl/etl.py

@@ -64,6 +64,8 @@
             "program", "access", "date_updated", "tcia_wiki_collection_id", "license_short_name", "active"
     ])}
 
+TOKENIZED_FIELDS = ["PatientID", "SeriesInstanceUID", "StudyInstanceUID"]
+
 ranges_needed = {
     'wbc_at_diagnosis': 'by_200',
     'event_free_survival_time_in_days': 'by_500',
@@ -424,7 +426,13 @@ def create_solr_params(schema_src, solr_src):
     schema = BigQuerySupport.get_table_schema(schema_src[0],schema_src[1],schema_src[2])
     solr_schema = []
     solr_index_strings = []
-    SCHEMA_BASE = '{"add-field": %s}'
+    field_types = ''
+    add_copy_field = ''
+    SCHEMA_BASE = '{{field_types}add-field": {fields}{add_copy_field}'
+    if len(TOKENIZED_FIELDS):
+        field_types = '"add-field-type": { "name":"tokenizedText", "class":"solr.TextField", "analyzer" : { "tokenizer": { "name":"nGram" }}}, '
+        copy_fields = ",".join(['{"source":"{field}","dest":"{field{}_tokenized"}'.format(field) for field in TOKENIZED_FIELDS])
+        add_copy_field = ', "add-copy-field": [{copy_fields}]'.format(copy_fields)
     CORE_CREATE_STRING = "sudo -u solr /opt/bitnami/solr/bin/solr create -c {solr_src} -s 2 -rf 2"
     SCHEMA_STRING = "curl -u {solr_user}:{solr_pwd} -X POST -H 'Content-type:application/json' --data-binary '{schema}' https://localhost:8983/solr/{solr_src}/schema --cacert solr-ssl.pem"
     INDEX_STRING = "curl -u {solr_user}:{solr_pwd} -X POST 'https://localhost:8983/solr/{solr_src}/update?commit=yes{params}' --data-binary @{file_name}.csv -H 'Content-type:application/csv' --cacert solr-ssl.pem"
@@ -441,11 +449,22 @@ def create_solr_params(schema_src, solr_src):
                 "stored": True
             }
             solr_schema.append(field_schema)
+            if TOKENIZED_FIELDS and field['name'] in TOKENIZED_FIELDS:
+                solr_schema.append({
+                    "name": "{}_tokenized".format(field["name"]),
+                    "type": "tokenizedText",
+                    "multiValued": False if field['name'] in SOLR_SINGLE_VAL.get(solr_src.aggregate_level,
+                                                                                 {}) else True,
+                    "stored": True
+                })
             if field_schema['multiValued']:
                 solr_index_strings.append("f.{}.split=true&f.{}.separator=|".format(field['name'],field['name']))
 
     with open("{}_solr_cmds.txt".format(solr_src.name), "w") as cmd_outfile:
-        schema_array = SCHEMA_BASE % solr_schema
+        schema_array = SCHEMA_BASE.format(
+            field_types=field_types,
+            add_copy_field=add_copy_field
+        )
         params = "&{}".format("&".join(solr_index_strings))
         cmd_outfile.write(CORE_CREATE_STRING.format(solr_src=solr_src.name))
         cmd_outfile.write("\n\n")

idc/views.py

@@ -372,16 +372,15 @@ def studymp(request):
     return JsonResponse(response, status=status)
 
 
-
-
 def populate_tables(request):
     response = {}
     status = 200
 
     try:
-        req = request.GET if request.GET else request.POST
+        req = request.GET if request.method == 'GET' else request.POST
         path_arr = [nstr for nstr in request.path.split('/') if nstr]
         table_type = path_arr[len(path_arr)-1]
+        table_search = req.get("table_search", "false").lower() == 'true'
         fields = None
         collapse_on = None
         filters = json.loads(req.get('filters', '{}'))
@@ -399,7 +398,7 @@ def populate_tables(request):
         sort = req.get('sort', 'PatientID')
         sortdir = req.get('sortdir', 'asc')
 
-        [cnt, tableRes]=get_table_data_with_cart_data(table_type, sort, sortdir, filters, filtergrp_list, partitions, limit, offset)
+        [cnt, tableRes]=get_table_data_with_cart_data(table_type, sort, sortdir, filters, filtergrp_list, partitions, limit, offset, table_search)
         response["res"] = tableRes
         response["cnt"] = cnt
         response["diff"] = []
@@ -415,10 +414,6 @@ def populate_tables(request):
     return JsonResponse(response, status=status)
 
 
-
-
-
-
 def populate_tables_old(request):
     response = {}
     status = 200

static/js/tables.js

@@ -560,7 +560,7 @@ define(['cartutils','filterutils','tippy','jquery', 'base'], function(cartutils,
     // recreates the cases table when a chevron is clicked in the projects table. Defines the chevron and cart selection actions.
     // Updates data structures such as casesCache, which caches case rows to limit calls to the server, and selProjects, which stores information
     // across tables about which items are added to the cart, and which chevrons are selected
-    window.updateCaseTable = function(rowsAdded,caseID) {
+    window.updateCaseTable = function(rowsAdded,caseID, table_search) {
         viewedProjects =Object.keys(window.openProjects);
         if ($('#cases_tab_wrapper').find('.dataTables_controls').length>0){
             pageRows = parseInt($('#cases_tab_wrapper').find('.dataTables_length select').val());
@@ -683,7 +683,7 @@ define(['cartutils','filterutils','tippy','jquery', 'base'], function(cartutils,
                                 }
 
                             }
-
+                        ndic['table_search'] = table_search;
 
                         var csrftoken = $.getCookie('csrftoken');
                         $.ajax({
@@ -748,7 +748,7 @@ define(['cartutils','filterutils','tippy','jquery', 'base'], function(cartutils,
     }
 
 
-    window.updateStudyTable = function(rowsAdded, studyID) {
+    window.updateStudyTable = function(rowsAdded, studyID, table_search) {
         let nonViewAbleModality= new Set([""]);
         var viewCases = [];
         for (projid in window.openCases) {
@@ -992,7 +992,6 @@ define(['cartutils','filterutils','tippy','jquery', 'base'], function(cartutils,
                               return '<i class="fa fa-download study-export export-button" data-series-count="'+row['unique_series']
                                   +'" data-uid="'+data+'"data-toggle="modal" data-target="#export-manifest-modal"></i>'
                           }
-
                       }
                 ],
                 "processing": true,
@@ -1035,9 +1034,9 @@ define(['cartutils','filterutils','tippy','jquery', 'base'], function(cartutils,
                             }
                             ndic['partitions'] = JSON.stringify(window.partitions);
                             ndic['filtergrp_list'] = JSON.stringify(window.filtergrp_list);
+                            ndic['table_search'] = table_search;
+                            let csrftoken = $.getCookie('csrftoken');
 
-
-                            var csrftoken = $.getCookie('csrftoken');
                             $.ajax({
                                 url: url,
                                 dataType: 'json',
@@ -1099,7 +1098,7 @@ define(['cartutils','filterutils','tippy','jquery', 'base'], function(cartutils,
         $('#studies_tab_wrapper').find('.dataTables_controls').find('.dataTables_paginate').after('<div class="dataTables_filter"><strong>Find by Study Instance UID:</strong><input class="studyID_inp" type="text-box" value="'+studyID+'"><button onclick="filterTable(\'studies_tab_wrapper\',\'studyID\')">Go</button></div>');
     }
 
-    window.updateSeriesTable = function(rowsAdded, seriesID) {
+    window.updateSeriesTable = function(rowsAdded, seriesID, table_search) {
         var nonViewAbleModality= new Set(["PR","SEG","RTSTRUCT","RTPLAN","RWV", "SR", "ANN"])
         var slimViewAbleModality=new Set(["SM"])
         viewStudies = []
@@ -1130,51 +1129,41 @@ define(['cartutils','filterutils','tippy','jquery', 'base'], function(cartutils,
 
                     $(row).attr('data-aws',  data['aws_bucket'])
 
-
-
                      if ('cart_series_in_collection' in data){
                        $(row).attr('cart_series_in_collection',  data['cart_series_in_collection']);
-                    }
-                    else{
+                    } else{
                         $(row).attr('cart_series_in_collection','0')
                     }
                     if ('filter_series_in_collection' in data){
                        $(row).attr('filter_series_in_collection',  data['filter_series_in_collection']);
-                    }
-                    else{
+                    } else {
                         $(row).attr('filter_series_in_collection','0')
                     }
                     if ('filter_cart_series_in_collection' in data){
                        $(row).attr('filter_cart_series_in_collection',  data['filter_cart_series_in_collection']);
-                    }
-                    else{
+                    } else {
                         $(row).attr('filter_cart_series_in_collection','0')
                     }
 
-
                     if ('cart_series_in_case' in data){
                        $(row).attr('cart_series_in_case',  data['cart_series_in_case']);
-                    }
-                    else{
+                    } else {
                         $(row).attr('cart_series_in_case','0')
                     }
                     if ('filter_series_in_case' in data){
                        $(row).attr('filter_series_in_case',  data['filter_series_in_case']);
-                    }
-                    else{
+                    } else {
                         $(row).attr('filter_series_in_case','0')
                     }
                     if ('filter_cart_series_in_case' in data){
                        $(row).attr('filter_cart_series_in_case',  data['filter_cart_series_in_case']);
-                    }
-                    else{
+                    } else {
                         $(row).attr('filter_cart_series_in_case','0')
                     }
 
                     if ('cart_series_in_study' in data){
                        $(row).attr('cart_series_in_study',  data['cart_series_in_study']);
-                    }
-                    else{
+                    } else {
                         $(row).attr('cart_series_in_study','0')
                     }
                     if ('filter_series_in_study' in data){
@@ -1385,6 +1374,7 @@ define(['cartutils','filterutils','tippy','jquery', 'base'], function(cartutils,
                         }
                          ndic['partitions'] = JSON.stringify(window.partitions);
                         ndic['filtergrp_list'] = JSON.stringify(window.filtergrp_list);
+                        ndic['table_search'] = table_search;
                         var csrftoken = $.getCookie('csrftoken');
                         $.ajax({
                             url: url,
@@ -2090,21 +2080,21 @@ define(['cartutils','filterutils','tippy','jquery', 'base'], function(cartutils,
             if ($('#cases_tab').find('.caseID-inp').length > 0) {
                 caseID = $('#studies_tab').find('.caseID-inp').val();
             }
-            updateCaseTable(rowsAdded, caseID);
+            updateCaseTable(rowsAdded, caseID, false);
         }
         if (studyTableUpdate) {
             var studyID = "";
             if ($('#studies_tab').find('.studyID-inp').length > 0) {
                 studyID = $('#studies_tab').find('.studyID-inp').val();
             }
-            updateStudyTable(rowsAdded, studyID);
+            updateStudyTable(rowsAdded, studyID, false);
         }
         if (seriesTableUpdate) {
             var seriesID = "";
             if ($('#series_tab').find('.seriesID-inp').length > 0) {
                 seriesID = $('#series_tab').find('.seriesID-inp').val();
             }
-            updateSeriesTable(rowsAdded, seriesID);
+            updateSeriesTable(rowsAdded, seriesID, false);
 
         }
 
@@ -2179,14 +2169,14 @@ define(['cartutils','filterutils','tippy','jquery', 'base'], function(cartutils,
         var elem=$('#'+wrapper);
         var varStr=elem.find('.dataTables_controls').find('.'+type+'_inp').val();
         if (type ==="seriesID") {
-            window.updateSeriesTable(false,varStr)
+            window.updateSeriesTable(false,varStr, true)
         }
 
         else if (type ==="studyID") {
-            window.updateStudyTable(false,  varStr)
+            window.updateStudyTable(false,  varStr, true)
         }
         else if (type==="caseID"){
-            window.updateCaseTable(false,varStr)
+            window.updateCaseTable(false,varStr, true)
         }
     }