Merge pull request #382 from ImagingDataCommons/idc-prod-wc

Idc prod wc

Author: s-paquette

cohorts/models.py

@@ -446,7 +446,7 @@ def get_operator(self):
         return self.OP_TO_STR[self.operator]
 
     def get_filter(self):
-        if self.operator not in [self.OR, self.BTW]:
+        if self.operator not in self.OP_TO_SUFFIX and self.operator != self.OR:
             return {
                 self.get_attr_name(): { 'op': self.get_operator(), 'values': self.value.split(self.value_delimiter) }
             }
@@ -463,7 +463,7 @@ def get_filter_flat(self):
         }
 
     def __repr__(self):
-        if self.operator not in [self.OR, self.BTW]:
+        if self.operator not in self.OP_TO_SUFFIX and self.operator != self.OR:
             return "{ %s: {'op': %s, 'values': %s }" % (self.get_attr_name(), self.get_operator(), "[{}]".format(self.value))
         return "{ %s }" % ("\"{}\": [{}]".format(self.get_attr_name(), self.value))
 

cohorts/utils.py

@@ -45,6 +45,7 @@ def _get_cohort_stats(cohort_id=0, filters=None, sources=None):
         'PatientID': 0,
         'StudyInstanceUID': 0,
         'SeriesInstanceUID': 0,
+        'total_instance_size': 0,
         'collections': []
     }
 

cohorts/utils_api_v2.py

@@ -17,16 +17,16 @@
 
 import logging
 import copy
-import json
 
 from django.conf import settings
 from idc_collections.models import ImagingDataCommonsVersion
 from idc_collections.collex_metadata_utils import get_bq_metadata, get_bq_string
-from google_helpers.bigquery.bq_support import BigQuerySupport
 
 logger = logging.getLogger('main_logger')
 DENYLIST_RE = settings.DENYLIST_RE
 
+NUMERIC_OPS = ('_btw', '_ebtw', '_btwe', '_ebtwe', '_gte', '_lte', '_gt', '_lt', '_eq')
+
 # All the filter values in the filterSet of a cohort are saved as strings. Particularly, a
 # filter value like [[35,45], [65,75]] is returned as ["[35,45]","[65,75]"] when you get
 # the filterSet. This script converts it back to numeric.
@@ -38,7 +38,10 @@ def to_numeric_list(item):
     # If the item is not a list, then we assume it is single string value or already a numeric
     elif isinstance(item, str):
           # if it is a string, then convert it to a float
-          item = float(item)
+          try:
+              item = int(item)
+          except ValueError:
+              item = float(item)
     return item
 
 
@@ -71,7 +74,7 @@ def _cohort_query_api(request, cohort, data, info):
             for collection in filters['collection_id']:
                 collections.append(collection.lower().replace('-', '_'))
             filters['collection_id'] = collections
-        if filter.endswith(('_lt', '_lte', '_ebtw', '_ebtwe', '_btw', '_btwe', '_gte' '_gt')):
+        if filter.endswith(NUMERIC_OPS):
            filters[filter] = to_numeric_list(value)
 
     data_version = cohort.get_data_versions()

cohorts/views/views_api_v2.py

@@ -19,14 +19,11 @@
 
 import json
 import logging
-import copy
 
-from django.contrib import messages
 from django.contrib.auth.models import User
 from django.conf import settings
 from django.core.exceptions import ObjectDoesNotExist, MultipleObjectsReturned
 from django.http import HttpResponse, JsonResponse
-from django.shortcuts import render, redirect
 from django.views.decorators.csrf import csrf_protect, csrf_exempt
 from django.views.decorators.http import require_http_methods
 from ..decorators import api_auth
@@ -37,6 +34,7 @@
     _cohort_preview_query_api, _cohort_query_api
 from ..views.views import _save_cohort,_delete_cohort
 
+NUMERIC_OPS = ('_btw', '_ebtw', '_btwe', '_ebtwe', '_gte', '_lte', '_gt', '_lt', '_eq')
 BQ_ATTEMPT_MAX = 10
 
 debug = settings.DEBUG # RO global for this file
@@ -67,7 +65,7 @@ def save_cohort_api(request):
             return JsonResponse(response)
 
         data = body["request_data"]
-        name = data['name']
+        cohort_name = data['name']
         description = data['description']
         filters = data['filters']
         # Create a cohort only against the current version
@@ -76,31 +74,28 @@ def save_cohort_api(request):
         # We first need to convert the filters to a form accepted by _save_cohorts
         filters_by_name = {}
         for filter, value in filters.items():
-            if filter.endswith(('_lt', '_lte', '_ebtw', '_ebtwe', '_btw', '_btwe', '_gte' '_gt')):
-                name = filter.rsplit('_', 1)[0]
-                op = filter.rsplit('_', 1)[-1]
-                filters_by_name[name] = dict(
+            if filter.endswith(NUMERIC_OPS):
+                attribute_name = filter.rsplit('_', 1)[0]
+                op = filter.rsplit('_', 1)[-1].upper()
+                filters_by_name[attribute_name] = dict(
                     op= op,
                     values = value
                 )
             else:
                 filters_by_name[filter] = value
         filters_by_id = {}
-        for attr in Attribute.objects.filter(name__in=list(filters.keys())).values('id', 'name'):
-            filters_by_id[str(attr['id'])] = filters[attr['name']]
-        response = _save_cohort(user, filters=filters_by_id, name=name, desc=description, version=version,
+        for attr in Attribute.objects.filter(name__in=list(filters_by_name.keys())).values('id', 'name'):
+            filters_by_id[str(attr['id'])] = filters_by_name[attr['name']]
+        response = _save_cohort(user, filters=filters_by_id, name=cohort_name, desc=description, version=version,
                                 no_stats=version.active==False)
         cohort_id = response['cohort_id']
         idc_data_version = Cohort.objects.get(id=cohort_id).get_data_versions()[0].version_number
-        # if request.GET['return_filter'] == 'True':
-        #     response["filterSet"] =  get_filterSet_api(cohort)
-        # response["filterSet"] =  get_filterSet_api(cohort)
 
         response['filterSet'] = {'idc_data_version': idc_data_version, 'filters': response.pop('filters')}
 
 
         for filter, value in response['filterSet']['filters'].items():
-            if filter.endswith(('_lt', '_lte', '_ebtw', '_ebtwe', '_btw', '_btwe', '_gte' '_gt')):
+            if filter.endswith(NUMERIC_OPS):
                 response['filterSet']['filters'][filter] = to_numeric_list(value)
 
         cohort_properties = dict(
@@ -130,11 +125,6 @@ def cohort_query_api(request, cohort_id=0):
         }
         return JsonResponse(info)
 
-    # if cohort_id == 0:
-    #     messages.error(request, 'Cohort requested does not exist.')
-    #     return redirect('/user_landing')
-
-    # print(request.GET.get('email', ''))
     try:
         cohort = Cohort.objects.get(id=cohort_id)
     except ObjectDoesNotExist as e:
@@ -150,9 +140,9 @@ def cohort_query_api(request, cohort_id=0):
         body = json.loads(request.body.decode('utf-8'))
         try:
             user = User.objects.get(email=body['email'])
-        except:
+        except Exception as exc:
             logger.error("[ERROR] While trying to save cohort: ")
-            logger.exception(e)
+            logger.exception(exc)
             info = {
                 "message": f"{body['email']} is not a known user",
                 "code": 401,
@@ -248,7 +238,9 @@ def cohort_list_api(request):
                 "filterSet": get_filterSet_api(cohort)
             }
             for filter, value in cohortMetadata['filterSet']['filters'].items():
-                if filter.endswith(('_lt', '_lte', '_ebtw', '_ebtwe', '_btw', '_btwe', '_gte' '_gt')):
+                # if filter.endswith(('_lt', '_lte', '_ebtw', '_ebtwe', '_btw', '_btwe', '_gte' '_gt')):
+                #     if filter.endswith(('_lt', '_lte', '_ebtw', '_ebtwe', '_btw', '_btwe', '_gte' '_gt')):
+                if filter.endswith(NUMERIC_OPS):
                     cohortMetadata['filterSet']['filters'][filter] = to_numeric_list(value)
             cohortList.append(cohortMetadata)
 
@@ -274,9 +266,9 @@ def delete_cohort_api(request):
         body = json.loads(request.body.decode('utf-8'))
         try:
             user = User.objects.get(email=body['email'])
-        except:
+        except Exception as exc:
             logger.error("[ERROR] While trying to save cohort: ")
-            logger.exception(e)
+            logger.exception(exc)
             response = {
                 "message": f"{body['email']} is not a known user",
                 "code": 401,

google_helpers/bigquery/bq_support.py

@@ -698,7 +698,7 @@ def build_bq_filter_and_params(filters, comb_with='AND', param_suffix=None, with
         # Standard query filters
         for attr, values in list(other_filters.items()):
             is_btw = re.search('_e?btwe?', attr.lower()) is not None
-            attr_name = attr[:attr.rfind('_')] if re.search('_[gl]te?|_e?btwe?', attr) else attr
+            attr_name = attr[:attr.rfind('_')] if re.search('_[gl]te?|_e?btwe?|_eq', attr) else attr
             if attr_name not in attr_filters:
                 attr_filters[attr_name] = {
                     'OP': 'OR',
@@ -745,15 +745,12 @@ def build_bq_filter_and_params(filters, comb_with='AND', param_suffix=None, with
                     # Single scalar param
                     query_param['parameterValue']['value'] = values[0]
                     if query_param['parameterType']['type'] == 'STRING':
-                        if '%' in values[0] or case_insens:
-                            filter_string += "LOWER({}{}) LIKE LOWER(@{})".format('' if not field_prefix else field_prefix, attr_name, param_name)
-                        else:
-                            filter_string += "{}{} = @{}".format('' if not field_prefix else field_prefix, attr,
-                                                                 param_name)
+                        filter_string += "{}{} = @{}".format('' if not field_prefix else field_prefix, attr,
+                                                             param_name)
                     elif query_param['parameterType']['type'] == 'NUMERIC':
                         operator = "{}{}".format(
                             ">" if re.search(r'_gte?',attr) else "<" if re.search(r'_lte?',attr) else "",
-                            '=' if re.search(r'_[lg]te',attr) or not re.search(r'_[lg]',attr) else ''
+                            '=' if re.search(r'_[lg]te',attr) or not re.search(r'_[lg]',attr) or attr.endswith('_eq') else ''
                         )
                         filter_string += "{}{} {} @{}".format(
                             '' if not field_prefix else field_prefix, attr_name,
@@ -844,41 +841,14 @@ def build_bq_filter_and_params(filters, comb_with='AND', param_suffix=None, with
                     filter_string += " OR ".join(btw_filter_strings)
                     query_param = query_params
                 else:
-                    # String param values that include the % or _char must be LIKE'd. We ignore that \%  or \_
-                    # don't need to be LIKE'd.
-                    query_params = []
-                    strings_filter_string = []
-                    values_copy = copy.deepcopy(values)
-                    if parameter_type == 'STRING':
-                        for index, value in enumerate(values_copy):
-                            if '%' in value or '_' in value:
-                                strings_filter_string.append(
-                                    "LOWER({}{}) LIKE LOWER(@{}_{})".format('' if not field_prefix else field_prefix, attr, param_name, index)
-                                )
-                                query_param_like = copy.deepcopy(query_param)
-                                query_param_like['name'] = f'{param_name}_{index}'
-                                query_param_like['parameterType']['type'] = 'STRING'
-                                query_param_like['parameterValue']['value'] = value.lower()
-                                query_params.append(query_param_like)
-                                values_copy[index] = ''
-
-                    # Squeeze out empty strings
-                    values_copy = [value for value in values_copy if value]
-                    if values_copy:
-                        # Simple array param
-                        query_param['parameterType']['type'] = "ARRAY"
-                        query_param['parameterType']['arrayType'] = {
-                            'type': parameter_type
-                        }
-
-
-                        query_param['parameterValue'] = {'arrayValues': [{'value': x.lower() if parameter_type == 'STRING' else x} for x in values_copy]}
-
-                        strings_filter_string.append("LOWER({}{}) IN UNNEST(@{})".format('' if not field_prefix else field_prefix, attr, param_name))
-                        query_params.append(query_param)
+                    # Simple array param
+                    query_param['parameterType']['type'] = "ARRAY"
+                    query_param['parameterType']['arrayType'] = {
+                        'type': parameter_type
+                    }
+                    query_param['parameterValue'] = {'arrayValues': [{'value': x.lower() if parameter_type == 'STRING' else x} for x in values]}
 
-                    query_param = query_params
-                    filter_string += ' OR '.join(strings_filter_string)
+                    filter_string += "LOWER({}{}) IN UNNEST(@{})".format('' if not field_prefix else field_prefix, attr, param_name)
 
             if with_count_toggle:
                 filter_string = "({}) OR @{}_filtering = 'not_filtering'".format(filter_string,param_name)

idc_collections/collex_metadata_utils.py

@@ -188,8 +188,8 @@ def _build_attr_by_source(attrs, data_version, source_type=DataSource.BIGQUERY,
 
         for attr in attrs:
             stripped_attr = attr if (not '_' in attr) else \
-                attr if not attr.rsplit('_', 1)[1] in ['gt', 'gte','ebtwe','ebtw','btwe', 'btw', 'lte', 'lt'] else \
-                attr.rsplit('_', 1)[0]
+                attr if not attr.rsplit('_', 1)[1] in ['gt', 'gte','ebtwe','ebtw','btwe', 'btw', 'lte', 'lt', 'eq'] else \
+                    attr.rsplit('_', 1)[0]
 
             for id, source in attr_data['sources'].items():
                 if stripped_attr in source['list']:

idc_collections/views/views_api_v2.py

@@ -30,6 +30,7 @@
 
 DENYLIST_RE = settings.DENYLIST_RE
 
+NUMERIC_OPS = ('_btw', '_ebtw', '_btwe', '_ebtwe', '_gte', '_lte', '_gt', '_lt', '_eq')
 # Return a list of defined IDC versions
 @api_auth
 @require_http_methods(["GET"])
@@ -152,12 +153,15 @@ def attributes_list_api(request):
                     # "active": attribute.active,
                     "units": attribute.units,
                 }
-                attributes_info.append(attribute_info)
                 if attribute_info['data_type'] == 'Continuous Numeric':
-                    for suffix in ['lt', 'lte', 'btw', 'ebtw', 'ebtwe', 'btwe', 'gte', 'gt']:
+                    # for suffix in ['lt', 'lte', 'btw', 'ebtw', 'ebtwe', 'btwe', 'gte', 'gt', 'eq']:
+                    for suffix in NUMERIC_OPS:
                         attribute_info_copy = dict(attribute_info)
-                        attribute_info_copy['name'] = '{}_{}'.format(attribute.name, suffix)
+                        attribute_info_copy['name'] = '{}{}'.format(attribute.name, suffix)
                         attributes_info.append(attribute_info_copy)
+                else:
+                    attributes_info.append(attribute_info)
+
             data_source = {
                 "data_source": source.name,
                 'filters': attributes_info