Merge branch 'master' of https://github.com/ImagingDataCommons/IDC-Common into idc-test

Author: s-paquette

google_helpers/bigquery/bq_support.py

@@ -587,6 +587,7 @@ def insert_job_batch_and_get_results(cls, query_set):
     # Support for specifying a set of continuous numeric attributes to be presumed for BETWEEN clauses
     #
     # TODO: add support for DATETIME eg 6/10/2010
+    @staticmethod
     def build_bq_filter_and_params(filters, comb_with='AND', param_suffix=None, with_count_toggle=False,
                                    field_prefix=None, type_schema=None, case_insens=True, continuous_numerics=None):
         if field_prefix and field_prefix[-1] != ".":
@@ -675,8 +676,13 @@ def build_bq_filter_and_params(filters, comb_with='AND', param_suffix=None, with
             is_btw = re.search('_e?btwe?', attr.lower()) is not None
             attr_name = attr[:attr.rfind('_')] if re.search('_[gl]te?|_e?btwe?|_eq', attr) else attr
             if attr_name not in attr_filters:
+                operator = 'OR'
+                if 'values' in values:
+                    # This is a fully qualified attribute which needs to have its definition broken out
+                    operator = values['op']
+                    values = values['values']
                 attr_filters[attr_name] = {
-                    'OP': 'OR',
+                    'OP': operator,
                     'filters': []
                 }
             attr_filter_set = attr_filters[attr_name]['filters']
@@ -820,17 +826,23 @@ def build_bq_filter_and_params(filters, comb_with='AND', param_suffix=None, with
                     filter_string += " OR ".join(btw_filter_strings)
                     query_param = query_params
                 else:
-                    # Simple array param
-                    query_param['parameterType']['type'] = "ARRAY"
-                    query_param['parameterType']['arrayType'] = {
-                        'type': parameter_type
-                    }
-                    query_param['parameterValue'] = {
-                        'arrayValues': [{'value': x.lower() if parameter_type == 'STRING' else x} for x in values]}
-
-                    clause_base = "%s IN UNNEST(@{})" % ("LOWER({}{})" if parameter_type == "STRING" else "{}{}")
-                    filter_string += clause_base.format('' if not field_prefix else field_prefix, attr,
-                                                                         param_name)
+                    if operator == 'AND' and len(values) > 1:
+                        # If an operator is to be AND'd with more than one value we must make an intersection statement
+                        # on the higher-level entity (i.e. select for studies which have series containing both values)
+                        # That cannot be performed here, as this is only a clause builder
+                        logger.warning("[WARNING] Multiple-value AND clauses require an intersection statement!")
+                    else:
+                        # Simple array param
+                        query_param['parameterType']['type'] = "ARRAY"
+                        query_param['parameterType']['arrayType'] = {
+                            'type': parameter_type
+                        }
+                        query_param['parameterValue'] = {
+                            'arrayValues': [{'value': x.lower() if parameter_type == 'STRING' else x} for x in values]}
+
+                        clause_base = "%s IN UNNEST(@{})" % ("LOWER({}{})" if parameter_type == "STRING" else "{}{}")
+                        filter_string += clause_base.format('' if not field_prefix else field_prefix, attr,
+                                                                             param_name)
 
             if with_count_toggle:
                 filter_string = "({}) OR @{}_filtering = 'not_filtering'".format(filter_string, param_name)

idc_collections/collex_metadata_utils.py

@@ -1831,7 +1831,7 @@ def get_table_data_with_cart_data(tabletype, sortarg, sortdir, current_filters,f
     return [num_found, table_arr]
 
 
-def get_cart_data_studylvl(filtergrp_list, partitions, limit, offset, length, mxseries,results_lvl='StudyInstanceUID', with_records=True, debug=False):
+def get_cart_data_studylvl(filtergrp_list, partitions, limit, offset, length, mxseries, results_lvl='StudyInstanceUID', with_records=True, debug=False):
     aggregate_level = "StudyInstanceUID"
     versions=ImagingDataCommonsVersion.objects.filter(
         active=True
@@ -1992,6 +1992,7 @@ def get_cart_data_studylvl(filtergrp_list, partitions, limit, offset, length, mx
     if debug:
         solr_result['response']['query_string'] = query_str
         solr_result['response']['query_string_series_lvl'] = query_str_series_lvl
+    print(solr_result['response'])
     return solr_result['response']
 
 
@@ -2206,7 +2207,6 @@ def get_cart_manifest(filtergrp_list, partitions, mxstudies, mxseries, field_lis
     if 'total_SeriesInstanceUID' in solr_result:
         manifest['total'] = solr_result['total_SeriesInstanceUID']
     elif 'total' in solr_result:
-
         manifest['total'] = solr_result['total']
 
     if ('total_instance_size' in  solr_result):
@@ -2818,13 +2818,11 @@ def get_bq_metadata(filters, fields, data_version, sources_and_attrs=None, group
         tables_in_query = []
         joins = []
         query_filters = []
-        non_related_filters = {}
         fields = [field_clauses[image_table]] if image_table in field_clauses else []
         if search_child_records_by:
             child_record_search_field = search_child_records_by
         if image_table in filter_attr_by_bq['sources']:
             filter_set = {x: filters[x] for x in filters if x in filter_attr_by_bq['sources'][image_table]['list']}
-            non_related_filters = filter_set
             if len(filter_set):
                 if may_need_intersect and len(filter_set.keys()) > 1:
                     for filter in filter_set: