Merge pull request #403 from ImagingDataCommons/idc-prod-sp

Cart

Author: s-paquette

cohorts/urls.py

@@ -19,9 +19,12 @@
 from . import views
 
 
-
 urlpatterns = [
     url(r'^$', views.cohorts_list, name='cohort_list'),
+    url(r'^manifests/fetch/(?P<file_name>[A-Za-z\-0-9]+\/manifest_[0-9_]+(aws|gcs|gcp)\.s5cmd)', views.fetch_user_manifest, name='fetch_user_manifest'),
+    url(r'^manifests/check/(?P<file_name>[A-Za-z\-0-9]+\/manifest_[0-9_]+(aws|gcs|gcp)\.s5cmd)', views.check_manifest_ready, name='check_user_manifest'),
+    url(r'^manifests/fetch/$', views.fetch_user_manifest, name='fetch_user_manifest_base'),
+    url(r'^manifests/check/$', views.check_manifest_ready, name='check_user_manifest_base'),
     url(r'^api/$', views.views_api_v1.cohort_list_api, name='cohort_list_api'),
     url(r'^api/v1/$', views.views_api_v1.cohort_list_api, name='cohort_list_api'),
     url(r'^api/v2/$', views.views_api_v2.cohort_list_api, name='cohort_list_api'),

cohorts/views/views.py

@@ -47,7 +47,6 @@
 
 logger = logging.getLogger('main_logger')
 
-USER_DATA_ON = settings.USER_DATA_ON
 
 @csrf_exempt
 @api_auth

cohorts/views/views_api_v1.py

@@ -44,7 +44,6 @@
 
 logger = logging.getLogger('main_logger')
 
-USER_DATA_ON = settings.USER_DATA_ON
 
 @csrf_exempt
 @api_auth

cohorts/views/views_api_v2.py

@@ -34,6 +34,7 @@
 import re
 #from projects.models import Program
 from google_helpers.bigquery.bq_support import BigQuerySupport
+from google_helpers.bigquery.utils import MOLECULAR_CATEGORIES
 
 #from django.conf import settings
 
@@ -49,23 +50,6 @@
 
 PREFORMATTED_ATTRIBUTES = {}
 
-# TODO: move this into a table, maybe metadata_attr?
-MOLECULAR_CATEGORIES = {
-    'nonsilent': {
-        'name': 'Non-silent',
-        'attrs': [
-            'Missense_Mutation',
-            'Nonsense_Mutation',
-            'Nonstop_Mutation',
-            'Frame_Shift_Del',
-            'Frame_Shift_Ins',
-            'In_Frame_Del',
-            'In_Frame_Ins',
-            'Translation_Start_Site',
-        ]
-    }
-}
-
 MOLECULAR_ATTR = [
     {'value': 'Missense_Mutation', 'displ_name': 'Missense Mutation'},
     {'value': 'Frame_Shift_Del', 'displ_name': 'Frame Shift - Deletion'},

google_helpers/bigquery/bq_support.py

@@ -9,7 +9,7 @@
 
 from idc_collections.models import Attribute, DataSource, Attribute_Ranges, DataSetType
 
-from metadata.query_helpers import MOLECULAR_CATEGORIES
+from google_helpers.bigquery.utils import MOLECULAR_CATEGORIES
 
 logger = logging.getLogger('main_logger')
 
@@ -112,9 +112,9 @@ def query_solr_and_format_result(query_settings, normalize_facets=True, normaliz
 
 # Execute a POST request to the solr server available available at settings.SOLR_URI
 def query_solr(collection=None, fields=None, query_string=None, fqs=None, facets=None, sort=None, counts_only=True,
-               collapse_on=None, offset=0, limit=1000, uniques=None, with_cursor=None, stats=None, totals=None):
-    query_uri = "{}{}/query".format(SOLR_URI, collection)
+               collapse_on=None, offset=0, limit=1000, uniques=None, with_cursor=None, stats=None, totals=None, op=None):
 
+    query_uri = "{}{}/query".format(SOLR_URI, collection)
     payload = {
         "query": query_string or "*:*",
         "limit": 0 if counts_only else limit,
@@ -124,6 +124,9 @@ def query_solr(collection=None, fields=None, query_string=None, fqs=None, facets
         }
     }
 
+    if op:
+        payload['params']['q.op'] = op
+
     if with_cursor:
         payload['params']['cursorMark'] = with_cursor
 
@@ -174,7 +177,6 @@ def query_solr(collection=None, fields=None, query_string=None, fqs=None, facets
 
     try:
         start = time.time()
-
         query_response = requests.post(query_uri, data=json.dumps(payload), headers={'Content-type': 'application/json'}, auth=(SOLR_LOGIN, SOLR_PASSWORD), verify=SOLR_CERT)
         stop = time.time()
 
@@ -392,7 +394,7 @@ def build_solr_facets(attrs, filter_tags=None, include_nulls=True, unique=None,
 # still want those records when filtering on this attribute.
 #
 def build_solr_query(filters, comb_with='AND', with_tags_for_ex=False, subq_join_field=None,
-                     search_child_records_by=None, global_value_op='OR'):
+                     search_child_records_by=None, global_value_op='OR', solr_default_op='OR'):
 
     # subq_join not currently used in IDC
     ranged_attrs = Attribute.get_ranged_attrs()
@@ -502,7 +504,10 @@ def build_solr_query(filters, comb_with='AND', with_tags_for_ex=False, subq_join
 
         # If it's looking for a single None value
         if len(values) == 1 and values[0] == 'None':
-            query_str += '(-%s:{* TO *})' % attr_name
+            if (solr_default_op == "OR"):
+                query_str += '(-%s:{* TO *})' % attr_name
+            else:
+                query_str += '(*:* NOT %s:{* TO *})' % attr_name
         # If it's a ranged value, calculate the bins
         elif attr_name == 'bmi':
             with_none = False
@@ -544,14 +549,19 @@ def build_solr_query(filters, comb_with='AND', with_tags_for_ex=False, subq_join
                     clause = rngTemp.format(attr_name, values[0], values[1])
                 else:
                     clause = "{}:{}".format(attr_name, values[0])
-
-            query_str += (('(-(-(%s) +(%s:{* TO *})))' % (clause, attr_name)) if with_none else "(+({}))".format(clause))
+            if (solr_default_op=="OR"):
+                query_str += (('(-(-(%s) +(%s:{* TO *})))' % (clause, attr_name)) if with_none else "(+({}))".format(clause))
+            else:
+                query_str += (('(%s OR (*:* NOT %s:{* TO *}))' % (clause, attr_name)) if with_none else "(+({}))".format(clause))
 
         else:
             vals = "\" {} \"".format(value_op).join(values)
             if 'None' in values:
                 values.remove('None')
-                query_str += '(-(-(%s:("%s")) +(%s:{* TO *})))' % (attr_name,vals, attr_name)
+                if (solr_default_op=="OR"):
+                    query_str += '(-(-(%s:("%s")) +(%s:{* TO *})))' % (attr_name,vals, attr_name)
+                else:
+                    query_str += '((%s:("%s")) OR (*:* NOT %s:{* TO *}))' % (attr_name, vals, attr_name)
             else:
                 query_str += '(+%s:("%s"))' % (attr_name, vals)