Merge pull request #370 from isb-cgc/prod-sp

Sprint 37

Author: s-paquette

.circleci/config.yml

@@ -113,7 +113,7 @@ jobs:
           command: |
             sudo -E /bin/bash ./shell/gcloud-pull-staging-files.sh
       - save_cache:
-          key: isb-cgc-api-{{ epoch }}
+          key: isb-cgc-api-{{ .Branch }}-{{ .Revision }}-{{ epoch }}
           paths:
             - ./apiv4
             - ./shell
@@ -125,7 +125,6 @@ jobs:
             - ./txt
             - ./json
             - ./.env
-
   deploy_job:
     environment:
       TZ: "/usr/share/zoneinfo/America/Los_Angeles"
@@ -135,7 +134,7 @@ jobs:
     steps:
       - restore_cache:
           keys:
-            - isb-cgc-api-
+            - isb-cgc-api-{{ .Branch }}-{{ .Revision }}-
       - restore_cache:
           keys:
             - isb-cgc-api-google-deps-

apiv4/api_logging.py

@@ -0,0 +1,8 @@
+from django.conf import settings
+from google_helpers.stackdriver import StackDriverLogger
+
+# Log all API activity to StackDriver
+st_logger = StackDriverLogger.build_from_django_settings()
+log_name = settings.API_ACTIVITY_LOG_NAME
+user_activity_message = "[USER API CALL] User {} performing method {} path {}"
+activity_message = "[API CALL] Saw method {} for path {}"

apiv4/auth.py

@@ -54,7 +54,7 @@ def auth_info():
         if 'email' not in user_info:
             raise UserValidationException("Couldn't obtain user email - the correct scopes may not have been provided during authorization!")
     else:
-        logger.info("[STATUS] No user encoded info found.")
+        logger.warn("[WARNING] No user encoded info found.")
         user_info = {'id': 'anonymous', 'email': 'Anonymous'}
 
     return user_info
@@ -71,6 +71,7 @@ def get_user(user_email=None):
     django.setup()
     try:
         user = Django_User.objects.get(email=user_email)
+        logger.info("[USER API AUTH] User {} seen in API".format(user_email))
     except ObjectDoesNotExist as e:
         logger.warn("User {} does not exist in our system.".format(user_email))
         raise UserValidationException(

apiv4/cohorts_routes.py

@@ -22,6 +22,7 @@
 from auth import auth_info, UserValidationException, validate_user
 from django.conf import settings
 from django.db import close_old_connections
+from api_logging import *
 
 logger = logging.getLogger(settings.LOGGER_NAME)
 
@@ -37,57 +38,60 @@ def cohort(cohort_id):
         user_info = auth_info()
         user = validate_user(user_info['email'], cohort_id)
 
-        response = None
         code = None
+        response_obj = None
 
         if not user:
-            response = jsonify({
-                'code': 500,
+            response_obj = {
                 'message': 'Encountered an error while attempting to identify this user.'
-            })
-            response.status_code = 500
+            }
+            code = 500
         else:
-            if request.method == 'GET':
-                include_barcodes = (request.args.get('include_barcodes', default="false", type=str).lower() == "true")
-                cohort_info = get_cohort_info(cohort_id, include_barcodes)
+            if cohort_id <= 0:
+                logger.warn("[WARNING] Invalid cohort ID {}".format(str(cohort_id)))
+                code = 400
+                response_obj = {
+                    'message': '"{}" is not a valid cohort ID.'.format(str(cohort_id))
+                }
             else:
-                cohort_info = edit_cohort(cohort_id, user, delete=(request.method == 'DELETE'))
+                st_logger.write_text_log_entry(log_name, user_activity_message.format(user_info['email'], request.method, request.full_path))
+                if request.method == 'GET':
+                    include_barcodes = (request.args.get('include_barcodes', default="false", type=str).lower() == "true")
+                    cohort_info = get_cohort_info(cohort_id, include_barcodes)
+                else:
+                    cohort_info = edit_cohort(cohort_id, user, delete=(request.method == 'DELETE'))
 
-            if cohort_info:
-                response_obj = {}
+                if cohort_info:
+                    response_obj['data'] = cohort_info
 
-                if 'message' in cohort_info:
-                    code = 400
+                    if 'message' in cohort_info:
+                        code = 400
+                    else:
+                        code = 200
                 else:
-                    code = 200
-                    
-                response_obj['data'] = cohort_info
-                response_obj['code'] = code
-                response = jsonify(response_obj)
-                response.status_code = code
-                
-            else:
-                response = jsonify({
-                    'code': 404,
-                    'message': "Cohort ID {} was not found.".format(str(cohort_id))})
-                response.status_code = 404
+                    response_obj = {
+                        'message': "Cohort ID {} was not found.".format(str(cohort_id))
+                    }
+                    code = 404
 
     except UserValidationException as e:
-        response = jsonify({
-            'code': 403,
+        response_obj = {
             'message': str(e)
-        })
-        response.status_code = 403
+        }
+        code = 403
     except Exception as e:
         logger.exception(e)
-        response = jsonify({
-            'code': 500,
+        response_obj = {
             'message': 'Encountered an error while attempting to retrieve this cohort\'s information.'
-        })
-        response.status_code = 500
+        }
+        code = 500
     finally:
         close_old_connections()
 
+    response_obj['code'] = code
+    response = jsonify(response_obj)
+    response.status_code = code
+
     return response
 
 
@@ -98,7 +102,7 @@ def cohorts():
     POST: Add a new cohort
     """
 
-    response = None
+    response_obj = None
     info = None
     code = None
 
@@ -107,58 +111,57 @@ def cohorts():
         user = validate_user(user_info['email'])
 
         if not user:
-            response = jsonify({
+            response_obj = {
                 'code': 500,
                 'message': 'Encountered an error while attempting to identify this user.'
-            })
-            response.status_code = 500
+            }
+            code = 500
         else:
+            st_logger.write_text_log_entry(log_name, user_activity_message.format(user_info['email'], request.method, request.full_path))
             if request.method == 'GET':
                 info = get_cohorts(user_info['email'])
             else:
                 info = create_cohort(user)
 
             if info:
                 response_obj = {}
-                
+
                 if 'message' in info:
                     code = 400
                 else:
                     code = 200
 
                 response_obj['data'] = info
-                response_obj['code'] = code
-                response = jsonify(response_obj)
-                response.status_code = code
 
             # Lack of a valid object means something went wrong on the server
             else:
-                response = jsonify({
-                    'code': 500,
+                response_obj = {
                     'message': "Error while attempting to {}.".format(
                         'retrieve the cohort list' if request.method == 'GET' else 'create this cohort'
                     )
-                })
-                response.status_code = 500
+                }
+                code = 500
 
     except UserValidationException as e:
-        response = jsonify({
-            'code': 403,
+        response_obj = {
             'message': str(e)
-        })
-        response.status_code = 403
+        }
+        code = 403
     except Exception as e:
         logger.exception(e)
-        response = jsonify({
-            'code': 500,
+        response_obj = {
             'message': 'Encountered an error while attempting to {}.'.format(
                 "retrieve a list of cohorts" if request.method == 'GET' else "create this cohort"
             )
-        })
-        response.status_code = 500
+        }
+        code = 500
     finally:
         close_old_connections()
-        
+
+    response_obj['code'] = code
+    response = jsonify(response_obj)
+    response.status_code = code
+
     return response
 
 
@@ -182,22 +185,30 @@ def cohort_file_manifest(cohort_id):
             }
             code = 500
         else:
-            file_manifest = get_file_manifest(cohort_id, user)
-            if file_manifest:
-                # Presence of a message means something went wrong with our request
-                if 'message' in file_manifest:
-                    response_obj = file_manifest
-                    code = 400
+            if cohort_id <= 0:
+                logger.warn("[WARNING] Invalid cohort ID {}".format(str(cohort_id)))
+                code = 400
+                response_obj = {
+                    'message': '"{}" is not a valid cohort ID.'.format(str(cohort_id))
+                }
+            else:
+                st_logger.write_text_log_entry(log_name, user_activity_message.format(user_info['email'], request.method, request.full_path))
+                file_manifest = get_file_manifest(cohort_id, user)
+                if file_manifest:
+                    # Presence of a message means something went wrong with our request
+                    if 'message' in file_manifest:
+                        response_obj = file_manifest
+                        code = 400
+                    else:
+                        code = 200
+                        response_obj = {
+                            'data': file_manifest
+                        }
                 else:
-                    code = 200
                     response_obj = {
-                        'data': file_manifest
+                        'message': "Error while attempting to retrieve file manifest for cohort {}.".format(str(cohort_id))
                     }
-            else:
-                response_obj = {
-                    'message': "Error while attempting to retrieve file manifest for cohort {}.".format(str(cohort_id))
-                }
-                code = 500
+                    code = 500
 
     except UserValidationException as e:
         response_obj = {
@@ -227,6 +238,8 @@ def cohort_preview():
     code = None
     response_obj = None
 
+    st_logger.write_text_log_entry(log_name, activity_message.format(request.method, request.full_path))
+
     try:
         cohort_counts = get_cohort_counts()
 

apiv4/cohorts_views.py

@@ -59,14 +59,17 @@ def get_file_manifest(cohort_id, user):
             'offset': {'default': 0, 'type': int, 'name': 'offset'},
             'page': {'default': 1, 'type': int, 'name': 'page'},
             'fetch_count': {'default': 5000, 'type': int, 'name': 'limit'},
-            'genomic_build': {'default': "HG19", 'type': str, 'name': 'build'}
+            'genomic_build': {'default': "HG19", 'type': str, 'name': 'build'},
+            'case_insensitive': {'default': "True", 'type': str, 'name': 'case_insensitive'}
         }
 
         for param, parameter in param_set.items():
             default = parameter['default']
             param_type = parameter['type']
             name = parameter['name']
             params[name] = request_data[param] if (request_data and param in request_data) else request.args.get(param, default=default, type=param_type) if param in request.args else default
+            if name == 'case_insensitive':
+                params[name] = bool(params[name] == "True")
 
             if request_data:
                 inc_filters = {
@@ -150,7 +153,9 @@ def get_cohort_counts():
                 'message': 'No filters were provided; ensure that the request body contains a \'filters\' property.'
             }
         else:
-            cohort_counts = get_sample_case_list_bq(None, request_data['filters'])
+            case_insensitive = request_data['case_insensitive'] if (request_data and 'case_insensitive' in request_data) else request.args.get('case_insensitive', default="True", type=str) if 'case_insensitive' in request.args else "True"
+            
+            cohort_counts = get_sample_case_list_bq(None, request_data['filters'], case_insens=bool(case_insensitive == "True"))
 
             if cohort_counts:
                 for prog in cohort_counts:
@@ -207,6 +212,10 @@ def create_cohort(user):
             }
 
         else:
+            case_insensitive = request_data['case_insensitive'] if (request_data and 'case_insensitive' in request_data) else request.args.get('case_insensitive', default="True", type=str) if 'case_insensitive' in request.args else "True"
+            request_data.pop('case_insensitive', None)
+            request_data['case_insens'] = bool(case_insensitive == 'True')
+
             result = make_cohort(user, **request_data)
 
             if 'message' in result:

apiv4/file_routes.py

@@ -22,6 +22,7 @@
 from django.db import close_old_connections
 from auth import validate_user, UserValidationException
 from file_views import get_file_paths, get_signed_uris
+from api_logging import *
 
 logger = logging.getLogger(settings.LOGGER_NAME)
 
@@ -122,6 +123,8 @@ def file_path(file_uuid):
     resp_obj = None
     code = None
 
+    st_logger.write_text_log_entry(log_name, activity_message.format(request.method, request.full_path))
+
     try:
         file_paths = get_file_paths(file_uuid)
 
@@ -163,6 +166,8 @@ def file_path_list():
 
     response_obj = None
 
+    st_logger.write_text_log_entry(log_name, activity_message.format(request.method, request.full_path))
+
     try:
         file_paths = get_file_paths()
 
@@ -199,4 +204,3 @@ def file_path_list():
     response.status_code = code
 
     return response
-

apiv4/main_routes.py

@@ -19,6 +19,7 @@
 from flask import jsonify, request, render_template
 from django.conf import settings
 from apiv4 import app
+from api_logging import *
 
 logger = logging.getLogger(settings.LOGGER_NAME)
 
@@ -28,6 +29,9 @@
 @app.route('/v4/about/', methods=['GET'], strict_slashes=False)
 def apiv4():
     """Base response"""
+
+    st_logger.write_text_log_entry(log_name, activity_message.format(request.method, request.full_path))
+
     response = jsonify({
         'code': 200,
         'message': 'Welcome to the ISB-CGC API, Version 4.',
@@ -41,6 +45,7 @@ def apiv4():
 # Swagger UI
 @app.route('/v4/swagger/', methods=['GET'], strict_slashes=False)
 def swagger():
+    st_logger.write_text_log_entry(log_name, activity_message.format(request.method, request.full_path))
     return render_template('swagger/index.html')