Do not delete CW logs for endpoints. (#165)

Keep the logs for failed tests only.

Author: lukmis

tests/integ/timeout.py

@@ -58,28 +58,42 @@ def handler(signum, frame):
 @contextmanager
 def timeout_and_delete_endpoint_by_name(endpoint_name, sagemaker_session, seconds=0, minutes=35, hours=0):
     with timeout(seconds=seconds, minutes=minutes, hours=hours) as t:
+        no_errors = False
         try:
             yield [t]
+            no_errors = True
         finally:
             try:
                 sagemaker_session.delete_endpoint(endpoint_name)
                 LOGGER.info('deleted endpoint {}'.format(endpoint_name))
-                _cleanup_endpoint_logs(endpoint_name, sagemaker_session)
+
+                _show_endpoint_logs(endpoint_name, sagemaker_session)
+                if no_errors:
+                    _cleanup_endpoint_logs(endpoint_name, sagemaker_session)
             except ClientError as ce:
                 if ce.response['Error']['Code'] == 'ValidationException':
                     # avoids the inner exception to be overwritten
                     pass
 
 
-def _cleanup_endpoint_logs(endpoint_name, sagemaker_session):
+def _show_endpoint_logs(endpoint_name, sagemaker_session):
     log_group = '/aws/sagemaker/Endpoints/{}'.format(endpoint_name)
     try:
         # print out logs before deletion for debuggability
         LOGGER.info('cloudwatch logs for log group {}:'.format(log_group))
         logs = AWSLogs(log_group_name=log_group, log_stream_name='ALL', start='1d',
                        aws_region=sagemaker_session.boto_session.region_name)
         logs.list_logs()
+    except Exception:
+        LOGGER.exception('Failure occurred while listing cloudwatch log group %s. ' +
+                         'Swallowing exception but printing stacktrace for debugging.', log_group)
+
 
+def _cleanup_endpoint_logs(endpoint_name, sagemaker_session):
+    log_group = '/aws/sagemaker/Endpoints/{}'.format(endpoint_name)
+    try:
+        # print out logs before deletion for debuggability
+        LOGGER.info('deleting cloudwatch log group {}:'.format(log_group))
         cwl_client = sagemaker_session.boto_session.client('logs')
         cwl_client.delete_log_group(logGroupName=log_group)
         LOGGER.info('deleted cloudwatch log group: {}'.format(log_group))