Merge branch 'master' into table_acls_in_chunks

Author: lorenzorubi-db

dbclient/ClustersClient.py

@@ -469,8 +469,11 @@ def import_instance_profiles(self, log_file='instance_profiles.log'):
                 if ip_arn not in list_of_profiles:
                     print("Importing arn: {0}".format(ip_arn))
                     resp = self.post('/instance-profiles/add', {'instance_profile_arn': ip_arn})
-                    if not logging_utils.log_response_error(error_logger, resp):
+                    if not logging_utils.check_error(resp):
                         import_profiles_count += 1
+                    else:
+                        logging.info(f"Failed instance profile import for {ip_arn}")
+                        logging_utils.log_response_error(error_logger, resp)
                 else:
                     logging.info("Skipping since profile already exists: {0}".format(ip_arn))
         return import_profiles_count

dbclient/WorkspaceClient.py

@@ -922,7 +922,7 @@ def create_repo(self, repo_str, error_logger, checkpoint_repo_set):
                 parent_directory = re.sub(r"^RESOURCE_DOES_NOT_EXIST: Parent directory ", '', resp.get('message'))
                 parent_directory = re.sub(r" does not exist.$", '', parent_directory)
                 if re.fullmatch(
-                    r'/Repos/.+\@.+', parent_directory
+                    r'/Repos/.+[^/]', parent_directory
                 ):
                     logging.info(f"Creating parent directory {parent_directory}")
                     resp2 = self.post('/workspace/mkdirs', {"path": parent_directory})

dbclient/dbclient.py

@@ -190,17 +190,21 @@ def req_session(self):
             self._local.session = session
         return self._local.session
 
-    def get(self, endpoint, json_params=None, version='2.0', print_json=False, do_not_throw=False):
+    def get(self, endpoint, json_params=None, version='2.0', print_json=False, do_not_throw=False, timeout=10):
         if version:
             ver = version
         while True:
             full_endpoint = self._url + '/api/{0}'.format(ver) + endpoint
             if self.is_verbose():
                 print("Get: {0}".format(full_endpoint))
             if json_params:
-                raw_results = self.req_session().get(full_endpoint, headers=self._token, params=json_params, verify=self._verify_ssl)
+                raw_results = self.req_session().get(
+                    full_endpoint, headers=self._token, params=json_params, verify=self._verify_ssl, timeout=timeout
+                )
             else:
-                raw_results = self.req_session().get(full_endpoint, headers=self._token, verify=self._verify_ssl)
+                raw_results = self.req_session().get(
+                    full_endpoint, headers=self._token, verify=self._verify_ssl, timeout=timeout
+                )
 
             if self._should_retry_with_new_token(raw_results):
                 continue
@@ -218,7 +222,7 @@ def get(self, endpoint, json_params=None, version='2.0', print_json=False, do_no
             results['http_status_code'] = http_status_code
             return results
 
-    def http_req(self, http_type, endpoint, json_params, version='2.0', print_json=False, files_json=None):
+    def http_req(self, http_type, endpoint, json_params, version='2.0', print_json=False, files_json=None, timeout=10):
         if version:
             ver = version
         while True:
@@ -228,17 +232,25 @@ def http_req(self, http_type, endpoint, json_params, version='2.0', print_json=F
             if json_params:
                 if http_type == 'post':
                     if files_json:
-                        raw_results = self.req_session().post(full_endpoint, headers=self._token,
-                                                    data=json_params, files=files_json, verify=self._verify_ssl)
+                        raw_results = self.req_session().post(
+                            full_endpoint, headers=self._token, data=json_params, files=files_json,
+                            verify=self._verify_ssl, timeout=timeout
+                        )
                     else:
-                        raw_results = self.req_session().post(full_endpoint, headers=self._token,
-                                                    json=json_params, verify=self._verify_ssl)
+                        raw_results = self.req_session().post(
+                            full_endpoint, headers=self._token, json=json_params, verify=self._verify_ssl,
+                            timeout=timeout
+                        )
                 if http_type == 'put':
-                    raw_results = self.req_session().put(full_endpoint, headers=self._token,
-                                               json=json_params, verify=self._verify_ssl)
+                    raw_results = self.req_session().put(
+                        full_endpoint, headers=self._token, json=json_params, verify=self._verify_ssl,
+                        timeout=timeout
+                    )
                 if http_type == 'patch':
-                    raw_results = self.req_session().patch(full_endpoint, headers=self._token,
-                                                 json=json_params, verify=self._verify_ssl)
+                    raw_results = self.req_session().patch(
+                        full_endpoint, headers=self._token, json=json_params, verify=self._verify_ssl,
+                        timeout=timeout
+                    )
             else:
                 print("Must have a payload in json_args param.")
                 return {}