diff --git a/python-connectors/pi-system_hierarchy/connector.py b/python-connectors/pi-system_hierarchy/connector.py index fac8d13..60fd4fb 100644 --- a/python-connectors/pi-system_hierarchy/connector.py +++ b/python-connectors/pi-system_hierarchy/connector.py @@ -30,7 +30,8 @@ def __init__(self, config, plugin_config): server_url, auth_type, username, password, is_ssl_check_disabled=is_ssl_check_disabled, is_debug_mode=is_debug_mode, - network_timer=self.network_timer + network_timer=self.network_timer, + nb_retries_on_504=3, ) self.use_batch_mode = config.get("use_batch_mode", False) self.batch_size = config.get("batch_size", 500) @@ -117,6 +118,7 @@ def batch_next_item(self, next_item, parent=None, type=None): batch_requests_parameters.append(request_kwargs) parent_of_batched_items.append(item.get("parent")) if not todo_list or len(batch_requests_parameters) > self.batch_size: + logger.info("Processing {} entries in batch".format(len(batch_requests_parameters))) json_responses = self.client._batch_requests(batch_requests_parameters) batch_requests_parameters = [] for parent_of_batched_item, json_response in zip(parent_of_batched_items, json_responses): diff --git a/python-lib/osisoft_client.py b/python-lib/osisoft_client.py index 8b10b9d..2ecd171 100644 --- a/python-lib/osisoft_client.py +++ b/python-lib/osisoft_client.py @@ -4,6 +4,7 @@ import json import simplejson from datetime import datetime +from time import sleep from requests_ntlm import HttpNtlmAuth from osisoft_constants import OSIsoftConstants from osisoft_endpoints import OSIsoftEndpoints @@ -26,7 +27,8 @@ class PISystemClientError(ValueError): class OSIsoftClient(object): - def __init__(self, server_url, auth_type, username, password, is_ssl_check_disabled=False, can_raise=True, is_debug_mode=False, network_timer=None): + def __init__(self, server_url, auth_type, username, password, is_ssl_check_disabled=False, + can_raise=True, is_debug_mode=False, network_timer=None, nb_retries_on_504=None, delay_on_504=None): if can_raise: assert_server_url_ok(server_url) self.session = requests.Session() @@ -39,6 +41,9 @@ def __init__(self, server_url, auth_type, username, password, is_ssl_check_disab self.is_debug_mode = is_debug_mode self.debug_level = None self.network_timer = network_timer + self.nb_retries_on_504 = nb_retries_on_504 + self.delay_on_504 = delay_on_504 or 60 + self.retries_on_504 = 0 def get_auth(self, auth_type, username, password): if auth_type == "basic": @@ -575,11 +580,13 @@ def post(self, url, headers, params, data, can_raise=True, error_source=None): logger.info("Trying to post to {}".format(url)) if self.network_timer: self.network_timer.start(url) - response = self.session.post( - url=url, - headers=headers, - json=data - ) + response = self.retry_init() + while self.should_retry(response): + response = self.session.post( + url=url, + headers=headers, + json=data + ) if self.network_timer: self.network_timer.stop() if self.is_debug_mode: @@ -588,6 +595,28 @@ def post(self, url, headers, params, data, can_raise=True, error_source=None): self.assert_valid_response(response, can_raise=can_raise, error_source=error_source) return response + def should_retry(self, response): + if response==-1: + return True + if isinstance(response, requests.Response): + status_code = response.status_code + if self.nb_retries_on_504 and status_code==504: + logger.warning("Status code is 504, should retry {} times".format(self.nb_retries_on_504)) + self.retries_on_504 += 1 + if self.retries_on_504 >= self.nb_retries_on_504: + logger.error("Max number of retries reached, giving up.") + return False + else: + logger.warning("Sleeping {}s...".format(self.delay_on_504)) + sleep(self.delay_on_504) + logger.warning("Retry ({}/{})...".format(self.retries_on_504, self.nb_retries_on_504)) + return True + return False + + def retry_init(self): + self.retries_on_504 = 0 + return -1 + def get_debug_level(self): if self.debug_level == 5000: self.debug_level = 1000 diff --git a/python-lib/osisoft_constants.py b/python-lib/osisoft_constants.py index ce9b74a..bb17ee4 100644 --- a/python-lib/osisoft_constants.py +++ b/python-lib/osisoft_constants.py @@ -405,7 +405,7 @@ class OSIsoftConstants(object): "Security": "{base_url}/eventframes/{webid}/security", "SecurityEntries": "{base_url}/eventframes/{webid}/securityentries" } - PLUGIN_VERSION = "1.4.1-beta.1" + PLUGIN_VERSION = "1.4.1-beta.2" VALUE_COLUMN_SUFFIX = "_val" WEB_API_PATH = "piwebapi" WRITE_HEADERS = {'X-Requested-With': 'XmlHttpRequest'}