diff --git a/custom-recipes/pi-system-retrieve-list/recipe.json b/custom-recipes/pi-system-retrieve-list/recipe.json index a742514..d30a1e4 100644 --- a/custom-recipes/pi-system-retrieve-list/recipe.json +++ b/custom-recipes/pi-system-retrieve-list/recipe.json @@ -105,7 +105,7 @@ "name": "use_batch_mode", "label": "Use batch mode", "type": "BOOLEAN", - "description": "", + "description": "Use to quickly retrieve small samples from multiple paths. ⚠️Not for large time ranges", "visibilityCondition": "model.show_advanced_parameters==true", "defaultValue": false }, diff --git a/parameter-sets/basic-auth/parameter-set.json b/parameter-sets/basic-auth/parameter-set.json index 13e76de..70c278c 100644 --- a/parameter-sets/basic-auth/parameter-set.json +++ b/parameter-sets/basic-auth/parameter-set.json @@ -50,6 +50,28 @@ "description": "(optional)", "defaultValue": "" }, + { + "name": "max_request_size", + "label": "Maximum request size", + "type": "INT", + "description": "", + "defaultValue": 1000 + }, + { + "name": "estimated_density", + "label": "Estimated point density", + "type": "DOUBLE", + "description": "points/hour", + "defaultValue": 2 + }, + { + "name": "maximum_points_returned", + "label": "Maximum points return", + "type": "INT", + "description": "Target optimum number of points returned by batch. Calculated based on point density.", + "defaultValue": 1000000, + "minI": 1 + }, { "name": "osisoft_basic", "type": "CREDENTIAL_REQUEST", diff --git a/python-lib/osisoft_plugin_common.py b/python-lib/osisoft_plugin_common.py index 2a160dd..a4aff59 100644 --- a/python-lib/osisoft_plugin_common.py +++ b/python-lib/osisoft_plugin_common.py @@ -56,6 +56,21 @@ def get_credentials(config, can_raise=True): return auth_type, username, password, server_url, is_ssl_check_disabled, error_message +def get_batch_parameters(config): + credentials = config.get("credentials", {}) + max_request_size = credentials.get("max_request_size", 1000) + estimated_density = credentials.get("estimated_density", 6000) + maximum_points_returned = credentials.get("maximum_points_returned", 1000000) + return max_request_size, estimated_density, maximum_points_returned + + +def compute_time_spent(start, end, bla): + # 2023-06-30T13:05:10.8692786Z->2024-06-30T13:05:10.9640942Z + start = iso_to_epoch(start) + end = iso_to_epoch(end) + return end - start + + def get_advanced_parameters(config): show_advanced_parameters = config.get('show_advanced_parameters', False) batch_size = 500 @@ -600,3 +615,25 @@ def get_worst_performers(self): for slowest_event, slowest_time in zip(self.slowest_events, self.slowest_times): worst_performers.append("{}: {}s".format(slowest_event, slowest_time)) return worst_performers + + +class BatchTimeCounter(object): + def __init__(self, max_time_to_retrieve_per_batch): + self.max_time_to_retrieve_per_batch = max_time_to_retrieve_per_batch * 60 * 60 + self.total_batch_time = 0 + # 2 points /h each line + # max 1 000 000 lines back -> 500k hours max + + def is_batch_full(self): + # return False + if self.max_time_to_retrieve_per_batch < 0: + return False + if self.total_batch_time > self.max_time_to_retrieve_per_batch: + logger.warning("batch contains {}s of request, needs to flush now".format(self.total_batch_time)) + self.total_batch_time = 0 + return True + logger.info("Batch below time threshold") + return False + + def add(self, start_time, end_time, interval): + self.total_batch_time += compute_time_spent(start_time, end_time, interval)