From b577ddb31e8839f798e6a0e7b039b60ec0bed8ba Mon Sep 17 00:00:00 2001 From: Alex Bourret Date: Mon, 30 Jun 2025 13:34:19 +0200 Subject: [PATCH 1/5] add warning to batch mode --- custom-recipes/pi-system-retrieve-list/recipe.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/custom-recipes/pi-system-retrieve-list/recipe.json b/custom-recipes/pi-system-retrieve-list/recipe.json index a742514..d30a1e4 100644 --- a/custom-recipes/pi-system-retrieve-list/recipe.json +++ b/custom-recipes/pi-system-retrieve-list/recipe.json @@ -105,7 +105,7 @@ "name": "use_batch_mode", "label": "Use batch mode", "type": "BOOLEAN", - "description": "", + "description": "Use to quickly retrieve small samples from multiple paths. ⚠️Not for large time ranges", "visibilityCondition": "model.show_advanced_parameters==true", "defaultValue": false }, From 2a81e81a898b2cf0e53d6f45c31c85096e490252 Mon Sep 17 00:00:00 2001 From: Alex Bourret Date: Mon, 30 Jun 2025 13:46:13 +0200 Subject: [PATCH 2/5] add mx request size selector in preset --- parameter-sets/basic-auth/parameter-set.json | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/parameter-sets/basic-auth/parameter-set.json b/parameter-sets/basic-auth/parameter-set.json index 13e76de..fb91c70 100644 --- a/parameter-sets/basic-auth/parameter-set.json +++ b/parameter-sets/basic-auth/parameter-set.json @@ -50,6 +50,13 @@ "description": "(optional)", "defaultValue": "" }, + { + "name": "max_request_size", + "label": "Maximum request size", + "type": "INT", + "description": "", + "defaultValue": 1000 + }, { "name": "osisoft_basic", "type": "CREDENTIAL_REQUEST", From 5fcd7ba10729bae3cd36c5e613470b21a37e7025 Mon Sep 17 00:00:00 2001 From: Alex Bourret Date: Wed, 30 Jul 2025 10:56:53 +0700 Subject: [PATCH 3/5] add batch time limitation to preset --- parameter-sets/basic-auth/parameter-set.json | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/parameter-sets/basic-auth/parameter-set.json b/parameter-sets/basic-auth/parameter-set.json index fb91c70..70c278c 100644 --- a/parameter-sets/basic-auth/parameter-set.json +++ b/parameter-sets/basic-auth/parameter-set.json @@ -57,6 +57,21 @@ "description": "", "defaultValue": 1000 }, + { + "name": "estimated_density", + "label": "Estimated point density", + "type": "DOUBLE", + "description": "points/hour", + "defaultValue": 2 + }, + { + "name": "maximum_points_returned", + "label": "Maximum points return", + "type": "INT", + "description": "Target optimum number of points returned by batch. Calculated based on point density.", + "defaultValue": 1000000, + "minI": 1 + }, { "name": "osisoft_basic", "type": "CREDENTIAL_REQUEST", From 7adfea1ae4a32dab450f0f0bbb750c381f9db3cf Mon Sep 17 00:00:00 2001 From: Alex Bourret Date: Mon, 4 Aug 2025 23:02:34 +0700 Subject: [PATCH 4/5] move BatchTimeCounter to commons --- python-lib/osisoft_plugin_common.py | 40 +++++++++++++++++++++++++++++ 1 file changed, 40 insertions(+) diff --git a/python-lib/osisoft_plugin_common.py b/python-lib/osisoft_plugin_common.py index 2a160dd..e7c920c 100644 --- a/python-lib/osisoft_plugin_common.py +++ b/python-lib/osisoft_plugin_common.py @@ -56,6 +56,21 @@ def get_credentials(config, can_raise=True): return auth_type, username, password, server_url, is_ssl_check_disabled, error_message +def get_batch_parameters(config): + credentials = config.get("credentials", {}) + max_request_size = credentials.get("max_request_size", 1000) + estimated_density = credentials.get("estimated_density", 6000) + maximum_points_returned = credentials.get("maximum_points_returned", 1000000) + return max_request_size, estimated_density, maximum_points_returned + + +def compute_time_spent(start, end, bla): + # 2023-06-30T13:05:10.8692786Z->2024-06-30T13:05:10.9640942Z + start = iso_to_epoch(start) + end = iso_to_epoch(end) + return end - start + + def get_advanced_parameters(config): show_advanced_parameters = config.get('show_advanced_parameters', False) batch_size = 500 @@ -600,3 +615,28 @@ def get_worst_performers(self): for slowest_event, slowest_time in zip(self.slowest_events, self.slowest_times): worst_performers.append("{}: {}s".format(slowest_event, slowest_time)) return worst_performers + + +class BatchTimeCounter(object): + def __init__(self, max_time_to_retrieve_per_batch): + logger.info("ALX:max_time_to_retrieve_per_batch:{}".format(max_time_to_retrieve_per_batch * 60 * 60)) + self.max_time_to_retrieve_per_batch = max_time_to_retrieve_per_batch * 60 * 60 + self.total_batch_time = 0 + # 2 points /h each line + # max 1 000 000 lines back -> 500k hours max + + def is_batch_full(self): + # return False + if self.max_time_to_retrieve_per_batch < 0: + return False + if self.total_batch_time > self.max_time_to_retrieve_per_batch: + logger.warning("batch contains {}s of request, needs to flush now".format(self.total_batch_time)) + self.total_batch_time = 0 + return True + logger.info("Batch below time threshold") + return False + + def add(self, start_time, end_time, interval): + print("ALX:add time {}, {}, {}, {}".format(start_time, end_time, interval, self.total_batch_time)) + self.total_batch_time += compute_time_spent(start_time, end_time, interval) + print("ALX:added time {}".format(self.total_batch_time)) From 938c041fd8a138258c9941ce999bd3a0bab812d7 Mon Sep 17 00:00:00 2001 From: Alex Bourret Date: Mon, 4 Aug 2025 23:04:07 +0700 Subject: [PATCH 5/5] cleaning --- python-lib/osisoft_plugin_common.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/python-lib/osisoft_plugin_common.py b/python-lib/osisoft_plugin_common.py index e7c920c..a4aff59 100644 --- a/python-lib/osisoft_plugin_common.py +++ b/python-lib/osisoft_plugin_common.py @@ -619,7 +619,6 @@ def get_worst_performers(self): class BatchTimeCounter(object): def __init__(self, max_time_to_retrieve_per_batch): - logger.info("ALX:max_time_to_retrieve_per_batch:{}".format(max_time_to_retrieve_per_batch * 60 * 60)) self.max_time_to_retrieve_per_batch = max_time_to_retrieve_per_batch * 60 * 60 self.total_batch_time = 0 # 2 points /h each line @@ -637,6 +636,4 @@ def is_batch_full(self): return False def add(self, start_time, end_time, interval): - print("ALX:add time {}, {}, {}, {}".format(start_time, end_time, interval, self.total_batch_time)) self.total_batch_time += compute_time_spent(start_time, end_time, interval) - print("ALX:added time {}".format(self.total_batch_time))