From 50993236caf0219deaf71a366a3d73714c43c049 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A9r=C3=B4me=20TERRIER?= Date: Thu, 5 Dec 2024 14:41:34 +0000 Subject: [PATCH 01/28] activate start time column --- custom-recipes/pi-system-retrieve-list/recipe.json | 12 ++++++------ custom-recipes/pi-system-retrieve-list/recipe.py | 6 ++---- 2 files changed, 8 insertions(+), 10 deletions(-) diff --git a/custom-recipes/pi-system-retrieve-list/recipe.json b/custom-recipes/pi-system-retrieve-list/recipe.json index 78f58a7..5d81ee0 100644 --- a/custom-recipes/pi-system-retrieve-list/recipe.json +++ b/custom-recipes/pi-system-retrieve-list/recipe.json @@ -125,8 +125,8 @@ { "name": "use_start_time_column", "label": "Use start time value per row", - "description": "Description here", - "visibilityCondition": "false && (['InterpolatedData', 'RecordedData', 'PlotData', 'EventFrames'].includes(model.data_type))", + "description": "", + "visibilityCondition": "['InterpolatedData', 'RecordedData', 'PlotData', 'EventFrames'].includes(model.data_type)", "type": "BOOLEAN", "defaultValue": false }, @@ -134,7 +134,7 @@ "visibilityCondition": "model.use_start_time_column==true && (['InterpolatedData', 'RecordedData', 'PlotData', 'EventFrames'].includes(model.data_type))", "name": "start_time_column", "label": "Start times' column", - "description": "Optional", + "description": "Column type must be string", "type": "COLUMN", "columnRole": "input_dataset" }, @@ -148,9 +148,9 @@ }, { "name": "use_end_time_column", - "label": "Use start time value per row", + "label": "Use end time value per row", "description": "", - "visibilityCondition": "false && (['InterpolatedData', 'RecordedData', 'PlotData', 'EventFrames'].includes(model.data_type))", + "visibilityCondition": "['InterpolatedData', 'RecordedData', 'PlotData', 'EventFrames'].includes(model.data_type)", "type": "BOOLEAN", "defaultValue": false }, @@ -158,7 +158,7 @@ "visibilityCondition": "model.use_end_time_column==true && (['InterpolatedData', 'RecordedData', 'PlotData', 'EventFrames'].includes(model.data_type))", "name": "end_time_column", "label": "End times' column", - "description": "Optional", + "description": "Column type must be string", "type": "COLUMN", "columnRole": "input_dataset" }, diff --git a/custom-recipes/pi-system-retrieve-list/recipe.py b/custom-recipes/pi-system-retrieve-list/recipe.py index b09c071..66b4412 100644 --- a/custom-recipes/pi-system-retrieve-list/recipe.py +++ b/custom-recipes/pi-system-retrieve-list/recipe.py @@ -82,10 +82,8 @@ # make sure all OSIsoft time string format are evaluated at the same time # rather than at every request, at least for start / end times set in the UI time_not_parsed = False - if not use_start_time_column: - start_time = client.parse_pi_time(start_time) - if not use_end_time_column: - end_time = client.parse_pi_time(end_time) + start_time = client.parse_pi_time(start_time) + end_time = client.parse_pi_time(end_time) sync_time = client.parse_pi_time(sync_time) object_id = input_parameters_row.get(path_column) From b6e0d7bf3e2cdd8324f735302d4ca3ab4a4ca2da Mon Sep 17 00:00:00 2001 From: Alexandre Bourret Date: Thu, 12 Dec 2024 15:00:07 +0100 Subject: [PATCH 02/28] Adding boundary type selector to recorded data type --- .../pi-system-retrieve-list/recipe.json | 2 +- .../pi-system-retrieve-list/recipe.py | 3 ++ python-lib/osisoft_client.py | 45 ++++++++++++------- python-lib/osisoft_plugin_common.py | 3 +- 4 files changed, 35 insertions(+), 18 deletions(-) diff --git a/custom-recipes/pi-system-retrieve-list/recipe.json b/custom-recipes/pi-system-retrieve-list/recipe.json index 78f58a7..010f52c 100644 --- a/custom-recipes/pi-system-retrieve-list/recipe.json +++ b/custom-recipes/pi-system-retrieve-list/recipe.json @@ -194,7 +194,7 @@ { "name": "boundary_type", "label": "Boundary type", - "visibilityCondition": "['InterpolatedData'].includes(model.data_type)", + "visibilityCondition": "['InterpolatedData','RecordedData'].includes(model.data_type)", "type": "SELECT", "selectChoices":[ {"value": "Inside", "label": "Inside"}, diff --git a/custom-recipes/pi-system-retrieve-list/recipe.py b/custom-recipes/pi-system-retrieve-list/recipe.py index b09c071..34a6415 100644 --- a/custom-recipes/pi-system-retrieve-list/recipe.py +++ b/custom-recipes/pi-system-retrieve-list/recipe.py @@ -48,6 +48,7 @@ end_time_column = config.get("end_time_column") server_url_column = config.get("server_url_column") interval, sync_time, boundary_type = get_interpolated_parameters(config) +record_boundary_type = config.get("boundary_type") if data_type == "RecordedData" else None summary_type, summary_duration = get_summary_parameters(config) network_timer = PerformanceTimer() @@ -102,6 +103,7 @@ interval=interval, sync_time=sync_time, boundary_type=boundary_type, + record_boundary_type=record_boundary_type, max_count=max_count, can_raise=False, object_id=object_id, @@ -117,6 +119,7 @@ interval=interval, sync_time=sync_time, boundary_type=boundary_type, + record_boundary_type=record_boundary_type, max_count=max_count, can_raise=False, endpoint_type="AF", diff --git a/python-lib/osisoft_client.py b/python-lib/osisoft_client.py index 6d66ad0..f1e63d8 100644 --- a/python-lib/osisoft_client.py +++ b/python-lib/osisoft_client.py @@ -48,7 +48,7 @@ def get_auth(self, auth_type, username, password): return None def recursive_get_rows_from_webid(self, webid, data_type, start_date=None, end_date=None, - interval=None, sync_time=None, boundary_type=None, selected_fields=None, + interval=None, sync_time=None, boundary_type=None, record_boundary_type=None, selected_fields=None, can_raise=True, endpoint_type="event_frames", search_full_hierarchy=None, max_count=None, summary_type=None, summary_duration=None): # Split the time range until no more HTTP 400 @@ -57,7 +57,8 @@ def recursive_get_rows_from_webid(self, webid, data_type, start_date=None, end_d while not done: logger.info("Attempting download webids from {} to {}".format(start_date, end_date)) rows = self.get_rows_from_webid(webid, data_type, start_date=start_date, end_date=end_date, - interval=interval, sync_time=sync_time, boundary_type=boundary_type, selected_fields=selected_fields, + interval=interval, sync_time=sync_time, boundary_type=boundary_type, + record_boundary_type=record_boundary_type, selected_fields=selected_fields, can_raise=can_raise, endpoint_type=endpoint_type, search_full_hierarchy=search_full_hierarchy, max_count=max_count, summary_type=summary_type, summary_duration=summary_duration) counter = 0 @@ -79,7 +80,8 @@ def recursive_get_rows_from_webid(self, webid, data_type, start_date=None, end_d start_timestamp, end_timestamp, half_time_iso = self.halve_time_range(start_date, end_date) first_half_rows = self.recursive_get_rows_from_webid( webid, data_type, start_date=start_timestamp, end_date=half_time_iso, - interval=interval, sync_time=sync_time, boundary_type=boundary_type, selected_fields=selected_fields, + interval=interval, sync_time=sync_time, boundary_type=boundary_type, + record_boundary_type=record_boundary_type, selected_fields=selected_fields, can_raise=can_raise, endpoint_type=endpoint_type, search_full_hierarchy=search_full_hierarchy, max_count=max_count, summary_type=summary_type, summary_duration=summary_duration ) @@ -88,7 +90,8 @@ def recursive_get_rows_from_webid(self, webid, data_type, start_date=None, end_d logger.info("Successfully retrieved first half ({} to {})".format(start_timestamp, half_time_iso)) second_half_rows = self.recursive_get_rows_from_webid( webid, data_type, start_date=half_time_iso, end_date=end_timestamp, - interval=interval, sync_time=sync_time, boundary_type=boundary_type, selected_fields=selected_fields, + interval=interval, sync_time=sync_time, boundary_type=boundary_type, + record_boundary_type=record_boundary_type, selected_fields=selected_fields, can_raise=can_raise, endpoint_type=endpoint_type, search_full_hierarchy=search_full_hierarchy, max_count=max_count, summary_type=summary_type, summary_duration=summary_duration ) @@ -109,7 +112,7 @@ def recursive_get_rows_from_webid(self, webid, data_type, start_date=None, end_d done = True def recursive_get_rows_from_item(self, item, data_type, start_date=None, end_date=None, - interval=None, sync_time=None, boundary_type=None, + interval=None, sync_time=None, boundary_type=None, record_boundary_type=None, can_raise=True, object_id=None, endpoint_type="event_frames", search_full_hierarchy=None, max_count=None, summary_type=None, summary_duration=None): # item can be an pi tag, a path to an element or event frame @@ -119,7 +122,8 @@ def recursive_get_rows_from_item(self, item, data_type, start_date=None, end_dat while not done: logger.info("Attempting download items from {} to {}".format(start_date, end_date)) rows = self.get_rows_from_item(item, data_type, start_date=start_date, end_date=end_date, interval=interval, - sync_time=sync_time, boundary_type=boundary_type, can_raise=True, object_id=object_id, + sync_time=sync_time, boundary_type=boundary_type, record_boundary_type=record_boundary_type, + can_raise=True, object_id=object_id, search_full_hierarchy=search_full_hierarchy, max_count=max_count, summary_type=summary_type, summary_duration=summary_duration) counter = 0 @@ -141,7 +145,8 @@ def recursive_get_rows_from_item(self, item, data_type, start_date=None, end_dat start_timestamp, end_timestamp, half_time_iso = self.halve_time_range(start_date, end_date) first_half_rows = self.recursive_get_rows_from_item( item, data_type, start_date=start_timestamp, end_date=half_time_iso, - interval=interval, sync_time=sync_time, boundary_type=boundary_type, can_raise=True, object_id=object_id, + interval=interval, sync_time=sync_time, boundary_type=boundary_type, + record_boundary_type=record_boundary_type, can_raise=True, object_id=object_id, search_full_hierarchy=search_full_hierarchy, max_count=max_count, summary_type=summary_type, summary_duration=summary_duration ) for row in first_half_rows: @@ -149,7 +154,8 @@ def recursive_get_rows_from_item(self, item, data_type, start_date=None, end_dat logger.info("Successfully retrieved first half ({} to {})".format(start_timestamp, half_time_iso)) second_half_rows = self.recursive_get_rows_from_item( item, data_type, start_date=half_time_iso, end_date=end_timestamp, - interval=interval, sync_time=sync_time, boundary_type=boundary_type, can_raise=True, object_id=object_id, + interval=interval, sync_time=sync_time, boundary_type=boundary_type, + record_boundary_type=record_boundary_type, can_raise=True, object_id=object_id, search_full_hierarchy=search_full_hierarchy, max_count=max_count, summary_type=summary_type, summary_duration=summary_duration ) for row in second_half_rows: @@ -219,7 +225,7 @@ def parse_pi_time(self, pi_time, to_epoch=False): return iso_timestamp def get_rows_from_webid(self, webid, data_type, start_date=None, end_date=None, - interval=None, sync_time=None, boundary_type=None, selected_fields=None, + interval=None, sync_time=None, boundary_type=None, record_boundary_type=None, selected_fields=None, can_raise=True, endpoint_type="event_frames", search_full_hierarchy=None, max_count=None, summary_type=None, summary_duration=None): @@ -234,6 +240,7 @@ def get_rows_from_webid(self, webid, data_type, start_date=None, end_date=None, interval=interval, sync_time=sync_time, boundary_type=boundary_type, + record_boundary_type=record_boundary_type, selected_fields=selected_fields, search_full_hierarchy=search_full_hierarchy, max_count=max_count, @@ -252,7 +259,7 @@ def get_rows_from_webid(self, webid, data_type, start_date=None, end_date=None, yield item def get_rows_from_webids(self, input_rows, data_type, start_date=None, end_date=None, - interval=None, sync_time=None, boundary_type=None, selected_fields=None, search_full_hierarchy=None, + interval=None, sync_time=None, boundary_type=None, record_boundary_type=None, selected_fields=None, search_full_hierarchy=None, max_count=None, can_raise=True, endpoint_type="event_frames", batch_size=500, summary_type=None, summary_duration=None): batch_requests_parameters = [] number_processed_webids = 0 @@ -318,7 +325,7 @@ def _batch_requests(self, batch_requests_parameters): yield batch_section.get("Content", {}) def generic_get_kwargs(self, start_date=None, end_date=None, interval=None, sync_time=None, - boundary_type=None, selected_fields=None, search_full_hierarchy=None, max_count=None, + boundary_type=None, record_boundary_type=None, selected_fields=None, search_full_hierarchy=None, max_count=None, summary_type=None, summary_duration=None, can_raise=None): headers = self.get_requests_headers() params = self.get_requests_params( @@ -327,6 +334,7 @@ def generic_get_kwargs(self, start_date=None, end_date=None, interval=None, sync interval=interval, sync_time=sync_time, boundary_type=boundary_type, + record_boundary_type=record_boundary_type, selected_fields=selected_fields, search_full_hierarchy=search_full_hierarchy, max_count=max_count, @@ -339,7 +347,7 @@ def generic_get_kwargs(self, start_date=None, end_date=None, interval=None, sync } def generic_get(self, url, start_date=None, end_date=None, interval=None, sync_time=None, - boundary_type=None, selected_fields=None, search_full_hierarchy=None, max_count=None, + boundary_type=None, record_boundary_type=None, selected_fields=None, search_full_hierarchy=None, max_count=None, can_raise=None, summary_type=None, summary_duration=None): headers = self.get_requests_headers() params = self.get_requests_params( @@ -348,6 +356,7 @@ def generic_get(self, url, start_date=None, end_date=None, interval=None, sync_t interval=interval, sync_time=sync_time, boundary_type=boundary_type, + record_boundary_type=record_boundary_type, selected_fields=selected_fields, search_full_hierarchy=search_full_hierarchy, max_count=max_count, @@ -363,7 +372,7 @@ def generic_get(self, url, start_date=None, end_date=None, interval=None, sync_t return json_response def get_rows_from_item(self, item, data_type, start_date=None, end_date=None, interval=None, - sync_time=None, boundary_type=None, can_raise=True, object_id=None, + sync_time=None, boundary_type=None, record_boundary_type=None, can_raise=True, object_id=None, search_full_hierarchy=None, max_count=None, summary_type=None, summary_duration=None): # item can be an pi tag, a path to an element or event frame @@ -378,6 +387,7 @@ def get_rows_from_item(self, item, data_type, start_date=None, end_date=None, in interval=interval, sync_time=sync_time, boundary_type=boundary_type, + record_boundary_type=record_boundary_type, max_count=max_count, search_full_hierarchy=search_full_hierarchy, can_raise=can_raise, @@ -392,7 +402,7 @@ def get_rows_from_item(self, item, data_type, start_date=None, end_date=None, in yield self.loop_sub_items(item) def get_link_from_item(self, item, data_type, start_date, end_date, interval=None, - sync_time=None, boundary_type=None, search_full_hierarchy=None, + sync_time=None, boundary_type=None, record_boundary_type=None, search_full_hierarchy=None, max_count=None, can_raise=True, summary_type=None, summary_duration=None): url = self.extract_link_with_key(item, data_type) @@ -404,7 +414,8 @@ def get_link_from_item(self, item, data_type, start_date, end_date, interval=Non headers = self.get_requests_headers() params = build_requests_params( start_time=start_date, end_time=end_date, interval=interval, - sync_time=sync_time, sync_time_boundary_type=boundary_type, search_full_hierarchy=search_full_hierarchy, + sync_time=sync_time, sync_time_boundary_type=boundary_type, record_boundary_type=record_boundary_type, + search_full_hierarchy=search_full_hierarchy, max_count=max_count, summary_type=summary_type, summary_duration=summary_duration ) json_response = self.get( @@ -625,7 +636,7 @@ def get_requests_headers(self): } def get_requests_params(self, start_date=None, end_date=None, interval=None, sync_time=None, - boundary_type=None, selected_fields=None, search_full_hierarchy=None, + boundary_type=None, record_boundary_type=None, selected_fields=None, search_full_hierarchy=None, max_count=None, summary_type=None, summary_duration=None): params = {} if start_date: @@ -638,6 +649,8 @@ def get_requests_params(self, start_date=None, end_date=None, interval=None, syn params.update({"syncTime": sync_time}) if boundary_type: params.update({"syncTimeBoundaryType": boundary_type}) + if record_boundary_type: + params.update({"boundaryType": record_boundary_type}) if selected_fields: params.update({"selectedFields": selected_fields}) if search_full_hierarchy: diff --git a/python-lib/osisoft_plugin_common.py b/python-lib/osisoft_plugin_common.py index aae0ac1..5a3fc07 100644 --- a/python-lib/osisoft_plugin_common.py +++ b/python-lib/osisoft_plugin_common.py @@ -133,6 +133,7 @@ def build_requests_params(**kwargs): "interval": "interval", "sync_time": "syncTime", "sync_time_boundary_type": "syncTimeBoundaryType", + "record_boundary_type": "boundaryType", "name_filter": "nameFilter", "category_name": "categoryName", "template_name": "templateName", @@ -482,7 +483,7 @@ class PerformanceTimer(): - adds up all start / stop intervals - count the number of intervals - compute the average event time - - provides a lists of the NUMBER_OF_SLOWEST_EVENTS_KEPT longest events by event id, for instance url + - provides a lists of the NUMBER_OF_SLOWEST_EVENTS_KEPT longest events by event id, for instance url """ NUMBER_OF_SLOWEST_EVENTS_KEPT = 5 From eee84cb7d8c21968e5411412fedaac546a8ca703 Mon Sep 17 00:00:00 2001 From: Alexandre Bourret Date: Thu, 12 Dec 2024 15:00:34 +0100 Subject: [PATCH 03/28] Adding boundary selector to connector --- .../pi-system_attribute-search/connector.json | 11 +++++++++++ .../pi-system_attribute-search/connector.py | 14 ++++++++------ 2 files changed, 19 insertions(+), 6 deletions(-) diff --git a/python-connectors/pi-system_attribute-search/connector.json b/python-connectors/pi-system_attribute-search/connector.json index c8707c4..9708871 100644 --- a/python-connectors/pi-system_attribute-search/connector.json +++ b/python-connectors/pi-system_attribute-search/connector.json @@ -360,6 +360,17 @@ ], "visibilityCondition": "((model.must_retrieve_metrics) && (model.data_type == 'SummaryData'))" }, + { + "name": "boundary_type", + "label": "Boundary type", + "visibilityCondition": "((model.must_retrieve_metrics) && ['InterpolatedData','RecordedData'].includes(model.data_type))", + "type": "SELECT", + "selectChoices":[ + {"value": "Inside", "label": "Inside"}, + {"value": "Outside", "label": "Outside"} + ], + "defaultValue": "Inside" + }, { "name": "summary_duration", "label": "Summary duration", diff --git a/python-connectors/pi-system_attribute-search/connector.py b/python-connectors/pi-system_attribute-search/connector.py index 7977436..28f448e 100644 --- a/python-connectors/pi-system_attribute-search/connector.py +++ b/python-connectors/pi-system_attribute-search/connector.py @@ -6,7 +6,8 @@ from osisoft_plugin_common import ( PISystemConnectorError, RecordsLimit, get_credentials, assert_time_format, remove_unwanted_columns, format_output, filter_columns_from_schema, is_child_attribute_path, - check_debug_mode, PerformanceTimer, get_max_count, get_summary_parameters, fields_selector + check_debug_mode, PerformanceTimer, get_max_count, get_summary_parameters, fields_selector, + get_interpolated_parameters ) from osisoft_constants import OSIsoftConstants @@ -36,9 +37,7 @@ def __init__(self, config, plugin_config): self.start_time = self.client.parse_pi_time(self.start_time) self.end_time = config.get("end_time") self.end_time = self.client.parse_pi_time(self.end_time) - is_interpolated_data = config.get("data_type", "").endswith("InterpolatedData") - self.interval = config.get("interval") if is_interpolated_data else None - self.sync_time = config.get("sync_time") if is_interpolated_data else None + self.interval, self.sync_time, self.boundary_type = get_interpolated_parameters(config) self.sync_time = self.client.parse_pi_time(self.sync_time) assert_time_format(self.start_time, error_source="start time") assert_time_format(self.end_time, error_source="end time") @@ -57,6 +56,8 @@ def __init__(self, config, plugin_config): self.config = config self.summary_type, self.summary_duration = get_summary_parameters(config) + self.record_boundary_type = config.get("boundary_type") if self.data_type == "RecordedData" else None + def extract_database_webid(self, database_endpoint): return database_endpoint.split("/")[-1] @@ -114,8 +115,9 @@ def generate_rows(self, dataset_schema=None, dataset_partitioning=None, selected_fields=fields_selector(self.data_type), max_count=self.max_count, summary_type=self.summary_type, - summary_duration=self.summary_duration - # boundary_type=self.boundary_type + summary_duration=self.summary_duration, + boundary_type=self.boundary_type, + record_boundary_type=self.record_boundary_type ): if limit.is_reached(): return From 20adf06941ab26462b95ace8a94cb4c191021fda Mon Sep 17 00:00:00 2001 From: Alexandre Bourret Date: Thu, 12 Dec 2024 15:01:02 +0100 Subject: [PATCH 04/28] v1.2.4 --- plugin.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugin.json b/plugin.json index 50903e2..88c4611 100644 --- a/plugin.json +++ b/plugin.json @@ -1,6 +1,6 @@ { "id": "pi-system", - "version": "1.2.3", + "version": "1.2.4", "meta": { "label": "PI System", "description": "Retrieve data from your OSIsoft PI System servers", From 81197c117ddb709138245b7e4296389f15cf11ed Mon Sep 17 00:00:00 2001 From: Alexandre Bourret Date: Thu, 12 Dec 2024 15:01:13 +0100 Subject: [PATCH 05/28] Updating changelog --- CHANGELOG.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 1a55f0c..7990e91 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,10 @@ # Changelog +## [Version 1.2.4](https://github.com/dataiku/dss-plugin-pi-server/releases/tag/v1.2.4) - Feature release - 2024-12-12 + +- Add boundary type selector to recorded data type +- Add boundary type selector to attribute search connector + ## [Version 1.2.3](https://github.com/dataiku/dss-plugin-pi-server/releases/tag/v1.2.3) - Feature release - 2024-09-26 - Add summaryDuration input (duration of each summary interval) From 07b7a1f904a6b1a0bded3490c876af7a388ef656 Mon Sep 17 00:00:00 2001 From: Alexandre Bourret Date: Thu, 12 Dec 2024 15:01:21 +0100 Subject: [PATCH 06/28] beta marker --- python-lib/osisoft_constants.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python-lib/osisoft_constants.py b/python-lib/osisoft_constants.py index 3266ce4..53e0e7f 100644 --- a/python-lib/osisoft_constants.py +++ b/python-lib/osisoft_constants.py @@ -403,7 +403,7 @@ class OSIsoftConstants(object): "Security": "{base_url}/eventframes/{webid}/security", "SecurityEntries": "{base_url}/eventframes/{webid}/securityentries" } - PLUGIN_VERSION = "1.2.3" + PLUGIN_VERSION = "1.2.3-beta.1" VALUE_COLUMN_SUFFIX = "_val" WEB_API_PATH = "piwebapi" WRITE_HEADERS = {'X-Requested-With': 'XmlHttpRequest'} From 9d59cda35c49a154590f97c2875f314b05e228d4 Mon Sep 17 00:00:00 2001 From: Alexandre Bourret Date: Thu, 12 Dec 2024 15:04:12 +0100 Subject: [PATCH 07/28] v1.2.4 --- python-lib/osisoft_constants.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python-lib/osisoft_constants.py b/python-lib/osisoft_constants.py index 53e0e7f..f061bf2 100644 --- a/python-lib/osisoft_constants.py +++ b/python-lib/osisoft_constants.py @@ -403,7 +403,7 @@ class OSIsoftConstants(object): "Security": "{base_url}/eventframes/{webid}/security", "SecurityEntries": "{base_url}/eventframes/{webid}/securityentries" } - PLUGIN_VERSION = "1.2.3-beta.1" + PLUGIN_VERSION = "1.2.4-beta.1" VALUE_COLUMN_SUFFIX = "_val" WEB_API_PATH = "piwebapi" WRITE_HEADERS = {'X-Requested-With': 'XmlHttpRequest'} From edd18f9cac6f54558eb3b25bd1bee62a19a8f5e9 Mon Sep 17 00:00:00 2001 From: Alexandre Bourret Date: Fri, 20 Dec 2024 16:19:16 +0100 Subject: [PATCH 08/28] add Items.Value.Value to fields selector --- python-lib/osisoft_plugin_common.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python-lib/osisoft_plugin_common.py b/python-lib/osisoft_plugin_common.py index 5a3fc07..6812a4e 100644 --- a/python-lib/osisoft_plugin_common.py +++ b/python-lib/osisoft_plugin_common.py @@ -449,7 +449,7 @@ def fields_selector(data_type): if data_type in ["Value", "EndValue"]: return "Links%3BTimestamp%3BValue%3BType%3BUnitsAbbreviation" else: - return "Links%3BItems.Timestamp%3BItems.Value%3BItems.Type" + return "Links%3BItems.Timestamp%3BItems.Value%3BItems.Type%3BItems.Value.Value" def get_next_page_url(json): From 7cb64f622fce9a1de1792f6b0110c41421f53694 Mon Sep 17 00:00:00 2001 From: Alexandre Bourret Date: Fri, 20 Dec 2024 16:19:37 +0100 Subject: [PATCH 09/28] beta 2 marker --- python-lib/osisoft_constants.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python-lib/osisoft_constants.py b/python-lib/osisoft_constants.py index f061bf2..eb22ac3 100644 --- a/python-lib/osisoft_constants.py +++ b/python-lib/osisoft_constants.py @@ -403,7 +403,7 @@ class OSIsoftConstants(object): "Security": "{base_url}/eventframes/{webid}/security", "SecurityEntries": "{base_url}/eventframes/{webid}/securityentries" } - PLUGIN_VERSION = "1.2.4-beta.1" + PLUGIN_VERSION = "1.2.4-beta.2" VALUE_COLUMN_SUFFIX = "_val" WEB_API_PATH = "piwebapi" WRITE_HEADERS = {'X-Requested-With': 'XmlHttpRequest'} From ca55b61e7bbbd2a6a36b8cdcf8feb76463b99838 Mon Sep 17 00:00:00 2001 From: Alexandre Bourret Date: Fri, 20 Dec 2024 16:19:47 +0100 Subject: [PATCH 10/28] update changelog --- CHANGELOG.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7990e91..0b14748 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,9 +1,10 @@ # Changelog -## [Version 1.2.4](https://github.com/dataiku/dss-plugin-pi-server/releases/tag/v1.2.4) - Feature release - 2024-12-12 +## [Version 1.2.4](https://github.com/dataiku/dss-plugin-pi-server/releases/tag/v1.2.4) - Feature and bug release - 2024-12-12 - Add boundary type selector to recorded data type - Add boundary type selector to attribute search connector +- Fix issue with recorded data type ## [Version 1.2.3](https://github.com/dataiku/dss-plugin-pi-server/releases/tag/v1.2.3) - Feature release - 2024-09-26 From 11c8779d810061d49db5438d4b4b0279691e2186 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A9r=C3=B4me=20TERRIER?= Date: Thu, 2 Jan 2025 08:24:11 +0000 Subject: [PATCH 11/28] fix recursive_get_rows_from_webid --- custom-recipes/pi-system-retrieve-list/recipe.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/custom-recipes/pi-system-retrieve-list/recipe.py b/custom-recipes/pi-system-retrieve-list/recipe.py index 34a6415..f60ac5b 100644 --- a/custom-recipes/pi-system-retrieve-list/recipe.py +++ b/custom-recipes/pi-system-retrieve-list/recipe.py @@ -111,7 +111,7 @@ summary_duration=summary_duration ) else: - rows = client.get_rows_from_webid( + rows = client.recursive_get_rows_from_webid( object_id, data_type, start_date=start_time, From 027203ca024d132590eb1c90a097117d83aaf30d Mon Sep 17 00:00:00 2001 From: Alex Bourret Date: Fri, 31 Jan 2025 16:43:05 +0100 Subject: [PATCH 12/28] Add option to copy input rows into recipe output [sc-227010] --- custom-recipes/pi-system-retrieve-list/recipe.json | 6 ++++++ custom-recipes/pi-system-retrieve-list/recipe.py | 9 +++++++++ 2 files changed, 15 insertions(+) diff --git a/custom-recipes/pi-system-retrieve-list/recipe.json b/custom-recipes/pi-system-retrieve-list/recipe.json index 010f52c..7b260f9 100644 --- a/custom-recipes/pi-system-retrieve-list/recipe.json +++ b/custom-recipes/pi-system-retrieve-list/recipe.json @@ -112,6 +112,12 @@ "type": "COLUMN", "columnRole": "input_dataset" }, + { + "name": "do_duplicate_input_row", + "label": " ", + "description": " Copy input parameters in the output dataset", + "type": "BOOLEAN" + }, { "type": "SEPARATOR", "description": "Time", diff --git a/custom-recipes/pi-system-retrieve-list/recipe.py b/custom-recipes/pi-system-retrieve-list/recipe.py index 34a6415..9ade67b 100644 --- a/custom-recipes/pi-system-retrieve-list/recipe.py +++ b/custom-recipes/pi-system-retrieve-list/recipe.py @@ -50,6 +50,7 @@ interval, sync_time, boundary_type = get_interpolated_parameters(config) record_boundary_type = config.get("boundary_type") if data_type == "RecordedData" else None summary_type, summary_duration = get_summary_parameters(config) +do_duplicate_input_row = config.get("do_duplicate_input_row", False) network_timer = PerformanceTimer() processing_timer = PerformanceTimer() @@ -64,6 +65,9 @@ client = None previous_server_url = "" time_not_parsed = True + +input_columns = list(input_parameters_dataframe.columns) if do_duplicate_input_row else [] + with output_dataset.get_writer() as writer: first_dataframe = True for index, input_parameters_row in input_parameters_dataframe.iterrows(): @@ -71,6 +75,9 @@ start_time = input_parameters_row.get(start_time_column, start_time) if use_start_time_column else start_time end_time = input_parameters_row.get(end_time_column, end_time) if use_end_time_column else end_time row_name = input_parameters_row.get("Name") + duplicate_initial_row = {} + for input_column in input_columns: + duplicate_initial_row[input_column] = input_parameters_row.get(input_column) if client is None or previous_server_url != server_url: client = OSIsoftClient( @@ -137,6 +144,8 @@ results.extend(extention) else: base = get_base_for_data_type(data_type, object_id) + if duplicate_initial_row: + base.update(duplicate_initial_row) base.update(row) extention = client.unnest_row(base) results.extend(extention) From c026bf3bbe1f792f74a70322db92a3c17786a489 Mon Sep 17 00:00:00 2001 From: Alex Bourret Date: Fri, 31 Jan 2025 16:43:16 +0100 Subject: [PATCH 13/28] update changelog --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7990e91..8b88c44 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,7 @@ - Add boundary type selector to recorded data type - Add boundary type selector to attribute search connector +- *Assets values downloader* Add option to copy each input row into output dataset ## [Version 1.2.3](https://github.com/dataiku/dss-plugin-pi-server/releases/tag/v1.2.3) - Feature release - 2024-09-26 From 601fb6fe2046418146bef398919fd010ae326d1e Mon Sep 17 00:00:00 2001 From: Alex Bourret Date: Fri, 31 Jan 2025 16:43:24 +0100 Subject: [PATCH 14/28] beta 2 --- python-lib/osisoft_constants.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python-lib/osisoft_constants.py b/python-lib/osisoft_constants.py index f061bf2..eb22ac3 100644 --- a/python-lib/osisoft_constants.py +++ b/python-lib/osisoft_constants.py @@ -403,7 +403,7 @@ class OSIsoftConstants(object): "Security": "{base_url}/eventframes/{webid}/security", "SecurityEntries": "{base_url}/eventframes/{webid}/securityentries" } - PLUGIN_VERSION = "1.2.4-beta.1" + PLUGIN_VERSION = "1.2.4-beta.2" VALUE_COLUMN_SUFFIX = "_val" WEB_API_PATH = "piwebapi" WRITE_HEADERS = {'X-Requested-With': 'XmlHttpRequest'} From 4b72df52ae4866273ca990ddaa42246e177283b1 Mon Sep 17 00:00:00 2001 From: Alex Bourret Date: Tue, 4 Feb 2025 09:24:30 +0100 Subject: [PATCH 15/28] beta 3 --- python-lib/osisoft_constants.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python-lib/osisoft_constants.py b/python-lib/osisoft_constants.py index eb22ac3..5267225 100644 --- a/python-lib/osisoft_constants.py +++ b/python-lib/osisoft_constants.py @@ -403,7 +403,7 @@ class OSIsoftConstants(object): "Security": "{base_url}/eventframes/{webid}/security", "SecurityEntries": "{base_url}/eventframes/{webid}/securityentries" } - PLUGIN_VERSION = "1.2.4-beta.2" + PLUGIN_VERSION = "1.2.4-beta.3" VALUE_COLUMN_SUFFIX = "_val" WEB_API_PATH = "piwebapi" WRITE_HEADERS = {'X-Requested-With': 'XmlHttpRequest'} From e44bcbab32d7b52ff5a5f6c8c84de61f282e5707 Mon Sep 17 00:00:00 2001 From: Alex Bourret Date: Tue, 4 Feb 2025 10:23:26 +0100 Subject: [PATCH 16/28] update changelog --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 2e83028..5a88117 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,7 @@ - Add boundary type selector to attribute search connector - *Assets values downloader* Add option to copy each input row into output dataset - Fix issue with recorded data type +- *Assets values downloader* Time range pagination when using webids ## [Version 1.2.3](https://github.com/dataiku/dss-plugin-pi-server/releases/tag/v1.2.3) - Feature release - 2024-09-26 From d8e072ed44514241ec7ff90d0eaad12ca6c4a6cc Mon Sep 17 00:00:00 2001 From: Alex Bourret Date: Tue, 4 Feb 2025 14:01:44 +0100 Subject: [PATCH 17/28] beta 4 --- python-lib/osisoft_constants.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python-lib/osisoft_constants.py b/python-lib/osisoft_constants.py index 5267225..abe3711 100644 --- a/python-lib/osisoft_constants.py +++ b/python-lib/osisoft_constants.py @@ -403,7 +403,7 @@ class OSIsoftConstants(object): "Security": "{base_url}/eventframes/{webid}/security", "SecurityEntries": "{base_url}/eventframes/{webid}/securityentries" } - PLUGIN_VERSION = "1.2.4-beta.3" + PLUGIN_VERSION = "1.2.4-beta.4" VALUE_COLUMN_SUFFIX = "_val" WEB_API_PATH = "piwebapi" WRITE_HEADERS = {'X-Requested-With': 'XmlHttpRequest'} From ae29b864d13754c80330dfdd1ad66efead2e0d03 Mon Sep 17 00:00:00 2001 From: Alex Bourret Date: Tue, 4 Feb 2025 14:01:49 +0100 Subject: [PATCH 18/28] update changelog --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5a88117..3951d16 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,7 @@ - *Assets values downloader* Add option to copy each input row into output dataset - Fix issue with recorded data type - *Assets values downloader* Time range pagination when using webids +- *Assets values downloader* Time can be selected from a column of the input dataset ## [Version 1.2.3](https://github.com/dataiku/dss-plugin-pi-server/releases/tag/v1.2.3) - Feature release - 2024-09-26 From 9c482c330b64ffcd18ab367e1c4659cb2917477c Mon Sep 17 00:00:00 2001 From: Alex Bourret Date: Tue, 4 Feb 2025 19:40:11 +0100 Subject: [PATCH 19/28] Add interpolated value for EF recipe and dataset --- .../recipe.json | 35 ++++++++++--------- .../connector.json | 32 +++++++++++++++++ .../connector.py | 9 +++-- 3 files changed, 57 insertions(+), 19 deletions(-) diff --git a/custom-recipes/pi-system-retrieve-event-frames/recipe.json b/custom-recipes/pi-system-retrieve-event-frames/recipe.json index e9cb714..3f09517 100644 --- a/custom-recipes/pi-system-retrieve-event-frames/recipe.json +++ b/custom-recipes/pi-system-retrieve-event-frames/recipe.json @@ -180,6 +180,24 @@ "type": "STRING", "defaultValue": "" }, + { + "type": "SEPARATOR", + "description": "Data to retrieve" + }, + { + "name": "data_type", + "label": "Data type", + "type": "SELECT", + "selectChoices":[ + {"value": "InterpolatedData", "label": "Interpolated"}, + {"value": "PlotData", "label": "Plot"}, + {"value": "RecordedData", "label": "Recorded"}, + {"value": "SummaryData", "label": "SummaryData"}, + {"value": "Value", "label": "Value"}, + {"value": "EndValue", "label": "EndValue"} + ], + "defaultValue": "SummaryData" + }, { "type": "SEPARATOR", "description": "Interpolation", @@ -212,23 +230,6 @@ ], "defaultValue": "Inside" }, - { - "type": "SEPARATOR", - "description": "Data to retrieve" - }, - { - "name": "data_type", - "label": "Data type", - "type": "SELECT", - "selectChoices":[ - {"value": "PlotData", "label": "Plot"}, - {"value": "RecordedData", "label": "Recorded"}, - {"value": "SummaryData", "label": "SummaryData"}, - {"value": "Value", "label": "Value"}, - {"value": "EndValue", "label": "EndValue"} - ], - "defaultValue": "SummaryData" - }, { "name": "summary_type", "label": "Summary type", diff --git a/python-connectors/pi-system_event-frames-search/connector.json b/python-connectors/pi-system_event-frames-search/connector.json index 4e5b9f0..0391175 100644 --- a/python-connectors/pi-system_event-frames-search/connector.json +++ b/python-connectors/pi-system_event-frames-search/connector.json @@ -200,6 +200,38 @@ ], "mandatory": true }, + { + "type": "SEPARATOR", + "description": "Interpolation", + "visibilityCondition": "(model.must_retrieve_metrics) && ['InterpolatedData'].includes(model.data_type)" + }, + { + "name": "interval", + "label": "Interval", + "visibilityCondition": "(model.must_retrieve_metrics) && ['InterpolatedData'].includes(model.data_type)", + "description": "Optional", + "type": "STRING", + "defaultValue": "" + }, + { + "name": "sync_time", + "label": "Sync time", + "visibilityCondition": "(model.must_retrieve_metrics) && ['InterpolatedData'].includes(model.data_type)", + "description": "Optional", + "type": "STRING", + "defaultValue": "" + }, + { + "name": "boundary_type", + "label": "Boundary type", + "visibilityCondition": "(model.must_retrieve_metrics) && ['InterpolatedData'].includes(model.data_type)", + "type": "SELECT", + "selectChoices":[ + {"value": "Inside", "label": "Inside"}, + {"value": "Outside", "label": "Outside"} + ], + "defaultValue": "Inside" + }, { "name": "summary_type", "label": "Summary type", diff --git a/python-connectors/pi-system_event-frames-search/connector.py b/python-connectors/pi-system_event-frames-search/connector.py index 8c1b4d4..574c064 100644 --- a/python-connectors/pi-system_event-frames-search/connector.py +++ b/python-connectors/pi-system_event-frames-search/connector.py @@ -7,7 +7,7 @@ from osisoft_plugin_common import ( PISystemConnectorError, RecordsLimit, get_credentials, build_requests_params, assert_time_format, get_advanced_parameters, check_debug_mode, - PerformanceTimer, get_max_count, get_summary_parameters + PerformanceTimer, get_max_count, get_summary_parameters, get_interpolated_parameters ) @@ -47,6 +47,7 @@ def __init__(self, config, plugin_config): self.end_time = self.client.parse_pi_time(self.end_time) if self.end_time: config["end_time"] = self.end_time + self.interval, self.sync_time, self.boundary_type = get_interpolated_parameters(config) self.search_mode = config.get("search_mode", None) self.output_type = config.get("output_type") assert_time_format(self.start_time, error_source="start time") @@ -79,7 +80,7 @@ def generate_rows(self, dataset_schema=None, dataset_partitioning=None, if self.object_id: for event_frame in self.client.get_rows_from_urls( self.object_id, self.data_type, start_date=self.start_time, - end_date=self.end_time, max_count=self.max_count): + end_date=self.end_time, interval=self.interval, sync_time=self.sync_time, max_count=self.max_count): self.yields_timer.start() yield event_frame self.yields_timer.stop() @@ -115,6 +116,9 @@ def generate_rows(self, dataset_schema=None, dataset_partitioning=None, batch_size=self.batch_size, summary_type=self.summary_type, summary_duration=self.summary_duration, + boundary_type=self.boundary_type, + interval=self.interval, + sync_time=self.sync_time, max_count=self.max_count ) for batch_row in batch_rows: @@ -145,6 +149,7 @@ def generate_rows(self, dataset_schema=None, dataset_partitioning=None, event_frame_id = event_frame.get("WebId") event_frame_metrics = self.client.get_rows_from_webid( event_frame_id, self.data_type, summary_type=self.summary_type, summary_duration=self.summary_duration, + interval=self.interval, sync_time=self.sync_time, boundary_type=self.boundary_type, search_full_hierarchy=self.search_full_hierarchy, max_count=self.max_count, can_raise=False ) From 2fa074e5fb349556b063c248edceaa15519d9d3a Mon Sep 17 00:00:00 2001 From: Alex Bourret Date: Thu, 13 Feb 2025 13:25:30 +0100 Subject: [PATCH 20/28] Fix for [sc-229162] --- python-lib/osisoft_plugin_common.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python-lib/osisoft_plugin_common.py b/python-lib/osisoft_plugin_common.py index 6812a4e..1089b34 100644 --- a/python-lib/osisoft_plugin_common.py +++ b/python-lib/osisoft_plugin_common.py @@ -264,7 +264,7 @@ def format_output(input_row, reference_row=None, is_enumeration_value=False): type_column = None if "Value" in output_row and isinstance(output_row.get("Value"), dict): type_column = output_row.get("Type") - output_row = output_row.get("Value") + output_row.update(output_row.get("Value")) output_row.pop("Good", None) output_row.pop("Questionable", None) output_row.pop("Substituted", None) From ecb01ed13814287a1bb4d15b7cf6a7607023fd33 Mon Sep 17 00:00:00 2001 From: Alex Bourret Date: Thu, 13 Feb 2025 13:25:35 +0100 Subject: [PATCH 21/28] beta 5 --- python-lib/osisoft_constants.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python-lib/osisoft_constants.py b/python-lib/osisoft_constants.py index abe3711..4957a01 100644 --- a/python-lib/osisoft_constants.py +++ b/python-lib/osisoft_constants.py @@ -403,7 +403,7 @@ class OSIsoftConstants(object): "Security": "{base_url}/eventframes/{webid}/security", "SecurityEntries": "{base_url}/eventframes/{webid}/securityentries" } - PLUGIN_VERSION = "1.2.4-beta.4" + PLUGIN_VERSION = "1.2.4-beta.5" VALUE_COLUMN_SUFFIX = "_val" WEB_API_PATH = "piwebapi" WRITE_HEADERS = {'X-Requested-With': 'XmlHttpRequest'} From aa0432799f92c26faf168fb7df2fa5dec135439b Mon Sep 17 00:00:00 2001 From: Mayeul Rousselet Date: Mon, 17 Feb 2025 17:09:24 +0100 Subject: [PATCH 22/28] Recursive get rows webid [sc-229599] --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5a88117..ed21a91 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,6 @@ # Changelog -## [Version 1.2.4](https://github.com/dataiku/dss-plugin-pi-server/releases/tag/v1.2.4) - Feature and bug release - 2024-12-12 +## [Version 1.2.4](https://github.com/dataiku/dss-plugin-pi-server/releases/tag/v1.2.4) - Feature and bug release - 2024-02-02 - Add boundary type selector to recorded data type - Add boundary type selector to attribute search connector From 9aa6f6d0d50373ce9811b55ff5cd831215aea1a5 Mon Sep 17 00:00:00 2001 From: Mayeul Rousselet Date: Mon, 17 Feb 2025 17:14:05 +0100 Subject: [PATCH 23/28] Activate start time col [sc-229601] --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3951d16..4f77b8a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,6 @@ # Changelog -## [Version 1.2.4](https://github.com/dataiku/dss-plugin-pi-server/releases/tag/v1.2.4) - Feature and bug release - 2024-12-12 +## [Version 1.2.4](https://github.com/dataiku/dss-plugin-pi-server/releases/tag/v1.2.4) - Feature and bug release - 2025-02-18 - Add boundary type selector to recorded data type - Add boundary type selector to attribute search connector From 74e6328fb70f7354e4035ac52548b66ad463961b Mon Sep 17 00:00:00 2001 From: Alex Bourret Date: Wed, 19 Feb 2025 17:31:11 +0100 Subject: [PATCH 24/28] add interpolate boundary type for record data type --- custom-recipes/pi-system-retrieve-list/recipe.json | 14 +++++++++++++- custom-recipes/pi-system-retrieve-list/recipe.py | 2 +- .../pi-system_attribute-search/connector.json | 14 +++++++++++++- .../pi-system_attribute-search/connector.py | 2 +- 4 files changed, 28 insertions(+), 4 deletions(-) diff --git a/custom-recipes/pi-system-retrieve-list/recipe.json b/custom-recipes/pi-system-retrieve-list/recipe.json index 860efd4..8660c6d 100644 --- a/custom-recipes/pi-system-retrieve-list/recipe.json +++ b/custom-recipes/pi-system-retrieve-list/recipe.json @@ -200,7 +200,7 @@ { "name": "boundary_type", "label": "Boundary type", - "visibilityCondition": "['InterpolatedData','RecordedData'].includes(model.data_type)", + "visibilityCondition": "((model.must_retrieve_metrics) && ['InterpolatedData'].includes(model.data_type))", "type": "SELECT", "selectChoices":[ {"value": "Inside", "label": "Inside"}, @@ -208,6 +208,18 @@ ], "defaultValue": "Inside" }, + { + "name": "record_boundary_type", + "label": "Boundary type", + "visibilityCondition": "((model.must_retrieve_metrics) && ['RecordedData'].includes(model.data_type))", + "type": "SELECT", + "selectChoices":[ + {"value": "Inside", "label": "Inside"}, + {"value": "Interpolated", "label": "Interpolated"}, + {"value": "Outside", "label": "Outside"} + ], + "defaultValue": "Inside" + }, { "type": "SEPARATOR", "description": "Data to retrieve" diff --git a/custom-recipes/pi-system-retrieve-list/recipe.py b/custom-recipes/pi-system-retrieve-list/recipe.py index ee0e6af..a4e3e93 100644 --- a/custom-recipes/pi-system-retrieve-list/recipe.py +++ b/custom-recipes/pi-system-retrieve-list/recipe.py @@ -48,7 +48,7 @@ end_time_column = config.get("end_time_column") server_url_column = config.get("server_url_column") interval, sync_time, boundary_type = get_interpolated_parameters(config) -record_boundary_type = config.get("boundary_type") if data_type == "RecordedData" else None +record_boundary_type = config.get("record_boundary_type") if data_type == "RecordedData" else None summary_type, summary_duration = get_summary_parameters(config) do_duplicate_input_row = config.get("do_duplicate_input_row", False) diff --git a/python-connectors/pi-system_attribute-search/connector.json b/python-connectors/pi-system_attribute-search/connector.json index 9708871..2702b7e 100644 --- a/python-connectors/pi-system_attribute-search/connector.json +++ b/python-connectors/pi-system_attribute-search/connector.json @@ -363,7 +363,7 @@ { "name": "boundary_type", "label": "Boundary type", - "visibilityCondition": "((model.must_retrieve_metrics) && ['InterpolatedData','RecordedData'].includes(model.data_type))", + "visibilityCondition": "((model.must_retrieve_metrics) && ['InterpolatedData'].includes(model.data_type))", "type": "SELECT", "selectChoices":[ {"value": "Inside", "label": "Inside"}, @@ -371,6 +371,18 @@ ], "defaultValue": "Inside" }, + { + "name": "record_boundary_type", + "label": "Boundary type", + "visibilityCondition": "((model.must_retrieve_metrics) && ['RecordedData'].includes(model.data_type))", + "type": "SELECT", + "selectChoices":[ + {"value": "Inside", "label": "Inside"}, + {"value": "Interpolated", "label": "Interpolated"}, + {"value": "Outside", "label": "Outside"} + ], + "defaultValue": "Inside" + }, { "name": "summary_duration", "label": "Summary duration", diff --git a/python-connectors/pi-system_attribute-search/connector.py b/python-connectors/pi-system_attribute-search/connector.py index 28f448e..50daa2b 100644 --- a/python-connectors/pi-system_attribute-search/connector.py +++ b/python-connectors/pi-system_attribute-search/connector.py @@ -56,7 +56,7 @@ def __init__(self, config, plugin_config): self.config = config self.summary_type, self.summary_duration = get_summary_parameters(config) - self.record_boundary_type = config.get("boundary_type") if self.data_type == "RecordedData" else None + self.record_boundary_type = config.get("record_boundary_type") if self.data_type == "RecordedData" else None def extract_database_webid(self, database_endpoint): return database_endpoint.split("/")[-1] From f8d9384825d14ffd70692729185256e42414684d Mon Sep 17 00:00:00 2001 From: Alex Bourret Date: Mon, 24 Feb 2025 10:45:55 +0100 Subject: [PATCH 25/28] UI update --- custom-recipes/pi-system-retrieve-list/recipe.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/custom-recipes/pi-system-retrieve-list/recipe.json b/custom-recipes/pi-system-retrieve-list/recipe.json index 7b260f9..28ede5a 100644 --- a/custom-recipes/pi-system-retrieve-list/recipe.json +++ b/custom-recipes/pi-system-retrieve-list/recipe.json @@ -114,8 +114,8 @@ }, { "name": "do_duplicate_input_row", - "label": " ", - "description": " Copy input parameters in the output dataset", + "label": "Copy other input columns", + "description": "Copy other input columns to the output dataset", "type": "BOOLEAN" }, { From 76a62b9a25caf8c542f60f2b6cc9d29c9eac8492 Mon Sep 17 00:00:00 2001 From: Alex Bourret Date: Fri, 7 Mar 2025 16:43:00 +0100 Subject: [PATCH 26/28] add warning of possible column overwritting --- custom-recipes/pi-system-retrieve-list/recipe.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/custom-recipes/pi-system-retrieve-list/recipe.json b/custom-recipes/pi-system-retrieve-list/recipe.json index 28ede5a..8a65c9e 100644 --- a/custom-recipes/pi-system-retrieve-list/recipe.json +++ b/custom-recipes/pi-system-retrieve-list/recipe.json @@ -115,7 +115,7 @@ { "name": "do_duplicate_input_row", "label": "Copy other input columns", - "description": "Copy other input columns to the output dataset", + "description": "(those with matching names will be overwritten by PI system data)", "type": "BOOLEAN" }, { From 5c06172ec455fc22a5463cff7728c4d7d0e45dda Mon Sep 17 00:00:00 2001 From: Alex Bourret Date: Fri, 7 Mar 2025 17:29:40 +0100 Subject: [PATCH 27/28] removing the lonely ' --- custom-recipes/pi-system-retrieve-event-frames/recipe.json | 4 ++-- custom-recipes/pi-system-retrieve-list/recipe.json | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/custom-recipes/pi-system-retrieve-event-frames/recipe.json b/custom-recipes/pi-system-retrieve-event-frames/recipe.json index e9cb714..c8a3374 100644 --- a/custom-recipes/pi-system-retrieve-event-frames/recipe.json +++ b/custom-recipes/pi-system-retrieve-event-frames/recipe.json @@ -143,7 +143,7 @@ { "visibilityCondition": "model.use_start_time_column==true && (['InterpolatedData', 'RecordedData', 'PlotData', 'EventFrames'].includes(model.data_type))", "name": "start_time_column", - "label": "Start times' column", + "label": "Start times column", "description": "Optional", "type": "COLUMN", "columnRole": "input_dataset" @@ -167,7 +167,7 @@ { "visibilityCondition": "model.use_end_time_column==true && (['InterpolatedData', 'RecordedData', 'PlotData', 'EventFrames'].includes(model.data_type))", "name": "end_time_column", - "label": "End times' column", + "label": "End times column", "description": "Optional", "type": "COLUMN", "columnRole": "input_dataset" diff --git a/custom-recipes/pi-system-retrieve-list/recipe.json b/custom-recipes/pi-system-retrieve-list/recipe.json index 860efd4..edc284a 100644 --- a/custom-recipes/pi-system-retrieve-list/recipe.json +++ b/custom-recipes/pi-system-retrieve-list/recipe.json @@ -139,7 +139,7 @@ { "visibilityCondition": "model.use_start_time_column==true && (['InterpolatedData', 'RecordedData', 'PlotData', 'EventFrames'].includes(model.data_type))", "name": "start_time_column", - "label": "Start times' column", + "label": "Start times column", "description": "Column type must be string", "type": "COLUMN", "columnRole": "input_dataset" @@ -163,7 +163,7 @@ { "visibilityCondition": "model.use_end_time_column==true && (['InterpolatedData', 'RecordedData', 'PlotData', 'EventFrames'].includes(model.data_type))", "name": "end_time_column", - "label": "End times' column", + "label": "End times column", "description": "Column type must be string", "type": "COLUMN", "columnRole": "input_dataset" From 31fc7b688fc65c1bd46fab04caea6958601594b7 Mon Sep 17 00:00:00 2001 From: Alex Bourret Date: Tue, 11 Mar 2025 11:29:47 +0100 Subject: [PATCH 28/28] fix boundary type for ef connector (selected ef) --- .../pi-system_event-frames-search/connector.py | 3 ++- python-lib/osisoft_client.py | 14 +++++++++----- 2 files changed, 11 insertions(+), 6 deletions(-) diff --git a/python-connectors/pi-system_event-frames-search/connector.py b/python-connectors/pi-system_event-frames-search/connector.py index 574c064..1775ca9 100644 --- a/python-connectors/pi-system_event-frames-search/connector.py +++ b/python-connectors/pi-system_event-frames-search/connector.py @@ -80,7 +80,8 @@ def generate_rows(self, dataset_schema=None, dataset_partitioning=None, if self.object_id: for event_frame in self.client.get_rows_from_urls( self.object_id, self.data_type, start_date=self.start_time, - end_date=self.end_time, interval=self.interval, sync_time=self.sync_time, max_count=self.max_count): + end_date=self.end_time, interval=self.interval, sync_time=self.sync_time, + boundary_type=self.boundary_type, max_count=self.max_count): self.yields_timer.start() yield event_frame self.yields_timer.stop() diff --git a/python-lib/osisoft_client.py b/python-lib/osisoft_client.py index f1e63d8..385b8fe 100644 --- a/python-lib/osisoft_client.py +++ b/python-lib/osisoft_client.py @@ -426,13 +426,13 @@ def get_link_from_item(self, item, data_type, start_date, end_date, interval=Non ) return json_response - def get_rows_from_url(self, url=None, start_date=None, end_date=None, interval=None, sync_time=None, max_count=None): + def get_rows_from_url(self, url=None, start_date=None, end_date=None, interval=None, sync_time=None, boundary_type=None, max_count=None): pagination = OffsetPagination() has_more = True while has_more: json_response, has_more = pagination.get_offset_paginated( self.get_link_from_url, - url, start_date=start_date, end_date=end_date, interval=interval, sync_time=sync_time, max_count=max_count + url, start_date=start_date, end_date=end_date, interval=interval, sync_time=sync_time, boundary_type=boundary_type, max_count=max_count ) items = json_response.get(OSIsoftConstants.API_ITEM_KEY, [json_response]) for item in items: @@ -443,15 +443,18 @@ def get_rows_from_url(self, url=None, start_date=None, end_date=None, interval=N else: yield item - def get_rows_from_urls(self, links=None, data_type=None, start_date=None, end_date=None, interval=None, sync_time=None, max_count=None): + def get_rows_from_urls(self, links=None, data_type=None, start_date=None, end_date=None, interval=None, sync_time=None, boundary_type=None, max_count=None): links = links or [] for link in links: url = link - rows = self.get_rows_from_url(url, start_date=start_date, end_date=end_date, interval=interval, sync_time=sync_time, max_count=max_count) + rows = self.get_rows_from_url( + url, start_date=start_date, end_date=end_date, interval=interval, + sync_time=sync_time, boundary_type=boundary_type, max_count=max_count + ) for row in rows: yield row - def get_link_from_url(self, url, start_date=None, end_date=None, interval=None, sync_time=None, start_index=None, max_count=None): + def get_link_from_url(self, url, start_date=None, end_date=None, interval=None, sync_time=None, start_index=None, boundary_type=None, max_count=None): if not url: url = self.endpoint.get_base_url() headers = self.get_requests_headers() @@ -461,6 +464,7 @@ def get_link_from_url(self, url, start_date=None, end_date=None, interval=None, interval=interval, sync_time=sync_time, start_index=start_index, + sync_time_boundary_type=boundary_type, max_count=max_count ) json_response = self.get(