Merge pull request #45 from dataiku/feature/sc-205296-add-interval-selection-for-summarydata

alexbourret · web-flow · commit 5ca92268fd01 · 2024-11-18T14:44:42.000+01:00
feat: [sc-205296] [pi-system plugin] Add interval selection for summarydata
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -1,5 +1,12 @@
 # Changelog
 
+## [Version 1.2.3](https://github.com/dataiku/dss-plugin-pi-server/releases/tag/v1.2.3) - Feature release - 2024-09-26
+
+- Add summaryDuration input (duration of each summary interval)
+- Fix issue when using `Starting After` search option in browse PI event frames
+- Fix issue with `Value` and `End value` data types in Search attributes dataset
+- Add pagination on element retrieval
+
 ## [Version 1.2.2](https://github.com/dataiku/dss-plugin-pi-server/releases/tag/v1.2.2) - Feature release - 2023-12-11
 
 - Time range pagination - no more `maxCount` and / or `maxReturnedItemsPerCall` limitation
diff --git a/custom-recipes/pi-system-retrieve-event-frames/recipe.json b/custom-recipes/pi-system-retrieve-event-frames/recipe.json
@@ -249,6 +249,14 @@
             ],
             "visibilityCondition": "model.data_type == 'SummaryData'"
         },
+        {
+            "name": "summary_duration",
+            "label": "Summary duration",
+            "visibilityCondition": "model.data_type == 'SummaryData'",
+            "description": "Optional",
+            "type": "STRING",
+            "defaultValue": ""
+        },
         {
             "name": "max_count",
             "label": "Max count",
diff --git a/custom-recipes/pi-system-retrieve-event-frames/recipe.py b/custom-recipes/pi-system-retrieve-event-frames/recipe.py
@@ -6,7 +6,8 @@
 from safe_logger import SafeLogger
 from osisoft_plugin_common import (
     get_credentials, get_interpolated_parameters,
-    get_advanced_parameters, check_debug_mode, PerformanceTimer, get_max_count, reorder_dataframe
+    get_advanced_parameters, check_debug_mode, PerformanceTimer,
+    get_max_count, reorder_dataframe, get_summary_parameters
 )
 from osisoft_constants import OSIsoftConstants
 from osisoft_client import OSIsoftClient
@@ -45,9 +46,9 @@
 end_time_column = config.get("end_time_column")
 server_url_column = config.get("server_url_column")
 search_full_hierarchy = config.get("search_full_hierarchy", None)
-summary_type = config.get("summary_type")
 use_batch_mode, batch_size = get_advanced_parameters(config)
 interval, sync_time, boundary_type = get_interpolated_parameters(config)
+summary_type, summary_duration = get_summary_parameters(config)
 
 network_timer = PerformanceTimer()
 processing_timer = PerformanceTimer()
@@ -109,7 +110,8 @@
                 search_full_hierarchy=search_full_hierarchy,
                 can_raise=False,
                 max_count=max_count,
-                summary_type=summary_type
+                summary_type=summary_type,
+                summary_duration=summary_duration
             )
         elif use_batch_mode:
             buffer.append({"WebId": object_id, "StartTime": event_frame_start_time, "EndTime": event_frame_end_time})
@@ -120,7 +122,8 @@
                     search_full_hierarchy=search_full_hierarchy,
                     can_raise=False,
                     batch_size=batch_size,
-                    summary_type=summary_type
+                    summary_type=summary_type,
+                    summary_duration=summary_duration
                 )
                 batch_buffer_size = 0
                 buffer = []
@@ -138,7 +141,8 @@
                 search_full_hierarchy=search_full_hierarchy,
                 max_count=max_count,
                 can_raise=False,
-                summary_type=summary_type
+                summary_type=summary_type,
+                summary_duration=summary_duration
             )
         unnested_items_rows = []
         row_count = 0
@@ -155,7 +159,7 @@
                 item_row = {} if use_batch_mode else {"event_frame_webid": event_frame_webid}
                 value = item.get("Value", {})
                 if isinstance(value, dict):
-                    item.pop("Value")
+                    item.pop("Value", None)
                     item_row.update(value)
                 item_row.update(base_row)
                 item_row.update(item)
@@ -164,7 +168,7 @@
                 item_row = {} if use_batch_mode else {"event_frame_webid": event_frame_webid}
                 value = base_row.get("Value", {})
                 if isinstance(value, dict):
-                    base_row.pop("Value")
+                    base_row.pop("Value", None)
                     base_row.update(value)
                 item_row.update(base_row)
                 unnested_items_rows.append(item_row)
diff --git a/custom-recipes/pi-system-retrieve-list/recipe.json b/custom-recipes/pi-system-retrieve-list/recipe.json
@@ -245,6 +245,14 @@
             ],
             "visibilityCondition": "model.data_type == 'SummaryData'"
         },
+        {
+            "name": "summary_duration",
+            "label": "Summary duration",
+            "visibilityCondition": "model.data_type == 'SummaryData'",
+            "description": "Optional",
+            "type": "STRING",
+            "defaultValue": ""
+        },
         {
             "name": "max_count",
             "label": "Max count",
diff --git a/custom-recipes/pi-system-retrieve-list/recipe.py b/custom-recipes/pi-system-retrieve-list/recipe.py
@@ -7,7 +7,7 @@
     get_credentials, get_interpolated_parameters, normalize_af_path,
     get_combined_description, get_base_for_data_type, check_debug_mode,
     PerformanceTimer, get_max_count, check_must_convert_object_to_string,
-    convert_schema_objects_to_string
+    convert_schema_objects_to_string, get_summary_parameters
 )
 from osisoft_client import OSIsoftClient
 from osisoft_constants import OSIsoftConstants
@@ -48,6 +48,7 @@
 end_time_column = config.get("end_time_column")
 server_url_column = config.get("server_url_column")
 interval, sync_time, boundary_type = get_interpolated_parameters(config)
+summary_type, summary_duration = get_summary_parameters(config)
 
 network_timer = PerformanceTimer()
 processing_timer = PerformanceTimer()
@@ -104,7 +105,8 @@
                 max_count=max_count,
                 can_raise=False,
                 object_id=object_id,
-                summary_type=summary_type
+                summary_type=summary_type,
+                summary_duration=summary_duration
             )
         else:
             rows = client.get_rows_from_webid(
@@ -118,7 +120,8 @@
                 max_count=max_count,
                 can_raise=False,
                 endpoint_type="AF",
-                summary_type=summary_type
+                summary_type=summary_type,
+                summary_duration=summary_duration
             )
         for row in rows:
             row["Name"] = row_name
diff --git a/plugin.json b/plugin.json
@@ -1,6 +1,6 @@
 {
     "id": "pi-system",
-    "version": "1.2.2",
+    "version": "1.2.3",
     "meta": {
         "label": "PI System",
         "description": "Retrieve data from your OSIsoft PI System servers",
diff --git a/python-connectors/pi-system_attribute-search/connector.json b/python-connectors/pi-system_attribute-search/connector.json
@@ -360,6 +360,14 @@
             ],
             "visibilityCondition": "((model.must_retrieve_metrics) && (model.data_type == 'SummaryData'))"
         },
+        {
+            "name": "summary_duration",
+            "label": "Summary duration",
+            "visibilityCondition": "((model.must_retrieve_metrics) && (model.data_type == 'SummaryData'))",
+            "description": "Optional",
+            "type": "STRING",
+            "defaultValue": ""
+        },
         {
             "name": "max_count",
             "label": "Max count",
diff --git a/python-connectors/pi-system_attribute-search/connector.py b/python-connectors/pi-system_attribute-search/connector.py
@@ -6,7 +6,7 @@
 from osisoft_plugin_common import (
     PISystemConnectorError, RecordsLimit, get_credentials, assert_time_format,
     remove_unwanted_columns, format_output, filter_columns_from_schema, is_child_attribute_path,
-    check_debug_mode, PerformanceTimer, get_max_count
+    check_debug_mode, PerformanceTimer, get_max_count, get_summary_parameters, fields_selector
 )
 from osisoft_constants import OSIsoftConstants
 
@@ -55,7 +55,7 @@ def __init__(self, config, plugin_config):
         self.must_filter_child_attributes = not (config.get("must_keep_child_attributes", False))
         self.max_count = get_max_count(config)
         self.config = config
-        self.summary_type = config.get("summary_type")
+        self.summary_type, self.summary_duration = get_summary_parameters(config)
 
     def extract_database_webid(self, database_endpoint):
         return database_endpoint.split("/")[-1]
@@ -111,9 +111,10 @@ def generate_rows(self, dataset_schema=None, dataset_partitioning=None,
                         interval=self.interval,
                         sync_time=self.sync_time,
                         endpoint_type="AF",
-                        selected_fields="Links%3BItems.Timestamp%3BItems.Value%3BItems.Type",
+                        selected_fields=fields_selector(self.data_type),
                         max_count=self.max_count,
-                        summary_type=self.summary_type
+                        summary_type=self.summary_type,
+                        summary_duration=self.summary_duration
                         # boundary_type=self.boundary_type
                     ):
                         if limit.is_reached():
diff --git a/python-connectors/pi-system_event-frames-search/connector.json b/python-connectors/pi-system_event-frames-search/connector.json
@@ -220,6 +220,14 @@
             ],
             "visibilityCondition": "((model.must_retrieve_metrics) && (model.database_name.length>=0) && (model.data_type == 'SummaryData'))"
         },
+        {
+            "name": "summary_duration",
+            "label": "Summary duration",
+            "visibilityCondition": "((model.must_retrieve_metrics) && (model.data_type == 'SummaryData'))",
+            "description": "Optional",
+            "type": "STRING",
+            "defaultValue": ""
+        },
         {
             "name": "max_count",
             "label": "Max count",
diff --git a/python-connectors/pi-system_event-frames-search/connector.py b/python-connectors/pi-system_event-frames-search/connector.py
@@ -7,7 +7,7 @@
 from osisoft_plugin_common import (
     PISystemConnectorError, RecordsLimit, get_credentials,
     build_requests_params, assert_time_format, get_advanced_parameters, check_debug_mode,
-    PerformanceTimer, get_max_count
+    PerformanceTimer, get_max_count, get_summary_parameters
 )
 
 
@@ -59,10 +59,10 @@ def __init__(self, config, plugin_config):
         if self.must_retrieve_metrics:
             self.search_full_hierarchy = config.get("search_full_hierarchy", None)
         self.data_type = config.get("data_type", "Recorded")
-        self.summary_type = config.get("summary_type", None)
         self.max_count = get_max_count(config)
         self.config = config
         self.use_batch_mode, self.batch_size = get_advanced_parameters(config)
+        self.summary_type, self.summary_duration = get_summary_parameters(config)
 
     def get_read_schema(self):
         return {
@@ -114,6 +114,7 @@ def generate_rows(self, dataset_schema=None, dataset_partitioning=None,
                                 can_raise=False,
                                 batch_size=self.batch_size,
                                 summary_type=self.summary_type,
+                                summary_duration=self.summary_duration,
                                 max_count=self.max_count
                             )
                         for batch_row in batch_rows:
@@ -143,7 +144,7 @@ def generate_rows(self, dataset_schema=None, dataset_partitioning=None,
                         for event_frame in event_frames:
                             event_frame_id = event_frame.get("WebId")
                             event_frame_metrics = self.client.get_rows_from_webid(
-                                event_frame_id, self.data_type, summary_type=self.summary_type,
+                                event_frame_id, self.data_type, summary_type=self.summary_type, summary_duration=self.summary_duration,
                                 search_full_hierarchy=self.search_full_hierarchy, max_count=self.max_count,
                                 can_raise=False
                             )
diff --git a/python-connectors/pi-system_pi-explorer/connector.py b/python-connectors/pi-system_pi-explorer/connector.py
@@ -45,7 +45,7 @@ def generate_rows(self, dataset_schema=None, dataset_partitioning=None,
         limit = RecordsLimit(records_limit)
         rows = self.client.get_rows_from_url(self.data_server_url)
         for row in rows:
-            row.pop("Links")
+            row.pop("Links", None)
             yield row
             if limit.is_reached():
                 return
diff --git a/python-connectors/pi-system_piwebapi-toolbox/connector.json b/python-connectors/pi-system_piwebapi-toolbox/connector.json
@@ -90,6 +90,14 @@
             ],
             "visibilityCondition": "model.data_type == 'SummaryData'"
         },
+        {
+            "name": "summary_duration",
+            "label": "Summary duration",
+            "visibilityCondition": "model.data_type === 'SummaryData'",
+            "description": "Optional",
+            "type": "STRING",
+            "defaultValue": ""
+        },
         {
             "name": "max_count",
             "label": "Max count",
diff --git a/python-connectors/pi-system_piwebapi-toolbox/connector.py b/python-connectors/pi-system_piwebapi-toolbox/connector.py
@@ -2,7 +2,8 @@
 from osisoft_client import OSIsoftClient, OSIsoftWriter
 from safe_logger import SafeLogger
 from osisoft_plugin_common import (
-    PISystemConnectorError, RecordsLimit, get_credentials, assert_time_format, get_schema_as_arrays, normalize_af_path, get_max_count
+    PISystemConnectorError, RecordsLimit, get_credentials, assert_time_format,
+    get_schema_as_arrays, normalize_af_path, get_max_count, get_summary_parameters
 )
 from osisoft_constants import OSIsoftConstants
 
@@ -29,14 +30,14 @@ def __init__(self, config, plugin_config):
         self.end_time = config.get("end_time")
         self.interval = config.get("interval")
         self.sync_time = config.get("sync_time")
-        self.summary_type = config.get("summary_type")
         assert_time_format(self.start_time, error_source="start time")
         assert_time_format(self.end_time, error_source="start time")
         self.item = None
         if self.client.is_resource_path(self.object_id):
             self.object_id = normalize_af_path(self.object_id)
             self.item = self.client.get_item_from_path(self.object_id)
         self.max_count = get_max_count(config)
+        self.summary_type, self.summary_duration = get_summary_parameters(config)
 
     def get_read_schema(self):
         return None
@@ -54,7 +55,8 @@ def generate_rows(self, dataset_schema=None, dataset_partitioning=None,
                 interval=self.interval,
                 sync_time=self.sync_time,
                 max_count=self.max_count,
-                summary_type=self.summary_type
+                summary_type=self.summary_type,
+                summary_duration=self.summary_duration
             ):
                 if limit.is_reached():
                     break
@@ -69,7 +71,8 @@ def generate_rows(self, dataset_schema=None, dataset_partitioning=None,
                 sync_time=self.sync_time,
                 endpoint_type="AF",
                 max_count=self.max_count,
-                summary_type=self.summary_type
+                summary_type=self.summary_type,
+                summary_duration=self.summary_duration
             ):
                 if limit.is_reached():
                     break
diff --git a/python-lib/osisoft_client.py b/python-lib/osisoft_client.py
diff --git a/python-lib/osisoft_constants.py b/python-lib/osisoft_constants.py
diff --git a/python-lib/osisoft_plugin_common.py b/python-lib/osisoft_plugin_common.py

Original file line number	Diff line number	Diff line change
`@@ -1,6 +1,6 @@`
`1`	`1`	`{`
`2`	`2`	`"id": "pi-system",`
`3`		`- "version": "1.2.2",`
	`3`	`+ "version": "1.2.3",`
`4`	`4`	`"meta": {`
`5`	`5`	`"label": "PI System",`
`6`	`6`	`"description": "Retrieve data from your OSIsoft PI System servers",`