Merge pull request #47 from dataiku/bug/dss13-sc-207988-pi-system-end-value-and-value-data-types

Bug/dss13 sc 207988 pi system end value and value data types

Author: alexbourret

CHANGELOG.md

@@ -2,8 +2,10 @@
 
 ## [Version 1.2.3](https://github.com/dataiku/dss-plugin-pi-server/releases/tag/v1.2.3) - Feature release - 2024-09-26
 
-- Add summaryDuration selector (duration of each summary interval)
+- Add summaryDuration input (duration of each summary interval)
 - Fix issue when using `Starting After` search option in browse PI event frames
+- Fix issue with `Value` and `End value` data types in Search attributes dataset
+- Add pagination on element retrieval
 
 ## [Version 1.2.2](https://github.com/dataiku/dss-plugin-pi-server/releases/tag/v1.2.2) - Feature release - 2023-12-11
 

python-connectors/pi-system_attribute-search/connector.py

@@ -6,7 +6,7 @@
 from osisoft_plugin_common import (
     PISystemConnectorError, RecordsLimit, get_credentials, assert_time_format,
     remove_unwanted_columns, format_output, filter_columns_from_schema, is_child_attribute_path,
-    check_debug_mode, PerformanceTimer, get_max_count, get_summary_parameters
+    check_debug_mode, PerformanceTimer, get_max_count, get_summary_parameters, fields_selector
 )
 from osisoft_constants import OSIsoftConstants
 
@@ -111,7 +111,7 @@ def generate_rows(self, dataset_schema=None, dataset_partitioning=None,
                         interval=self.interval,
                         sync_time=self.sync_time,
                         endpoint_type="AF",
-                        selected_fields="Links%3BItems.Timestamp%3BItems.Value%3BItems.Type",
+                        selected_fields=fields_selector(self.data_type),
                         max_count=self.max_count,
                         summary_type=self.summary_type,
                         summary_duration=self.summary_duration

python-lib/osisoft_client.py

@@ -10,7 +10,7 @@
 from osisoft_plugin_common import (
     assert_server_url_ok, build_requests_params,
     is_filtered_out, is_server_throttling, escape, epoch_to_iso,
-    iso_to_epoch, RecordsLimit, is_iso8601
+    iso_to_epoch, RecordsLimit, is_iso8601, get_next_page_url
 )
 from osisoft_pagination import OffsetPagination
 from safe_logger import SafeLogger
@@ -763,9 +763,15 @@ def search_attributes(self, database_webid, **kwargs):
         json_response = self.get(url=search_attributes_base_url, headers=headers, params=params)
         if OSIsoftConstants.DKU_ERROR_KEY in json_response:
             yield json_response
-        items = json_response.get(OSIsoftConstants.API_ITEM_KEY, [])
-        for item in items:
-            yield item
+        while json_response:
+            next_page_url = get_next_page_url(json_response)
+            items = json_response.get(OSIsoftConstants.API_ITEM_KEY, [])
+            for item in items:
+                yield item
+            if next_page_url:
+                json_response = self.get(url=next_page_url, headers={}, params={})
+            else:
+                json_response = None
 
     def build_element_query(self, **kwargs):
         element_query_keys = {

python-lib/osisoft_constants.py

@@ -403,7 +403,7 @@ class OSIsoftConstants(object):
         "Security": "{base_url}/eventframes/{webid}/security",
         "SecurityEntries": "{base_url}/eventframes/{webid}/securityentries"
     }
-    PLUGIN_VERSION = "1.2.3-beta.3"
+    PLUGIN_VERSION = "1.2.3-beta.4"
     VALUE_COLUMN_SUFFIX = "_val"
     WEB_API_PATH = "piwebapi"
     WRITE_HEADERS = {'X-Requested-With': 'XmlHttpRequest'}

python-lib/osisoft_plugin_common.py

@@ -443,6 +443,25 @@ def reorder_dataframe(unnested_items_rows, first_elements):
     return unnested_items_rows
 
 
+def fields_selector(data_type):
+    # specifies the fields to be returned for each data type
+    if data_type in ["Value", "EndValue"]:
+        return "Links%3BTimestamp%3BValue%3BType%3BUnitsAbbreviation"
+    else:
+        return "Links%3BItems.Timestamp%3BItems.Value%3BItems.Type"
+
+
+def get_next_page_url(json):
+    if not json:
+        return None
+    next_page_url = json.get("Links", {}).get("Next", "").replace('&', '&')
+    if next_page_url:
+        logger.info("Next page's url is {}".format(next_page_url))
+    else:
+        logger.info("No more pages available")
+    return next_page_url
+
+
 class RecordsLimit():
     def __init__(self, records_limit=-1):
         self.has_no_limit = (records_limit == -1)