Skip to content

Commit 9afa9ea

Browse files
alexbourretalexbourret
authored
Merge pull request #58 from dataiku/bug/sc-220666-some-value-not-available
Bug/sc 220666 some value not available
2 parents 9d59cda + 26a213e commit 9afa9ea

File tree

11 files changed

+131
-47
lines changed

11 files changed

+131
-47
lines changed

‎CHANGELOG.md‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,13 @@
11
# Changelog
22

3-
## [Version 1.2.4](https://github.com/dataiku/dss-plugin-pi-server/releases/tag/v1.2.4) - Feature release - 2024-12-12
3+
## [Version 1.2.4](https://github.com/dataiku/dss-plugin-pi-server/releases/tag/v1.2.4) - Feature and bug release - 2025-02-18
44

55
- Add boundary type selector to recorded data type
66
- Add boundary type selector to attribute search connector
7+
- *Assets values downloader* Add option to copy each input row into output dataset
8+
- Fix issue with recorded data type
9+
- *Assets values downloader* Time range pagination when using webids
10+
- *Assets values downloader* Time can be selected from a column of the input dataset
711

812
## [Version 1.2.3](https://github.com/dataiku/dss-plugin-pi-server/releases/tag/v1.2.3) - Feature release - 2024-09-26
913

‎custom-recipes/pi-system-retrieve-event-frames/recipe.json‎

Lines changed: 20 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -143,7 +143,7 @@
143143
{
144144
"visibilityCondition": "model.use_start_time_column==true && (['InterpolatedData', 'RecordedData', 'PlotData', 'EventFrames'].includes(model.data_type))",
145145
"name": "start_time_column",
146-
"label": "Start times' column",
146+
"label": "Start times column",
147147
"description": "Optional",
148148
"type": "COLUMN",
149149
"columnRole": "input_dataset"
@@ -167,7 +167,7 @@
167167
{
168168
"visibilityCondition": "model.use_end_time_column==true && (['InterpolatedData', 'RecordedData', 'PlotData', 'EventFrames'].includes(model.data_type))",
169169
"name": "end_time_column",
170-
"label": "End times' column",
170+
"label": "End times column",
171171
"description": "Optional",
172172
"type": "COLUMN",
173173
"columnRole": "input_dataset"
@@ -180,6 +180,24 @@
180180
"type": "STRING",
181181
"defaultValue": ""
182182
},
183+
{
184+
"type": "SEPARATOR",
185+
"description": "Data to retrieve"
186+
},
187+
{
188+
"name": "data_type",
189+
"label": "Data type",
190+
"type": "SELECT",
191+
"selectChoices":[
192+
{"value": "InterpolatedData", "label": "Interpolated"},
193+
{"value": "PlotData", "label": "Plot"},
194+
{"value": "RecordedData", "label": "Recorded"},
195+
{"value": "SummaryData", "label": "SummaryData"},
196+
{"value": "Value", "label": "Value"},
197+
{"value": "EndValue", "label": "EndValue"}
198+
],
199+
"defaultValue": "SummaryData"
200+
},
183201
{
184202
"type": "SEPARATOR",
185203
"description": "Interpolation",
@@ -212,23 +230,6 @@
212230
],
213231
"defaultValue": "Inside"
214232
},
215-
{
216-
"type": "SEPARATOR",
217-
"description": "Data to retrieve"
218-
},
219-
{
220-
"name": "data_type",
221-
"label": "Data type",
222-
"type": "SELECT",
223-
"selectChoices":[
224-
{"value": "PlotData", "label": "Plot"},
225-
{"value": "RecordedData", "label": "Recorded"},
226-
{"value": "SummaryData", "label": "SummaryData"},
227-
{"value": "Value", "label": "Value"},
228-
{"value": "EndValue", "label": "EndValue"}
229-
],
230-
"defaultValue": "SummaryData"
231-
},
232233
{
233234
"name": "summary_type",
234235
"label": "Summary type",

‎custom-recipes/pi-system-retrieve-list/recipe.json‎

Lines changed: 27 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -112,6 +112,12 @@
112112
"type": "COLUMN",
113113
"columnRole": "input_dataset"
114114
},
115+
{
116+
"name": "do_duplicate_input_row",
117+
"label": "Copy other input columns",
118+
"description": "(those with matching names will be overwritten by PI system data)",
119+
"type": "BOOLEAN"
120+
},
115121
{
116122
"type": "SEPARATOR",
117123
"description": "Time",
@@ -125,16 +131,16 @@
125131
{
126132
"name": "use_start_time_column",
127133
"label": "Use start time value per row",
128-
"description": "Description here",
129-
"visibilityCondition": "false && (['InterpolatedData', 'RecordedData', 'PlotData', 'EventFrames'].includes(model.data_type))",
134+
"description": "",
135+
"visibilityCondition": "['InterpolatedData', 'RecordedData', 'PlotData', 'EventFrames'].includes(model.data_type)",
130136
"type": "BOOLEAN",
131137
"defaultValue": false
132138
},
133139
{
134140
"visibilityCondition": "model.use_start_time_column==true && (['InterpolatedData', 'RecordedData', 'PlotData', 'EventFrames'].includes(model.data_type))",
135141
"name": "start_time_column",
136-
"label": "Start times' column",
137-
"description": "Optional",
142+
"label": "Start times column",
143+
"description": "Column type must be string",
138144
"type": "COLUMN",
139145
"columnRole": "input_dataset"
140146
},
@@ -148,17 +154,17 @@
148154
},
149155
{
150156
"name": "use_end_time_column",
151-
"label": "Use start time value per row",
157+
"label": "Use end time value per row",
152158
"description": "",
153-
"visibilityCondition": "false && (['InterpolatedData', 'RecordedData', 'PlotData', 'EventFrames'].includes(model.data_type))",
159+
"visibilityCondition": "['InterpolatedData', 'RecordedData', 'PlotData', 'EventFrames'].includes(model.data_type)",
154160
"type": "BOOLEAN",
155161
"defaultValue": false
156162
},
157163
{
158164
"visibilityCondition": "model.use_end_time_column==true && (['InterpolatedData', 'RecordedData', 'PlotData', 'EventFrames'].includes(model.data_type))",
159165
"name": "end_time_column",
160-
"label": "End times' column",
161-
"description": "Optional",
166+
"label": "End times column",
167+
"description": "Column type must be string",
162168
"type": "COLUMN",
163169
"columnRole": "input_dataset"
164170
},
@@ -194,10 +200,22 @@
194200
{
195201
"name": "boundary_type",
196202
"label": "Boundary type",
197-
"visibilityCondition": "['InterpolatedData','RecordedData'].includes(model.data_type)",
203+
"visibilityCondition": "((model.must_retrieve_metrics) && ['InterpolatedData'].includes(model.data_type))",
204+
"type": "SELECT",
205+
"selectChoices":[
206+
{"value": "Inside", "label": "Inside"},
207+
{"value": "Outside", "label": "Outside"}
208+
],
209+
"defaultValue": "Inside"
210+
},
211+
{
212+
"name": "record_boundary_type",
213+
"label": "Boundary type",
214+
"visibilityCondition": "((model.must_retrieve_metrics) && ['RecordedData'].includes(model.data_type))",
198215
"type": "SELECT",
199216
"selectChoices":[
200217
{"value": "Inside", "label": "Inside"},
218+
{"value": "Interpolated", "label": "Interpolated"},
201219
{"value": "Outside", "label": "Outside"}
202220
],
203221
"defaultValue": "Inside"

‎custom-recipes/pi-system-retrieve-list/recipe.py‎

Lines changed: 13 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -48,8 +48,9 @@
4848
end_time_column = config.get("end_time_column")
4949
server_url_column = config.get("server_url_column")
5050
interval, sync_time, boundary_type = get_interpolated_parameters(config)
51-
record_boundary_type = config.get("boundary_type") if data_type == "RecordedData" else None
51+
record_boundary_type = config.get("record_boundary_type") if data_type == "RecordedData" else None
5252
summary_type, summary_duration = get_summary_parameters(config)
53+
do_duplicate_input_row = config.get("do_duplicate_input_row", False)
5354

5455
network_timer = PerformanceTimer()
5556
processing_timer = PerformanceTimer()
@@ -64,13 +65,19 @@
6465
client = None
6566
previous_server_url = ""
6667
time_not_parsed = True
68+
69+
input_columns = list(input_parameters_dataframe.columns) if do_duplicate_input_row else []
70+
6771
with output_dataset.get_writer() as writer:
6872
first_dataframe = True
6973
for index, input_parameters_row in input_parameters_dataframe.iterrows():
7074
server_url = input_parameters_row.get(server_url_column, server_url) if use_server_url_column else server_url
7175
start_time = input_parameters_row.get(start_time_column, start_time) if use_start_time_column else start_time
7276
end_time = input_parameters_row.get(end_time_column, end_time) if use_end_time_column else end_time
7377
row_name = input_parameters_row.get("Name")
78+
duplicate_initial_row = {}
79+
for input_column in input_columns:
80+
duplicate_initial_row[input_column] = input_parameters_row.get(input_column)
7481

7582
if client is None or previous_server_url != server_url:
7683
client = OSIsoftClient(
@@ -83,10 +90,8 @@
8390
# make sure all OSIsoft time string format are evaluated at the same time
8491
# rather than at every request, at least for start / end times set in the UI
8592
time_not_parsed = False
86-
if not use_start_time_column:
87-
start_time = client.parse_pi_time(start_time)
88-
if not use_end_time_column:
89-
end_time = client.parse_pi_time(end_time)
93+
start_time = client.parse_pi_time(start_time)
94+
end_time = client.parse_pi_time(end_time)
9095
sync_time = client.parse_pi_time(sync_time)
9196

9297
object_id = input_parameters_row.get(path_column)
@@ -111,7 +116,7 @@
111116
summary_duration=summary_duration
112117
)
113118
else:
114-
rows = client.get_rows_from_webid(
119+
rows = client.recursive_get_rows_from_webid(
115120
object_id,
116121
data_type,
117122
start_date=start_time,
@@ -137,6 +142,8 @@
137142
results.extend(extention)
138143
else:
139144
base = get_base_for_data_type(data_type, object_id)
145+
if duplicate_initial_row:
146+
base.update(duplicate_initial_row)
140147
base.update(row)
141148
extention = client.unnest_row(base)
142149
results.extend(extention)

‎python-connectors/pi-system_attribute-search/connector.json‎

Lines changed: 13 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -363,14 +363,26 @@
363363
{
364364
"name": "boundary_type",
365365
"label": "Boundary type",
366-
"visibilityCondition": "((model.must_retrieve_metrics) && ['InterpolatedData','RecordedData'].includes(model.data_type))",
366+
"visibilityCondition": "((model.must_retrieve_metrics) && ['InterpolatedData'].includes(model.data_type))",
367367
"type": "SELECT",
368368
"selectChoices":[
369369
{"value": "Inside", "label": "Inside"},
370370
{"value": "Outside", "label": "Outside"}
371371
],
372372
"defaultValue": "Inside"
373373
},
374+
{
375+
"name": "record_boundary_type",
376+
"label": "Boundary type",
377+
"visibilityCondition": "((model.must_retrieve_metrics) && ['RecordedData'].includes(model.data_type))",
378+
"type": "SELECT",
379+
"selectChoices":[
380+
{"value": "Inside", "label": "Inside"},
381+
{"value": "Interpolated", "label": "Interpolated"},
382+
{"value": "Outside", "label": "Outside"}
383+
],
384+
"defaultValue": "Inside"
385+
},
374386
{
375387
"name": "summary_duration",
376388
"label": "Summary duration",

‎python-connectors/pi-system_attribute-search/connector.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -56,7 +56,7 @@ def __init__(self, config, plugin_config):
5656
self.config = config
5757
self.summary_type, self.summary_duration = get_summary_parameters(config)
5858

59-
self.record_boundary_type = config.get("boundary_type") if self.data_type == "RecordedData" else None
59+
self.record_boundary_type = config.get("record_boundary_type") if self.data_type == "RecordedData" else None
6060

6161
def extract_database_webid(self, database_endpoint):
6262
return database_endpoint.split("/")[-1]

‎python-connectors/pi-system_event-frames-search/connector.json‎

Lines changed: 32 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -200,6 +200,38 @@
200200
],
201201
"mandatory": true
202202
},
203+
{
204+
"type": "SEPARATOR",
205+
"description": "Interpolation",
206+
"visibilityCondition": "(model.must_retrieve_metrics) && ['InterpolatedData'].includes(model.data_type)"
207+
},
208+
{
209+
"name": "interval",
210+
"label": "Interval",
211+
"visibilityCondition": "(model.must_retrieve_metrics) && ['InterpolatedData'].includes(model.data_type)",
212+
"description": "Optional",
213+
"type": "STRING",
214+
"defaultValue": ""
215+
},
216+
{
217+
"name": "sync_time",
218+
"label": "Sync time",
219+
"visibilityCondition": "(model.must_retrieve_metrics) && ['InterpolatedData'].includes(model.data_type)",
220+
"description": "Optional",
221+
"type": "STRING",
222+
"defaultValue": ""
223+
},
224+
{
225+
"name": "boundary_type",
226+
"label": "Boundary type",
227+
"visibilityCondition": "(model.must_retrieve_metrics) && ['InterpolatedData'].includes(model.data_type)",
228+
"type": "SELECT",
229+
"selectChoices":[
230+
{"value": "Inside", "label": "Inside"},
231+
{"value": "Outside", "label": "Outside"}
232+
],
233+
"defaultValue": "Inside"
234+
},
203235
{
204236
"name": "summary_type",
205237
"label": "Summary type",

‎python-connectors/pi-system_event-frames-search/connector.py‎

Lines changed: 8 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -7,7 +7,7 @@
77
from osisoft_plugin_common import (
88
PISystemConnectorError, RecordsLimit, get_credentials,
99
build_requests_params, assert_time_format, get_advanced_parameters, check_debug_mode,
10-
PerformanceTimer, get_max_count, get_summary_parameters
10+
PerformanceTimer, get_max_count, get_summary_parameters, get_interpolated_parameters
1111
)
1212

1313

@@ -47,6 +47,7 @@ def __init__(self, config, plugin_config):
4747
self.end_time = self.client.parse_pi_time(self.end_time)
4848
if self.end_time:
4949
config["end_time"] = self.end_time
50+
self.interval, self.sync_time, self.boundary_type = get_interpolated_parameters(config)
5051
self.search_mode = config.get("search_mode", None)
5152
self.output_type = config.get("output_type")
5253
assert_time_format(self.start_time, error_source="start time")
@@ -79,7 +80,8 @@ def generate_rows(self, dataset_schema=None, dataset_partitioning=None,
7980
if self.object_id:
8081
for event_frame in self.client.get_rows_from_urls(
8182
self.object_id, self.data_type, start_date=self.start_time,
82-
end_date=self.end_time, max_count=self.max_count):
83+
end_date=self.end_time, interval=self.interval, sync_time=self.sync_time,
84+
boundary_type=self.boundary_type, max_count=self.max_count):
8385
self.yields_timer.start()
8486
yield event_frame
8587
self.yields_timer.stop()
@@ -115,6 +117,9 @@ def generate_rows(self, dataset_schema=None, dataset_partitioning=None,
115117
batch_size=self.batch_size,
116118
summary_type=self.summary_type,
117119
summary_duration=self.summary_duration,
120+
boundary_type=self.boundary_type,
121+
interval=self.interval,
122+
sync_time=self.sync_time,
118123
max_count=self.max_count
119124
)
120125
for batch_row in batch_rows:
@@ -145,6 +150,7 @@ def generate_rows(self, dataset_schema=None, dataset_partitioning=None,
145150
event_frame_id = event_frame.get("WebId")
146151
event_frame_metrics = self.client.get_rows_from_webid(
147152
event_frame_id, self.data_type, summary_type=self.summary_type, summary_duration=self.summary_duration,
153+
interval=self.interval, sync_time=self.sync_time, boundary_type=self.boundary_type,
148154
search_full_hierarchy=self.search_full_hierarchy, max_count=self.max_count,
149155
can_raise=False
150156
)

‎python-lib/osisoft_client.py‎

Lines changed: 9 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -426,13 +426,13 @@ def get_link_from_item(self, item, data_type, start_date, end_date, interval=Non
426426
)
427427
return json_response
428428

429-
def get_rows_from_url(self, url=None, start_date=None, end_date=None, interval=None, sync_time=None, max_count=None):
429+
def get_rows_from_url(self, url=None, start_date=None, end_date=None, interval=None, sync_time=None, boundary_type=None, max_count=None):
430430
pagination = OffsetPagination()
431431
has_more = True
432432
while has_more:
433433
json_response, has_more = pagination.get_offset_paginated(
434434
self.get_link_from_url,
435-
url, start_date=start_date, end_date=end_date, interval=interval, sync_time=sync_time, max_count=max_count
435+
url, start_date=start_date, end_date=end_date, interval=interval, sync_time=sync_time, boundary_type=boundary_type, max_count=max_count
436436
)
437437
items = json_response.get(OSIsoftConstants.API_ITEM_KEY, [json_response])
438438
for item in items:
@@ -443,15 +443,18 @@ def get_rows_from_url(self, url=None, start_date=None, end_date=None, interval=N
443443
else:
444444
yield item
445445

446-
def get_rows_from_urls(self, links=None, data_type=None, start_date=None, end_date=None, interval=None, sync_time=None, max_count=None):
446+
def get_rows_from_urls(self, links=None, data_type=None, start_date=None, end_date=None, interval=None, sync_time=None, boundary_type=None, max_count=None):
447447
links = links or []
448448
for link in links:
449449
url = link
450-
rows = self.get_rows_from_url(url, start_date=start_date, end_date=end_date, interval=interval, sync_time=sync_time, max_count=max_count)
450+
rows = self.get_rows_from_url(
451+
url, start_date=start_date, end_date=end_date, interval=interval,
452+
sync_time=sync_time, boundary_type=boundary_type, max_count=max_count
453+
)
451454
for row in rows:
452455
yield row
453456

454-
def get_link_from_url(self, url, start_date=None, end_date=None, interval=None, sync_time=None, start_index=None, max_count=None):
457+
def get_link_from_url(self, url, start_date=None, end_date=None, interval=None, sync_time=None, start_index=None, boundary_type=None, max_count=None):
455458
if not url:
456459
url = self.endpoint.get_base_url()
457460
headers = self.get_requests_headers()
@@ -461,6 +464,7 @@ def get_link_from_url(self, url, start_date=None, end_date=None, interval=None,
461464
interval=interval,
462465
sync_time=sync_time,
463466
start_index=start_index,
467+
sync_time_boundary_type=boundary_type,
464468
max_count=max_count
465469
)
466470
json_response = self.get(

‎python-lib/osisoft_constants.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -403,7 +403,7 @@ class OSIsoftConstants(object):
403403
"Security": "{base_url}/eventframes/{webid}/security",
404404
"SecurityEntries": "{base_url}/eventframes/{webid}/securityentries"
405405
}
406-
PLUGIN_VERSION = "1.2.4-beta.1"
406+
PLUGIN_VERSION = "1.2.4-beta.5"
407407
VALUE_COLUMN_SUFFIX = "_val"
408408
WEB_API_PATH = "piwebapi"
409409
WRITE_HEADERS = {'X-Requested-With': 'XmlHttpRequest'}

0 commit comments

Comments
 (0)