Skip to content

Commit 17ed4a8

Browse files
alexbourretalexbourret
authored
Merge pull request #55 from dataiku/feature/sc-219335-add-boundary-type-selector-everywhere
feat: [sc-219335] [pi-system plugin] Add boundary type selector to recorded type data
2 parents 5ca9226 + 9afa9ea commit 17ed4a8

File tree

12 files changed

+186
-67
lines changed

12 files changed

+186
-67
lines changed

‎CHANGELOG.md‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,14 @@
11
# Changelog
22

3+
## [Version 1.2.4](https://github.com/dataiku/dss-plugin-pi-server/releases/tag/v1.2.4) - Feature and bug release - 2025-02-18
4+
5+
- Add boundary type selector to recorded data type
6+
- Add boundary type selector to attribute search connector
7+
- *Assets values downloader* Add option to copy each input row into output dataset
8+
- Fix issue with recorded data type
9+
- *Assets values downloader* Time range pagination when using webids
10+
- *Assets values downloader* Time can be selected from a column of the input dataset
11+
312
## [Version 1.2.3](https://github.com/dataiku/dss-plugin-pi-server/releases/tag/v1.2.3) - Feature release - 2024-09-26
413

514
- Add summaryDuration input (duration of each summary interval)

‎custom-recipes/pi-system-retrieve-event-frames/recipe.json‎

Lines changed: 20 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -143,7 +143,7 @@
143143
{
144144
"visibilityCondition": "model.use_start_time_column==true && (['InterpolatedData', 'RecordedData', 'PlotData', 'EventFrames'].includes(model.data_type))",
145145
"name": "start_time_column",
146-
"label": "Start times' column",
146+
"label": "Start times column",
147147
"description": "Optional",
148148
"type": "COLUMN",
149149
"columnRole": "input_dataset"
@@ -167,7 +167,7 @@
167167
{
168168
"visibilityCondition": "model.use_end_time_column==true && (['InterpolatedData', 'RecordedData', 'PlotData', 'EventFrames'].includes(model.data_type))",
169169
"name": "end_time_column",
170-
"label": "End times' column",
170+
"label": "End times column",
171171
"description": "Optional",
172172
"type": "COLUMN",
173173
"columnRole": "input_dataset"
@@ -180,6 +180,24 @@
180180
"type": "STRING",
181181
"defaultValue": ""
182182
},
183+
{
184+
"type": "SEPARATOR",
185+
"description": "Data to retrieve"
186+
},
187+
{
188+
"name": "data_type",
189+
"label": "Data type",
190+
"type": "SELECT",
191+
"selectChoices":[
192+
{"value": "InterpolatedData", "label": "Interpolated"},
193+
{"value": "PlotData", "label": "Plot"},
194+
{"value": "RecordedData", "label": "Recorded"},
195+
{"value": "SummaryData", "label": "SummaryData"},
196+
{"value": "Value", "label": "Value"},
197+
{"value": "EndValue", "label": "EndValue"}
198+
],
199+
"defaultValue": "SummaryData"
200+
},
183201
{
184202
"type": "SEPARATOR",
185203
"description": "Interpolation",
@@ -212,23 +230,6 @@
212230
],
213231
"defaultValue": "Inside"
214232
},
215-
{
216-
"type": "SEPARATOR",
217-
"description": "Data to retrieve"
218-
},
219-
{
220-
"name": "data_type",
221-
"label": "Data type",
222-
"type": "SELECT",
223-
"selectChoices":[
224-
{"value": "PlotData", "label": "Plot"},
225-
{"value": "RecordedData", "label": "Recorded"},
226-
{"value": "SummaryData", "label": "SummaryData"},
227-
{"value": "Value", "label": "Value"},
228-
{"value": "EndValue", "label": "EndValue"}
229-
],
230-
"defaultValue": "SummaryData"
231-
},
232233
{
233234
"name": "summary_type",
234235
"label": "Summary type",

‎custom-recipes/pi-system-retrieve-list/recipe.json‎

Lines changed: 27 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -112,6 +112,12 @@
112112
"type": "COLUMN",
113113
"columnRole": "input_dataset"
114114
},
115+
{
116+
"name": "do_duplicate_input_row",
117+
"label": "Copy other input columns",
118+
"description": "(those with matching names will be overwritten by PI system data)",
119+
"type": "BOOLEAN"
120+
},
115121
{
116122
"type": "SEPARATOR",
117123
"description": "Time",
@@ -125,16 +131,16 @@
125131
{
126132
"name": "use_start_time_column",
127133
"label": "Use start time value per row",
128-
"description": "Description here",
129-
"visibilityCondition": "false && (['InterpolatedData', 'RecordedData', 'PlotData', 'EventFrames'].includes(model.data_type))",
134+
"description": "",
135+
"visibilityCondition": "['InterpolatedData', 'RecordedData', 'PlotData', 'EventFrames'].includes(model.data_type)",
130136
"type": "BOOLEAN",
131137
"defaultValue": false
132138
},
133139
{
134140
"visibilityCondition": "model.use_start_time_column==true && (['InterpolatedData', 'RecordedData', 'PlotData', 'EventFrames'].includes(model.data_type))",
135141
"name": "start_time_column",
136-
"label": "Start times' column",
137-
"description": "Optional",
142+
"label": "Start times column",
143+
"description": "Column type must be string",
138144
"type": "COLUMN",
139145
"columnRole": "input_dataset"
140146
},
@@ -148,17 +154,17 @@
148154
},
149155
{
150156
"name": "use_end_time_column",
151-
"label": "Use start time value per row",
157+
"label": "Use end time value per row",
152158
"description": "",
153-
"visibilityCondition": "false && (['InterpolatedData', 'RecordedData', 'PlotData', 'EventFrames'].includes(model.data_type))",
159+
"visibilityCondition": "['InterpolatedData', 'RecordedData', 'PlotData', 'EventFrames'].includes(model.data_type)",
154160
"type": "BOOLEAN",
155161
"defaultValue": false
156162
},
157163
{
158164
"visibilityCondition": "model.use_end_time_column==true && (['InterpolatedData', 'RecordedData', 'PlotData', 'EventFrames'].includes(model.data_type))",
159165
"name": "end_time_column",
160-
"label": "End times' column",
161-
"description": "Optional",
166+
"label": "End times column",
167+
"description": "Column type must be string",
162168
"type": "COLUMN",
163169
"columnRole": "input_dataset"
164170
},
@@ -194,10 +200,22 @@
194200
{
195201
"name": "boundary_type",
196202
"label": "Boundary type",
197-
"visibilityCondition": "['InterpolatedData'].includes(model.data_type)",
203+
"visibilityCondition": "((model.must_retrieve_metrics) && ['InterpolatedData'].includes(model.data_type))",
204+
"type": "SELECT",
205+
"selectChoices":[
206+
{"value": "Inside", "label": "Inside"},
207+
{"value": "Outside", "label": "Outside"}
208+
],
209+
"defaultValue": "Inside"
210+
},
211+
{
212+
"name": "record_boundary_type",
213+
"label": "Boundary type",
214+
"visibilityCondition": "((model.must_retrieve_metrics) && ['RecordedData'].includes(model.data_type))",
198215
"type": "SELECT",
199216
"selectChoices":[
200217
{"value": "Inside", "label": "Inside"},
218+
{"value": "Interpolated", "label": "Interpolated"},
201219
{"value": "Outside", "label": "Outside"}
202220
],
203221
"defaultValue": "Inside"

‎custom-recipes/pi-system-retrieve-list/recipe.py‎

Lines changed: 15 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -48,7 +48,9 @@
4848
end_time_column = config.get("end_time_column")
4949
server_url_column = config.get("server_url_column")
5050
interval, sync_time, boundary_type = get_interpolated_parameters(config)
51+
record_boundary_type = config.get("record_boundary_type") if data_type == "RecordedData" else None
5152
summary_type, summary_duration = get_summary_parameters(config)
53+
do_duplicate_input_row = config.get("do_duplicate_input_row", False)
5254

5355
network_timer = PerformanceTimer()
5456
processing_timer = PerformanceTimer()
@@ -63,13 +65,19 @@
6365
client = None
6466
previous_server_url = ""
6567
time_not_parsed = True
68+
69+
input_columns = list(input_parameters_dataframe.columns) if do_duplicate_input_row else []
70+
6671
with output_dataset.get_writer() as writer:
6772
first_dataframe = True
6873
for index, input_parameters_row in input_parameters_dataframe.iterrows():
6974
server_url = input_parameters_row.get(server_url_column, server_url) if use_server_url_column else server_url
7075
start_time = input_parameters_row.get(start_time_column, start_time) if use_start_time_column else start_time
7176
end_time = input_parameters_row.get(end_time_column, end_time) if use_end_time_column else end_time
7277
row_name = input_parameters_row.get("Name")
78+
duplicate_initial_row = {}
79+
for input_column in input_columns:
80+
duplicate_initial_row[input_column] = input_parameters_row.get(input_column)
7381

7482
if client is None or previous_server_url != server_url:
7583
client = OSIsoftClient(
@@ -82,10 +90,8 @@
8290
# make sure all OSIsoft time string format are evaluated at the same time
8391
# rather than at every request, at least for start / end times set in the UI
8492
time_not_parsed = False
85-
if not use_start_time_column:
86-
start_time = client.parse_pi_time(start_time)
87-
if not use_end_time_column:
88-
end_time = client.parse_pi_time(end_time)
93+
start_time = client.parse_pi_time(start_time)
94+
end_time = client.parse_pi_time(end_time)
8995
sync_time = client.parse_pi_time(sync_time)
9096

9197
object_id = input_parameters_row.get(path_column)
@@ -102,21 +108,23 @@
102108
interval=interval,
103109
sync_time=sync_time,
104110
boundary_type=boundary_type,
111+
record_boundary_type=record_boundary_type,
105112
max_count=max_count,
106113
can_raise=False,
107114
object_id=object_id,
108115
summary_type=summary_type,
109116
summary_duration=summary_duration
110117
)
111118
else:
112-
rows = client.get_rows_from_webid(
119+
rows = client.recursive_get_rows_from_webid(
113120
object_id,
114121
data_type,
115122
start_date=start_time,
116123
end_date=end_time,
117124
interval=interval,
118125
sync_time=sync_time,
119126
boundary_type=boundary_type,
127+
record_boundary_type=record_boundary_type,
120128
max_count=max_count,
121129
can_raise=False,
122130
endpoint_type="AF",
@@ -134,6 +142,8 @@
134142
results.extend(extention)
135143
else:
136144
base = get_base_for_data_type(data_type, object_id)
145+
if duplicate_initial_row:
146+
base.update(duplicate_initial_row)
137147
base.update(row)
138148
extention = client.unnest_row(base)
139149
results.extend(extention)

‎plugin.json‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
{
22
"id": "pi-system",
3-
"version": "1.2.3",
3+
"version": "1.2.4",
44
"meta": {
55
"label": "PI System",
66
"description": "Retrieve data from your OSIsoft PI System servers",

‎python-connectors/pi-system_attribute-search/connector.json‎

Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -360,6 +360,29 @@
360360
],
361361
"visibilityCondition": "((model.must_retrieve_metrics) && (model.data_type == 'SummaryData'))"
362362
},
363+
{
364+
"name": "boundary_type",
365+
"label": "Boundary type",
366+
"visibilityCondition": "((model.must_retrieve_metrics) && ['InterpolatedData'].includes(model.data_type))",
367+
"type": "SELECT",
368+
"selectChoices":[
369+
{"value": "Inside", "label": "Inside"},
370+
{"value": "Outside", "label": "Outside"}
371+
],
372+
"defaultValue": "Inside"
373+
},
374+
{
375+
"name": "record_boundary_type",
376+
"label": "Boundary type",
377+
"visibilityCondition": "((model.must_retrieve_metrics) && ['RecordedData'].includes(model.data_type))",
378+
"type": "SELECT",
379+
"selectChoices":[
380+
{"value": "Inside", "label": "Inside"},
381+
{"value": "Interpolated", "label": "Interpolated"},
382+
{"value": "Outside", "label": "Outside"}
383+
],
384+
"defaultValue": "Inside"
385+
},
363386
{
364387
"name": "summary_duration",
365388
"label": "Summary duration",

‎python-connectors/pi-system_attribute-search/connector.py‎

Lines changed: 8 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,8 @@
66
from osisoft_plugin_common import (
77
PISystemConnectorError, RecordsLimit, get_credentials, assert_time_format,
88
remove_unwanted_columns, format_output, filter_columns_from_schema, is_child_attribute_path,
9-
check_debug_mode, PerformanceTimer, get_max_count, get_summary_parameters, fields_selector
9+
check_debug_mode, PerformanceTimer, get_max_count, get_summary_parameters, fields_selector,
10+
get_interpolated_parameters
1011
)
1112
from osisoft_constants import OSIsoftConstants
1213

@@ -36,9 +37,7 @@ def __init__(self, config, plugin_config):
3637
self.start_time = self.client.parse_pi_time(self.start_time)
3738
self.end_time = config.get("end_time")
3839
self.end_time = self.client.parse_pi_time(self.end_time)
39-
is_interpolated_data = config.get("data_type", "").endswith("InterpolatedData")
40-
self.interval = config.get("interval") if is_interpolated_data else None
41-
self.sync_time = config.get("sync_time") if is_interpolated_data else None
40+
self.interval, self.sync_time, self.boundary_type = get_interpolated_parameters(config)
4241
self.sync_time = self.client.parse_pi_time(self.sync_time)
4342
assert_time_format(self.start_time, error_source="start time")
4443
assert_time_format(self.end_time, error_source="end time")
@@ -57,6 +56,8 @@ def __init__(self, config, plugin_config):
5756
self.config = config
5857
self.summary_type, self.summary_duration = get_summary_parameters(config)
5958

59+
self.record_boundary_type = config.get("record_boundary_type") if self.data_type == "RecordedData" else None
60+
6061
def extract_database_webid(self, database_endpoint):
6162
return database_endpoint.split("/")[-1]
6263

@@ -114,8 +115,9 @@ def generate_rows(self, dataset_schema=None, dataset_partitioning=None,
114115
selected_fields=fields_selector(self.data_type),
115116
max_count=self.max_count,
116117
summary_type=self.summary_type,
117-
summary_duration=self.summary_duration
118-
# boundary_type=self.boundary_type
118+
summary_duration=self.summary_duration,
119+
boundary_type=self.boundary_type,
120+
record_boundary_type=self.record_boundary_type
119121
):
120122
if limit.is_reached():
121123
return

‎python-connectors/pi-system_event-frames-search/connector.json‎

Lines changed: 32 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -200,6 +200,38 @@
200200
],
201201
"mandatory": true
202202
},
203+
{
204+
"type": "SEPARATOR",
205+
"description": "Interpolation",
206+
"visibilityCondition": "(model.must_retrieve_metrics) && ['InterpolatedData'].includes(model.data_type)"
207+
},
208+
{
209+
"name": "interval",
210+
"label": "Interval",
211+
"visibilityCondition": "(model.must_retrieve_metrics) && ['InterpolatedData'].includes(model.data_type)",
212+
"description": "Optional",
213+
"type": "STRING",
214+
"defaultValue": ""
215+
},
216+
{
217+
"name": "sync_time",
218+
"label": "Sync time",
219+
"visibilityCondition": "(model.must_retrieve_metrics) && ['InterpolatedData'].includes(model.data_type)",
220+
"description": "Optional",
221+
"type": "STRING",
222+
"defaultValue": ""
223+
},
224+
{
225+
"name": "boundary_type",
226+
"label": "Boundary type",
227+
"visibilityCondition": "(model.must_retrieve_metrics) && ['InterpolatedData'].includes(model.data_type)",
228+
"type": "SELECT",
229+
"selectChoices":[
230+
{"value": "Inside", "label": "Inside"},
231+
{"value": "Outside", "label": "Outside"}
232+
],
233+
"defaultValue": "Inside"
234+
},
203235
{
204236
"name": "summary_type",
205237
"label": "Summary type",

0 commit comments

Comments
 (0)