Skip to content

Commit c35fc72

Browse files
author
Sushanth Sathish Kumar
committed
fix: fix dashboard tests, TODO: fix assertion error in test_automatic_data_quality_dashboard()
1 parent f5dbad9 commit c35fc72

File tree

2 files changed

+39
-68
lines changed

2 files changed

+39
-68
lines changed

‎src/sagemaker/model_monitor/dashboards.py

Lines changed: 15 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -12,9 +12,9 @@
1212
# language governing permissions and limitations under the License.
1313
"""This module contains code containing wrapper classes for dashboard structures in CloudWatch.
1414
15-
These classes assist with creating dashboards in Python3 and then using boto3 CloudWatch client
16-
to publish the generated dashboards. To be used to aid dashboard creation in the create_monitoring_schedule
17-
and update_monitoring_schedule methods in model_monitoring.py
15+
These classes assist with creating dashboards in Python3 and then using boto3 CloudWatch client
16+
to publish the generated dashboards. To be used to aid dashboard creation in the create_monitoring_schedule
17+
and update_monitoring_schedule methods in model_monitoring.py
1818
"""
1919

2020
import json
@@ -136,7 +136,7 @@ def __init__(self, endpoint_name, monitoring_schedule_name, batch_transform_inpu
136136
estimated_unique_values_widget = self._generate_estimated_unique_values_widget()
137137
completeness_widget = self._generate_completeness_widget()
138138
baseline_drift_widget = self._generate_baseline_drift_widget()
139-
139+
140140
self.dashboard = {
141141
"variables": variables,
142142
"widgets": [
@@ -149,7 +149,7 @@ def __init__(self, endpoint_name, monitoring_schedule_name, batch_transform_inpu
149149
}
150150

151151
def _generate_variables(self):
152-
if self.batch_transform:
152+
if self.batch_transform is not None:
153153
return [
154154
Variable(
155155
variable_type="property",
@@ -175,14 +175,14 @@ def _generate_variables(self):
175175
]
176176

177177
def _generate_type_counts_widget(self):
178-
if self.batch_transform:
178+
if self.batch_transform is not None:
179179
type_counts_widget_properties = WidgetProperties(
180180
view="timeSeries",
181181
stacked=False,
182182
metrics=[
183183
[
184184
{
185-
"expression": f"SEARCH( '{AutomaticDataQualityDashboard.DATA_QUALITY_METRICS_BATCH_NAMESPACE} %^(feature_fractional_counts_|feature_string_counts_|feature_integral_counts_|feature_boolean_counts_|feature_unknown_counts_).*% Feature=\"_\" MonitoringSchedule=\"{self.monitoring_schedule}\" ', 'Average')"
185+
"expression": f"SEARCH( '{AutomaticDataQualityDashboard.DATA_QUALITY_METRICS_BATCH_NAMESPACE} %^feature_fractional_counts_.*% OR %^feature_integral_counts_.*% OR %^feature_string_counts_.*% OR %^feature_boolean_counts_.*% OR %^feature_unknown_counts_.*% Feature=\"_\" MonitoringSchedule=\"{self.monitoring_schedule}\" ', 'Average')"
186186
}
187187
]
188188
],
@@ -196,7 +196,7 @@ def _generate_type_counts_widget(self):
196196
metrics=[
197197
[
198198
{
199-
"expression": f'SEARCH( \'{AutomaticDataQualityDashboard.DATA_QUALITY_METRICS_ENDPOINT_NAMESPACE} %^(feature_fractional_counts_|feature_string_counts_|feature_integral_counts_|feature_boolean_counts_|feature_unknown_counts_).*% Endpoint="{self.endpoint}" Feature="_" MonitoringSchedule="{self.monitoring_schedule}" \', \'Average\')'
199+
"expression": f'SEARCH( \'{AutomaticDataQualityDashboard.DATA_QUALITY_METRICS_ENDPOINT_NAMESPACE} %^feature_fractional_counts_.*% OR %^feature_integral_counts_.*% OR %^feature_string_counts_.*% OR %^feature_boolean_counts_.*% OR %^feature_unknown_counts_.*% Endpoint="{self.endpoint}" Feature="_" MonitoringSchedule="{self.monitoring_schedule}" \', \'Average\')'
200200
}
201201
]
202202
],
@@ -208,14 +208,14 @@ def _generate_type_counts_widget(self):
208208
)
209209

210210
def _generate_null_counts_widget(self):
211-
if self.batch_transform:
211+
if self.batch_transform is not None:
212212
null_counts_widget_properties = WidgetProperties(
213213
view="timeSeries",
214214
stacked=False,
215215
metrics=[
216216
[
217217
{
218-
"expression": f"SEARCH( '{AutomaticDataQualityDashboard.DATA_QUALITY_METRICS_BATCH_NAMESPACE} %^(feature_null_|feature_non_null_).*% Feature=\"_\" MonitoringSchedule=\"{self.monitoring_schedule}\" ', 'Average')"
218+
"expression": f"SEARCH( '{AutomaticDataQualityDashboard.DATA_QUALITY_METRICS_BATCH_NAMESPACE} %^feature_null_.*% OR %^feature_non_null_.*% Feature=\"_\" MonitoringSchedule=\"{self.monitoring_schedule}\" ', 'Average')"
219219
}
220220
]
221221
],
@@ -229,7 +229,7 @@ def _generate_null_counts_widget(self):
229229
metrics=[
230230
[
231231
{
232-
"expression": f'SEARCH( \'{AutomaticDataQualityDashboard.DATA_QUALITY_METRICS_ENDPOINT_NAMESPACE} %^(feature_null_|feature_non_null_).*% Endpoint="{self.endpoint}" Feature="_" MonitoringSchedule="{self.monitoring_schedule}" \', \'Average\')'
232+
"expression": f'SEARCH( \'{AutomaticDataQualityDashboard.DATA_QUALITY_METRICS_ENDPOINT_NAMESPACE} %^feature_null_.*% OR %^feature_non_null_.*% Endpoint="{self.endpoint}" Feature="_" MonitoringSchedule="{self.monitoring_schedule}" \', \'Average\')'
233233
}
234234
]
235235
],
@@ -241,7 +241,7 @@ def _generate_null_counts_widget(self):
241241
)
242242

243243
def _generate_estimated_unique_values_widget(self):
244-
if self.batch_transform:
244+
if self.batch_transform is not None:
245245
estimated_unique_vals_widget_properties = WidgetProperties(
246246
view="timeSeries",
247247
stacked=False,
@@ -278,7 +278,7 @@ def _generate_estimated_unique_values_widget(self):
278278
)
279279

280280
def _generate_completeness_widget(self):
281-
if self.batch_transform:
281+
if self.batch_transform is not None:
282282
completeness_widget_properties = WidgetProperties(
283283
view="timeSeries",
284284
stacked=False,
@@ -311,7 +311,7 @@ def _generate_completeness_widget(self):
311311
)
312312

313313
def _generate_baseline_drift_widget(self):
314-
if self.batch_transform:
314+
if self.batch_transform is not None:
315315
baseline_drift_widget_properties = WidgetProperties(
316316
view="timeSeries",
317317
stacked=False,
@@ -350,4 +350,4 @@ def to_dict(self):
350350
}
351351

352352
def to_json(self):
353-
return json.dumps(self.to_dict(), indent=4)
353+
return json.dumps(self.to_dict(), indent=4)

‎tests/unit/test_dashboard_methods.py

Lines changed: 24 additions & 53 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,3 @@
1-
import pytest
21
from unittest.mock import patch
32
from sagemaker.model_monitor.dashboards import (
43
Variable,
@@ -13,9 +12,9 @@ def test_variable_to_dict():
1312
variable_type="property",
1413
variable_property="Feature",
1514
inputType="select",
16-
id="Feature",
15+
variable_id="Feature",
1716
label="Feature",
18-
search=AutomaticDataQualityDashboard.DATA_QUALITY_METRICS_ENDPOINT_NAMESPACE,
17+
search="{aws/sagemaker/Endpoints/data-metrics,Endpoint,Feature,MonitoringSchedule}",
1918
populateFrom="Feature",
2019
)
2120
expected_dict = {
@@ -24,7 +23,7 @@ def test_variable_to_dict():
2423
"inputType": "select",
2524
"id": "Feature",
2625
"label": "Feature",
27-
"search": AutomaticDataQualityDashboard.DATA_QUALITY_METRICS_ENDPOINT_NAMESPACE,
26+
"search": "{aws/sagemaker/Endpoints/data-metrics,Endpoint,Feature,MonitoringSchedule}",
2827
"populateFrom": "Feature",
2928
}
3029
assert var.to_dict() == expected_dict
@@ -37,7 +36,7 @@ def test_widget_properties_to_dict():
3736
metrics=[
3837
[
3938
{
40-
"expression": f'SEARCH( \'{AutomaticDataQualityDashboard.DATA_QUALITY_METRICS_ENDPOINT_NAMESPACE} %^(feature_null_|feature_non_null_).*% Endpoint="{self.endpoint}" Feature="_" MonitoringSchedule="{self.monitoring_schedule}" \', \'Average\')'
39+
"expression": f'SEARCH( \'aws/sagemaker/Endpoints/data-metrics,Endpoint,Feature,MonitoringSchedule %^(feature_null_|feature_non_null_).*% \', \'Average\')'
4140
}
4241
]
4342
],
@@ -50,7 +49,7 @@ def test_widget_properties_to_dict():
5049
"metrics": [
5150
[
5251
{
53-
"expression": f'SEARCH( \'{AutomaticDataQualityDashboard.DATA_QUALITY_METRICS_ENDPOINT_NAMESPACE} %^(feature_null_|feature_non_null_).*% Endpoint="{self.endpoint}" Feature="_" MonitoringSchedule="{self.monitoring_schedule}" \', \'Average\')'
52+
"expression": f'SEARCH( \'aws/sagemaker/Endpoints/data-metrics,Endpoint,Feature,MonitoringSchedule %^(feature_null_|feature_non_null_).*% \', \'Average\')'
5453
}
5554
]
5655
],
@@ -67,7 +66,7 @@ def test_widget_to_dict():
6766
metrics=[
6867
[
6968
{
70-
"expression": f'SEARCH( \'{AutomaticDataQualityDashboard.DATA_QUALITY_METRICS_ENDPOINT_NAMESPACE} %^(feature_null_|feature_non_null_).*% Endpoint="{self.endpoint}" Feature="_" MonitoringSchedule="{self.monitoring_schedule}" \', \'Average\')'
69+
"expression": f'SEARCH( \'aws/sagemaker/Endpoints/data-metrics,Endpoint,Feature,MonitoringSchedule %^(feature_null_|feature_non_null_).*% \', \'Average\')'
7170
}
7271
]
7372
],
@@ -85,7 +84,7 @@ def test_widget_to_dict():
8584
"metrics": [
8685
[
8786
{
88-
"expression": f'SEARCH( \'{AutomaticDataQualityDashboard.DATA_QUALITY_METRICS_ENDPOINT_NAMESPACE} %^(feature_null_|feature_non_null_).*% Endpoint="{self.endpoint}" Feature="_" MonitoringSchedule="{self.monitoring_schedule}" \', \'Average\')'
87+
"expression": f'SEARCH( \'aws/sagemaker/Endpoints/data-metrics,Endpoint,Feature,MonitoringSchedule %^(feature_null_|feature_non_null_).*% \', \'Average\')'
8988
}
9089
]
9190
],
@@ -96,75 +95,47 @@ def test_widget_to_dict():
9695
assert widget.to_dict() == expected_dict
9796

9897

99-
@patch("sagemaker.model_monitor.dashboards.AutomaticDataQualityDashboard._generate_variables")
100-
@patch(
101-
"sagemaker.model_monitor.dashboards.AutomaticDataQualityDashboard._generate_type_counts_widget"
102-
)
103-
@patch(
104-
"sagemaker.model_monitor.dashboards.AutomaticDataQualityDashboard._generate_null_counts_widget"
105-
)
106-
@patch(
107-
"sagemaker.model_monitor.dashboards.AutomaticDataQualityDashboard._generate_estimated_unique_values_widget"
108-
)
109-
@patch(
110-
"sagemaker.model_monitor.dashboards.AutomaticDataQualityDashboard._generate_completeness_widget"
111-
)
112-
@patch(
113-
"sagemaker.model_monitor.dashboards.AutomaticDataQualityDashboard._generate_baseline_drift_widget"
114-
)
115-
def test_automatic_data_quality_dashboard(
116-
mock_generate_variables,
117-
mock_generate_type_counts_widget,
118-
mock_generate_null_counts_widget,
119-
mock_generate_estimated_unique_values_widget,
120-
mock_generate_completeness_widget,
121-
mock_generate_baseline_drift_widget,
122-
):
123-
mock_generate_variables.return_value = [
98+
def test_automatic_data_quality_dashboard():
99+
mock_generate_variables = [
124100
Variable(
125101
variable_type="property",
126102
variable_property="Feature",
127103
inputType="select",
128-
id="Feature",
104+
variable_id="Feature",
129105
label="Feature",
130106
search=AutomaticDataQualityDashboard.DATA_QUALITY_METRICS_ENDPOINT_NAMESPACE,
131107
populateFrom="Feature",
132108
)
133109
]
134-
mock_generate_type_counts_widget.return_value = Widget(
110+
mock_generate_type_counts_widget = Widget(
135111
height=8, width=12, widget_type="metric", properties=WidgetProperties()
136112
)
137-
mock_generate_null_counts_widget.return_value = Widget(
113+
mock_generate_null_counts_widget = Widget(
138114
height=8, width=12, widget_type="metric", properties=WidgetProperties()
139115
)
140-
mock_generate_estimated_unique_values_widget.return_value = Widget(
116+
mock_generate_estimated_unique_values_widget = Widget(
141117
height=8, width=12, widget_type="metric", properties=WidgetProperties()
142118
)
143-
mock_generate_completeness_widget.return_value = Widget(
119+
mock_generate_completeness_widget = Widget(
144120
height=8, width=12, widget_type="metric", properties=WidgetProperties()
145121
)
146-
mock_generate_baseline_drift_widget.return_value = Widget(
122+
mock_generate_baseline_drift_widget = Widget(
147123
height=8, width=12, widget_type="metric", properties=WidgetProperties()
148124
)
149125

150-
dashboard = AutomaticDataQualityDashboard(
151-
endpoint_name="my-endpoint",
152-
monitoring_schedule_name="my-schedule",
153-
batch_transform_input=False,
154-
region_name="us-east-1",
155-
)
126+
dashboard = AutomaticDataQualityDashboard("endpoint", "monitoring_schedule", None, "us-west-2")
156127

157128
expected_dashboard = {
158-
"variables": [var.to_dict() for var in mock_generate_variables.return_value],
129+
"variables": [var.to_dict() for var in mock_generate_variables],
159130
"widgets": [
160-
widget.to_dict()
161-
for widget in [
162-
mock_generate_type_counts_widget.return_value,
163-
mock_generate_null_counts_widget.return_value,
164-
mock_generate_estimated_unique_values_widget.return_value,
165-
mock_generate_completeness_widget.return_value,
166-
mock_generate_baseline_drift_widget.return_value,
131+
widget.to_dict() for widget in [
132+
mock_generate_type_counts_widget,
133+
mock_generate_null_counts_widget,
134+
mock_generate_estimated_unique_values_widget,
135+
mock_generate_completeness_widget,
136+
mock_generate_baseline_drift_widget,
167137
]
168138
],
169139
}
140+
170141
assert dashboard.to_dict() == expected_dashboard

0 commit comments

Comments
 (0)