Skip to content

Commit 524c220

Browse files
author
Sushanth Sathish Kumar
committed
tests: Add integration tests to publish dashboard, TODO: Verify dashboards are published using boto3
1 parent 5521550 commit 524c220

File tree

7 files changed

+183
-48
lines changed

7 files changed

+183
-48
lines changed

‎src/sagemaker/dashboard/dashboard_variables.py

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -25,12 +25,12 @@ class DashboardVariable:
2525
2626
Attributes:
2727
variable_type (str): Type of dashboard variable ('property' or 'pattern').
28-
variable_property (str): Property affected by the variable, such as a JSON property or metric dimension.
28+
variable_property (str): Property affected by the variable, such as metric dimension.
2929
inputType (str): Type of input field ('input', 'select', or 'radio') for user interaction.
3030
id (str): Identifier for the variable, up to 32 characters.
3131
label (str): Label displayed for the input field (optional, defaults based on context).
32-
search (str): Metric search expression to populate input options (required for 'select' or 'radio').
33-
populateFrom (str): Dimension name used to populate input options from search results.
32+
search (str): Metric search expression to populate fields (required for 'select').
33+
populateFrom (str): Dimension name used to populate fields from search results.
3434
"""
3535

3636
def __init__(
@@ -40,12 +40,12 @@ def __init__(
4040
4141
Args:
4242
variable_type (str): Type of dashboard variable ('property' or 'pattern').
43-
variable_property (str): Property affected by the variable, such as a JSON property or metric dimension.
44-
inputType (str): Type of input field ('input', 'select', or 'radio') for user interaction.
43+
variable_property (str): Property affected by the variable, such as metric dimension.
44+
inputType (str): Type of input field ('input', 'select', or 'radio').
4545
variable_id (str): Identifier for the variable, up to 32 characters.
4646
label (str, optional): Label displayed for the input field (default is None).
4747
search (str, optional): Metric search expression to populate input options.
48-
populateFrom (str, optional): Dimension name used to populate input options from search results.
48+
populateFrom (str, optional): Dimension name used to populate field.
4949
"""
5050
self.variable_type = variable_type
5151
self.variable_property = variable_property

‎src/sagemaker/dashboard/dashboard_widgets.py

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -24,8 +24,8 @@ class DashboardWidgetProperties:
2424
"""Represents properties of a dashboard widget used for metrics in CloudWatch.
2525
2626
Attributes:
27-
view (str): Type of visualization ('timeSeries', 'singleValue', 'gauge', 'bar', 'pie', 'table').
28-
stacked (bool): Whether to display the graph as stacked lines (applies to 'timeSeries' view).
27+
view (str): Type of visualization ('timeSeries', 'bar', 'pie', 'table').
28+
stacked (bool): Whether to display graph as stacked lines (applies to 'timeSeries' view).
2929
metrics (list): Array of metrics configurations for the widget.
3030
region (str): Region associated with the metrics.
3131
period (int): Period in seconds for data points on the graph.
@@ -46,8 +46,8 @@ def __init__(
4646
"""Initializes DashboardWidgetProperties instance.
4747
4848
Args:
49-
view (str, optional): Type of visualization ('timeSeries', 'singleValue', 'gauge', 'bar', 'pie', 'table').
50-
stacked (bool, optional): Whether to display the graph as stacked lines (applies to 'timeSeries' view).
49+
view (str, optional): Type of visualization ('timeSeries', 'bar', 'pie', 'table').
50+
stacked (bool, optional): Whether to display the graph as stacked lines.
5151
metrics (list, optional): Array of metrics configurations for the widget.
5252
region (str, optional): Region associated with the metrics.
5353
period (int, optional): Period in seconds for data points on the graph.
@@ -111,7 +111,7 @@ def __init__(self, height, width, widget_type, properties=None):
111111
height (int): Height of the widget.
112112
width (int): Width of the widget.
113113
widget_type (str): Type of the widget.
114-
properties (DashboardWidgetProperties, optional): Properties specific to the widget type.
114+
properties (DashboardWidgetProperties, optional): Properties of the widget type.
115115
"""
116116
self.height = height
117117
self.width = width

‎src/sagemaker/dashboard/data_quality_dashboard.py

Lines changed: 19 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -28,8 +28,8 @@ class AutomaticDataQualityDashboard:
2828
schedule provided.
2929
3030
Attributes:
31-
DATA_QUALITY_METRICS_ENDPOINT_NAMESPACE (str): Namespace for endpoint data quality metrics.
32-
DATA_QUALITY_METRICS_BATCH_NAMESPACE (str): Namespace for batch transform data quality metrics.
31+
DATA_QUALITY_METRICS_ENDPOINT_NAMESPACE (str): Namespace for endpoint.
32+
DATA_QUALITY_METRICS_BATCH_NAMESPACE (str): Namespace for batch transform.
3333
3434
Methods:
3535
__init__(self, endpoint_name, monitoring_schedule_name, batch_transform_input, region_name):
@@ -39,19 +39,19 @@ class AutomaticDataQualityDashboard:
3939
Generates variables for the dashboard based on whether batch transform is used or not.
4040
4141
_generate_type_counts_widget(self):
42-
Generates a widget for displaying type counts based on endpoint or batch transform.
42+
Generates a widget for displaying type counts.
4343
4444
_generate_null_counts_widget(self):
45-
Generates a widget for displaying null and non-null counts based on endpoint or batch transform.
45+
Generates a widget for displaying null and non-null counts.
4646
4747
_generate_estimated_unique_values_widget(self):
48-
Generates a widget for displaying estimated unique values based on endpoint or batch transform.
48+
Generates a widget for displaying estimated unique values.
4949
5050
_generate_completeness_widget(self):
51-
Generates a widget for displaying completeness based on endpoint or batch transform.
51+
Generates a widget for displaying completeness.
5252
5353
_generate_baseline_drift_widget(self):
54-
Generates a widget for displaying baseline drift based on endpoint or batch transform.
54+
Generates a widget for displaying baseline drift.
5555
5656
to_dict(self):
5757
Converts the dashboard configuration to a dictionary representation.
@@ -151,7 +151,7 @@ def _generate_type_counts_widget(self):
151151
[
152152
{
153153
"expression": (
154-
f"SEARCH( '{AutomaticDataQualityDashboard.DATA_QUALITY_METRICS_BATCH_NAMESPACE} "
154+
f"SEARCH( '{self.DATA_QUALITY_METRICS_BATCH_NAMESPACE} "
155155
f"%^feature_fractional_counts_.*% OR "
156156
f"%^feature_integral_counts_.*% OR "
157157
f"%^feature_string_counts_.*% OR "
@@ -176,7 +176,7 @@ def _generate_type_counts_widget(self):
176176
[
177177
{
178178
"expression": (
179-
f"SEARCH( '{AutomaticDataQualityDashboard.DATA_QUALITY_METRICS_ENDPOINT_NAMESPACE} "
179+
f"SEARCH( '{self.DATA_QUALITY_METRICS_ENDPOINT_NAMESPACE} "
180180
f"%^feature_fractional_counts_.*% OR "
181181
f"%^feature_integral_counts_.*% OR "
182182
f"%^feature_string_counts_.*% OR "
@@ -199,7 +199,7 @@ def _generate_type_counts_widget(self):
199199
)
200200

201201
def _generate_null_counts_widget(self):
202-
"""Generates a widget for displaying null and non-null counts based on endpoint or batch transform.
202+
"""Generates a widget for displaying null and non-null counts.
203203
204204
Returns:
205205
DashboardWidget: A DashboardWidget object configured for null counts.
@@ -213,7 +213,7 @@ def _generate_null_counts_widget(self):
213213
[
214214
{
215215
"expression": (
216-
f"SEARCH( '{AutomaticDataQualityDashboard.DATA_QUALITY_METRICS_BATCH_NAMESPACE} "
216+
f"SEARCH( '{self.DATA_QUALITY_METRICS_BATCH_NAMESPACE} "
217217
f"%^feature_null_.*% OR %^feature_non_null_.*% "
218218
f'Feature="_" '
219219
f'MonitoringSchedule="{self.monitoring_schedule}" \', '
@@ -234,7 +234,7 @@ def _generate_null_counts_widget(self):
234234
[
235235
{
236236
"expression": (
237-
f"SEARCH( '{AutomaticDataQualityDashboard.DATA_QUALITY_METRICS_ENDPOINT_NAMESPACE} "
237+
f"SEARCH( '{self.DATA_QUALITY_METRICS_ENDPOINT_NAMESPACE} "
238238
f"%^feature_null_.*% OR %^feature_non_null_.*% "
239239
f'Endpoint="{self.endpoint}" '
240240
f'Feature="_" '
@@ -252,7 +252,7 @@ def _generate_null_counts_widget(self):
252252
)
253253

254254
def _generate_estimated_unique_values_widget(self):
255-
"""Generates a widget for displaying estimated unique values based on endpoint or batch transform.
255+
"""Generates a widget for displaying estimated unique values.
256256
257257
Returns:
258258
DashboardWidget: A DashboardWidget object configured for estimated unique values.
@@ -266,7 +266,7 @@ def _generate_estimated_unique_values_widget(self):
266266
[
267267
{
268268
"expression": (
269-
f"SEARCH( '{AutomaticDataQualityDashboard.DATA_QUALITY_METRICS_BATCH_NAMESPACE} "
269+
f"SEARCH( '{self.DATA_QUALITY_METRICS_BATCH_NAMESPACE} "
270270
f"%^feature_estimated_unique_values_.*% "
271271
f'Feature="_" '
272272
f'MonitoringSchedule="{self.monitoring_schedule}" \', '
@@ -287,7 +287,7 @@ def _generate_estimated_unique_values_widget(self):
287287
[
288288
{
289289
"expression": (
290-
f"SEARCH( '{AutomaticDataQualityDashboard.DATA_QUALITY_METRICS_ENDPOINT_NAMESPACE} "
290+
f"SEARCH( '{self.DATA_QUALITY_METRICS_ENDPOINT_NAMESPACE} "
291291
f"%^feature_estimated_unique_values_.*% "
292292
f'Endpoint="{self.endpoint}" '
293293
f'Feature="_" '
@@ -323,7 +323,7 @@ def _generate_completeness_widget(self):
323323
[
324324
{
325325
"expression": (
326-
f"SEARCH( '{AutomaticDataQualityDashboard.DATA_QUALITY_METRICS_BATCH_NAMESPACE} "
326+
f"SEARCH( '{self.DATA_QUALITY_METRICS_BATCH_NAMESPACE} "
327327
f"%^feature_completeness_.*% "
328328
f'Feature="_" '
329329
f'MonitoringSchedule="{self.monitoring_schedule}" \', '
@@ -344,7 +344,7 @@ def _generate_completeness_widget(self):
344344
[
345345
{
346346
"expression": (
347-
f"SEARCH( '{AutomaticDataQualityDashboard.DATA_QUALITY_METRICS_ENDPOINT_NAMESPACE} "
347+
f"SEARCH( '{self.DATA_QUALITY_METRICS_ENDPOINT_NAMESPACE} "
348348
f"%^feature_completeness_.*% "
349349
f'Endpoint="{self.endpoint}" '
350350
f'Feature="_" '
@@ -377,7 +377,7 @@ def _generate_baseline_drift_widget(self):
377377
[
378378
{
379379
"expression": (
380-
f"SEARCH( '{AutomaticDataQualityDashboard.DATA_QUALITY_METRICS_BATCH_NAMESPACE} "
380+
f"SEARCH( '{self.DATA_QUALITY_METRICS_BATCH_NAMESPACE} "
381381
f"%^feature_baseline_drift_.*% "
382382
f'Feature="_" '
383383
f'MonitoringSchedule="{self.monitoring_schedule}" \', '
@@ -398,7 +398,7 @@ def _generate_baseline_drift_widget(self):
398398
[
399399
{
400400
"expression": (
401-
f"SEARCH( '{AutomaticDataQualityDashboard.DATA_QUALITY_METRICS_ENDPOINT_NAMESPACE} "
401+
f"SEARCH( '{self.DATA_QUALITY_METRICS_ENDPOINT_NAMESPACE} "
402402
f"%^feature_baseline_drift_.*% "
403403
f'Endpoint="{self.endpoint}" '
404404
f'Feature="_" '

‎src/sagemaker/dashboard/model_quality_dashboard.py

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -24,7 +24,8 @@ class AutomaticModelQualityDashboard:
2424
"""Represents a dashboard for automatic model quality metrics in Amazon SageMaker.
2525
2626
Methods:
27-
__init__(self, endpoint_name, monitoring_schedule_name, batch_transform_input, problem_type, region_name):
27+
__init__(self, endpoint_name, monitoring_schedule_name,
28+
batch_transform_input, problem_type, region_name):
2829
Initializes an AutomaticModelQualityDashboard instance.
2930
3031
_generate_widgets(self):
@@ -111,7 +112,8 @@ def __init__(
111112
endpoint_name (str): Name of the SageMaker endpoint.
112113
monitoring_schedule_name (str): Name of the monitoring schedule.
113114
batch_transform_input (str): Batch transform input (can be None).
114-
problem_type (str): Type of problem ('Regression', 'BinaryClassification', or 'MulticlassClassification').
115+
problem_type (str): Type of problem
116+
('Regression', 'BinaryClassification', 'MulticlassClassification').
115117
region_name (str): AWS region name.
116118
"""
117119
self.endpoint = endpoint_name

‎src/sagemaker/model_monitor/model_monitoring.py

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1599,7 +1599,7 @@ def _check_automatic_dashboard_validity(
15991599
dashboard_exists = True
16001600
try:
16011601
cw_client.get_dashboard(DashboardName=dashboard_name)
1602-
except Exception as _: # noqa: F841
1602+
except ClientError as _: # noqa: F841
16031603
dashboard_exists = False
16041604

16051605
if dashboard_exists:

‎tests/integ/test_model_monitor.py

Lines changed: 60 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -63,6 +63,8 @@
6363
encrypt_inter_container_traffic=True,
6464
)
6565
ENABLE_CLOUDWATCH_METRICS = True
66+
ENABLE_AUTOMATIC_DASHBOARD = True
67+
DASHBOARD_NAME = "DASHBOARD NAME"
6668

6769
DEFAULT_BASELINING_MAX_RUNTIME_IN_SECONDS = 86400
6870
DEFAULT_EXECUTION_MAX_RUNTIME_IN_SECONDS = 3600
@@ -958,16 +960,19 @@ def test_default_monitor_create_stop_and_start_monitoring_schedule_with_customiz
958960
my_default_monitor.stop_monitoring_schedule()
959961

960962
_wait_for_schedule_changes_to_apply(monitor=my_default_monitor)
961-
962963

963964
@pytest.mark.skipif(
964965
tests.integ.test_region() in tests.integ.NO_MODEL_MONITORING_REGIONS,
965966
reason="ModelMonitoring is not yet supported in this region.",
966967
)
967-
def test_default_monitor_create_stop_and_start_monitoring_schedule_with_dashboards(
968-
sagemaker_session, output_kms_key, volume_kms_key, predictor
968+
def test_default_monitor_create_and_update_schedule_config_with_customizations(
969+
sagemaker_session,
970+
predictor,
971+
volume_kms_key,
972+
output_kms_key,
973+
updated_volume_kms_key,
974+
updated_output_kms_key,
969975
):
970-
971976
my_default_monitor = DefaultModelMonitor(
972977
role=ROLE,
973978
instance_count=INSTANCE_COUNT,
@@ -985,7 +990,7 @@ def test_default_monitor_create_stop_and_start_monitoring_schedule_with_dashboar
985990
output_s3_uri = os.path.join(
986991
"s3://",
987992
sagemaker_session.default_bucket(),
988-
INTEG_TEST_MONITORING_OUTPUT_BUCKET,
993+
"integ-test-monitoring-output-bucket",
989994
str(uuid.uuid4()),
990995
)
991996

@@ -1019,32 +1024,69 @@ def test_default_monitor_create_stop_and_start_monitoring_schedule_with_dashboar
10191024
network_config=NETWORK_CONFIG,
10201025
)
10211026

1022-
_wait_for_schedule_changes_to_apply(monitor=my_default_monitor)
1027+
statistics = Statistics.from_file_path(
1028+
statistics_file_path=os.path.join(tests.integ.DATA_DIR, "monitor/statistics.json"),
1029+
sagemaker_session=sagemaker_session,
1030+
)
10231031

1024-
my_default_monitor.stop_monitoring_schedule()
1032+
constraints = Constraints.from_file_path(
1033+
constraints_file_path=os.path.join(tests.integ.DATA_DIR, "monitor/constraints.json"),
1034+
sagemaker_session=sagemaker_session,
1035+
)
10251036

10261037
_wait_for_schedule_changes_to_apply(monitor=my_default_monitor)
10271038

1028-
stopped_schedule_description = my_default_monitor.describe_schedule()
1029-
assert stopped_schedule_description["MonitoringScheduleStatus"] == "Stopped"
1039+
my_default_monitor.update_monitoring_schedule(
1040+
output_s3_uri=output_s3_uri,
1041+
statistics=statistics,
1042+
constraints=constraints,
1043+
schedule_cron_expression=CronExpressionGenerator.hourly(),
1044+
instance_count=UPDATED_INSTANCE_COUNT,
1045+
instance_type=UPDATED_INSTANCE_TYPE,
1046+
volume_size_in_gb=UPDATED_VOLUME_SIZE_IN_GB,
1047+
volume_kms_key=updated_volume_kms_key,
1048+
output_kms_key=updated_output_kms_key,
1049+
max_runtime_in_seconds=UPDATED_MAX_RUNTIME_IN_SECONDS,
1050+
env=UPDATED_ENVIRONMENT,
1051+
network_config=UPDATED_NETWORK_CONFIG,
1052+
enable_cloudwatch_metrics=DISABLE_CLOUDWATCH_METRICS,
1053+
role=UPDATED_ROLE,
1054+
)
10301055

1031-
my_default_monitor.start_monitoring_schedule()
1056+
_wait_for_schedule_changes_to_apply(my_default_monitor)
10321057

1033-
_wait_for_schedule_changes_to_apply(monitor=my_default_monitor)
1058+
schedule_description = my_default_monitor.describe_schedule()
1059+
_verify_default_monitoring_schedule(
1060+
sagemaker_session=sagemaker_session,
1061+
schedule_description=schedule_description,
1062+
statistics=statistics,
1063+
constraints=constraints,
1064+
output_kms_key=updated_output_kms_key,
1065+
volume_kms_key=updated_volume_kms_key,
1066+
cron_expression=CronExpressionGenerator.hourly(),
1067+
instant_count=UPDATED_INSTANCE_COUNT,
1068+
instant_type=UPDATED_INSTANCE_TYPE,
1069+
volume_size_in_gb=UPDATED_VOLUME_SIZE_IN_GB,
1070+
network_config=UPDATED_NETWORK_CONFIG,
1071+
max_runtime_in_seconds=UPDATED_MAX_RUNTIME_IN_SECONDS,
1072+
publish_cloudwatch_metrics="Disabled",
1073+
env_key=UPDATED_ENV_KEY_1,
1074+
env_value=UPDATED_ENV_VALUE_1,
1075+
)
10341076

1035-
started_schedule_description = my_default_monitor.describe_schedule()
1036-
assert started_schedule_description["MonitoringScheduleStatus"] == "Scheduled"
1077+
_wait_for_schedule_changes_to_apply(monitor=my_default_monitor)
10371078

10381079
my_default_monitor.stop_monitoring_schedule()
10391080

10401081
_wait_for_schedule_changes_to_apply(monitor=my_default_monitor)
10411082

1083+
assert len(predictor.list_monitors()) > 0
10421084

10431085
@pytest.mark.skipif(
10441086
tests.integ.test_region() in tests.integ.NO_MODEL_MONITORING_REGIONS,
10451087
reason="ModelMonitoring is not yet supported in this region.",
10461088
)
1047-
def test_default_monitor_create_and_update_schedule_config_with_customizations(
1089+
def test_default_monitor_create_and_update_schedule_config_with_dashboards(
10481090
sagemaker_session,
10491091
predictor,
10501092
volume_kms_key,
@@ -1090,6 +1132,8 @@ def test_default_monitor_create_and_update_schedule_config_with_customizations(
10901132
constraints=constraints,
10911133
schedule_cron_expression=CronExpressionGenerator.daily(),
10921134
enable_cloudwatch_metrics=ENABLE_CLOUDWATCH_METRICS,
1135+
enable_automatic_dashboard=ENABLE_AUTOMATIC_DASHBOARD,
1136+
dashboard_name=DASHBOARD_NAME,
10931137
)
10941138

10951139
schedule_description = my_default_monitor.describe_schedule()
@@ -1128,7 +1172,8 @@ def test_default_monitor_create_and_update_schedule_config_with_customizations(
11281172
max_runtime_in_seconds=UPDATED_MAX_RUNTIME_IN_SECONDS,
11291173
env=UPDATED_ENVIRONMENT,
11301174
network_config=UPDATED_NETWORK_CONFIG,
1131-
enable_cloudwatch_metrics=DISABLE_CLOUDWATCH_METRICS,
1175+
enable_cloudwatch_metrics=ENABLE_CLOUDWATCH_METRICS,
1176+
enable_automatic_dashboard=ENABLE_AUTOMATIC_DASHBOARD,
11321177
role=UPDATED_ROLE,
11331178
)
11341179

0 commit comments

Comments
 (0)