tests: Add integration tests to publish dashboard, TODO: Verify dashboards are published using boto3

Author: Sushanth Sathish Kumar

src/sagemaker/dashboard/dashboard_variables.py

@@ -25,12 +25,12 @@ class DashboardVariable:
 
     Attributes:
         variable_type (str): Type of dashboard variable ('property' or 'pattern').
-        variable_property (str): Property affected by the variable, such as a JSON property or metric dimension.
+        variable_property (str): Property affected by the variable, such as metric dimension.
         inputType (str): Type of input field ('input', 'select', or 'radio') for user interaction.
         id (str): Identifier for the variable, up to 32 characters.
         label (str): Label displayed for the input field (optional, defaults based on context).
-        search (str): Metric search expression to populate input options (required for 'select' or 'radio').
-        populateFrom (str): Dimension name used to populate input options from search results.
+        search (str): Metric search expression to populate fields (required for 'select').
+        populateFrom (str): Dimension name used to populate fields from search results.
     """
 
     def __init__(
@@ -40,12 +40,12 @@ def __init__(
 
         Args:
             variable_type (str): Type of dashboard variable ('property' or 'pattern').
-            variable_property (str): Property affected by the variable, such as a JSON property or metric dimension.
-            inputType (str): Type of input field ('input', 'select', or 'radio') for user interaction.
+            variable_property (str): Property affected by the variable, such as metric dimension.
+            inputType (str): Type of input field ('input', 'select', or 'radio').
             variable_id (str): Identifier for the variable, up to 32 characters.
             label (str, optional): Label displayed for the input field (default is None).
             search (str, optional): Metric search expression to populate input options.
-            populateFrom (str, optional): Dimension name used to populate input options from search results.
+            populateFrom (str, optional): Dimension name used to populate field.
         """
         self.variable_type = variable_type
         self.variable_property = variable_property

src/sagemaker/dashboard/dashboard_widgets.py

@@ -24,8 +24,8 @@ class DashboardWidgetProperties:
     """Represents properties of a dashboard widget used for metrics in CloudWatch.
 
     Attributes:
-        view (str): Type of visualization ('timeSeries', 'singleValue', 'gauge', 'bar', 'pie', 'table').
-        stacked (bool): Whether to display the graph as stacked lines (applies to 'timeSeries' view).
+        view (str): Type of visualization ('timeSeries', 'bar', 'pie', 'table').
+        stacked (bool): Whether to display graph as stacked lines (applies to 'timeSeries' view).
         metrics (list): Array of metrics configurations for the widget.
         region (str): Region associated with the metrics.
         period (int): Period in seconds for data points on the graph.
@@ -46,8 +46,8 @@ def __init__(
         """Initializes DashboardWidgetProperties instance.
 
         Args:
-            view (str, optional): Type of visualization ('timeSeries', 'singleValue', 'gauge', 'bar', 'pie', 'table').
-            stacked (bool, optional): Whether to display the graph as stacked lines (applies to 'timeSeries' view).
+            view (str, optional): Type of visualization ('timeSeries', 'bar', 'pie', 'table').
+            stacked (bool, optional): Whether to display the graph as stacked lines.
             metrics (list, optional): Array of metrics configurations for the widget.
             region (str, optional): Region associated with the metrics.
             period (int, optional): Period in seconds for data points on the graph.
@@ -111,7 +111,7 @@ def __init__(self, height, width, widget_type, properties=None):
             height (int): Height of the widget.
             width (int): Width of the widget.
             widget_type (str): Type of the widget.
-            properties (DashboardWidgetProperties, optional): Properties specific to the widget type.
+            properties (DashboardWidgetProperties, optional): Properties of the widget type.
         """
         self.height = height
         self.width = width

src/sagemaker/dashboard/data_quality_dashboard.py

@@ -28,8 +28,8 @@ class AutomaticDataQualityDashboard:
     schedule provided.
 
     Attributes:
-        DATA_QUALITY_METRICS_ENDPOINT_NAMESPACE (str): Namespace for endpoint data quality metrics.
-        DATA_QUALITY_METRICS_BATCH_NAMESPACE (str): Namespace for batch transform data quality metrics.
+        DATA_QUALITY_METRICS_ENDPOINT_NAMESPACE (str): Namespace for endpoint.
+        DATA_QUALITY_METRICS_BATCH_NAMESPACE (str): Namespace for batch transform.
 
     Methods:
         __init__(self, endpoint_name, monitoring_schedule_name, batch_transform_input, region_name):
@@ -39,19 +39,19 @@ class AutomaticDataQualityDashboard:
             Generates variables for the dashboard based on whether batch transform is used or not.
 
         _generate_type_counts_widget(self):
-            Generates a widget for displaying type counts based on endpoint or batch transform.
+            Generates a widget for displaying type counts.
 
         _generate_null_counts_widget(self):
-            Generates a widget for displaying null and non-null counts based on endpoint or batch transform.
+            Generates a widget for displaying null and non-null counts.
 
         _generate_estimated_unique_values_widget(self):
-            Generates a widget for displaying estimated unique values based on endpoint or batch transform.
+            Generates a widget for displaying estimated unique values.
 
         _generate_completeness_widget(self):
-            Generates a widget for displaying completeness based on endpoint or batch transform.
+            Generates a widget for displaying completeness.
 
         _generate_baseline_drift_widget(self):
-            Generates a widget for displaying baseline drift based on endpoint or batch transform.
+            Generates a widget for displaying baseline drift.
 
         to_dict(self):
             Converts the dashboard configuration to a dictionary representation.
@@ -151,7 +151,7 @@ def _generate_type_counts_widget(self):
                     [
                         {
                             "expression": (
-                                f"SEARCH( '{AutomaticDataQualityDashboard.DATA_QUALITY_METRICS_BATCH_NAMESPACE} "
+                                f"SEARCH( '{self.DATA_QUALITY_METRICS_BATCH_NAMESPACE} "
                                 f"%^feature_fractional_counts_.*% OR "
                                 f"%^feature_integral_counts_.*% OR "
                                 f"%^feature_string_counts_.*% OR "
@@ -176,7 +176,7 @@ def _generate_type_counts_widget(self):
                     [
                         {
                             "expression": (
-                                f"SEARCH( '{AutomaticDataQualityDashboard.DATA_QUALITY_METRICS_ENDPOINT_NAMESPACE} "
+                                f"SEARCH( '{self.DATA_QUALITY_METRICS_ENDPOINT_NAMESPACE} "
                                 f"%^feature_fractional_counts_.*% OR "
                                 f"%^feature_integral_counts_.*% OR "
                                 f"%^feature_string_counts_.*% OR "
@@ -199,7 +199,7 @@ def _generate_type_counts_widget(self):
         )
 
     def _generate_null_counts_widget(self):
-        """Generates a widget for displaying null and non-null counts based on endpoint or batch transform.
+        """Generates a widget for displaying null and non-null counts.
 
         Returns:
             DashboardWidget: A DashboardWidget object configured for null counts.
@@ -213,7 +213,7 @@ def _generate_null_counts_widget(self):
                     [
                         {
                             "expression": (
-                                f"SEARCH( '{AutomaticDataQualityDashboard.DATA_QUALITY_METRICS_BATCH_NAMESPACE} "
+                                f"SEARCH( '{self.DATA_QUALITY_METRICS_BATCH_NAMESPACE} "
                                 f"%^feature_null_.*% OR %^feature_non_null_.*% "
                                 f'Feature="_" '
                                 f'MonitoringSchedule="{self.monitoring_schedule}" \', '
@@ -234,7 +234,7 @@ def _generate_null_counts_widget(self):
                     [
                         {
                             "expression": (
-                                f"SEARCH( '{AutomaticDataQualityDashboard.DATA_QUALITY_METRICS_ENDPOINT_NAMESPACE} "
+                                f"SEARCH( '{self.DATA_QUALITY_METRICS_ENDPOINT_NAMESPACE} "
                                 f"%^feature_null_.*% OR %^feature_non_null_.*% "
                                 f'Endpoint="{self.endpoint}" '
                                 f'Feature="_" '
@@ -252,7 +252,7 @@ def _generate_null_counts_widget(self):
         )
 
     def _generate_estimated_unique_values_widget(self):
-        """Generates a widget for displaying estimated unique values based on endpoint or batch transform.
+        """Generates a widget for displaying estimated unique values.
 
         Returns:
             DashboardWidget: A DashboardWidget object configured for estimated unique values.
@@ -266,7 +266,7 @@ def _generate_estimated_unique_values_widget(self):
                     [
                         {
                             "expression": (
-                                f"SEARCH( '{AutomaticDataQualityDashboard.DATA_QUALITY_METRICS_BATCH_NAMESPACE} "
+                                f"SEARCH( '{self.DATA_QUALITY_METRICS_BATCH_NAMESPACE} "
                                 f"%^feature_estimated_unique_values_.*% "
                                 f'Feature="_" '
                                 f'MonitoringSchedule="{self.monitoring_schedule}" \', '
@@ -287,7 +287,7 @@ def _generate_estimated_unique_values_widget(self):
                     [
                         {
                             "expression": (
-                                f"SEARCH( '{AutomaticDataQualityDashboard.DATA_QUALITY_METRICS_ENDPOINT_NAMESPACE} "
+                                f"SEARCH( '{self.DATA_QUALITY_METRICS_ENDPOINT_NAMESPACE} "
                                 f"%^feature_estimated_unique_values_.*% "
                                 f'Endpoint="{self.endpoint}" '
                                 f'Feature="_" '
@@ -323,7 +323,7 @@ def _generate_completeness_widget(self):
                     [
                         {
                             "expression": (
-                                f"SEARCH( '{AutomaticDataQualityDashboard.DATA_QUALITY_METRICS_BATCH_NAMESPACE} "
+                                f"SEARCH( '{self.DATA_QUALITY_METRICS_BATCH_NAMESPACE} "
                                 f"%^feature_completeness_.*% "
                                 f'Feature="_" '
                                 f'MonitoringSchedule="{self.monitoring_schedule}" \', '
@@ -344,7 +344,7 @@ def _generate_completeness_widget(self):
                     [
                         {
                             "expression": (
-                                f"SEARCH( '{AutomaticDataQualityDashboard.DATA_QUALITY_METRICS_ENDPOINT_NAMESPACE} "
+                                f"SEARCH( '{self.DATA_QUALITY_METRICS_ENDPOINT_NAMESPACE} "
                                 f"%^feature_completeness_.*% "
                                 f'Endpoint="{self.endpoint}" '
                                 f'Feature="_" '
@@ -377,7 +377,7 @@ def _generate_baseline_drift_widget(self):
                     [
                         {
                             "expression": (
-                                f"SEARCH( '{AutomaticDataQualityDashboard.DATA_QUALITY_METRICS_BATCH_NAMESPACE} "
+                                f"SEARCH( '{self.DATA_QUALITY_METRICS_BATCH_NAMESPACE} "
                                 f"%^feature_baseline_drift_.*% "
                                 f'Feature="_" '
                                 f'MonitoringSchedule="{self.monitoring_schedule}" \', '
@@ -398,7 +398,7 @@ def _generate_baseline_drift_widget(self):
                     [
                         {
                             "expression": (
-                                f"SEARCH( '{AutomaticDataQualityDashboard.DATA_QUALITY_METRICS_ENDPOINT_NAMESPACE} "
+                                f"SEARCH( '{self.DATA_QUALITY_METRICS_ENDPOINT_NAMESPACE} "
                                 f"%^feature_baseline_drift_.*% "
                                 f'Endpoint="{self.endpoint}" '
                                 f'Feature="_" '

src/sagemaker/dashboard/model_quality_dashboard.py

@@ -24,7 +24,8 @@ class AutomaticModelQualityDashboard:
     """Represents a dashboard for automatic model quality metrics in Amazon SageMaker.
 
     Methods:
-        __init__(self, endpoint_name, monitoring_schedule_name, batch_transform_input, problem_type, region_name):
+        __init__(self, endpoint_name, monitoring_schedule_name,
+                batch_transform_input, problem_type, region_name):
             Initializes an AutomaticModelQualityDashboard instance.
 
         _generate_widgets(self):
@@ -111,7 +112,8 @@ def __init__(
             endpoint_name (str): Name of the SageMaker endpoint.
             monitoring_schedule_name (str): Name of the monitoring schedule.
             batch_transform_input (str): Batch transform input (can be None).
-            problem_type (str): Type of problem ('Regression', 'BinaryClassification', or 'MulticlassClassification').
+            problem_type (str): Type of problem
+                                ('Regression', 'BinaryClassification', 'MulticlassClassification').
             region_name (str): AWS region name.
         """
         self.endpoint = endpoint_name

src/sagemaker/model_monitor/model_monitoring.py

@@ -1599,7 +1599,7 @@ def _check_automatic_dashboard_validity(
         dashboard_exists = True
         try:
             cw_client.get_dashboard(DashboardName=dashboard_name)
-        except Exception as _:  # noqa: F841
+        except ClientError as _:  # noqa: F841
             dashboard_exists = False
 
         if dashboard_exists:

tests/integ/test_model_monitor.py

@@ -63,6 +63,8 @@
     encrypt_inter_container_traffic=True,
 )
 ENABLE_CLOUDWATCH_METRICS = True
+ENABLE_AUTOMATIC_DASHBOARD = True
+DASHBOARD_NAME = "DASHBOARD NAME"
 
 DEFAULT_BASELINING_MAX_RUNTIME_IN_SECONDS = 86400
 DEFAULT_EXECUTION_MAX_RUNTIME_IN_SECONDS = 3600
@@ -958,16 +960,19 @@ def test_default_monitor_create_stop_and_start_monitoring_schedule_with_customiz
     my_default_monitor.stop_monitoring_schedule()
 
     _wait_for_schedule_changes_to_apply(monitor=my_default_monitor)
-    
 
 @pytest.mark.skipif(
     tests.integ.test_region() in tests.integ.NO_MODEL_MONITORING_REGIONS,
     reason="ModelMonitoring is not yet supported in this region.",
 )
-def test_default_monitor_create_stop_and_start_monitoring_schedule_with_dashboards(
-    sagemaker_session, output_kms_key, volume_kms_key, predictor
+def test_default_monitor_create_and_update_schedule_config_with_customizations(
+    sagemaker_session,
+    predictor,
+    volume_kms_key,
+    output_kms_key,
+    updated_volume_kms_key,
+    updated_output_kms_key,
 ):
-
     my_default_monitor = DefaultModelMonitor(
         role=ROLE,
         instance_count=INSTANCE_COUNT,
@@ -985,7 +990,7 @@ def test_default_monitor_create_stop_and_start_monitoring_schedule_with_dashboar
     output_s3_uri = os.path.join(
         "s3://",
         sagemaker_session.default_bucket(),
-        INTEG_TEST_MONITORING_OUTPUT_BUCKET,
+        "integ-test-monitoring-output-bucket",
         str(uuid.uuid4()),
     )
 
@@ -1019,32 +1024,69 @@ def test_default_monitor_create_stop_and_start_monitoring_schedule_with_dashboar
         network_config=NETWORK_CONFIG,
     )
 
-    _wait_for_schedule_changes_to_apply(monitor=my_default_monitor)
+    statistics = Statistics.from_file_path(
+        statistics_file_path=os.path.join(tests.integ.DATA_DIR, "monitor/statistics.json"),
+        sagemaker_session=sagemaker_session,
+    )
 
-    my_default_monitor.stop_monitoring_schedule()
+    constraints = Constraints.from_file_path(
+        constraints_file_path=os.path.join(tests.integ.DATA_DIR, "monitor/constraints.json"),
+        sagemaker_session=sagemaker_session,
+    )
 
     _wait_for_schedule_changes_to_apply(monitor=my_default_monitor)
 
-    stopped_schedule_description = my_default_monitor.describe_schedule()
-    assert stopped_schedule_description["MonitoringScheduleStatus"] == "Stopped"
+    my_default_monitor.update_monitoring_schedule(
+        output_s3_uri=output_s3_uri,
+        statistics=statistics,
+        constraints=constraints,
+        schedule_cron_expression=CronExpressionGenerator.hourly(),
+        instance_count=UPDATED_INSTANCE_COUNT,
+        instance_type=UPDATED_INSTANCE_TYPE,
+        volume_size_in_gb=UPDATED_VOLUME_SIZE_IN_GB,
+        volume_kms_key=updated_volume_kms_key,
+        output_kms_key=updated_output_kms_key,
+        max_runtime_in_seconds=UPDATED_MAX_RUNTIME_IN_SECONDS,
+        env=UPDATED_ENVIRONMENT,
+        network_config=UPDATED_NETWORK_CONFIG,
+        enable_cloudwatch_metrics=DISABLE_CLOUDWATCH_METRICS,
+        role=UPDATED_ROLE,
+    )
 
-    my_default_monitor.start_monitoring_schedule()
+    _wait_for_schedule_changes_to_apply(my_default_monitor)
 
-    _wait_for_schedule_changes_to_apply(monitor=my_default_monitor)
+    schedule_description = my_default_monitor.describe_schedule()
+    _verify_default_monitoring_schedule(
+        sagemaker_session=sagemaker_session,
+        schedule_description=schedule_description,
+        statistics=statistics,
+        constraints=constraints,
+        output_kms_key=updated_output_kms_key,
+        volume_kms_key=updated_volume_kms_key,
+        cron_expression=CronExpressionGenerator.hourly(),
+        instant_count=UPDATED_INSTANCE_COUNT,
+        instant_type=UPDATED_INSTANCE_TYPE,
+        volume_size_in_gb=UPDATED_VOLUME_SIZE_IN_GB,
+        network_config=UPDATED_NETWORK_CONFIG,
+        max_runtime_in_seconds=UPDATED_MAX_RUNTIME_IN_SECONDS,
+        publish_cloudwatch_metrics="Disabled",
+        env_key=UPDATED_ENV_KEY_1,
+        env_value=UPDATED_ENV_VALUE_1,
+    )
 
-    started_schedule_description = my_default_monitor.describe_schedule()
-    assert started_schedule_description["MonitoringScheduleStatus"] == "Scheduled"
+    _wait_for_schedule_changes_to_apply(monitor=my_default_monitor)
 
     my_default_monitor.stop_monitoring_schedule()
 
     _wait_for_schedule_changes_to_apply(monitor=my_default_monitor)
 
+    assert len(predictor.list_monitors()) > 0
 
 @pytest.mark.skipif(
     tests.integ.test_region() in tests.integ.NO_MODEL_MONITORING_REGIONS,
     reason="ModelMonitoring is not yet supported in this region.",
 )
-def test_default_monitor_create_and_update_schedule_config_with_customizations(
+def test_default_monitor_create_and_update_schedule_config_with_dashboards(
     sagemaker_session,
     predictor,
     volume_kms_key,
@@ -1090,6 +1132,8 @@ def test_default_monitor_create_and_update_schedule_config_with_customizations(
         constraints=constraints,
         schedule_cron_expression=CronExpressionGenerator.daily(),
         enable_cloudwatch_metrics=ENABLE_CLOUDWATCH_METRICS,
+        enable_automatic_dashboard=ENABLE_AUTOMATIC_DASHBOARD,
+        dashboard_name=DASHBOARD_NAME,
     )
 
     schedule_description = my_default_monitor.describe_schedule()
@@ -1128,7 +1172,8 @@ def test_default_monitor_create_and_update_schedule_config_with_customizations(
         max_runtime_in_seconds=UPDATED_MAX_RUNTIME_IN_SECONDS,
         env=UPDATED_ENVIRONMENT,
         network_config=UPDATED_NETWORK_CONFIG,
-        enable_cloudwatch_metrics=DISABLE_CLOUDWATCH_METRICS,
+        enable_cloudwatch_metrics=ENABLE_CLOUDWATCH_METRICS,
+        enable_automatic_dashboard=ENABLE_AUTOMATIC_DASHBOARD,
         role=UPDATED_ROLE,
     )