change: modify schedule cleanup to abide by latest validations (#1154)

This commit also stops monitoring schedules as part of the tests as
soon as the schedules no longer need to provide jobs and
prevents ModelMonitoring tests from running in me-south-1.

Author: knakad

tests/integ/__init__.py

@@ -77,6 +77,7 @@
 NO_LDA_REGIONS = ["eu-west-3", "eu-north-1", "sa-east-1", "ap-east-1", "me-south-1"]
 NO_MARKET_PLACE_REGIONS = ["eu-west-3", "eu-north-1", "sa-east-1", "ap-east-1", "me-south-1"]
 NO_AUTO_ML_REGIONS = ["sa-east-1", "me-south-1", "ap-east-1", "eu-west-3"]
+NO_MODEL_MONITORING_REGIONS = ["me-south-1"]
 
 EFS_TEST_ENABLED_REGION = []
 

tests/integ/test_model_monitor.py

@@ -266,6 +266,10 @@ def updated_output_kms_key(sagemaker_session):
     )
 
 
+@pytest.mark.skipif(
+    tests.integ.test_region() in tests.integ.NO_MODEL_MONITORING_REGIONS,
+    reason="ModelMonitoring is not yet supported in this region.",
+)
 def test_default_monitor_suggest_baseline_and_create_monitoring_schedule_with_customizations(
     sagemaker_session, output_kms_key, volume_kms_key, predictor
 ):
@@ -464,6 +468,10 @@ def test_default_monitor_suggest_baseline_and_create_monitoring_schedule_with_cu
     assert len(summary["MonitoringScheduleSummaries"]) > 0
 
 
+@pytest.mark.skipif(
+    tests.integ.test_region() in tests.integ.NO_MODEL_MONITORING_REGIONS,
+    reason="ModelMonitoring is not yet supported in this region.",
+)
 def test_default_monitor_suggest_baseline_and_create_monitoring_schedule_without_customizations(
     sagemaker_session, predictor
 ):
@@ -637,6 +645,10 @@ def test_default_monitor_suggest_baseline_and_create_monitoring_schedule_without
     assert len(summary["MonitoringScheduleSummaries"]) > 0
 
 
+@pytest.mark.skipif(
+    tests.integ.test_region() in tests.integ.NO_MODEL_MONITORING_REGIONS,
+    reason="ModelMonitoring is not yet supported in this region.",
+)
 def test_default_monitor_create_stop_and_start_monitoring_schedule_with_customizations(
     sagemaker_session, output_kms_key, volume_kms_key, predictor
 ):
@@ -792,7 +804,15 @@ def test_default_monitor_create_stop_and_start_monitoring_schedule_with_customiz
     started_schedule_description = my_default_monitor.describe_schedule()
     assert started_schedule_description["MonitoringScheduleStatus"] == "Scheduled"
 
+    my_default_monitor.stop_monitoring_schedule()
+
+    _wait_for_schedule_changes_to_apply(monitor=my_default_monitor)
 
+
+@pytest.mark.skipif(
+    tests.integ.test_region() in tests.integ.NO_MODEL_MONITORING_REGIONS,
+    reason="ModelMonitoring is not yet supported in this region.",
+)
 def test_default_monitor_create_and_update_schedule_config_with_customizations(
     sagemaker_session,
     predictor,
@@ -1061,9 +1081,20 @@ def test_default_monitor_create_and_update_schedule_config_with_customizations(
         ]["EnableNetworkIsolation"]
         == UPDATED_NETWORK_CONFIG.enable_network_isolation
     )
+
+    _wait_for_schedule_changes_to_apply(monitor=my_default_monitor)
+
+    my_default_monitor.stop_monitoring_schedule()
+
+    _wait_for_schedule_changes_to_apply(monitor=my_default_monitor)
+
     assert len(predictor.list_monitors()) > 0
 
 
+@pytest.mark.skipif(
+    tests.integ.test_region() in tests.integ.NO_MODEL_MONITORING_REGIONS,
+    reason="ModelMonitoring is not yet supported in this region.",
+)
 def test_default_monitor_create_and_update_schedule_config_without_customizations(
     sagemaker_session, predictor
 ):
@@ -1277,7 +1308,17 @@ def test_default_monitor_create_and_update_schedule_config_without_customization
         is None
     )
 
+    _wait_for_schedule_changes_to_apply(monitor=my_default_monitor)
+
+    my_default_monitor.stop_monitoring_schedule()
+
+    _wait_for_schedule_changes_to_apply(monitor=my_default_monitor)
+
 
+@pytest.mark.skipif(
+    tests.integ.test_region() in tests.integ.NO_MODEL_MONITORING_REGIONS,
+    reason="ModelMonitoring is not yet supported in this region.",
+)
 def test_default_monitor_attach_followed_by_baseline_and_update_monitoring_schedule(
     sagemaker_session,
     default_monitoring_schedule_name,
@@ -1421,11 +1462,20 @@ def test_default_monitor_attach_followed_by_baseline_and_update_monitoring_sched
         == UPDATED_NETWORK_CONFIG.enable_network_isolation
     )
 
+    _wait_for_schedule_changes_to_apply(monitor=my_attached_monitor)
+
+    my_attached_monitor.stop_monitoring_schedule()
+
+    _wait_for_schedule_changes_to_apply(monitor=my_attached_monitor)
+
 
+@pytest.mark.skipif(
+    tests.integ.test_region() in tests.integ.NO_MODEL_MONITORING_REGIONS,
+    reason="ModelMonitoring is not yet supported in this region.",
+)
 def test_default_monitor_monitoring_execution_interactions(
     sagemaker_session, default_monitoring_schedule_name
 ):
-
     my_attached_monitor = DefaultModelMonitor.attach(
         monitor_schedule_name=default_monitoring_schedule_name, sagemaker_session=sagemaker_session
     )
@@ -1462,6 +1512,10 @@ def test_default_monitor_monitoring_execution_interactions(
     assert constraint_violations.body_dict["violations"][0]["feature_name"] == "store_and_fwd_flag"
 
 
+@pytest.mark.skipif(
+    tests.integ.test_region() in tests.integ.NO_MODEL_MONITORING_REGIONS,
+    reason="ModelMonitoring is not yet supported in this region.",
+)
 def test_byoc_monitor_suggest_baseline_and_create_monitoring_schedule_with_customizations(
     sagemaker_session, output_kms_key, volume_kms_key, predictor
 ):
@@ -1666,10 +1720,20 @@ def test_byoc_monitor_suggest_baseline_and_create_monitoring_schedule_with_custo
         == NETWORK_CONFIG.enable_network_isolation
     )
 
+    _wait_for_schedule_changes_to_apply(monitor=my_byoc_monitor)
+
+    my_byoc_monitor.stop_monitoring_schedule()
+
+    _wait_for_schedule_changes_to_apply(monitor=my_byoc_monitor)
+
     summary = sagemaker_session.list_monitoring_schedules()
     assert len(summary["MonitoringScheduleSummaries"]) > 0
 
 
+@pytest.mark.skipif(
+    tests.integ.test_region() in tests.integ.NO_MODEL_MONITORING_REGIONS,
+    reason="ModelMonitoring is not yet supported in this region.",
+)
 def test_byoc_monitor_suggest_baseline_and_create_monitoring_schedule_without_customizations(
     sagemaker_session, predictor
 ):
@@ -1850,10 +1914,20 @@ def test_byoc_monitor_suggest_baseline_and_create_monitoring_schedule_without_cu
         is None
     )
 
+    _wait_for_schedule_changes_to_apply(monitor=my_byoc_monitor)
+
+    my_byoc_monitor.stop_monitoring_schedule()
+
+    _wait_for_schedule_changes_to_apply(monitor=my_byoc_monitor)
+
     summary = sagemaker_session.list_monitoring_schedules()
     assert len(summary["MonitoringScheduleSummaries"]) > 0
 
 
+@pytest.mark.skipif(
+    tests.integ.test_region() in tests.integ.NO_MODEL_MONITORING_REGIONS,
+    reason="ModelMonitoring is not yet supported in this region.",
+)
 def test_byoc_monitor_create_and_update_schedule_config_with_customizations(
     sagemaker_session,
     predictor,
@@ -2123,9 +2197,20 @@ def test_byoc_monitor_create_and_update_schedule_config_with_customizations(
         ]["EnableNetworkIsolation"]
         == UPDATED_NETWORK_CONFIG.enable_network_isolation
     )
+
+    _wait_for_schedule_changes_to_apply(monitor=my_byoc_monitor)
+
+    my_byoc_monitor.stop_monitoring_schedule()
+
+    _wait_for_schedule_changes_to_apply(monitor=my_byoc_monitor)
+
     assert len(predictor.list_monitors()) > 0
 
 
+@pytest.mark.skipif(
+    tests.integ.test_region() in tests.integ.NO_MODEL_MONITORING_REGIONS,
+    reason="ModelMonitoring is not yet supported in this region.",
+)
 def test_byoc_monitor_attach_followed_by_baseline_and_update_monitoring_schedule(
     sagemaker_session,
     predictor,
@@ -2337,7 +2422,17 @@ def test_byoc_monitor_attach_followed_by_baseline_and_update_monitoring_schedule
         == UPDATED_NETWORK_CONFIG.enable_network_isolation
     )
 
+    _wait_for_schedule_changes_to_apply(monitor=my_attached_monitor)
+
+    my_attached_monitor.stop_monitoring_schedule()
+
+    _wait_for_schedule_changes_to_apply(monitor=my_attached_monitor)
+
 
+@pytest.mark.skipif(
+    tests.integ.test_region() in tests.integ.NO_MODEL_MONITORING_REGIONS,
+    reason="ModelMonitoring is not yet supported in this region.",
+)
 def test_byoc_monitor_monitoring_execution_interactions(
     sagemaker_session, byoc_monitoring_schedule_name
 ):

tests/integ/timeout.py

@@ -20,6 +20,8 @@
 from botocore.exceptions import ClientError
 import stopit
 
+from sagemaker import RealTimePredictor
+from tests.integ.retry import retries
 
 LOGGER = logging.getLogger("timeout")
 
@@ -115,12 +117,41 @@ def timeout_and_delete_model_with_transformer(
 
 
 def _delete_schedules_associated_with_endpoint(sagemaker_session, endpoint_name):
-    response = sagemaker_session.list_monitoring_schedules(endpoint_name=endpoint_name)
-    schedule_list = response["MonitoringScheduleSummaries"]
-    for schedule in schedule_list:
-        sagemaker_session.delete_monitoring_schedule(
-            monitoring_schedule_name=schedule["MonitoringScheduleName"]
-        )
+    """Deletes schedules associated with a given endpoint. Per latest validation, ensures the
+    schedule is stopped and no executions are running, before deleting (otherwise latest
+    server-side validations will prevent deletes).
+
+    Args:
+        sagemaker_session (sagemaker.session.Session): A SageMaker Session
+            object, used for SageMaker interactions (default: None). If not
+            specified, one is created using the default AWS configuration
+            chain.
+        endpoint_name (str): The name of the endpoint to delete schedules from.
+
+    """
+    predictor = RealTimePredictor(endpoint=endpoint_name, sagemaker_session=sagemaker_session)
+    monitors = predictor.list_monitors()
+    for monitor in monitors:
+        try:
+            monitor._wait_for_schedule_changes_to_apply()
+            # Stop the schedules to prevent new executions from triggering.
+            monitor.stop_monitoring_schedule()
+            executions = monitor.list_executions()
+            for execution in executions:
+                execution.stop()
+            # Wait for all executions to completely stop.
+            # Schedules can't be deleted with running executions.
+            for execution in executions:
+                for _ in retries(60, "Waiting for executions to stop", seconds_to_sleep=5):
+                    status = execution.describe()["ProcessingJobStatus"]
+                    if status == "Stopped":
+                        break
+            # Delete schedules.
+            monitor.delete_monitoring_schedule()
+        except Exception as e:
+            LOGGER.warning(
+                "Failed to delete monitor {}".format(monitor.monitoring_schedule_name), e
+            )
 
 
 def _show_logs(resource_name, resource_type, sagemaker_session):