fix: create auto ml job for tests that based on existing job (#1151)

* fix: create auto ml job for integ tests that based on existing auto ml job

* skip automl tests in unavailable regions

Author: chuyang-deng

tests/integ/__init__.py

@@ -76,6 +76,7 @@
 
 NO_LDA_REGIONS = ["eu-west-3", "eu-north-1", "sa-east-1", "ap-east-1", "me-south-1"]
 NO_MARKET_PLACE_REGIONS = ["eu-west-3", "eu-north-1", "sa-east-1", "ap-east-1", "me-south-1"]
+NO_AUTO_ML_REGIONS = ["sa-east-1", "me-south-1", "ap-east-1", "eu-west-3"]
 
 EFS_TEST_ENABLED_REGION = []
 

tests/integ/auto_ml_utils.py

@@ -0,0 +1,42 @@
+# Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"). You
+# may not use this file except in compliance with the License. A copy of
+# the License is located at
+#
+#     http://aws.amazon.com/apache2.0/
+#
+# or in the "license" file accompanying this file. This file is
+# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF
+# ANY KIND, either express or implied. See the License for the specific
+# language governing permissions and limitations under the License.
+from __future__ import absolute_import
+
+import os
+
+from sagemaker import AutoML
+from tests.integ import DATA_DIR, AUTO_ML_DEFAULT_TIMEMOUT_MINUTES
+from tests.integ.timeout import timeout
+
+ROLE = "SageMakerRole"
+DATA_DIR = os.path.join(DATA_DIR, "automl", "data")
+PREFIX = "sagemaker/beta-automl-xgboost"
+TRAINING_DATA = os.path.join(DATA_DIR, "iris_training.csv")
+TARGET_ATTRIBUTE_NAME = "virginica"
+
+
+def create_auto_ml_job_if_not_exist(sagemaker_session):
+    auto_ml_job_name = "python-sdk-integ-test-base-job"
+    auto_ml = AutoML(
+        role=ROLE,
+        target_attribute_name=TARGET_ATTRIBUTE_NAME,
+        sagemaker_session=sagemaker_session,
+        max_candidates=3,
+    )
+
+    try:
+        auto_ml.describe_auto_ml_job(job_name=auto_ml_job_name)
+    except Exception as e:  # noqa: F841
+        inputs = sagemaker_session.upload_data(path=TRAINING_DATA, key_prefix=PREFIX + "/input")
+        with timeout(minutes=AUTO_ML_DEFAULT_TIMEMOUT_MINUTES):
+            auto_ml.fit(inputs, job_name=auto_ml_job_name)

tests/integ/test_auto_ml.py

@@ -16,11 +16,12 @@
 import time
 
 import pytest
+import tests.integ
 from sagemaker import AutoML, CandidateEstimator, AutoMLInput
 
 from sagemaker.exceptions import UnexpectedStatusException
 from sagemaker.utils import unique_name_from_base
-from tests.integ import DATA_DIR, AUTO_ML_DEFAULT_TIMEMOUT_MINUTES
+from tests.integ import DATA_DIR, AUTO_ML_DEFAULT_TIMEMOUT_MINUTES, auto_ml_utils
 from tests.integ.timeout import timeout
 
 DEV_ACCOUNT = 142577830533
@@ -38,7 +39,7 @@
 JOB_NAME = "auto-ml-{}".format(time.strftime("%y%m%d-%H%M%S"))
 
 # use a succeeded AutoML job to test describe and list candidates method, otherwise tests will run too long
-AUTO_ML_JOB_NAME = "sagemaker-auto-gamma-ml-test"
+AUTO_ML_JOB_NAME = "python-sdk-integ-test-base-job"
 
 EXPECTED_DEFAULT_INPUT_CONFIG = [
     {
@@ -62,6 +63,10 @@
 }
 
 
+@pytest.mark.skipif(
+    tests.integ.test_region() in tests.integ.NO_AUTO_ML_REGIONS,
+    reason="AutoML is not supported in the region yet.",
+)
 def test_auto_ml_fit(sagemaker_session):
     auto_ml = AutoML(
         role=ROLE,
@@ -75,6 +80,10 @@ def test_auto_ml_fit(sagemaker_session):
         auto_ml.fit(inputs)
 
 
+@pytest.mark.skipif(
+    tests.integ.test_region() in tests.integ.NO_AUTO_ML_REGIONS,
+    reason="AutoML is not supported in the region yet.",
+)
 def test_auto_ml_fit_local_input(sagemaker_session):
     auto_ml = AutoML(
         role=ROLE,
@@ -88,6 +97,10 @@ def test_auto_ml_fit_local_input(sagemaker_session):
         auto_ml.fit(inputs)
 
 
+@pytest.mark.skipif(
+    tests.integ.test_region() in tests.integ.NO_AUTO_ML_REGIONS,
+    reason="AutoML is not supported in the region yet.",
+)
 def test_auto_ml_input_object_fit(sagemaker_session):
     auto_ml = AutoML(
         role=ROLE,
@@ -101,6 +114,10 @@ def test_auto_ml_input_object_fit(sagemaker_session):
         auto_ml.fit(inputs)
 
 
+@pytest.mark.skipif(
+    tests.integ.test_region() in tests.integ.NO_AUTO_ML_REGIONS,
+    reason="AutoML is not supported in the region yet.",
+)
 def test_auto_ml_fit_optional_args(sagemaker_session):
     output_path = "s3://sagemaker-us-east-2-{}/{}".format(DEV_ACCOUNT, "specified_ouput_path")
     problem_type = "MulticlassClassification"
@@ -126,6 +143,10 @@ def test_auto_ml_fit_optional_args(sagemaker_session):
     assert auto_ml_desc["OutputDataConfig"]["S3OutputPath"] == output_path
 
 
+@pytest.mark.skipif(
+    tests.integ.test_region() in tests.integ.NO_AUTO_ML_REGIONS,
+    reason="AutoML is not supported in the region yet.",
+)
 def test_auto_ml_invalid_target_attribute(sagemaker_session):
     auto_ml = AutoML(
         role=ROLE, target_attribute_name="y", sagemaker_session=sagemaker_session, max_candidates=1
@@ -137,7 +158,13 @@ def test_auto_ml_invalid_target_attribute(sagemaker_session):
         auto_ml.fit(inputs)
 
 
+@pytest.mark.skipif(
+    tests.integ.test_region() in tests.integ.NO_AUTO_ML_REGIONS,
+    reason="AutoML is not supported in the region yet.",
+)
 def test_auto_ml_describe_auto_ml_job(sagemaker_session):
+    auto_ml_utils.create_auto_ml_job_if_not_exist(sagemaker_session)
+
     auto_ml = AutoML(
         role=ROLE, target_attribute_name=TARGET_ATTRIBUTE_NAME, sagemaker_session=sagemaker_session
     )
@@ -151,7 +178,13 @@ def test_auto_ml_describe_auto_ml_job(sagemaker_session):
     assert desc["OutputDataConfig"] == EXPECTED_DEFAULT_OUTPUT_CONFIG
 
 
+@pytest.mark.skipif(
+    tests.integ.test_region() in tests.integ.NO_AUTO_ML_REGIONS,
+    reason="AutoML is not supported in the region yet.",
+)
 def test_list_candidates(sagemaker_session):
+    auto_ml_utils.create_auto_ml_job_if_not_exist(sagemaker_session)
+
     auto_ml = AutoML(
         role=ROLE, target_attribute_name=TARGET_ATTRIBUTE_NAME, sagemaker_session=sagemaker_session
     )
@@ -160,7 +193,13 @@ def test_list_candidates(sagemaker_session):
     assert len(candidates) == 3
 
 
+@pytest.mark.skipif(
+    tests.integ.test_region() in tests.integ.NO_AUTO_ML_REGIONS,
+    reason="AutoML is not supported in the region yet.",
+)
 def test_best_candidate(sagemaker_session):
+    auto_ml_utils.create_auto_ml_job_if_not_exist(sagemaker_session)
+
     auto_ml = AutoML(
         role=ROLE, target_attribute_name=TARGET_ATTRIBUTE_NAME, sagemaker_session=sagemaker_session
     )
@@ -170,7 +209,13 @@ def test_best_candidate(sagemaker_session):
     assert best_candidate["CandidateStatus"] == "Completed"
 
 
+@pytest.mark.skipif(
+    tests.integ.test_region() in tests.integ.NO_AUTO_ML_REGIONS,
+    reason="AutoML is not supported in the region yet.",
+)
 def test_deploy_best_candidate(sagemaker_session):
+    auto_ml_utils.create_auto_ml_job_if_not_exist(sagemaker_session)
+
     auto_ml = AutoML(
         role=ROLE, target_attribute_name=TARGET_ATTRIBUTE_NAME, sagemaker_session=sagemaker_session
     )
@@ -192,7 +237,13 @@ def test_deploy_best_candidate(sagemaker_session):
     sagemaker_session.sagemaker_client.delete_endpoint(EndpointName=endpoint_name)
 
 
+@pytest.mark.skipif(
+    tests.integ.test_region() in tests.integ.NO_AUTO_ML_REGIONS,
+    reason="AutoML is not supported in the region yet.",
+)
 def test_candidate_estimator_default_rerun_and_deploy(sagemaker_session):
+    auto_ml_utils.create_auto_ml_job_if_not_exist(sagemaker_session)
+
     auto_ml = AutoML(
         role=ROLE, target_attribute_name=TARGET_ATTRIBUTE_NAME, sagemaker_session=sagemaker_session
     )
@@ -219,7 +270,13 @@ def test_candidate_estimator_default_rerun_and_deploy(sagemaker_session):
     sagemaker_session.sagemaker_client.delete_endpoint(EndpointName=endpoint_name)
 
 
+@pytest.mark.skipif(
+    tests.integ.test_region() in tests.integ.NO_AUTO_ML_REGIONS,
+    reason="AutoML is not supported in the region yet.",
+)
 def test_candidate_estimator_rerun_with_optional_args(sagemaker_session):
+    auto_ml_utils.create_auto_ml_job_if_not_exist(sagemaker_session)
+
     auto_ml = AutoML(
         role=ROLE, target_attribute_name=TARGET_ATTRIBUTE_NAME, sagemaker_session=sagemaker_session
     )
@@ -246,7 +303,13 @@ def test_candidate_estimator_rerun_with_optional_args(sagemaker_session):
     sagemaker_session.sagemaker_client.delete_endpoint(EndpointName=endpoint_name)
 
 
+@pytest.mark.skipif(
+    tests.integ.test_region() in tests.integ.NO_AUTO_ML_REGIONS,
+    reason="AutoML is not supported in the region yet.",
+)
 def test_candidate_estimator_get_steps(sagemaker_session):
+    auto_ml_utils.create_auto_ml_job_if_not_exist(sagemaker_session)
+
     auto_ml = AutoML(
         role=ROLE, target_attribute_name=TARGET_ATTRIBUTE_NAME, sagemaker_session=sagemaker_session
     )