change: region build from staging pr (#979)

* Add eu-west-3, eu-north-1, sa-east-1 and ap-east-1 to the no-p2 regions and no-p3 regions.

* making changes for regions without m4 instances. Adding account for HKG. Squash commits.

* unit test for asimov hkg account image uri

* keep skipping broken tests

* add new region support

Author: chuyang-deng

src/sagemaker/amazon/amazon_estimator.py

@@ -386,6 +386,10 @@ def registry(region_name, algorithm=None):
             "eu-west-2": "644912444149",
             "us-west-1": "632365934929",
             "us-iso-east-1": "490574956308",
+            "ap-east-1": "286214385809",
+            "eu-north-1": "669576153137",
+            "eu-west-3": "749696950732",
+            "sa-east-1": "855470959533",
         }[region_name]
     elif algorithm in ["lda"]:
         account_id = {
@@ -422,6 +426,10 @@ def registry(region_name, algorithm=None):
             "eu-west-2": "644912444149",
             "us-west-1": "632365934929",
             "us-iso-east-1": "490574956308",
+            "ap-east-1": "286214385809",
+            "eu-north-1": "669576153137",
+            "eu-west-3": "749696950732",
+            "sa-east-1": "855470959533",
         }[region_name]
     elif algorithm in [
         "xgboost",
@@ -447,6 +455,10 @@ def registry(region_name, algorithm=None):
             "eu-west-2": "644912444149",
             "us-west-1": "632365934929",
             "us-iso-east-1": "490574956308",
+            "ap-east-1": "286214385809",
+            "eu-north-1": "669576153137",
+            "eu-west-3": "749696950732",
+            "sa-east-1": "855470959533",
         }[region_name]
     elif algorithm in ["image-classification-neo", "xgboost-neo"]:
         account_id = NEO_IMAGE_ACCOUNT[region_name]

src/sagemaker/fw_registry.py

@@ -32,6 +32,10 @@
     "ca-central-1": {"sparkml-serving": "341280168497", "scikit-learn": "341280168497"},
     "us-gov-west-1": {"sparkml-serving": "414596584902", "scikit-learn": "414596584902"},
     "us-iso-east-1": {"sparkml-serving": "833128469047", "scikit-learn": "833128469047"},
+    "ap-east-1": {"sparkml-serving": "651117190479", "scikit-learn": "651117190479"},
+    "sa-east-1": {"sparkml-serving": "737474898029", "scikit-learn": "737474898029"},
+    "eu-north-1": {"sparkml-serving": "662702820516", "scikit-learn": "662702820516"},
+    "eu-west-3": {"sparkml-serving": "659782779980", "scikit-learn": "659782779980"},
 }
 
 

src/sagemaker/fw_utils.py

@@ -28,7 +28,6 @@
 
 UploadedCode = namedtuple("UserCode", ["s3_prefix", "script_name"])
 """sagemaker.fw_utils.UserCode: An object containing the S3 prefix and script name.
-
 This is for the source code used for the entry point with an ``Estimator``. It can be
 instantiated with positional or keyword arguments.
 """
@@ -54,6 +53,8 @@
 VALID_PY_VERSIONS = ["py2", "py3"]
 VALID_EIA_FRAMEWORKS = ["tensorflow", "tensorflow-serving", "mxnet", "mxnet-serving"]
 VALID_ACCOUNTS_BY_REGION = {"us-gov-west-1": "246785580436", "us-iso-east-1": "744548109606"}
+OPT_IN_ACCOUNTS_BY_REGION = {"ap-east-1": "057415533634"}
+ASIMOV_OPT_IN_ACCOUNTS_BY_REGION = {"ap-east-1": "871362719292"}
 
 MERGED_FRAMEWORKS_REPO_MAP = {
     "tensorflow-scriptmode": "tensorflow-training",
@@ -73,12 +74,10 @@
 def is_version_equal_or_higher(lowest_version, framework_version):
     """Determine whether the ``framework_version`` is equal to or higher than
     ``lowest_version``
-
     Args:
         lowest_version (List[int]): lowest version represented in an integer
             list
         framework_version (str): framework version string
-
     Returns:
         bool: Whether or not framework_version is equal to or higher than
         lowest_version
@@ -125,7 +124,11 @@ def _registry_id(region, framework, py_version, account, accelerator_type, frame
         framework_version:
     """
     if _using_merged_images(region, framework, py_version, accelerator_type, framework_version):
+        if region in ASIMOV_OPT_IN_ACCOUNTS_BY_REGION:
+            return ASIMOV_OPT_IN_ACCOUNTS_BY_REGION.get(region)
         return "763104351884"
+    if region in OPT_IN_ACCOUNTS_BY_REGION:
+        return OPT_IN_ACCOUNTS_BY_REGION.get(region)
     return VALID_ACCOUNTS_BY_REGION.get(region, account)
 
 
@@ -140,7 +143,6 @@ def create_image_uri(
     optimized_families=None,
 ):
     """Return the ECR URI of an image.
-
     Args:
         region (str): AWS region where the image is uploaded.
         framework (str): framework used by the image.
@@ -155,7 +157,6 @@ def create_image_uri(
         accelerator_type (str): SageMaker Elastic Inference accelerator type.
         optimized_families (str): Instance families for which there exist
             specific optimized images.
-
     Returns:
         str: The appropriate image URI based on the given parameters.
     """
@@ -249,11 +250,9 @@ def _accelerator_type_valid_for_framework(
 
 def validate_source_dir(script, directory):
     """Validate that the source directory exists and it contains the user script
-
     Args:
         script (str): Script filename.
         directory (str): Directory containing the source file.
-
     Raises:
         ValueError: If ``directory`` does not exist, is not a directory, or does
             not contain ``script``.
@@ -272,18 +271,14 @@ def tar_and_upload_dir(
 ):
     """Package source files and upload a compress tar file to S3. The S3
     location will be ``s3://<bucket>/s3_key_prefix/sourcedir.tar.gz``.
-
     If directory is an S3 URI, an UploadedCode object will be returned, but
     nothing will be uploaded to S3 (this allow reuse of code already in S3).
-
     If directory is None, the script will be added to the archive at
     ``./<basename of script>``.
-
     If directory is not None, the (recursive) contents of the directory will
     be added to the archive. directory is treated as the base path of the
     archive, and the script name is assumed to be a filename or relative path
     inside the directory.
-
     Args:
         session (boto3.Session): Boto session used to access S3.
         bucket (str): S3 bucket to which the compressed file is uploaded.
@@ -296,7 +291,6 @@ def tar_and_upload_dir(
             copied into /opt/ml/lib
         kms_key (str): Optional. KMS key ID used to upload objects to the bucket
             (default: None).
-
     Returns:
         sagemaker.fw_utils.UserCode: An object with the S3 bucket and key (S3 prefix) and
             script name.
@@ -343,7 +337,6 @@ def _list_files_to_compress(script, directory):
 def framework_name_from_image(image_name):
     # noinspection LongLine
     """Extract the framework and Python version from the image name.
-
     Args:
         image_name (str): Image URI, which should be one of the following forms:
             legacy:
@@ -354,7 +347,6 @@ def framework_name_from_image(image_name):
             '<account>.dkr.ecr.<region>.amazonaws.com/sagemaker-<fw>:<fw_version>-<device>-<py_ver>'
             current:
             '<account>.dkr.ecr.<region>.amazonaws.com/sagemaker-rl-<fw>:<rl_toolkit><rl_version>-<device>-<py_ver>'
-
     Returns:
         tuple: A tuple containing:
             str: The framework name str: The Python version str: The image tag
@@ -390,11 +382,9 @@ def framework_name_from_image(image_name):
 
 def framework_version_from_tag(image_tag):
     """Extract the framework version from the image tag.
-
     Args:
         image_tag (str): Image tag, which should take the form
             '<framework_version>-<device>-<py_version>'
-
     Returns:
         str: The framework version.
     """
@@ -406,10 +396,8 @@ def framework_version_from_tag(image_tag):
 def parse_s3_url(url):
     """Returns an (s3 bucket, key name/prefix) tuple from a url with an s3
     scheme
-
     Args:
         url (str):
-
     Returns:
         tuple: A tuple containing:
             str: S3 bucket name str: S3 key
@@ -422,16 +410,13 @@ def parse_s3_url(url):
 
 def model_code_key_prefix(code_location_key_prefix, model_name, image):
     """Returns the s3 key prefix for uploading code during model deployment
-
     The location returned is a potential concatenation of 2 parts
         1. code_location_key_prefix if it exists
         2. model_name or a name derived from the image
-
     Args:
         code_location_key_prefix (str): the s3 key prefix from code_location
         model_name (str): the name of the model
         image (str): the image from which a default name can be extracted
-
     Returns:
         str: the key prefix to be used in uploading code
     """

tests/conftest.py

@@ -17,6 +17,7 @@
 
 import boto3
 import pytest
+import tests.integ
 from botocore.config import Config
 
 from sagemaker import Session
@@ -30,6 +31,8 @@
 
 DEFAULT_REGION = "us-west-2"
 
+NO_M4_REGIONS = ["eu-west-3", "eu-north-1", "ap-east-1"]
+
 
 def pytest_addoption(parser):
     parser.addoption("--sagemaker-client-config", action="store", default=None)
@@ -242,3 +245,33 @@ def tf_full_version(request):
 @pytest.fixture(scope="module")
 def ei_tf_full_version(request):
     return request.config.getoption("--ei-tf-full-version")
+
+
+@pytest.fixture(scope="session")
+def cpu_instance_type(sagemaker_session, request):
+    region = sagemaker_session.boto_session.region_name
+    if region in NO_M4_REGIONS:
+        return "ml.m5.xlarge"
+    else:
+        return "ml.m4.xlarge"
+
+
+@pytest.fixture(scope="session")
+def cpu_instance_family(cpu_instance_type):
+    "_".join(cpu_instance_type.split(".")[0:2])
+
+
+def pytest_generate_tests(metafunc):
+    if "instance_type" in metafunc.fixturenames:
+        boto_config = metafunc.config.getoption("--boto-config")
+        parsed_config = json.loads(boto_config) if boto_config else {}
+        region = parsed_config.get("region_name", DEFAULT_REGION)
+        cpu_instance_type = "ml.m5.xlarge" if region in NO_M4_REGIONS else "ml.m4.xlarge"
+
+        params = [cpu_instance_type]
+        if not (
+            region in tests.integ.HOSTING_NO_P2_REGIONS
+            or region in tests.integ.TRAINING_NO_P2_REGIONS
+        ):
+            params.append("ml.p2.xlarge")
+        metafunc.parametrize("instance_type", params, scope="session")

tests/integ/__init__.py

@@ -25,7 +25,16 @@
 PYTHON_VERSION = "py" + str(sys.version_info.major)
 
 # these regions have some p2 and p3 instances, but not enough for continuous testing
-HOSTING_NO_P2_REGIONS = ["ca-central-1", "eu-central-1", "eu-west-2", "us-west-1"]
+HOSTING_NO_P2_REGIONS = [
+    "ca-central-1",
+    "eu-central-1",
+    "eu-west-2",
+    "us-west-1",
+    "eu-west-3",
+    "eu-north-1",
+    "sa-east-1",
+    "ap-east-1",
+]
 HOSTING_NO_P3_REGIONS = [
     "ap-southeast-1",
     "ap-southeast-2",
@@ -34,8 +43,19 @@
     "eu-central-1",
     "eu-west-2",
     "us-west-1",
+    "eu-west-3",
+    "eu-north-1",
+    "sa-east-1",
+    "ap-east-1",
+]
+TRAINING_NO_P2_REGIONS = [
+    "ap-southeast-1",
+    "ap-southeast-2",
+    "eu-west-3",
+    "eu-north-1",
+    "sa-east-1",
+    "ap-east-1",
 ]
-TRAINING_NO_P2_REGIONS = ["ap-southeast-1", "ap-southeast-2"]
 
 # EI is currently only supported in the following regions
 # regions were derived from https://aws.amazon.com/machine-learning/elastic-inference/pricing/
@@ -48,6 +68,9 @@
     "ap-northeast-2",
 ]
 
+NO_LDA_REGIONS = ["eu-west-3", "eu-north-1", "sa-east-1", "ap-east-1"]
+NO_MARKET_PLACE_REGIONS = ["eu-west-3", "eu-north-1", "sa-east-1", "ap-east-1"]
+
 logging.getLogger("boto3").setLevel(logging.INFO)
 logging.getLogger("botocore").setLevel(logging.INFO)
 

tests/integ/conftest.py

@@ -14,8 +14,6 @@
 
 import os
 
-import pytest
-
 
 def create_sagemaker_local_network():
     """
@@ -28,8 +26,3 @@ def create_sagemaker_local_network():
 
 
 create_sagemaker_local_network()
-
-
-@pytest.fixture(scope="session", params=["local", "ml.c4.xlarge"])
-def instance_type(request):
-    return request.param

tests/integ/marketplace_utils.py

@@ -26,4 +26,8 @@
     "ca-central-1": "470592106596",
     "eu-west-2": "856760150666",
     "us-west-1": "382657785993",
+    "eu-west-3": "843114510376",
+    "eu-north-1": "136758871317",
+    "sa-east-1": "270155090741",
+    "ap-east-1": "822005858737",
 }

tests/integ/test_byo_estimator.py

@@ -41,7 +41,7 @@ def fm_serializer(data):
 
 
 @pytest.mark.canary_quick
-def test_byo_estimator(sagemaker_session, region):
+def test_byo_estimator(sagemaker_session, region, cpu_instance_type):
     """Use Factorization Machines algorithm as an example here.
 
     First we need to prepare data for training. We take standard data set, convert it to the
@@ -74,7 +74,7 @@ def test_byo_estimator(sagemaker_session, region):
             image_name=image_name,
             role="SageMakerRole",
             train_instance_count=1,
-            train_instance_type="ml.c4.xlarge",
+            train_instance_type=cpu_instance_type,
             sagemaker_session=sagemaker_session,
         )
 
@@ -87,7 +87,7 @@ def test_byo_estimator(sagemaker_session, region):
 
     with timeout_and_delete_endpoint_by_name(job_name, sagemaker_session):
         model = estimator.create_model()
-        predictor = model.deploy(1, "ml.m4.xlarge", endpoint_name=job_name)
+        predictor = model.deploy(1, cpu_instance_type, endpoint_name=job_name)
         predictor.serializer = fm_serializer
         predictor.content_type = "application/json"
         predictor.deserializer = sagemaker.predictor.json_deserializer
@@ -99,7 +99,7 @@ def test_byo_estimator(sagemaker_session, region):
             assert prediction["score"] is not None
 
 
-def test_async_byo_estimator(sagemaker_session, region):
+def test_async_byo_estimator(sagemaker_session, region, cpu_instance_type):
     image_name = registry(region) + "/factorization-machines:1"
     endpoint_name = unique_name_from_base("byo")
     training_data_path = os.path.join(DATA_DIR, "dummy_tensor")
@@ -123,7 +123,7 @@ def test_async_byo_estimator(sagemaker_session, region):
             image_name=image_name,
             role="SageMakerRole",
             train_instance_count=1,
-            train_instance_type="ml.c4.xlarge",
+            train_instance_type=cpu_instance_type,
             sagemaker_session=sagemaker_session,
         )
 
@@ -139,7 +139,7 @@ def test_async_byo_estimator(sagemaker_session, region):
             training_job_name=job_name, sagemaker_session=sagemaker_session
         )
         model = estimator.create_model()
-        predictor = model.deploy(1, "ml.m4.xlarge", endpoint_name=endpoint_name)
+        predictor = model.deploy(1, cpu_instance_type, endpoint_name=endpoint_name)
         predictor.serializer = fm_serializer
         predictor.content_type = "application/json"
         predictor.deserializer = sagemaker.predictor.json_deserializer