feature: add XGBoost Estimator as new framework (#980)

This PR adds a new XGBoost estimator to release XGBoost as a framework container.

Author: iyerr3

README.rst

@@ -49,18 +49,19 @@ Table of Contents
 5. `Chainer SageMaker Estimators <#chainer-sagemaker-estimators>`__
 6. `PyTorch SageMaker Estimators <#pytorch-sagemaker-estimators>`__
 7. `Scikit-learn SageMaker Estimators <#scikit-learn-sagemaker-estimators>`__
-8. `SageMaker Reinforcement Learning Estimators <#sagemaker-reinforcement-learning-estimators>`__
-9. `SageMaker SparkML Serving <#sagemaker-sparkml-serving>`__
-10. `AWS SageMaker Estimators <#aws-sagemaker-estimators>`__
-11. `Using SageMaker AlgorithmEstimators <https://sagemaker.readthedocs.io/en/stable/overview.html#using-sagemaker-algorithmestimators>`__
-12. `Consuming SageMaker Model Packages <https://sagemaker.readthedocs.io/en/stable/overview.html#consuming-sagemaker-model-packages>`__
-13. `BYO Docker Containers with SageMaker Estimators <https://sagemaker.readthedocs.io/en/stable/overview.html#byo-docker-containers-with-sagemaker-estimators>`__
-14. `SageMaker Automatic Model Tuning <https://sagemaker.readthedocs.io/en/stable/overview.html#sagemaker-automatic-model-tuning>`__
-15. `SageMaker Batch Transform <https://sagemaker.readthedocs.io/en/stable/overview.html#sagemaker-batch-transform>`__
-16. `Secure Training and Inference with VPC <https://sagemaker.readthedocs.io/en/stable/overview.html#secure-training-and-inference-with-vpc>`__
-17. `BYO Model <https://sagemaker.readthedocs.io/en/stable/overview.html#byo-model>`__
-18. `Inference Pipelines <https://sagemaker.readthedocs.io/en/stable/overview.html#inference-pipelines>`__
-19. `SageMaker Workflow <#sagemaker-workflow>`__
+8. `XGBoost SageMaker Estimators <#xgboost-sagemaker-estimators>`__
+9. `SageMaker Reinforcement Learning Estimators <#sagemaker-reinforcement-learning-estimators>`__
+10. `SageMaker SparkML Serving <#sagemaker-sparkml-serving>`__
+11. `AWS SageMaker Estimators <#aws-sagemaker-estimators>`__
+12. `Using SageMaker AlgorithmEstimators <https://sagemaker.readthedocs.io/en/stable/overview.html#using-sagemaker-algorithmestimators>`__
+13. `Consuming SageMaker Model Packages <https://sagemaker.readthedocs.io/en/stable/overview.html#consuming-sagemaker-model-packages>`__
+14. `BYO Docker Containers with SageMaker Estimators <https://sagemaker.readthedocs.io/en/stable/overview.html#byo-docker-containers-with-sagemaker-estimators>`__
+15. `SageMaker Automatic Model Tuning <https://sagemaker.readthedocs.io/en/stable/overview.html#sagemaker-automatic-model-tuning>`__
+16. `SageMaker Batch Transform <https://sagemaker.readthedocs.io/en/stable/overview.html#sagemaker-batch-transform>`__
+17. `Secure Training and Inference with VPC <https://sagemaker.readthedocs.io/en/stable/overview.html#secure-training-and-inference-with-vpc>`__
+18. `BYO Model <https://sagemaker.readthedocs.io/en/stable/overview.html#byo-model>`__
+19. `Inference Pipelines <https://sagemaker.readthedocs.io/en/stable/overview.html#inference-pipelines>`__
+20. `SageMaker Workflow <#sagemaker-workflow>`__
 
 
 Installing the SageMaker Python SDK
@@ -247,6 +248,21 @@ For more information about Scikit-learn SageMaker Estimators, see `Using Scikit-
 
 .. _Using Scikit-learn with the SageMaker Python SDK: https://sagemaker.readthedocs.io/en/stable/using_sklearn.html
 
+XGBoost SageMaker Estimators
+----------------------------
+
+With XGBoost SageMaker Estimators, you can train and host XGBoost models on Amazon SageMaker.
+
+Supported versions of XGBoost: ``0.90-1``.
+
+We recommend that you use the latest supported version, because that's where we focus most of our development efforts.
+
+For more information about XGBoost, see https://xgboost.readthedocs.io/en/latest/
+
+For more information about XGBoost SageMaker Estimators, see `Using XGBoost with the SageMaker Python SDK`_.
+
+.. _Using XGBoost with the SageMaker Python SDK: https://sagemaker.readthedocs.io/en/stable/using_xgboost.html
+
 
 SageMaker Reinforcement Learning Estimators
 -------------------------------------------

src/sagemaker/amazon/amazon_estimator.py

@@ -26,6 +26,8 @@
 from sagemaker.model import NEO_IMAGE_ACCOUNT
 from sagemaker.session import s3_input
 from sagemaker.utils import sagemaker_timestamp, get_ecr_image_uri_prefix
+from sagemaker.xgboost.estimator import get_xgboost_image_uri
+from sagemaker.xgboost.defaults import XGBOOST_LATEST_VERSION
 
 logger = logging.getLogger(__name__)
 
@@ -479,5 +481,15 @@ def get_image_uri(region_name, repo_name, repo_version=1):
         repo_name:
         repo_version:
     """
+    if repo_name == "xgboost":
+        if repo_version in ["0.90", "0.90-1", "0.90-1-cpu-py3"]:
+            return get_xgboost_image_uri(region_name, XGBOOST_LATEST_VERSION)
+        logging.warning(
+            "There is a more up to date SageMaker XGBoost image."
+            "To use the newer image, please set 'repo_version'="
+            "'0.90-1. For example:\n"
+            "\tget_image_uri(region, 'xgboost', %s).",
+            XGBOOST_LATEST_VERSION,
+        )
     repo = "{}:{}".format(repo_name, repo_version)
     return "{}/{}".format(registry(region_name, repo_name), repo)

src/sagemaker/fw_registry.py

@@ -17,25 +17,101 @@
 from sagemaker.utils import get_ecr_image_uri_prefix
 
 image_registry_map = {
-    "us-west-1": {"sparkml-serving": "746614075791", "scikit-learn": "746614075791"},
-    "us-west-2": {"sparkml-serving": "246618743249", "scikit-learn": "246618743249"},
-    "us-east-1": {"sparkml-serving": "683313688378", "scikit-learn": "683313688378"},
-    "us-east-2": {"sparkml-serving": "257758044811", "scikit-learn": "257758044811"},
-    "ap-northeast-1": {"sparkml-serving": "354813040037", "scikit-learn": "354813040037"},
-    "ap-northeast-2": {"sparkml-serving": "366743142698", "scikit-learn": "366743142698"},
-    "ap-southeast-1": {"sparkml-serving": "121021644041", "scikit-learn": "121021644041"},
-    "ap-southeast-2": {"sparkml-serving": "783357654285", "scikit-learn": "783357654285"},
-    "ap-south-1": {"sparkml-serving": "720646828776", "scikit-learn": "720646828776"},
-    "eu-west-1": {"sparkml-serving": "141502667606", "scikit-learn": "141502667606"},
-    "eu-west-2": {"sparkml-serving": "764974769150", "scikit-learn": "764974769150"},
-    "eu-central-1": {"sparkml-serving": "492215442770", "scikit-learn": "492215442770"},
-    "ca-central-1": {"sparkml-serving": "341280168497", "scikit-learn": "341280168497"},
-    "us-gov-west-1": {"sparkml-serving": "414596584902", "scikit-learn": "414596584902"},
-    "us-iso-east-1": {"sparkml-serving": "833128469047", "scikit-learn": "833128469047"},
-    "ap-east-1": {"sparkml-serving": "651117190479", "scikit-learn": "651117190479"},
-    "sa-east-1": {"sparkml-serving": "737474898029", "scikit-learn": "737474898029"},
-    "eu-north-1": {"sparkml-serving": "662702820516", "scikit-learn": "662702820516"},
-    "eu-west-3": {"sparkml-serving": "659782779980", "scikit-learn": "659782779980"},
+    "us-west-1": {
+        "sparkml-serving": "746614075791",
+        "scikit-learn": "746614075791",
+        "xgboost": "746614075791",
+    },
+    "us-west-2": {
+        "sparkml-serving": "246618743249",
+        "scikit-learn": "246618743249",
+        "xgboost": "246618743249",
+    },
+    "us-east-1": {
+        "sparkml-serving": "683313688378",
+        "scikit-learn": "683313688378",
+        "xgboost": "683313688378",
+    },
+    "us-east-2": {
+        "sparkml-serving": "257758044811",
+        "scikit-learn": "257758044811",
+        "xgboost": "257758044811",
+    },
+    "ap-northeast-1": {
+        "sparkml-serving": "354813040037",
+        "scikit-learn": "354813040037",
+        "xgboost": "354813040037",
+    },
+    "ap-northeast-2": {
+        "sparkml-serving": "366743142698",
+        "scikit-learn": "366743142698",
+        "xgboost": "366743142698",
+    },
+    "ap-southeast-1": {
+        "sparkml-serving": "121021644041",
+        "scikit-learn": "121021644041",
+        "xgboost": "121021644041",
+    },
+    "ap-southeast-2": {
+        "sparkml-serving": "783357654285",
+        "scikit-learn": "783357654285",
+        "xgboost": "783357654285",
+    },
+    "ap-south-1": {
+        "sparkml-serving": "720646828776",
+        "scikit-learn": "720646828776",
+        "xgboost": "720646828776",
+    },
+    "eu-west-1": {
+        "sparkml-serving": "141502667606",
+        "scikit-learn": "141502667606",
+        "xgboost": "141502667606",
+    },
+    "eu-west-2": {
+        "sparkml-serving": "764974769150",
+        "scikit-learn": "764974769150",
+        "xgboost": "764974769150",
+    },
+    "eu-central-1": {
+        "sparkml-serving": "492215442770",
+        "scikit-learn": "492215442770",
+        "xgboost": "492215442770",
+    },
+    "ca-central-1": {
+        "sparkml-serving": "341280168497",
+        "scikit-learn": "341280168497",
+        "xgboost": "341280168497",
+    },
+    "us-gov-west-1": {
+        "sparkml-serving": "414596584902",
+        "scikit-learn": "414596584902",
+        "xgboost": "414596584902",
+    },
+    "us-iso-east-1": {
+        "sparkml-serving": "833128469047",
+        "scikit-learn": "833128469047",
+        "xgboost": "833128469047",
+    },
+    "ap-east-1": {
+        "sparkml-serving": "651117190479",
+        "scikit-learn": "651117190479",
+        "xgboost": "651117190479",
+    },
+    "sa-east-1": {
+        "sparkml-serving": "737474898029",
+        "scikit-learn": "737474898029",
+        "xgboost": "737474898029",
+    },
+    "eu-north-1": {
+        "sparkml-serving": "662702820516",
+        "scikit-learn": "662702820516",
+        "xgboost": "662702820516",
+    },
+    "eu-west-3": {
+        "sparkml-serving": "659782779980",
+        "scikit-learn": "659782779980",
+        "xgboost": "659782779980",
+    },
 }
 
 

src/sagemaker/fw_utils.py

@@ -49,6 +49,9 @@
     "framework_version is required for script mode estimator. "
     "Please add framework_version={} to your constructor to avoid this error."
 )
+UNSUPPORTED_FRAMEWORK_VERSION_ERROR = (
+    "{} framework does not support version {}. Please use one of the following: {}."
+)
 
 VALID_PY_VERSIONS = ["py2", "py3"]
 VALID_EIA_FRAMEWORKS = ["tensorflow", "tensorflow-serving", "mxnet", "mxnet-serving"]
@@ -359,7 +362,7 @@ def framework_name_from_image(image_name):
     # extract framework, python version and image tag
     # We must support both the legacy and current image name format.
     name_pattern = re.compile(
-        r"^(?:sagemaker(?:-rl)?-)?(tensorflow|mxnet|chainer|pytorch|scikit-learn)(?:-)?(scriptmode|training)?:(.*)-(.*?)-(py2|py3)$"  # noqa: E501 # pylint: disable=line-too-long
+        r"^(?:sagemaker(?:-rl)?-)?(tensorflow|mxnet|chainer|pytorch|scikit-learn|xgboost)(?:-)?(scriptmode|training)?:(.*)-(.*?)-(py2|py3)$"  # noqa: E501 # pylint: disable=line-too-long
     )
     legacy_name_pattern = re.compile(r"^sagemaker-(tensorflow|mxnet)-(py2|py3)-(cpu|gpu):(.*)$")
 
@@ -436,6 +439,25 @@ def empty_framework_version_warning(default_version, latest_version):
     return " ".join(msgs)
 
 
+def get_unsupported_framework_version_error(
+    framework_name, unsupported_version, supported_versions
+):
+    """Return error message for unsupported framework version.
+
+    This should also return the supported versions for customers.
+
+    :param framework_name:
+    :param unsupported_version:
+    :param supported_versions:
+    :return:
+    """
+    return UNSUPPORTED_FRAMEWORK_VERSION_ERROR.format(
+        framework_name,
+        unsupported_version,
+        ", ".join('"{}"'.format(version) for version in supported_versions),
+    )
+
+
 def python_deprecation_warning(framework):
     """
     Args:

src/sagemaker/xgboost/README.rst

@@ -0,0 +1,83 @@
+============================================
+XGBoost SageMaker Estimators and Models
+============================================
+
+With XGBoost Estimators, you can train and host XGBoost models on Amazon SageMaker.
+
+Supported versions of SageMaker XGBoost: ``0.90-1``
+
+Note that the first part of the version refers to the upstream module version (aka, 0.90), while the second
+part refers to the SageMaker version for the container.
+
+You can visit the XGBoost repository at https://github.com/dmlc/xgboost
+
+For information about using XGBoost with the SageMaker Python SDK, see https://sagemaker.readthedocs.io/en/stable/using_xgboost.html.
+
+XGBoost Training Examples
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Amazon provides an example Jupyter notebook that demonstrate end-to-end training on Amazon SageMaker using XGBoost.
+Please refer to:
+
+https://github.com/awslabs/amazon-sagemaker-examples/tree/master/sagemaker-python-sdk
+
+These are also available in SageMaker Notebook Instance hosted Jupyter notebooks under the "sample notebooks" folder.
+
+
+SageMaker XGBoost Docker Containers
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+When training and deploying training scripts, SageMaker runs your Python script in a Docker container with several
+libraries installed. When creating the Estimator and calling deploy to create the SageMaker Endpoint, you can control
+the environment your script runs in.
+
+SageMaker runs XGBoost Estimator scripts in either Python 2.7 or Python 3.5. You can select the Python version by
+passing a py_version keyword arg to the XGBoost Estimator constructor. Setting this to py3 (the default) will cause
+your training script to be run on Python 3.5. Setting this to py2 will cause your training script to be run on Python 2.7
+This Python version applies to both the Training Job, created by fit, and the Endpoint, created by deploy.
+
+The XGBoost Docker images have the following dependencies installed:
+
++-----------------------------+-------------+
+| Dependencies                | Version     |
++-----------------------------+-------------+
+| xgboost                     | 0.90.0      |
++-----------------------------+-------------+
+| matplotlib                  | 3.0.3+      |
++-----------------------------+-------------+
+| numpy                       | 1.16.4+     |
++-----------------------------+-------------+
+| pandas                      | 0.24.2+     |
++-----------------------------+-------------+
+| psutils                     | 5.6.3+      |
++-----------------------------+-------------+
+| PyYAML                      | < 4.3       |
++-----------------------------+-------------+
+| requests                    | < 2.21      |
++-----------------------------+-------------+
+| retrying                    | 1.3.3       |
++-----------------------------+-------------+
+| scikit-learn                | 0.21.2+     |
++-----------------------------+-------------+
+| scipy                       | 1.3.0+      |
++-----------------------------+-------------+
+| sagemaker-containers        | 2.5.1+      |
++-----------------------------+-------------+
+| urllib3                     | < 1.25      |
++-----------------------------+-------------+
+| Python                      | 2.7 or 3.5  |
++-----------------------------+-------------+
+
+You can see the full list by calling ``pip freeze`` from the running Docker image.
+
+The Docker images extend Ubuntu 16.04.
+
+You can select version of XGBoost by passing a framework_version keyword arg to the XGBoost Estimator constructor.
+Currently supported versions are listed in the above table. You can also set framework_version to only specify major and
+minor version, which will cause your training script to be run on the latest supported patch version of that minor
+version.
+
+Alternatively, you can build your own image by following the instructions in the SageMaker XGBoost containers
+repository, and passing ``image_name`` to the XGBoost Estimator constructor.
+
+You can visit the SageMaker XGBoost containers repository here: https://github.com/aws/sagemaker-xgboost-container

src/sagemaker/xgboost/__init__.py

@@ -0,0 +1,16 @@
+# Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"). You
+# may not use this file except in compliance with the License. A copy of
+# the License is located at
+#
+#     http://aws.amazon.com/apache2.0/
+#
+# or in the "license" file accompanying this file. This file is
+# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF
+# ANY KIND, either express or implied. See the License for the specific
+# language governing permissions and limitations under the License.
+"""Placeholder docstring"""
+from sagemaker.xgboost.defaults import XGBOOST_NAME, XGBOOST_LATEST_VERSION  # noqa: F401
+from sagemaker.xgboost.estimator import XGBoost  # noqa: F401
+from sagemaker.xgboost.model import XGBoostModel, XGBoostPredictor  # noqa: F401

src/sagemaker/xgboost/defaults.py

@@ -0,0 +1,18 @@
+# Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"). You
+# may not use this file except in compliance with the License. A copy of
+# the License is located at
+#
+#     http://aws.amazon.com/apache2.0/
+#
+# or in the "license" file accompanying this file. This file is
+# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF
+# ANY KIND, either express or implied. See the License for the specific
+# language governing permissions and limitations under the License.
+"""Placeholder docstring"""
+from __future__ import absolute_import
+
+XGBOOST_NAME = "xgboost"
+XGBOOST_LATEST_VERSION = "0.90-1"
+XGBOOST_SUPPORTED_VERSIONS = [XGBOOST_LATEST_VERSION]