feature: add xgboost framework version 1.2-1 (#1899)

* feature: add xgboost framework version 1.2-1

* Add copywrite to xgboost utils file

* Add absolute_import to xgboost utils file

* Remove xgboost_latest_py_version fixture from airflow config test

Co-authored-by: Chuyang <[email protected]>
Co-authored-by: icywang86rui <[email protected]>

Author: 3 people

src/sagemaker/image_uri_config/xgboost.json

@@ -125,6 +125,35 @@
                 "us-west-2": "246618743249"
             },
             "repository": "sagemaker-xgboost"
+        },
+        "1.2-1": {
+            "registries": {
+                "af-south-1": "510948584623",
+                "ap-east-1": "651117190479",
+                "ap-northeast-1": "354813040037",
+                "ap-northeast-2": "366743142698",
+                "ap-south-1": "720646828776",
+                "ap-southeast-1": "121021644041",
+                "ap-southeast-2": "783357654285",
+                "ca-central-1": "341280168497",
+                "cn-north-1": "450853457545",
+                "cn-northwest-1": "451049120500",
+                "eu-central-1": "492215442770",
+                "eu-north-1": "662702820516",
+                "eu-west-1": "141502667606",
+                "eu-west-2": "764974769150",
+                "eu-west-3": "659782779980",
+                "eu-south-1": "978288397137",
+                "me-south-1": "801668240914",
+                "sa-east-1": "737474898029",
+                "us-east-1": "683313688378",
+                "us-east-2": "257758044811",
+                "us-gov-west-1": "414596584902",
+                "us-iso-east-1": "833128469047",
+                "us-west-1": "746614075791",
+                "us-west-2": "246618743249"
+            },
+            "repository": "sagemaker-xgboost"
         }
     }
 }

src/sagemaker/image_uris.py

@@ -252,4 +252,4 @@ def _validate_arg(arg, available_options, arg_name):
 
 def _format_tag(tag_prefix, processor, py_version, container_version):
     """Creates a tag for the image URI."""
-    return "-".join([x for x in (tag_prefix, processor, py_version, container_version) if x])
+    return "-".join(x for x in (tag_prefix, processor, py_version, container_version) if x)

src/sagemaker/xgboost/defaults.py

@@ -14,3 +14,11 @@
 from __future__ import absolute_import
 
 XGBOOST_NAME = "xgboost"
+XGBOOST_UNSUPPORTED_VERSIONS = {
+    "1.1": (
+        "XGBoost 1.1 is not supported on SageMaker because XGBoost 1.1 has broken capability to "
+        "run prediction when the test input has fewer features than the training data in LIBSVM "
+        "inputs. This capability has been restored in XGBoost 1.2 "
+        "(https://github.com/dmlc/xgboost/pull/5955). Consider using SageMaker XGBoost 1.2-1."
+    ),
+}

src/sagemaker/xgboost/estimator.py

@@ -26,6 +26,7 @@
 from sagemaker.vpc_utils import VPC_CONFIG_DEFAULT
 from sagemaker.xgboost import defaults
 from sagemaker.xgboost.model import XGBoostModel
+from sagemaker.xgboost.utils import validate_py_version, validate_framework_version
 
 logger = logging.getLogger("sagemaker")
 
@@ -101,6 +102,9 @@ def __init__(
         self.py_version = py_version
         self.framework_version = framework_version
 
+        validate_py_version(py_version)
+        validate_framework_version(framework_version)
+
         if image_uri is None:
             self.image_uri = image_uris.retrieve(
                 self._framework_name,

src/sagemaker/xgboost/model.py

@@ -23,6 +23,7 @@
 from sagemaker.predictor import Predictor
 from sagemaker.serializers import LibSVMSerializer
 from sagemaker.xgboost.defaults import XGBOOST_NAME
+from sagemaker.xgboost.utils import validate_py_version, validate_framework_version
 
 logger = logging.getLogger("sagemaker")
 
@@ -105,13 +106,15 @@ def __init__(
         self.framework_version = framework_version
         self.model_server_workers = model_server_workers
 
+        validate_py_version(py_version)
+        validate_framework_version(framework_version)
+
     def prepare_container_def(self, instance_type=None, accelerator_type=None):
         """Return a container definition with framework configuration
         set in model environment variables.
 
         Args:
             instance_type (str): The EC2 instance type to deploy this Model to.
-                This parameter is unused because XGBoost supports only CPU.
             accelerator_type (str): The Elastic Inference accelerator type to deploy to the
             instance for loading and making inferences to the model. This parameter is
                 unused because accelerator types are not supported by XGBoostModel.
@@ -148,7 +151,5 @@ def serving_image_uri(self, region_name, instance_type):
             self._framework_name,
             region_name,
             version=self.framework_version,
-            py_version=self.py_version,
             instance_type=instance_type,
-            image_scope="inference",
         )

src/sagemaker/xgboost/utils.py

@@ -0,0 +1,31 @@
+# Copyright 2018-2020 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"). You
+# may not use this file except in compliance with the License. A copy of
+# the License is located at
+#
+#     http://aws.amazon.com/apache2.0/
+#
+# or in the "license" file accompanying this file. This file is
+# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF
+# ANY KIND, either express or implied. See the License for the specific
+# language governing permissions and limitations under the License.
+"""Placeholder docstring"""
+from __future__ import absolute_import
+
+from sagemaker.xgboost import defaults
+
+
+def validate_py_version(py_version):
+    """Placeholder docstring"""
+    if py_version != "py3":
+        raise ValueError("Unsupported Python version: {}.".format(py_version))
+
+
+def validate_framework_version(framework_version):
+    """Placeholder docstring"""
+
+    xgboost_version = framework_version.split("-")[0]
+    if xgboost_version in defaults.XGBOOST_UNSUPPORTED_VERSIONS:
+        msg = defaults.XGBOOST_UNSUPPORTED_VERSIONS[xgboost_version]
+        raise ValueError(msg)

tests/conftest.py

@@ -184,6 +184,15 @@ def xgboost_framework_version(xgboost_version):
     return xgboost_version
 
 
+@pytest.fixture(scope="module")
+def xgboost_gpu_framework_version(xgboost_version):
+    if xgboost_version in ("1", "latest"):
+        pytest.skip("Skipping XGBoost algorithm version.")
+    if Version(xgboost_version) < Version("1.2"):
+        pytest.skip("Skipping XGBoost cpu-only version.")
+    return xgboost_version
+
+
 @pytest.fixture(scope="module", params=["py2", "py3"])
 def tensorflow_training_py_version(tensorflow_training_version, request):
     return _tf_py_version(tensorflow_training_version, request)

tests/integ/test_airflow_config.py

@@ -552,13 +552,13 @@ def test_tf_airflow_config_uploads_data_source_to_s3(
 
 @pytest.mark.canary_quick
 def test_xgboost_airflow_config_uploads_data_source_to_s3(
-    sagemaker_session, cpu_instance_type, xgboost_latest_version, xgboost_latest_py_version
+    sagemaker_session, cpu_instance_type, xgboost_latest_version
 ):
     with timeout(seconds=AIRFLOW_CONFIG_TIMEOUT_IN_SECONDS):
         xgboost = XGBoost(
             entry_point=os.path.join(DATA_DIR, "dummy_script.py"),
             framework_version=xgboost_latest_version,
-            py_version=xgboost_latest_py_version,
+            py_version="py3",
             role=ROLE,
             sagemaker_session=sagemaker_session,
             instance_type=cpu_instance_type,

tests/unit/sagemaker/image_uris/expected_uris.py

@@ -25,9 +25,7 @@
 
 def framework_uri(repo, fw_version, account, py_version=None, processor="cpu", region=REGION):
     domain = ALTERNATE_DOMAINS.get(region, DOMAIN)
-    tag = "{}-{}".format(fw_version, processor)
-    if py_version:
-        tag = "-".join((tag, py_version))
+    tag = "-".join(x for x in (fw_version, processor, py_version) if x)
 
     return IMAGE_URI_FORMAT.format(account, region, domain, repo, tag)
 

tests/unit/sagemaker/image_uris/test_xgboost.py

@@ -44,7 +44,8 @@
     "us-west-2": "433757028032",
 }
 ALGO_VERSIONS = ("1", "latest")
-XGBOOST_FRAMEWORK_VERSIONS = ("0.90-2", "0.90-1", "1.0-1")
+XGBOOST_FRAMEWORK_CPU_ONLY_VERSIONS = ("0.90-2", "0.90-1", "1.0-1")
+XGBOOST_FRAMEWORK_CPU_GPU_VERSIONS = ("1.2-1",)
 
 FRAMEWORK_REGISTRIES = {
     "af-south-1": "510948584623",
@@ -74,22 +75,42 @@
 }
 
 
-@pytest.mark.parametrize("xgboost_framework_version", XGBOOST_FRAMEWORK_VERSIONS)
+@pytest.mark.parametrize("xgboost_framework_version", XGBOOST_FRAMEWORK_CPU_GPU_VERSIONS)
 def test_xgboost_framework(xgboost_framework_version):
     for region in regions.regions():
         uri = image_uris.retrieve(
             framework="xgboost",
             region=region,
             version=xgboost_framework_version,
-            py_version="py3",
         )
 
         expected = expected_uris.framework_uri(
             "sagemaker-xgboost",
             xgboost_framework_version,
             FRAMEWORK_REGISTRIES[region],
-            py_version="py3",
+            py_version=None,
+            processor=None,
+            region=region,
+        )
+        assert expected == uri
+
+
+@pytest.mark.parametrize("xgboost_framework_version", XGBOOST_FRAMEWORK_CPU_ONLY_VERSIONS)
+def test_xgboost_framework_cpu_only(xgboost_framework_version):
+    for region in regions.regions():
+        uri = image_uris.retrieve(
+            framework="xgboost",
             region=region,
+            version=xgboost_framework_version,
+        )
+
+        expected = expected_uris.framework_uri(
+            "sagemaker-xgboost",
+            xgboost_framework_version,
+            FRAMEWORK_REGISTRIES[region],
+            region=region,
+            py_version="py3",
+            processor="cpu",
         )
         assert expected == uri