|
| 1 | +# Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved. |
| 2 | +# |
| 3 | +# Licensed under the Apache License, Version 2.0 (the "License"). You |
| 4 | +# may not use this file except in compliance with the License. A copy of |
| 5 | +# the License is located at |
| 6 | +# |
| 7 | +# http://aws.amazon.com/apache2.0/ |
| 8 | +# |
| 9 | +# or in the "license" file accompanying this file. This file is |
| 10 | +# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF |
| 11 | +# ANY KIND, either express or implied. See the License for the specific |
| 12 | +# language governing permissions and limitations under the License. |
| 13 | +from __future__ import absolute_import |
| 14 | + |
| 15 | +import logging |
| 16 | + |
| 17 | +from sagemaker.estimator import Framework |
| 18 | +from sagemaker.fw_registry import default_framework_uri |
| 19 | +from sagemaker.fw_utils import framework_name_from_image, empty_framework_version_warning |
| 20 | +from sagemaker.sklearn.defaults import SKLEARN_VERSION |
| 21 | +from sagemaker.sklearn.model import SKLearnModel |
| 22 | +from sagemaker.vpc_utils import VPC_CONFIG_DEFAULT |
| 23 | + |
| 24 | +logging.basicConfig() |
| 25 | +logger = logging.getLogger('sagemaker') |
| 26 | + |
| 27 | + |
| 28 | +class SKLearn(Framework): |
| 29 | + """Handle end-to-end training and deployment of custom Scikit-learn code.""" |
| 30 | + |
| 31 | + __framework_name__ = "scikit-learn" |
| 32 | + |
| 33 | + def __init__(self, entry_point, framework_version=SKLEARN_VERSION, source_dir=None, hyperparameters=None, |
| 34 | + py_version='py3', image_name=None, **kwargs): |
| 35 | + """ |
| 36 | + This ``Estimator`` executes an Scikit-learn script in a managed Scikit-learn execution environment, within a |
| 37 | + SageMaker Training Job. The managed Scikit-learn environment is an Amazon-built Docker container that executes |
| 38 | + functions defined in the supplied ``entry_point`` Python script. |
| 39 | +
|
| 40 | + Training is started by calling :meth:`~sagemaker.amazon.estimator.Framework.fit` on this Estimator. |
| 41 | + After training is complete, calling :meth:`~sagemaker.amazon.estimator.Framework.deploy` creates a |
| 42 | + hosted SageMaker endpoint and returns an :class:`~sagemaker.amazon.sklearn.model.SKLearnPredictor` instance |
| 43 | + that can be used to perform inference against the hosted model. |
| 44 | +
|
| 45 | + Technical documentation on preparing Scikit-learn scripts for SageMaker training and using the Scikit-learn |
| 46 | + Estimator is available on the project home-page: https://github.com/aws/sagemaker-python-sdk |
| 47 | +
|
| 48 | + Args: |
| 49 | + entry_point (str): Path (absolute or relative) to the Python source file which should be executed |
| 50 | + as the entry point to training. This should be compatible with either Python 2.7 or Python 3.5. |
| 51 | + source_dir (str): Path (absolute or relative) to a directory with any other training |
| 52 | + source code dependencies aside from tne entry point file (default: None). Structure within this |
| 53 | + directory are preserved when training on Amazon SageMaker. |
| 54 | + hyperparameters (dict): Hyperparameters that will be used for training (default: None). |
| 55 | + The hyperparameters are made accessible as a dict[str, str] to the training code on SageMaker. |
| 56 | + For convenience, this accepts other types for keys and values, but ``str()`` will be called |
| 57 | + to convert them before training. |
| 58 | + py_version (str): Python version you want to use for executing your model training code (default: 'py2'). |
| 59 | + One of 'py2' or 'py3'. |
| 60 | + framework_version (str): Scikit-learn version you want to use for executing your model training code. |
| 61 | + List of supported versions https://github.com/aws/sagemaker-python-sdk#sklearn-sagemaker-estimators |
| 62 | + image_name (str): If specified, the estimator will use this image for training and hosting, instead of |
| 63 | + selecting the appropriate SageMaker official image based on framework_version and py_version. It can |
| 64 | + be an ECR url or dockerhub image and tag. |
| 65 | + Examples: |
| 66 | + 123.dkr.ecr.us-west-2.amazonaws.com/my-custom-image:1.0 |
| 67 | + custom-image:latest. |
| 68 | + **kwargs: Additional kwargs passed to the :class:`~sagemaker.estimator.Framework` constructor. |
| 69 | + """ |
| 70 | + # SciKit-Learn does not support distributed training or training on GPU instance types. Fail fast. |
| 71 | + train_instance_type = kwargs.get('train_instance_type') |
| 72 | + _validate_not_gpu_instance_type(train_instance_type) |
| 73 | + |
| 74 | + train_instance_count = kwargs.get('train_instance_count') |
| 75 | + if train_instance_count: |
| 76 | + if train_instance_count != 1: |
| 77 | + raise AttributeError("SciKit-Learn does not support distributed training. " |
| 78 | + "Please remove the 'train_instance_count' argument or set " |
| 79 | + "'train_instance_count=1' when initializing SKLearn.") |
| 80 | + super(SKLearn, self).__init__(entry_point, source_dir, hyperparameters, image_name=image_name, |
| 81 | + **dict(kwargs, train_instance_count=1)) |
| 82 | + |
| 83 | + self.py_version = py_version |
| 84 | + |
| 85 | + if framework_version is None: |
| 86 | + logger.warning(empty_framework_version_warning(SKLEARN_VERSION, SKLEARN_VERSION)) |
| 87 | + self.framework_version = framework_version or SKLEARN_VERSION |
| 88 | + |
| 89 | + if image_name is None: |
| 90 | + image_tag = "{}-{}-{}".format(framework_version, "cpu", py_version) |
| 91 | + self.image_name = default_framework_uri( |
| 92 | + SKLearn.__framework_name__, |
| 93 | + self.sagemaker_session.boto_region_name, |
| 94 | + image_tag) |
| 95 | + |
| 96 | + def create_model(self, model_server_workers=None, role=None, |
| 97 | + vpc_config_override=VPC_CONFIG_DEFAULT, **kwargs): |
| 98 | + """Create a SageMaker ``SKLearnModel`` object that can be deployed to an ``Endpoint``. |
| 99 | +
|
| 100 | + Args: |
| 101 | + role (str): The ``ExecutionRoleArn`` IAM Role ARN for the ``Model``, which is also used during |
| 102 | + transform jobs. If not specified, the role from the Estimator will be used. |
| 103 | + model_server_workers (int): Optional. The number of worker processes used by the inference server. |
| 104 | + If None, server will use one worker per vCPU. |
| 105 | + vpc_config_override (dict[str, list[str]]): Optional override for VpcConfig set on the model. |
| 106 | + Default: use subnets and security groups from this Estimator. |
| 107 | + * 'Subnets' (list[str]): List of subnet ids. |
| 108 | + * 'SecurityGroupIds' (list[str]): List of security group ids. |
| 109 | + **kwargs: Passed to initialization of ``SKLearnModel``. |
| 110 | +
|
| 111 | + Returns: |
| 112 | + sagemaker.sklearn.model.SKLearnModel: A SageMaker ``SKLearnModel`` object. |
| 113 | + See :func:`~sagemaker.sklearn.model.SKLearnModel` for full details. |
| 114 | + """ |
| 115 | + role = role or self.role |
| 116 | + return SKLearnModel(self.model_data, role, self.entry_point, source_dir=self._model_source_dir(), |
| 117 | + enable_cloudwatch_metrics=self.enable_cloudwatch_metrics, name=self._current_job_name, |
| 118 | + container_log_level=self.container_log_level, code_location=self.code_location, |
| 119 | + py_version=self.py_version, framework_version=self.framework_version, |
| 120 | + model_server_workers=model_server_workers, image=self.image_name, |
| 121 | + sagemaker_session=self.sagemaker_session, |
| 122 | + vpc_config=self.get_vpc_config(vpc_config_override), |
| 123 | + **kwargs) |
| 124 | + |
| 125 | + @classmethod |
| 126 | + def _prepare_init_params_from_job_description(cls, job_details, model_channel_name=None): |
| 127 | + """Convert the job description to init params that can be handled by the class constructor |
| 128 | +
|
| 129 | + Args: |
| 130 | + job_details: the returned job details from a describe_training_job API call. |
| 131 | +
|
| 132 | + Returns: |
| 133 | + dictionary: The transformed init_params |
| 134 | +
|
| 135 | + """ |
| 136 | + init_params = super(SKLearn, cls)._prepare_init_params_from_job_description(job_details) |
| 137 | + |
| 138 | + image_name = init_params.pop('image') |
| 139 | + framework, py_version, _ = framework_name_from_image(image_name) |
| 140 | + init_params['py_version'] = py_version |
| 141 | + |
| 142 | + if framework and framework != cls.__framework_name__: |
| 143 | + training_job_name = init_params['base_job_name'] |
| 144 | + raise ValueError("Training job: {} didn't use image for requested framework".format(training_job_name)) |
| 145 | + elif not framework: |
| 146 | + # If we were unable to parse the framework name from the image it is not one of our |
| 147 | + # officially supported images, in this case just add the image to the init params. |
| 148 | + init_params['image_name'] = image_name |
| 149 | + return init_params |
| 150 | + |
| 151 | + |
| 152 | +def _validate_not_gpu_instance_type(training_instance_type): |
| 153 | + gpu_instance_types = ['ml.p2.xlarge', 'ml.p2.8xlarge', 'ml.p2.16xlarge', |
| 154 | + 'ml.p3.xlarge', 'ml.p3.8xlarge', 'ml.p3.16xlarge'] |
| 155 | + |
| 156 | + if training_instance_type in gpu_instance_types: |
| 157 | + raise ValueError("GPU training in not supported for SciKit-Learn. " |
| 158 | + "Please pick a different instance type from here: " |
| 159 | + "https://aws.amazon.com/ec2/instance-types/") |
0 commit comments