Merge branch 'master' into processing-job-codeartifact-support

Author: akuma12

CHANGELOG.md

@@ -1,5 +1,89 @@
 # Changelog
 
+## v2.197.0 (2023-11-07)
+
+### Features
+
+ * PT2.1 SM Training/Inference DLC Release
+
+### Bug Fixes and Other Changes
+
+ * Release HuggingFace PT Neuronx training image 1.13.1
+ * HuggingFace PT Neuronx release in SDK
+
+## v2.196.0 (2023-10-27)
+
+### Features
+
+ * inference instance type conditioned on training instance type
+
+### Bug Fixes and Other Changes
+
+ * improved jumpstart tagging
+
+## v2.195.1 (2023-10-26)
+
+### Bug Fixes and Other Changes
+
+ * Allow either instance_type or instance_group to be defined in…
+ * enhance image_uris unit tests
+
+## v2.195.0 (2023-10-25)
+
+### Features
+
+ * jumpstart gated model artifacts
+ * jumpstart extract generated text from response
+ * jumpstart contruct payload utility
+
+### Bug Fixes and Other Changes
+
+ * relax upper bound on urllib in local mode requirements
+ * bump urllib3 version
+ * allow smdistributed to be enabled with torch_distributed.
+ * fix URL links
+
+### Documentation Changes
+
+ * remove python 2 reference
+ * update framework version links
+
+## v2.194.0 (2023-10-19)
+
+### Features
+
+ * Added register step in Jumpstart model
+ * jumpstart instance specific metric definitions
+
+### Bug Fixes and Other Changes
+
+ * Updates for DJL 0.24.0 Release
+ * use getter for resource-metadata dict
+ * add method to Model class to check if repack is needed
+
+## v2.193.0 (2023-10-18)
+
+### Features
+
+ * jumpstart model artifact instance type variants
+ * jumpstart instance specific hyperparameters
+ * Feature Processor event based triggers (#1132)
+ * Support job checkpoint in remote function
+ * jumpstart model package arn instance type variants
+
+### Bug Fixes and Other Changes
+
+ * Fix hyperlinks in feature_processor.scheduler parameter descriptions
+ * add image_uris_unit_test pytest mark
+ * bump apache-airflow to `v2.7.2`
+ * clone distribution in validate_distribution
+ * fix flaky Inference Recommender integration tests
+
+### Documentation Changes
+
+ * Update PipelineModel.register documentation
+ * specify that input_shape in no longer required for torch 2.0 mod…
+
 ## v2.192.1 (2023-10-13)
 
 ### Bug Fixes and Other Changes

VERSION

@@ -1 +1 @@
-2.192.2.dev0
+2.197.1.dev0

doc/amazon_sagemaker_featurestore.rst

@@ -230,9 +230,11 @@ The following code from the fraud detection example shows a minimal
        enable_online_store=True
    )
 
-Creating a feature group takes time as the data is loaded. You will need
-to wait until it is created before you can use it. You can check status
-using the following method.
+Creating a feature group takes time as the data is loaded. You will
+need to wait until it is created before you can use it. You can
+check status using the following method. Note that it can take
+approximately 10-15 minutes to provision an online ``FeatureGroup``
+with the ``InMemory`` ``StorageType``.
 
 .. code:: python
 
@@ -480,7 +482,9 @@ Feature Store `DatasetBuilder API Reference
 .. rubric:: Delete a feature group
    :name: bCe9CA61b78
 
-You can delete a feature group with the ``delete`` function.
+You can delete a feature group with the ``delete`` function. Note that it
+can take approximately 10-15 minutes to delete an online ``FeatureGroup``
+with the ``InMemory`` ``StorageType``.
 
 .. code:: python
 

doc/overview.rst

@@ -32,7 +32,7 @@ After you train a model, you can save it, and then serve the model as an endpoin
 Prepare a Training script
 =========================
 
-Your training script must be a Python 2.7 or 3.6 compatible source file.
+Your training script must be a 3.6 compatible source file.
 
 The training script is very similar to a training script you might run outside of SageMaker, but you can access useful properties about the training environment through various environment variables, including the following:
 

requirements/extras/local_requirements.txt

@@ -1,3 +1,3 @@
-urllib3>=1.26.8,<1.26.15
+urllib3>=1.26.8,<3.0.0
 docker>=5.0.2,<7.0.0
 PyYAML>=5.4.1,<7

requirements/extras/test_requirements.txt

@@ -12,7 +12,7 @@ awslogs==0.14.0
 black==22.3.0
 stopit==1.1.2
 # Update tox.ini to have correct version of airflow constraints file
-apache-airflow==2.7.1
+apache-airflow==2.7.2
 apache-airflow-providers-amazon==7.2.1
 attrs>=23.1.0,<24
 fabric==2.6.0
@@ -24,7 +24,7 @@ pandas>=1.3.5,<1.5
 scikit-learn==1.3.0
 cloudpickle==2.2.1
 scipy==1.10.1
-urllib3>=1.26.8,<1.26.15
+urllib3>=1.26.8,<3.0.0
 docker>=5.0.2,<7.0.0
 PyYAML==6.0
 pyspark==3.3.1

src/sagemaker/chainer/estimator.py

@@ -108,7 +108,7 @@ def __init__(
             framework_version (str): Chainer version you want to use for
                 executing your model training code. Defaults to ``None``. Required unless
                 ``image_uri`` is provided. List of supported versions:
-                https://github.com/aws/sagemaker-python-sdk#chainer-sagemaker-estimators.
+                https://sagemaker.readthedocs.io/en/stable/frameworks/chainer/using_chainer.html#using-chainer-with-the-sagemaker-python-sdk.
             image_uri (str): If specified, the estimator will use this image
                 for training and hosting, instead of selecting the appropriate
                 SageMaker official image based on framework_version and

src/sagemaker/djl_inference/model.py

@@ -781,7 +781,7 @@ def serving_image_uri(self, region_name):
             str: The appropriate image URI based on the given parameters.
         """
         if not self.djl_version:
-            self.djl_version = "0.23.0"
+            self.djl_version = "0.24.0"
 
         return image_uris.retrieve(
             self._framework(),

src/sagemaker/estimator.py

@@ -71,7 +71,7 @@
 from sagemaker.utils import instance_supports_kms
 from sagemaker.job import _Job
 from sagemaker.jumpstart.utils import (
-    add_jumpstart_tags,
+    add_jumpstart_uri_tags,
     get_jumpstart_base_name_if_jumpstart_model,
     update_inference_tags_with_jumpstart_training_tags,
 )
@@ -101,6 +101,7 @@
 )
 from sagemaker.workflow import is_pipeline_variable
 from sagemaker.workflow.entities import PipelineVariable
+from sagemaker.workflow.parameters import ParameterString
 from sagemaker.workflow.pipeline_context import PipelineSession, runnable_by_pipeline
 
 logger = logging.getLogger(__name__)
@@ -576,9 +577,7 @@ def __init__(
         self.entry_point = entry_point
         self.dependencies = dependencies or []
         self.uploaded_code: Optional[UploadedCode] = None
-        self.tags = add_jumpstart_tags(
-            tags=tags, training_model_uri=self.model_uri, training_script_uri=self.source_dir
-        )
+
         if self.instance_type in ("local", "local_gpu"):
             if self.instance_type == "local_gpu" and self.instance_count > 1:
                 raise RuntimeError("Distributed Training in Local GPU is not supported")
@@ -591,6 +590,15 @@ def __init__(
         else:
             self.sagemaker_session = sagemaker_session or Session()
 
+        self.tags = (
+            add_jumpstart_uri_tags(
+                tags=tags, training_model_uri=self.model_uri, training_script_uri=self.source_dir
+            )
+            if getattr(self.sagemaker_session, "settings", None) is not None
+            and self.sagemaker_session.settings.include_jumpstart_tags
+            else tags
+        )
+
         self.base_job_name = base_job_name
         self._current_job_name = None
         if (
@@ -3198,6 +3206,7 @@ class Framework(EstimatorBase):
     """
 
     _framework_name = None
+    UNSUPPORTED_DLC_IMAGE_FOR_SM_PARALLELISM = ("2.0.1-gpu-py310-cu121", "2.0-gpu-py310-cu121")
 
     def __init__(
         self,
@@ -3816,6 +3825,7 @@ def _distribution_configuration(self, distribution):
 
         mpi_enabled = False
         smdataparallel_enabled = False
+        p5_enabled = False
         if "instance_groups" in distribution:
             distribution_config["sagemaker_distribution_instance_groups"] = distribution[
                 "instance_groups"
@@ -3843,16 +3853,44 @@ def _distribution_configuration(self, distribution):
                 "custom_mpi_options", ""
             )
 
-            if get_mp_parameters(distribution):
-                distribution_config["mp_parameters"] = get_mp_parameters(distribution)
-
-        elif "modelparallel" in distribution.get("smdistributed", {}):
-            raise ValueError("Cannot use Model Parallelism without MPI enabled!")
-
         if "smdistributed" in distribution:
             # smdistributed strategy selected
+            if get_mp_parameters(distribution):
+                distribution_config["mp_parameters"] = get_mp_parameters(distribution)
+            # first make sure torch_distributed is enabled if instance type is p5
+            torch_distributed_enabled = False
+            if "torch_distributed" in distribution:
+                torch_distributed_enabled = distribution.get("torch_distributed").get(
+                    "enabled", False
+                )
             smdistributed = distribution["smdistributed"]
             smdataparallel_enabled = smdistributed.get("dataparallel", {}).get("enabled", False)
+            if isinstance(self.instance_type, ParameterString):
+                p5_enabled = "p5.48xlarge" in self.instance_type.default_value
+            elif isinstance(self.instance_type, str):
+                p5_enabled = "p5.48xlarge" in self.instance_type
+            else:
+                for instance in self.instance_groups:
+                    if "p5.48xlarge" in instance._to_request_dict().get("InstanceType", ()):
+                        p5_enabled = True
+                        break
+
+            img_uri = "" if self.image_uri is None else self.image_uri
+            for unsupported_image in Framework.UNSUPPORTED_DLC_IMAGE_FOR_SM_PARALLELISM:
+                if (
+                    unsupported_image in img_uri and not torch_distributed_enabled
+                ):  # disabling DLC images with CUDA12
+                    raise ValueError(
+                        f"SMDistributed is currently incompatible with DLC image: {img_uri}. "
+                        "(Could be due to CUDA version being greater than 11.)"
+                    )
+            if (
+                not torch_distributed_enabled and p5_enabled
+            ):  # disabling p5 when torch distributed is disabled
+                raise ValueError(
+                    "SMModelParallel and SMDataParallel currently do not support p5 instances."
+                )
+            # smdistributed strategy selected with supported instance type
             distribution_config[self.LAUNCH_SM_DDP_ENV_NAME] = smdataparallel_enabled
             distribution_config[self.INSTANCE_TYPE] = self.instance_type
             if smdataparallel_enabled:

src/sagemaker/feature_store/feature_processor/__init__.py

@@ -30,8 +30,16 @@
     to_pipeline,
     schedule,
     describe,
+    put_trigger,
+    delete_trigger,
+    enable_trigger,
+    disable_trigger,
     delete_schedule,
     list_pipelines,
     execute,
     TransformationCode,
+    FeatureProcessorPipelineEvents,
+)
+from sagemaker.feature_store.feature_processor._enums import (  # noqa: F401
+    FeatureProcessorPipelineExecutionStatus,
 )