Merge branch 'master' into add_target_container_hostname_to_predict

Author: kwnath

CHANGELOG.md

@@ -1,5 +1,17 @@
 # Changelog
 
+## v2.247.0 (2025-06-13)
+
+### Features
+
+ * Add support for MetricDefinitions in ModelTrainer
+
+### Bug Fixes and Other Changes
+
+ * update jumpstart region_config, update image_uri_configs 06-12-2025 07:18:12 PST
+ * Add ignore_patterns in ModelTrainer to ignore specific files/folders
+ * Allow import failure for internal _hashlib module
+
 ## v2.246.0 (2025-06-04)
 
 ### Features

VERSION

@@ -1 +1 @@
-2.246.1.dev0
+2.247.1.dev0

src/sagemaker/image_uri_config/pytorch.json

@@ -1705,7 +1705,8 @@
             "2.3": "2.3.0",
             "2.4": "2.4.0",
             "2.5": "2.5.1",
-            "2.6": "2.6.0"
+            "2.6": "2.6.0",
+            "2.7": "2.7.1"
         },
         "versions": {
             "0.4.0": {
@@ -2946,6 +2947,51 @@
                     "us-west-2": "763104351884"
                 },
                 "repository": "pytorch-training"
+            },
+            "2.7.1": {
+                "py_versions": [
+                    "py312"
+                ],
+                "registries": {
+                    "af-south-1": "626614931356",
+                    "ap-east-1": "871362719292",
+                    "ap-east-2": "975050140332",
+                    "ap-northeast-1": "763104351884",
+                    "ap-northeast-2": "763104351884",
+                    "ap-northeast-3": "364406365360",
+                    "ap-south-1": "763104351884",
+                    "ap-south-2": "772153158452",
+                    "ap-southeast-1": "763104351884",
+                    "ap-southeast-2": "763104351884",
+                    "ap-southeast-3": "907027046896",
+                    "ap-southeast-4": "457447274322",
+                    "ap-southeast-5": "550225433462",
+                    "ap-southeast-7": "590183813437",
+                    "ca-central-1": "763104351884",
+                    "ca-west-1": "204538143572",
+                    "cn-north-1": "727897471807",
+                    "cn-northwest-1": "727897471807",
+                    "eu-central-1": "763104351884",
+                    "eu-central-2": "380420809688",
+                    "eu-north-1": "763104351884",
+                    "eu-south-1": "692866216735",
+                    "eu-south-2": "503227376785",
+                    "eu-west-1": "763104351884",
+                    "eu-west-2": "763104351884",
+                    "eu-west-3": "763104351884",
+                    "il-central-1": "780543022126",
+                    "me-central-1": "914824155844",
+                    "me-south-1": "217643126080",
+                    "mx-central-1": "637423239942",
+                    "sa-east-1": "763104351884",
+                    "us-east-1": "763104351884",
+                    "us-east-2": "763104351884",
+                    "us-gov-east-1": "446045086412",
+                    "us-gov-west-1": "442386744353",
+                    "us-west-1": "763104351884",
+                    "us-west-2": "763104351884"
+                },
+                "repository": "pytorch-training"
             }
         }
     }

src/sagemaker/jumpstart/region_config.json

@@ -7,6 +7,10 @@
         "content_bucket": "jumpstart-cache-prod-ap-east-1",
         "gated_content_bucket": "jumpstart-private-cache-prod-ap-east-1"
     },
+    "ap-east-2": {
+        "content_bucket": "jumpstart-cache-prod-ap-east-2",
+        "gated_content_bucket": "jumpstart-private-cache-prod-ap-east-2"
+    },
     "ap-northeast-1": {
         "content_bucket": "jumpstart-cache-prod-ap-northeast-1",
         "gated_content_bucket": "jumpstart-private-cache-prod-ap-northeast-1",

src/sagemaker/modules/configs.py

@@ -21,7 +21,7 @@
 
 from __future__ import absolute_import
 
-from typing import Optional, Union
+from typing import Optional, Union, List
 from pydantic import BaseModel, model_validator, ConfigDict
 
 import sagemaker_core.shapes as shapes
@@ -42,6 +42,7 @@
     RemoteDebugConfig,
     SessionChainingConfig,
     InstanceGroup,
+    MetricDefinition,
 )
 
 from sagemaker.modules.utils import convert_unassigned_to_none
@@ -68,6 +69,7 @@
     "Compute",
     "Networking",
     "InputData",
+    "MetricDefinition",
 ]
 
 
@@ -96,12 +98,23 @@ class SourceCode(BaseConfig):
         command (Optional[str]):
             The command(s) to execute in the training job container. Example: "python my_script.py".
             If not specified, entry_script must be provided.
+        ignore_patterns: (Optional[List[str]]) :
+            The ignore patterns to ignore specific files/folders when uploading to S3. If not specified,
+            default to: ['.env', '.git', '__pycache__', '.DS_Store', '.cache', '.ipynb_checkpoints'].
     """
 
     source_dir: Optional[str] = None
     requirements: Optional[str] = None
     entry_script: Optional[str] = None
     command: Optional[str] = None
+    ignore_patterns: Optional[List[str]] = [
+        ".env",
+        ".git",
+        "__pycache__",
+        ".DS_Store",
+        ".cache",
+        ".ipynb_checkpoints",
+    ]
 
 
 class Compute(shapes.ResourceConfig):

src/sagemaker/modules/train/model_trainer.py

@@ -66,6 +66,7 @@
     RemoteDebugConfig,
     SessionChainingConfig,
     InputData,
+    MetricDefinition,
 )
 
 from sagemaker.modules.local_core.local_container import _LocalContainer
@@ -119,7 +120,8 @@ class ModelTrainer(BaseModel):
         from sagemaker.modules.train import ModelTrainer
         from sagemaker.modules.configs import SourceCode, Compute, InputData
 
-        source_code = SourceCode(source_dir="source", entry_script="train.py")
+        ignore_patterns = ['.env', '.git', '__pycache__', '.DS_Store', 'data']
+        source_code = SourceCode(source_dir="source", entry_script="train.py", ignore_patterns=ignore_patterns)
         training_image = "123456789012.dkr.ecr.us-west-2.amazonaws.com/my-training-image"
         model_trainer = ModelTrainer(
             training_image=training_image,
@@ -238,6 +240,7 @@ class ModelTrainer(BaseModel):
     _infra_check_config: Optional[InfraCheckConfig] = PrivateAttr(default=None)
     _session_chaining_config: Optional[SessionChainingConfig] = PrivateAttr(default=None)
     _remote_debug_config: Optional[RemoteDebugConfig] = PrivateAttr(default=None)
+    _metric_definitions: Optional[List[MetricDefinition]] = PrivateAttr(default=None)
 
     _temp_recipe_train_dir: Optional[TemporaryDirectory] = PrivateAttr(default=None)
 
@@ -654,6 +657,7 @@ def train(
                     channel_name=SM_CODE,
                     data_source=self.source_code.source_dir,
                     key_prefix=input_data_key_prefix,
+                    ignore_patterns=self.source_code.ignore_patterns,
                 )
                 final_input_data_config.append(source_code_channel)
 
@@ -675,6 +679,7 @@ def train(
                 channel_name=SM_DRIVERS,
                 data_source=tmp_dir.name,
                 key_prefix=input_data_key_prefix,
+                ignore_patterns=self.source_code.ignore_patterns,
             )
             final_input_data_config.append(sm_drivers_channel)
 
@@ -693,6 +698,7 @@ def train(
             training_image_config=self.training_image_config,
             container_entrypoint=container_entrypoint,
             container_arguments=container_arguments,
+            metric_definitions=self._metric_definitions,
         )
 
         resource_config = self.compute._to_resource_config()
@@ -755,7 +761,11 @@ def train(
             local_container.train(wait)
 
     def create_input_data_channel(
-        self, channel_name: str, data_source: DataSourceType, key_prefix: Optional[str] = None
+        self,
+        channel_name: str,
+        data_source: DataSourceType,
+        key_prefix: Optional[str] = None,
+        ignore_patterns: Optional[List[str]] = None,
     ) -> Channel:
         """Create an input data channel for the training job.
 
@@ -771,6 +781,10 @@ def create_input_data_channel(
 
                 If specified, local data will be uploaded to:
                 ``s3://<default_bucket_path>/<key_prefix>/<channel_name>/``
+            ignore_patterns: (Optional[List[str]]) :
+                The ignore patterns to ignore specific files/folders when uploading to S3.
+                If not specified, default to: ['.env', '.git', '__pycache__', '.DS_Store',
+                '.cache', '.ipynb_checkpoints'].
         """
         channel = None
         if isinstance(data_source, str):
@@ -810,11 +824,28 @@ def create_input_data_channel(
                     )
                     if self.sagemaker_session.default_bucket_prefix:
                         key_prefix = f"{self.sagemaker_session.default_bucket_prefix}/{key_prefix}"
-                    s3_uri = self.sagemaker_session.upload_data(
-                        path=data_source,
-                        bucket=self.sagemaker_session.default_bucket(),
-                        key_prefix=key_prefix,
-                    )
+                    if ignore_patterns and _is_valid_path(data_source, path_type="Directory"):
+                        tmp_dir = TemporaryDirectory()
+                        copied_path = os.path.join(
+                            tmp_dir.name, os.path.basename(os.path.normpath(data_source))
+                        )
+                        shutil.copytree(
+                            data_source,
+                            copied_path,
+                            dirs_exist_ok=True,
+                            ignore=shutil.ignore_patterns(*ignore_patterns),
+                        )
+                        s3_uri = self.sagemaker_session.upload_data(
+                            path=copied_path,
+                            bucket=self.sagemaker_session.default_bucket(),
+                            key_prefix=key_prefix,
+                        )
+                    else:
+                        s3_uri = self.sagemaker_session.upload_data(
+                            path=data_source,
+                            bucket=self.sagemaker_session.default_bucket(),
+                            key_prefix=key_prefix,
+                        )
                     channel = Channel(
                         channel_name=channel_name,
                         data_source=DataSource(
@@ -861,7 +892,9 @@ def _get_input_data_config(
                 channels.append(input_data)
             elif isinstance(input_data, InputData):
                 channel = self.create_input_data_channel(
-                    input_data.channel_name, input_data.data_source, key_prefix=key_prefix
+                    input_data.channel_name,
+                    input_data.data_source,
+                    key_prefix=key_prefix,
                 )
                 channels.append(channel)
             else:
@@ -1260,3 +1293,33 @@ def with_checkpoint_config(
         """
         self.checkpoint_config = checkpoint_config or configs.CheckpointConfig()
         return self
+
+    def with_metric_definitions(
+        self, metric_definitions: List[MetricDefinition]
+    ) -> "ModelTrainer":  # noqa: D412
+        """Set the metric definitions for the training job.
+
+        Example:
+
+        .. code:: python
+
+            from sagemaker.modules.train import ModelTrainer
+            from sagemaker.modules.configs import MetricDefinition
+
+            metric_definitions = [
+                MetricDefinition(
+                    name="loss",
+                    regex="Loss: (.*?)",
+                )
+            ]
+
+            model_trainer = ModelTrainer(
+                ...
+            ).with_metric_definitions(metric_definitions)
+
+        Args:
+            metric_definitions (List[MetricDefinition]):
+                The metric definitions for the training job.
+        """
+        self._metric_definitions = metric_definitions
+        return self

src/sagemaker/session.py

@@ -7509,7 +7509,7 @@ def get_model_package_args(
     if source_uri is not None:
         model_package_args["source_uri"] = source_uri
     if model_life_cycle is not None:
-        model_package_args["model_life_cycle"] = model_life_cycle
+        model_package_args["model_life_cycle"] = model_life_cycle._to_request_dict()
     if model_card is not None:
         original_req = model_card._create_request_args()
         if original_req.get("ModelCardName") is not None:

src/sagemaker/workflow/utilities.py

@@ -21,7 +21,15 @@
 import hashlib
 from urllib.parse import unquote, urlparse
 from contextlib import contextmanager
-from _hashlib import HASH as Hash
+
+try:
+    # _hashlib is an internal python module, and is not present in
+    # statically linked interpreters.
+    from _hashlib import HASH as Hash
+except ImportError:
+    import typing
+
+    Hash = typing.Any
 
 from sagemaker.utils import base_from_name
 from sagemaker.workflow.parameters import Parameter

tests/integ/sagemaker/workflow/test_model_create_and_registration.py

@@ -48,6 +48,7 @@
 from sagemaker.s3 import S3Uploader
 from sagemaker.sklearn import SKLearnModel, SKLearnProcessor
 from sagemaker.mxnet.model import MXNetModel
+from sagemaker.model_life_cycle import ModelLifeCycle
 from sagemaker.workflow.condition_step import ConditionStep
 from sagemaker.workflow.parameters import ParameterInteger, ParameterString
 from sagemaker.workflow.pipeline import Pipeline
@@ -1005,11 +1006,11 @@ def test_model_registration_with_model_life_cycle_object(
         py_version="py3",
         role=role,
     )
-    create_model_life_cycle = {
-        "Stage": "Development",
-        "StageStatus": "In-Progress",
-        "StageDescription": "Development In Progress",
-    }
+    create_model_life_cycle = ModelLifeCycle(
+        stage="Development",
+        stage_status="In-Progress",
+        stage_description="Development In Progress",
+    )
 
     step_register = RegisterModel(
         name="MyRegisterModelStep",

tests/integ/test_model_package.py

@@ -103,7 +103,7 @@ def test_update_model_life_cycle_model_package(sagemaker_session):
         inference_instances=["ml.m5.large"],
         transform_instances=["ml.m5.large"],
         model_package_group_name=model_group_name,
-        model_life_cycle=create_model_life_cycle._to_request_dict(),
+        model_life_cycle=create_model_life_cycle,
     )
 
     desc_model_package = sagemaker_session.sagemaker_client.describe_model_package(