Merge branch 'master' into isort

Author: sage-maker

CHANGELOG.md

@@ -1,5 +1,17 @@
 # Changelog
 
+## v2.251.0 (2025-08-21)
+
+### Features
+
+ * support pipeline versioning
+
+### Bug Fixes and Other Changes
+
+ * GPT OSS Hotfix
+ * dockerfile stuck on interactive shell
+ * add sleep for model deployment
+
 ## v2.250.0 (2025-08-08)
 
 ### Features

VERSION

@@ -1 +1 @@
-2.250.1.dev0
+2.251.1.dev0

pyproject.toml

@@ -32,7 +32,7 @@ classifiers = [
 ]
 dependencies = [
   "attrs>=24,<26",
-  "boto3>=1.35.36,<2.0",
+  "boto3>=1.39.5,<2.0",
   "cloudpickle>=2.2.1",
   "docker",
   "fastapi",

src/sagemaker/image_uri_config/huggingface-tei-cpu.json

@@ -197,6 +197,53 @@
                 "container_version": {
                     "cpu": "ubuntu22.04"
                 }
+            },
+            "1.8.0":{
+                "py_versions": [
+                    "py310"
+                ],
+                "registries": {
+                    "af-south-1": "510948584623",
+                    "ap-east-1": "651117190479",
+                    "ap-northeast-1": "354813040037",
+                    "ap-northeast-2": "366743142698",
+                    "ap-northeast-3": "867004704886",
+                    "ap-south-1": "720646828776",
+                    "ap-south-2": "628508329040",
+                    "ap-southeast-1": "121021644041",
+                    "ap-southeast-2": "783357654285",
+                    "ap-southeast-3": "951798379941",
+                    "ap-southeast-4": "106583098589",
+                    "ca-central-1": "341280168497",
+                    "ca-west-1": "190319476487",
+                    "cn-north-1": "450853457545",
+                    "cn-northwest-1": "451049120500",
+                    "eu-central-1": "492215442770",
+                    "eu-central-2": "680994064768",
+                    "eu-north-1": "662702820516",
+                    "eu-south-1": "978288397137",
+                    "eu-south-2": "104374241257",
+                    "eu-west-1": "141502667606",
+                    "eu-west-2": "764974769150",
+                    "eu-west-3": "659782779980",
+                    "il-central-1": "898809789911",
+                    "me-central-1": "272398656194",
+                    "me-south-1": "801668240914",
+                    "sa-east-1": "737474898029",
+                    "us-east-1": "683313688378",
+                    "us-east-2": "257758044811",
+                    "us-gov-east-1": "237065988967",
+                    "us-gov-west-1": "414596584902",
+                    "us-iso-east-1": "833128469047",
+                    "us-isob-east-1": "281123927165",
+                    "us-west-1": "746614075791",
+                    "us-west-2": "246618743249"
+                },
+                "tag_prefix": "2.0.1-tei1.8.0",
+                "repository": "tei-cpu",
+                "container_version": {
+                    "cpu": "ubuntu22.04"
+                }
             }
         }
     }

src/sagemaker/image_uri_config/huggingface-tei.json

@@ -197,6 +197,53 @@
                 "container_version": {
                     "gpu": "cu122-ubuntu22.04"
                 }
+            },
+            "1.8.0": {
+                "py_versions": [
+                    "py310"
+                ],
+                "registries": {
+                    "af-south-1": "510948584623",
+                    "ap-east-1": "651117190479",
+                    "ap-northeast-1": "354813040037",
+                    "ap-northeast-2": "366743142698",
+                    "ap-northeast-3": "867004704886",
+                    "ap-south-1": "720646828776",
+                    "ap-south-2": "628508329040",
+                    "ap-southeast-1": "121021644041",
+                    "ap-southeast-2": "783357654285",
+                    "ap-southeast-3": "951798379941",
+                    "ap-southeast-4": "106583098589",
+                    "ca-central-1": "341280168497",
+                    "ca-west-1": "190319476487",
+                    "cn-north-1": "450853457545",
+                    "cn-northwest-1": "451049120500",
+                    "eu-central-1": "492215442770",
+                    "eu-central-2": "680994064768",
+                    "eu-north-1": "662702820516",
+                    "eu-south-1": "978288397137",
+                    "eu-south-2": "104374241257",
+                    "eu-west-1": "141502667606",
+                    "eu-west-2": "764974769150",
+                    "eu-west-3": "659782779980",
+                    "il-central-1": "898809789911",
+                    "me-central-1": "272398656194",
+                    "me-south-1": "801668240914",
+                    "sa-east-1": "737474898029",
+                    "us-east-1": "683313688378",
+                    "us-east-2": "257758044811",
+                    "us-gov-east-1": "237065988967",
+                    "us-gov-west-1": "414596584902",
+                    "us-iso-east-1": "833128469047",
+                    "us-isob-east-1": "281123927165",
+                    "us-west-1": "746614075791",
+                    "us-west-2": "246618743249"
+                },
+                "tag_prefix": "2.0.1-tei1.8.0",
+                "repository": "tei",
+                "container_version": {
+                    "gpu": "cu122-ubuntu22.04"
+                }
             }
         }
     }

src/sagemaker/modules/train/sm_recipes/utils.py

@@ -136,6 +136,7 @@ def _get_trainining_recipe_gpu_model_name_and_script(model_type: str):
         "mistral": ("mistral", "mistral_pretrain.py"),
         "mixtral": ("mixtral", "mixtral_pretrain.py"),
         "deepseek": ("deepseek", "deepseek_pretrain.py"),
+        "gpt_oss": ("custom_model", "custom_pretrain.py"),
     }
 
     for key in model_type_to_script:

src/sagemaker/pytorch/estimator.py

@@ -99,6 +99,7 @@ def _get_training_recipe_gpu_script(code_dir, recipe, source_dir):
         "mistral": ("mistral", "mistral_pretrain.py"),
         "mixtral": ("mixtral", "mixtral_pretrain.py"),
         "deepseek": ("deepseek", "deepseek_pretrain.py"),
+        "gpt_oss": ("custom_model", "custom_pretrain.py"),
     }
 
     if "model" not in recipe:

src/sagemaker/workflow/pipeline.py

@@ -125,6 +125,15 @@ def __init__(
             self.sagemaker_session.boto_session.client("scheduler"),
         )
 
+    @property
+    def latest_pipeline_version_id(self):
+        """Retrieves the latest version id of this pipeline"""
+        summaries = self.list_pipeline_versions(max_results=1)["PipelineVersionSummaries"]
+        if not summaries:
+            return None
+        else:
+            return summaries[0].get("PipelineVersionId")
+
     def create(
         self,
         role_arn: str = None,
@@ -166,7 +175,8 @@ def create(
             kwargs,
             Tags=tags,
         )
-        return self.sagemaker_session.sagemaker_client.create_pipeline(**kwargs)
+        response = self.sagemaker_session.sagemaker_client.create_pipeline(**kwargs)
+        return response
 
     def _create_args(
         self, role_arn: str, description: str, parallelism_config: ParallelismConfiguration
@@ -214,15 +224,21 @@ def _create_args(
         )
         return kwargs
 
-    def describe(self) -> Dict[str, Any]:
+    def describe(self, pipeline_version_id: int = None) -> Dict[str, Any]:
         """Describes a Pipeline in the Workflow service.
 
+        Args:
+            pipeline_version_id (Optional[str]): version ID of the pipeline to describe.
+
         Returns:
             Response dict from the service. See `boto3 client documentation
             <https://boto3.amazonaws.com/v1/documentation/api/latest/reference/services/\
 sagemaker.html#SageMaker.Client.describe_pipeline>`_
         """
-        return self.sagemaker_session.sagemaker_client.describe_pipeline(PipelineName=self.name)
+        kwargs = dict(PipelineName=self.name)
+        if pipeline_version_id:
+            kwargs["PipelineVersionId"] = pipeline_version_id
+        return self.sagemaker_session.sagemaker_client.describe_pipeline(**kwargs)
 
     def update(
         self,
@@ -257,7 +273,8 @@ def update(
             return self.sagemaker_session.sagemaker_client.update_pipeline(self, description)
 
         kwargs = self._create_args(role_arn, description, parallelism_config)
-        return self.sagemaker_session.sagemaker_client.update_pipeline(**kwargs)
+        response = self.sagemaker_session.sagemaker_client.update_pipeline(**kwargs)
+        return response
 
     def upsert(
         self,
@@ -332,6 +349,7 @@ def start(
         execution_description: str = None,
         parallelism_config: ParallelismConfiguration = None,
         selective_execution_config: SelectiveExecutionConfig = None,
+        pipeline_version_id: int = None,
     ):
         """Starts a Pipeline execution in the Workflow service.
 
@@ -345,6 +363,8 @@ def start(
                 over the parallelism configuration of the parent pipeline.
             selective_execution_config (Optional[SelectiveExecutionConfig]): The configuration for
                 selective step execution.
+            pipeline_version_id (Optional[str]): version ID of the pipeline to start the execution from. If not
+                specified, uses the latest version ID.
 
         Returns:
             A `_PipelineExecution` instance, if successful.
@@ -366,6 +386,7 @@ def start(
             PipelineExecutionDisplayName=execution_display_name,
             ParallelismConfiguration=parallelism_config,
             SelectiveExecutionConfig=selective_execution_config,
+            PipelineVersionId=pipeline_version_id,
         )
         if self.sagemaker_session.local_mode:
             update_args(kwargs, PipelineParameters=parameters)
@@ -461,6 +482,32 @@ def list_executions(
             if key in response
         }
 
+    def list_pipeline_versions(
+        self, sort_order: str = None, max_results: int = None, next_token: str = None
+    ) -> str:
+        """Lists a pipeline's versions.
+
+        Args:
+            sort_order (str): The sort order for results (Ascending/Descending).
+            max_results (int): The maximum number of pipeline executions to return in the response.
+            next_token (str):  If the result of the previous `ListPipelineExecutions` request was
+                truncated, the response includes a `NextToken`. To retrieve the next set of pipeline
+                executions, use the token in the next request.
+
+        Returns:
+            List of Pipeline Version Summaries. See
+            boto3 client list_pipeline_versions
+            https://boto3.amazonaws.com/v1/documentation/api/latest/reference/services/sagemaker/client/list_pipeline_versions.html#
+        """
+        kwargs = dict(PipelineName=self.name)
+        update_args(
+            kwargs,
+            SortOrder=sort_order,
+            NextToken=next_token,
+            MaxResults=max_results,
+        )
+        return self.sagemaker_session.sagemaker_client.list_pipeline_versions(**kwargs)
+
     def _get_latest_execution_arn(self):
         """Retrieves the latest execution of this pipeline"""
         response = self.list_executions(
@@ -855,7 +902,7 @@ def describe(self):
 sagemaker.html#SageMaker.Client.describe_pipeline_execution>`_.
         """
         return self.sagemaker_session.sagemaker_client.describe_pipeline_execution(
-            PipelineExecutionArn=self.arn,
+            PipelineExecutionArn=self.arn
         )
 
     def list_steps(self):

tests/integ/sagemaker/conftest.py

@@ -14,16 +14,16 @@
 
 import base64
 import os
-import subprocess
-import shutil
-import pytest
-import docker
 import re
+import shutil
+import subprocess
 import sys
 
+import docker
+import pytest
 from docker.errors import BuildError
 
-from sagemaker.utils import sagemaker_timestamp, _tmpdir, sts_regional_endpoint
+from sagemaker.utils import _tmpdir, sagemaker_timestamp, sts_regional_endpoint
 
 REPO_ACCOUNT_ID = "033110030271"
 
@@ -68,7 +68,7 @@
     "RUN curl 'https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip' -o 'awscliv2.zip' \
         && unzip awscliv2.zip \
         && ./aws/install\n\n"
-    "RUN apt install sudo\n"
+    "RUN apt install -y sudo\n"
     "RUN useradd -ms /bin/bash integ-test-user\n"
     # Add the user to sudo group
     "RUN usermod -aG sudo integ-test-user\n"

tests/integ/sagemaker/workflow/test_workflow.py

@@ -312,6 +312,7 @@ def test_three_step_definition(
             rf"arn:aws:sagemaker:{region_name}:\d{{12}}:pipeline/{pipeline_name}",
             create_arn,
         )
+        assert pipeline.latest_pipeline_version_id == 1
     finally:
         try:
             pipeline.delete()
@@ -937,7 +938,6 @@ def test_large_pipeline(sagemaker_session_for_pipeline, role, pipeline_name, reg
             rf"arn:aws:sagemaker:{region_name}:\d{{12}}:pipeline/{pipeline_name}",
             create_arn,
         )
-        response = pipeline.describe()
         assert len(json.loads(pipeline.describe()["PipelineDefinition"])["Steps"]) == 2000
 
         pipeline.parameters = [ParameterInteger(name="InstanceCount", default_value=1)]
@@ -1387,3 +1387,56 @@ def test_caching_behavior(
         except Exception:
             os.remove(script_dir + "/dummy_script.py")
             pass
+
+
+def test_pipeline_versioning(pipeline_session, role, pipeline_name, script_dir):
+    sklearn_train = SKLearn(
+        framework_version="0.20.0",
+        entry_point=os.path.join(script_dir, "train.py"),
+        instance_type="ml.m5.xlarge",
+        sagemaker_session=pipeline_session,
+        role=role,
+    )
+
+    step1 = TrainingStep(
+        name="my-train-1",
+        display_name="TrainingStep",
+        description="description for Training step",
+        step_args=sklearn_train.fit(),
+    )
+
+    step2 = TrainingStep(
+        name="my-train-2",
+        display_name="TrainingStep",
+        description="description for Training step",
+        step_args=sklearn_train.fit(),
+    )
+    pipeline = Pipeline(
+        name=pipeline_name,
+        steps=[step1],
+        sagemaker_session=pipeline_session,
+    )
+
+    try:
+        pipeline.create(role)
+
+        assert pipeline.latest_pipeline_version_id == 1
+
+        describe_response = pipeline.describe(pipeline_version_id=1)
+        assert len(json.loads(describe_response["PipelineDefinition"])["Steps"]) == 1
+
+        pipeline.steps.append(step2)
+        pipeline.upsert(role)
+
+        assert pipeline.latest_pipeline_version_id == 2
+
+        describe_response = pipeline.describe(pipeline_version_id=2)
+        assert len(json.loads(describe_response["PipelineDefinition"])["Steps"]) == 2
+
+        assert len(pipeline.list_pipeline_versions()["PipelineVersionSummaries"]) == 2
+
+    finally:
+        try:
+            pipeline.delete()
+        except Exception:
+            pass