Merge branch 'master' of github.com:IshaChid76/sagemaker-python-sdk

Author: isha chidrawar

src/sagemaker/_studio.py

@@ -65,7 +65,10 @@ def _find_config(working_dir=None):
         wd = Path(working_dir) if working_dir else Path.cwd()
 
         path = None
-        while path is None and not wd.match("/"):
+
+        # Get the root of the current working directory for both Windows and Unix-like systems
+        root = Path(wd.anchor)
+        while path is None and wd != root:
             candidate = wd / STUDIO_PROJECT_CONFIG
             if Path.exists(candidate):
                 path = candidate

src/sagemaker/image_uri_config/pytorch.json

@@ -85,7 +85,8 @@
             "2.2": "2.2.0",
             "2.3": "2.3.0",
             "2.4": "2.4.0",
-            "2.5": "2.5.1"
+            "2.5": "2.5.1",
+            "2.6": "2.6.0"
         },
         "versions": {
             "0.4.0": {
@@ -1253,6 +1254,50 @@
                     "us-west-2": "763104351884"
                 },
                 "repository": "pytorch-inference"
+            },
+            "2.6.0": {
+                "py_versions": [
+                    "py312"
+                ],
+                "registries": {
+                    "af-south-1": "626614931356",
+                    "ap-east-1": "871362719292",
+                    "ap-northeast-1": "763104351884",
+                    "ap-northeast-2": "763104351884",
+                    "ap-northeast-3": "364406365360",
+                    "ap-south-1": "763104351884",
+                    "ap-south-2": "772153158452",
+                    "ap-southeast-1": "763104351884",
+                    "ap-southeast-2": "763104351884",
+                    "ap-southeast-3": "907027046896",
+                    "ap-southeast-4": "457447274322",
+                    "ap-southeast-5": "550225433462",
+                    "ap-southeast-7": "590183813437",
+                    "ca-central-1": "763104351884",
+                    "ca-west-1": "204538143572",
+                    "cn-north-1": "727897471807",
+                    "cn-northwest-1": "727897471807",
+                    "eu-central-1": "763104351884",
+                    "eu-central-2": "380420809688",
+                    "eu-north-1": "763104351884",
+                    "eu-south-1": "692866216735",
+                    "eu-south-2": "503227376785",
+                    "eu-west-1": "763104351884",
+                    "eu-west-2": "763104351884",
+                    "eu-west-3": "763104351884",
+                    "il-central-1": "780543022126",
+                    "me-central-1": "914824155844",
+                    "me-south-1": "217643126080",
+                    "mx-central-1": "637423239942",
+                    "sa-east-1": "763104351884",
+                    "us-east-1": "763104351884",
+                    "us-east-2": "763104351884",
+                    "us-gov-east-1": "446045086412",
+                    "us-gov-west-1": "442386744353",
+                    "us-west-1": "763104351884",
+                    "us-west-2": "763104351884"
+                },
+                "repository": "pytorch-inference"
             }
         }
     },
@@ -1628,7 +1673,8 @@
             "2.2": "2.2.0",
             "2.3": "2.3.0",
             "2.4": "2.4.0",
-            "2.5": "2.5.1"
+            "2.5": "2.5.1",
+            "2.6": "2.6.0"
         },
         "versions": {
             "0.4.0": {
@@ -2801,6 +2847,50 @@
                     "us-west-2": "763104351884"
                 },
                 "repository": "pytorch-training"
+            },
+            "2.6.0": {
+                "py_versions": [
+                    "py312"
+                ],
+                "registries": {
+                    "af-south-1": "626614931356",
+                    "ap-east-1": "871362719292",
+                    "ap-northeast-1": "763104351884",
+                    "ap-northeast-2": "763104351884",
+                    "ap-northeast-3": "364406365360",
+                    "ap-south-1": "763104351884",
+                    "ap-south-2": "772153158452",
+                    "ap-southeast-1": "763104351884",
+                    "ap-southeast-2": "763104351884",
+                    "ap-southeast-3": "907027046896",
+                    "ap-southeast-4": "457447274322",
+                    "ap-southeast-5": "550225433462",
+                    "ap-southeast-7": "590183813437",
+                    "ca-central-1": "763104351884",
+                    "ca-west-1": "204538143572",
+                    "cn-north-1": "727897471807",
+                    "cn-northwest-1": "727897471807",
+                    "eu-central-1": "763104351884",
+                    "eu-central-2": "380420809688",
+                    "eu-north-1": "763104351884",
+                    "eu-south-1": "692866216735",
+                    "eu-south-2": "503227376785",
+                    "eu-west-1": "763104351884",
+                    "eu-west-2": "763104351884",
+                    "eu-west-3": "763104351884",
+                    "il-central-1": "780543022126",
+                    "me-central-1": "914824155844",
+                    "me-south-1": "217643126080",
+                    "mx-central-1": "637423239942",
+                    "sa-east-1": "763104351884",
+                    "us-east-1": "763104351884",
+                    "us-east-2": "763104351884",
+                    "us-gov-east-1": "446045086412",
+                    "us-gov-west-1": "442386744353",
+                    "us-west-1": "763104351884",
+                    "us-west-2": "763104351884"
+                },
+                "repository": "pytorch-training"
             }
         }
     }

src/sagemaker/jumpstart/factory/estimator.py

@@ -56,6 +56,7 @@
     JUMPSTART_LOGGER,
     TRAINING_ENTRY_POINT_SCRIPT_NAME,
     SAGEMAKER_GATED_MODEL_S3_URI_TRAINING_ENV_VAR_KEY,
+    JUMPSTART_MODEL_HUB_NAME,
 )
 from sagemaker.jumpstart.enums import JumpStartScriptScope, JumpStartModelType
 from sagemaker.jumpstart.factory import model
@@ -313,16 +314,31 @@ def _add_hub_access_config_to_kwargs_inputs(
 ):
     """Adds HubAccessConfig to kwargs inputs"""
 
+    dataset_uri = kwargs.specs.default_training_dataset_uri
     if isinstance(kwargs.inputs, str):
-        kwargs.inputs = TrainingInput(s3_data=kwargs.inputs, hub_access_config=hub_access_config)
+        if dataset_uri is not None and dataset_uri == kwargs.inputs:
+            kwargs.inputs = TrainingInput(
+                s3_data=kwargs.inputs, hub_access_config=hub_access_config
+            )
     elif isinstance(kwargs.inputs, TrainingInput):
-        kwargs.inputs.add_hub_access_config(hub_access_config=hub_access_config)
+        if (
+            dataset_uri is not None
+            and dataset_uri == kwargs.inputs.config["DataSource"]["S3DataSource"]["S3Uri"]
+        ):
+            kwargs.inputs.add_hub_access_config(hub_access_config=hub_access_config)
     elif isinstance(kwargs.inputs, dict):
         for k, v in kwargs.inputs.items():
             if isinstance(v, str):
-                kwargs.inputs[k] = TrainingInput(s3_data=v, hub_access_config=hub_access_config)
+                training_input = TrainingInput(s3_data=v)
+                if dataset_uri is not None and dataset_uri == v:
+                    training_input.add_hub_access_config(hub_access_config=hub_access_config)
+                kwargs.inputs[k] = training_input
             elif isinstance(kwargs.inputs, TrainingInput):
-                kwargs.inputs[k].add_hub_access_config(hub_access_config=hub_access_config)
+                if (
+                    dataset_uri is not None
+                    and dataset_uri == kwargs.inputs.config["DataSource"]["S3DataSource"]["S3Uri"]
+                ):
+                    kwargs.inputs[k].add_hub_access_config(hub_access_config=hub_access_config)
 
     return kwargs
 
@@ -616,8 +632,13 @@ def _add_model_reference_arn_to_kwargs(
 
 def _add_model_uri_to_kwargs(kwargs: JumpStartEstimatorInitKwargs) -> JumpStartEstimatorInitKwargs:
     """Sets model uri in kwargs based on default or override, returns full kwargs."""
-
-    if _model_supports_training_model_uri(**get_model_info_default_kwargs(kwargs)):
+    # hub_arn is by default None unless the user specifies the hub_name
+    # If no hub_name is specified, it is assumed the public hub
+    is_private_hub = JUMPSTART_MODEL_HUB_NAME not in kwargs.hub_arn if kwargs.hub_arn else False
+    if (
+        _model_supports_training_model_uri(**get_model_info_default_kwargs(kwargs))
+        or is_private_hub
+    ):
         default_model_uri = model_uris.retrieve(
             model_scope=JumpStartScriptScope.TRAINING,
             instance_type=kwargs.instance_type,

src/sagemaker/jumpstart/hub/hub.py

@@ -16,35 +16,25 @@
 from datetime import datetime
 import logging
 from typing import Optional, Dict, List, Any, Union
-from botocore import exceptions
 
 from sagemaker.jumpstart.constants import JUMPSTART_MODEL_HUB_NAME
 from sagemaker.jumpstart.enums import JumpStartScriptScope
 from sagemaker.session import Session
 
-from sagemaker.jumpstart.constants import (
-    JUMPSTART_LOGGER,
-)
 from sagemaker.jumpstart.types import (
     HubContentType,
 )
 from sagemaker.jumpstart.filters import Constant, Operator, BooleanValues
 from sagemaker.jumpstart.hub.utils import (
     get_hub_model_version,
     get_info_from_hub_resource_arn,
-    create_hub_bucket_if_it_does_not_exist,
-    generate_default_hub_bucket_name,
-    create_s3_object_reference_from_uri,
     construct_hub_arn_from_name,
 )
 
 from sagemaker.jumpstart.notebook_utils import (
     list_jumpstart_models,
 )
 
-from sagemaker.jumpstart.hub.types import (
-    S3ObjectLocation,
-)
 from sagemaker.jumpstart.hub.interfaces import (
     DescribeHubResponse,
     DescribeHubContentResponse,
@@ -66,8 +56,8 @@ class Hub:
     def __init__(
         self,
         hub_name: str,
+        sagemaker_session: Session,
         bucket_name: Optional[str] = None,
-        sagemaker_session: Optional[Session] = None,
     ) -> None:
         """Instantiates a SageMaker ``Hub``.
 
@@ -78,41 +68,11 @@ def __init__(
         """
         self.hub_name = hub_name
         self.region = sagemaker_session.boto_region_name
+        self.bucket_name = bucket_name
         self._sagemaker_session = (
             sagemaker_session
             or utils.get_default_jumpstart_session_with_user_agent_suffix(is_hub_content=True)
         )
-        self.hub_storage_location = self._generate_hub_storage_location(bucket_name)
-
-    def _fetch_hub_bucket_name(self) -> str:
-        """Retrieves hub bucket name from Hub config if exists"""
-        try:
-            hub_response = self._sagemaker_session.describe_hub(hub_name=self.hub_name)
-            hub_output_location = hub_response["S3StorageConfig"].get("S3OutputPath")
-            if hub_output_location:
-                location = create_s3_object_reference_from_uri(hub_output_location)
-                return location.bucket
-            default_bucket_name = generate_default_hub_bucket_name(self._sagemaker_session)
-            JUMPSTART_LOGGER.warning(
-                "There is not a Hub bucket associated with %s. Using %s",
-                self.hub_name,
-                default_bucket_name,
-            )
-            return default_bucket_name
-        except exceptions.ClientError:
-            hub_bucket_name = generate_default_hub_bucket_name(self._sagemaker_session)
-            JUMPSTART_LOGGER.warning(
-                "There is not a Hub bucket associated with %s. Using %s",
-                self.hub_name,
-                hub_bucket_name,
-            )
-            return hub_bucket_name
-
-    def _generate_hub_storage_location(self, bucket_name: Optional[str] = None) -> None:
-        """Generates an ``S3ObjectLocation`` given a Hub name."""
-        hub_bucket_name = bucket_name or self._fetch_hub_bucket_name()
-        curr_timestamp = datetime.now().timestamp()
-        return S3ObjectLocation(bucket=hub_bucket_name, key=f"{self.hub_name}-{curr_timestamp}")
 
     def _get_latest_model_version(self, model_id: str) -> str:
         """Populates the lastest version of a model from specs no matter what is passed.
@@ -132,19 +92,22 @@ def create(
         tags: Optional[str] = None,
     ) -> Dict[str, str]:
         """Creates a hub with the given description"""
+        curr_timestamp = datetime.now().timestamp()
 
-        create_hub_bucket_if_it_does_not_exist(
-            self.hub_storage_location.bucket, self._sagemaker_session
-        )
+        request = {
+            "hub_name": self.hub_name,
+            "hub_description": description,
+            "hub_display_name": display_name,
+            "hub_search_keywords": search_keywords,
+            "tags": tags,
+        }
 
-        return self._sagemaker_session.create_hub(
-            hub_name=self.hub_name,
-            hub_description=description,
-            hub_display_name=display_name,
-            hub_search_keywords=search_keywords,
-            s3_storage_config={"S3OutputPath": self.hub_storage_location.get_uri()},
-            tags=tags,
-        )
+        if self.bucket_name:
+            request["s3_storage_config"] = {
+                "S3OutputPath": (f"s3://{self.bucket_name}/{self.hub_name}-{curr_timestamp}")
+            }
+
+        return self._sagemaker_session.create_hub(**request)
 
     def describe(self, hub_name: Optional[str] = None) -> DescribeHubResponse:
         """Returns descriptive information about the Hub"""

src/sagemaker/jumpstart/hub/interfaces.py

@@ -630,7 +630,6 @@ def from_json(self, json_obj: Dict[str, Any]) -> None:
             if json_obj.get("ValidationSupported")
             else None
         )
-        self.default_training_dataset_uri: Optional[str] = json_obj.get("DefaultTrainingDatasetUri")
         self.resource_name_base: Optional[str] = json_obj.get("ResourceNameBase")
         self.gated_bucket: bool = bool(json_obj.get("GatedBucket", False))
         self.default_payloads: Optional[Dict[str, JumpStartSerializablePayload]] = (
@@ -671,6 +670,9 @@ def from_json(self, json_obj: Dict[str, Any]) -> None:
         )
 
         if self.training_supported:
+            self.default_training_dataset_uri: Optional[str] = json_obj.get(
+                "DefaultTrainingDatasetUri"
+            )
             self.training_model_package_artifact_uri: Optional[str] = json_obj.get(
                 "TrainingModelPackageArtifactUri"
             )

src/sagemaker/jumpstart/hub/parsers.py

@@ -279,4 +279,10 @@ def make_model_specs_from_describe_hub_content_response(
         specs["training_instance_type_variants"] = (
             hub_model_document.training_instance_type_variants
         )
+        if hub_model_document.default_training_dataset_uri:
+            _, default_training_dataset_key = parse_s3_url(  # pylint: disable=unused-variable
+                hub_model_document.default_training_dataset_uri
+            )
+            specs["default_training_dataset_key"] = default_training_dataset_key
+            specs["default_training_dataset_uri"] = hub_model_document.default_training_dataset_uri
     return JumpStartModelSpecs(_to_json(specs), is_hub_content=True)