Merge branch 'master' into master

Author: varunmoris

src/sagemaker/image_uri_config/sagemaker-distribution.json

@@ -0,0 +1,37 @@
+{
+    "processors": ["cpu", "gpu"],
+    "scope": ["inference"],
+    "version_aliases": {
+            "3.0": "3.0.0"
+        },
+    "versions": {
+        "3.0.0": {
+            "registries": {
+                "us-east-1": "885854791233",
+                "us-east-2": "137914896644",
+                "us-west-1": "053634841547",
+                "us-west-2": "542918446943",
+                "af-south-1": "238384257742",
+                "ap-east-1": "523751269255",
+                "ap-south-1": "245090515133",
+                "ap-northeast-2": "064688005998",
+                "ap-southeast-1": "022667117163",
+                "ap-southeast-2": "648430277019",
+                "ap-northeast-1": "010972774902",
+                "ca-central-1": "481561238223",
+                "eu-central-1": "545423591354",
+                "eu-west-1": "819792524951",
+                "eu-west-2": "021081402939",
+                "eu-west-3": "856416204555",
+                "eu-north-1": "175620155138",
+                "eu-south-1": "810671768855",
+                "sa-east-1": "567556641782",
+                "ap-northeast-3": "564864627153",
+                "ap-southeast-3": "370607712162",
+                "me-south-1": "523774347010",
+                "me-central-1": "358593528301"
+            },
+            "repository": "sagemaker-distribution-prod"
+        }
+    }
+}

src/sagemaker/modules/configs.py

@@ -88,7 +88,8 @@ class SourceCode(BaseConfig):
 
     Parameters:
         source_dir (Optional[str]):
-            The local directory containing the source code to be used in the training job container.
+            The local directory, s3 uri, or path to tar.gz file stored locally or in s3 that contains
+            the source code to be used in the training job container.
         requirements (Optional[str]):
             The path within ``source_dir`` to a ``requirements.txt`` file. If specified, the listed
             requirements will be installed in the training job container.

src/sagemaker/modules/train/model_trainer.py

@@ -407,28 +407,45 @@ def _validate_source_code(self, source_code: Optional[SourceCode]):
                         "If 'requirements' or 'entry_script' is provided in 'source_code', "
                         + "'source_dir' must also be provided.",
                     )
-                if not _is_valid_path(source_dir, path_type="Directory"):
+                if not (
+                    _is_valid_path(source_dir, path_type="Directory")
+                    or _is_valid_s3_uri(source_dir, path_type="Directory")
+                    or (
+                        _is_valid_path(source_dir, path_type="File")
+                        and source_dir.endswith(".tar.gz")
+                    )
+                    or (
+                        _is_valid_s3_uri(source_dir, path_type="File")
+                        and source_dir.endswith(".tar.gz")
+                    )
+                ):
                     raise ValueError(
-                        f"Invalid 'source_dir' path: {source_dir}. " + "Must be a valid directory.",
+                        f"Invalid 'source_dir' path: {source_dir}. "
+                        + "Must be a valid local directory, "
+                        "s3 uri or path to tar.gz file stored locally or in s3.",
                     )
                 if requirements:
-                    if not _is_valid_path(
-                        f"{source_dir}/{requirements}",
-                        path_type="File",
-                    ):
-                        raise ValueError(
-                            f"Invalid 'requirements': {requirements}. "
-                            + "Must be a valid file within the 'source_dir'.",
-                        )
+                    if not source_dir.endswith(".tar.gz"):
+                        if not _is_valid_path(
+                            f"{source_dir}/{requirements}", path_type="File"
+                        ) and not _is_valid_s3_uri(
+                            f"{source_dir}/{requirements}", path_type="File"
+                        ):
+                            raise ValueError(
+                                f"Invalid 'requirements': {requirements}. "
+                                + "Must be a valid file within the 'source_dir'.",
+                            )
                 if entry_script:
-                    if not _is_valid_path(
-                        f"{source_dir}/{entry_script}",
-                        path_type="File",
-                    ):
-                        raise ValueError(
-                            f"Invalid 'entry_script': {entry_script}. "
-                            + "Must be a valid file within the 'source_dir'.",
-                        )
+                    if not source_dir.endswith(".tar.gz"):
+                        if not _is_valid_path(
+                            f"{source_dir}/{entry_script}", path_type="File"
+                        ) and not _is_valid_s3_uri(
+                            f"{source_dir}/{entry_script}", path_type="File"
+                        ):
+                            raise ValueError(
+                                f"Invalid 'entry_script': {entry_script}. "
+                                + "Must be a valid file within the 'source_dir'.",
+                            )
 
     def model_post_init(self, __context: Any):
         """Post init method to perform custom validation and set default values."""
@@ -838,12 +855,17 @@ def _prepare_train_script(
 
         install_requirements = ""
         if source_code.requirements:
-            install_requirements = "echo 'Installing requirements'\n"
-            install_requirements = f"$SM_PIP_CMD install -r {source_code.requirements}"
+            install_requirements = (
+                "echo 'Installing requirements'\n"
+                + f"$SM_PIP_CMD install -r {source_code.requirements}"
+            )
 
         working_dir = ""
         if source_code.source_dir:
-            working_dir = f"cd {SM_CODE_CONTAINER_PATH}"
+            working_dir = f"cd {SM_CODE_CONTAINER_PATH} \n"
+            if source_code.source_dir.endswith(".tar.gz"):
+                tarfile_name = os.path.basename(source_code.source_dir)
+                working_dir += f"tar --strip-components=1 -xzf {tarfile_name} \n"
 
         if base_command:
             execute_driver = EXECUTE_BASE_COMMANDS.format(base_command=base_command)