add default ignore_patterns, fix minor path issue when uploaded to s3

Author: mollyheamazon

src/sagemaker/modules/configs.py

@@ -97,15 +97,15 @@ class SourceCode(BaseConfig):
             The command(s) to execute in the training job container. Example: "python my_script.py".
             If not specified, entry_script must be provided.
         ignore_patterns: (Optional[List[str]]) :
-            The ignore patterns to ignore specific files/folders when uploading to S3. Example:
-            ['.env', '.git', 'data', '__pycache__'].
+            The ignore patterns to ignore specific files/folders when uploading to S3. If not specified,
+            default to: ['.env', '.git', '__pycache__', '.DS_Store'].
     """
 
     source_dir: Optional[str] = None
     requirements: Optional[str] = None
     entry_script: Optional[str] = None
     command: Optional[str] = None
-    ignore_patterns: Optional[List[str]] = None
+    ignore_patterns: Optional[List[str]] = [".env", ".git", "__pycache__", ".DS_Store"]
 
 
 class Compute(shapes.ResourceConfig):

src/sagemaker/modules/train/model_trainer.py

@@ -119,7 +119,7 @@ class ModelTrainer(BaseModel):
         from sagemaker.modules.train import ModelTrainer
         from sagemaker.modules.configs import SourceCode, Compute, InputData
 
-        ignore_patterns = ['.env', '.git', 'data', '__pycache__']
+        ignore_patterns = ['.env', '.git', '__pycache__', '.DS_Store', 'data']
         source_code = SourceCode(source_dir="source", entry_script="train.py", ignore_patterns=ignore_patterns)
         training_image = "123456789012.dkr.ecr.us-west-2.amazonaws.com/my-training-image"
         model_trainer = ModelTrainer(
@@ -677,6 +677,7 @@ def train(
                 channel_name=SM_DRIVERS,
                 data_source=tmp_dir.name,
                 key_prefix=input_data_key_prefix,
+                ignore_patterns=self.source_code.ignore_patterns,
             )
             final_input_data_config.append(sm_drivers_channel)
 
@@ -779,7 +780,7 @@ def create_input_data_channel(
                 ``s3://<default_bucket_path>/<key_prefix>/<channel_name>/``
             ignore_patterns: (Optional[List[str]]) :
                 The ignore patterns to ignore specific files/folders when uploading to S3.
-                Example: ['.env', '.git', 'data', '__pycache__'].
+                If not specified, default to: ['.env', '.git', '__pycache__', '.DS_Store'].
         """
         channel = None
         if isinstance(data_source, str):
@@ -819,16 +820,19 @@ def create_input_data_channel(
                     )
                     if self.sagemaker_session.default_bucket_prefix:
                         key_prefix = f"{self.sagemaker_session.default_bucket_prefix}/{key_prefix}"
-                    if ignore_patterns:
+                    if ignore_patterns and _is_valid_path(data_source, path_type="Directory"):
                         tmp_dir = TemporaryDirectory()
+                        copied_path = os.path.join(
+                            tmp_dir.name, os.path.basename(os.path.normpath(data_source))
+                        )
                         shutil.copytree(
                             data_source,
-                            os.path.join(tmp_dir.name, os.path.basename(data_source)),
+                            copied_path,
                             dirs_exist_ok=True,
                             ignore=shutil.ignore_patterns(*ignore_patterns),
                         )
                         s3_uri = self.sagemaker_session.upload_data(
-                            path=tmp_dir.name,
+                            path=copied_path,
                             bucket=self.sagemaker_session.default_bucket(),
                             key_prefix=key_prefix,
                         )
@@ -884,7 +888,10 @@ def _get_input_data_config(
                 channels.append(input_data)
             elif isinstance(input_data, InputData):
                 channel = self.create_input_data_channel(
-                    input_data.channel_name, input_data.data_source, key_prefix=key_prefix
+                    input_data.channel_name,
+                    input_data.data_source,
+                    key_prefix=key_prefix,
+                    ignore_patterns=self.source_code.ignore_patterns,
                 )
                 channels.append(channel)
             else:

tests/unit/sagemaker/modules/train/test_model_trainer.py

@@ -208,7 +208,7 @@ def model_trainer():
                 "source_code": SourceCode(
                     source_dir=DEFAULT_SOURCE_DIR,
                     command="python custom_script.py",
-                    ignore_patterns=["data"]
+                    ignore_patterns=["data"],
                 ),
             },
             "should_throw": False,
@@ -224,7 +224,7 @@ def model_trainer():
         "supported_source_code_local_tar_file",
         "supported_source_code_s3_dir",
         "supported_source_code_s3_tar_file",
-        "supported_source_code_ignore_patterns"
+        "supported_source_code_ignore_patterns",
     ],
 )
 def test_model_trainer_param_validation(test_case, modules_session):