Properly support multiple submodules in git packager (#154)

Signed-off-by: Hemil Desai <[email protected]>

Author: hemildesai

src/nemo_run/core/packaging/git.py

@@ -120,8 +120,14 @@ def package(self, path: Path, job_dir: str, name: str) -> str:
         git_archive_cmd = (
             f"git archive --format=tar --output={output_file}.tmp {self.ref}:{git_sub_path}"
         )
+        if os.uname().sysname == "Linux":
+            tar_submodule_cmd = f"tar Af {output_file}.tmp $sha1.tmp && rm $sha1.tmp"
+        else:
+            tar_submodule_cmd = f"cat $sha1.tmp >> {output_file}.tmp && rm $sha1.tmp"
+
         git_submodule_cmd = f"""git submodule foreach --recursive \
-'git archive --format=tar --prefix=$sm_path/ --output=$sha1.tmp HEAD && cat $sha1.tmp >> {output_file}.tmp && rm $sha1.tmp'"""
+'git archive --format=tar --prefix=$sm_path/ --output=$sha1.tmp HEAD && {tar_submodule_cmd}'"""
+
         with ctx.cd(git_base_path):
             ctx.run(git_archive_cmd)
             if self.include_submodules:

test/core/packaging/test_git.py

@@ -342,17 +342,28 @@ def test_untracked_files_raises_exception(temp_repo):
 @patch("nemo_run.core.packaging.git.Context", MockContext)
 def test_package_with_include_submodules(packager, temp_repo):
     temp_repo = Path(temp_repo)
-    # Create a submodule
+    # Create first submodule
     submodule_path = temp_repo / "submodule"
     submodule_path.mkdir()
     os.chdir(str(submodule_path))
     subprocess.check_call(["git", "init", "--initial-branch=main"])
     open("submodule_file.txt", "w").write("Submodule file")
     subprocess.check_call(["git", "add", "."])
     subprocess.check_call(["git", "commit", "-m", "Initial submodule commit"])
+
+    # Create second submodule
+    submodule2_path = temp_repo / "submodule2"
+    submodule2_path.mkdir()
+    os.chdir(str(submodule2_path))
+    subprocess.check_call(["git", "init", "--initial-branch=main"])
+    open("submodule2_file.txt", "w").write("Second submodule file")
+    subprocess.check_call(["git", "add", "."])
+    subprocess.check_call(["git", "commit", "-m", "Initial submodule2 commit"])
+
     os.chdir(str(temp_repo))
     subprocess.check_call(["git", "submodule", "add", str(submodule_path)])
-    subprocess.check_call(["git", "commit", "-m", "Add submodule"])
+    subprocess.check_call(["git", "submodule", "add", str(submodule2_path)])
+    subprocess.check_call(["git", "commit", "-m", "Add submodules"])
 
     packager = GitArchivePackager(ref="HEAD", include_submodules=True)
     with tempfile.TemporaryDirectory() as job_dir:
@@ -364,13 +375,22 @@ def test_package_with_include_submodules(packager, temp_repo):
                 f"tar -xvzf {output_file} -C {os.path.join(job_dir, 'extracted_output')} --ignore-zeros"
             ),
         )
+        # Check first submodule
         cmp = filecmp.dircmp(
             os.path.join(temp_repo, "submodule"),
             os.path.join(job_dir, "extracted_output", "submodule"),
         )
         assert cmp.left_list == cmp.right_list
         assert not cmp.diff_files
 
+        # Check second submodule
+        cmp2 = filecmp.dircmp(
+            os.path.join(temp_repo, "submodule2"),
+            os.path.join(job_dir, "extracted_output", "submodule2"),
+        )
+        assert cmp2.left_list == cmp2.right_list
+        assert not cmp2.diff_files
+
 
 @patch("nemo_run.core.packaging.git.Context", MockContext)
 def test_package_without_include_submodules(packager, temp_repo):