Make GitArchivePackager behavior consistent with subpath and include_pattern (#101)

* Make git packager consistent and add docs

Signed-off-by: Hemil Desai <[email protected]>

* DockerExecutor

Signed-off-by: Hemil Desai <[email protected]>

---------

Signed-off-by: Hemil Desai <[email protected]>

Author: hemildesai

docs/source/guides/execution.md

@@ -4,6 +4,7 @@ After configuring NeMo-Run, the next step is to execute it. Nemo-Run decouples c
 
 Each execution of a single configured task requires an executor. Nemo-Run provides `run.Executor`, which are APIs to configure your remote executor and set up the packaging of your code. Currently we support:
 - `run.LocalExecutor`
+- `run.DockerExecutor`
 - `run.SlurmExecutor` with an optional `SSHTunnel` for executing on Slurm clusters from your local machine
 - `run.SkypilotExecutor` (available under the optional feature `skypilot` in the python package).
 
@@ -36,10 +37,13 @@ The packager support matrix is described below:
 | Executor | Packagers |
 |----------|----------|
 | LocalExecutor | run.Packager |
-| SlurmExecutor | run.GitArchivePackager |
-| SkypilotExecutor | run.GitArchivePackager |
+| DockerExecutor | run.Packager, run.GitArchivePackager, run.PatternPackager |
+| SlurmExecutor | run.Packager, run.GitArchivePackager, run.PatternPackager |
+| SkypilotExecutor | run.Packager, run.GitArchivePackager, run.PatternPackager |
 
-`run.Packager` is a passthrough base packager. `run.GitArchivePackager` uses `git archive` to package your code. Refer to the API reference for `run.GitArchivePackager` to see the exact mechanics of packaging using `git archive`.
+`run.Packager` is a passthrough base packager.
+
+`run.GitArchivePackager` uses `git archive` to package your code. Refer to the API reference for `run.GitArchivePackager` to see the exact mechanics of packaging using `git archive`.
 At a high level, it works in the following way:
 1. base_path = `git rev-parse --show-toplevel`.
 2. Optionally define a subpath as `base_path/GitArchivePackager.subpath` by setting `subpath` attribute on `GitArchivePackager`.
@@ -60,6 +64,20 @@ If you're executing a Python function, this working directory will automatically
 
 > **_NOTE:_** git archive doesn't package uncommitted changes. In the future, we may add support for including uncommitted changes while honoring `.gitignore`.
 
+`run.PatternPackager` is a packager that uses a pattern to package your code. It is useful for packaging code that is not under version control. For example, if you have a directory structure like this:
+```
+- docs
+- src
+  - your_library
+```
+
+You can use `run.PatternPackager` to package your code by specifying `include_pattern` as `src/**` and `relative_path` as `os.getcwd()`. This will package the entire `src` directory. The command used to get the list of files to package is:
+
+```bash
+# relative_include_pattern = os.path.relpath(self.include_pattern, self.relative_path)
+cd {relative_path} && find {relative_include_pattern} -type f
+```
+
 ### Defining Executors
 Next, We'll describe details on setting up each of the executors below.
 
@@ -69,6 +87,27 @@ The LocalExecutor is the simplest executor. It executes your task locally in a s
 
 The easiest way to define one is to call `run.LocalExecutor()`.
 
+#### DockerExecutor
+
+The DockerExecutor enables launching a task using `docker` on your local machine. It requires `docker` to be installed and running as a prerequisite.
+
+The DockerExecutor uses the [docker python client](https://docker-py.readthedocs.io/en/stable/) and most of the options are passed directly to the client.
+
+Below is an example of configuring a Docker Executor
+
+```python
+run.DockerExecutor(
+    container_image="python:3.12",
+    num_gpus=-1,
+    runtime="nvidia",
+    ipc_mode="host",
+    shm_size="30g",
+    volumes=["/local/path:/path/in/container"],
+    env_vars={"PYTHONUNBUFFERED": "1"},
+    packager=run.Packager(),
+)
+```
+
 #### SlurmExecutor
 
 The SlurmExecutor enables launching the configured task on a Slurm Cluster with Pyxis.  Additionally, you can configure a `run.SSHTunnel`, which enables you to execute tasks on the Slurm cluster from your local machine while NeMo-Run manages the SSH connection for you. This setup supports use cases such as launching the same task on multiple Slurm clusters.

src/nemo_run/core/packaging/git.py

@@ -107,6 +107,8 @@ def package(self, path: Path, job_dir: str, name: str) -> str:
             assert not bool(
                 untracked_files
             ), "Your repo has untracked files. Please track your files via git or set check_untracked_files to False to proceed with packaging."
+
+        ctx = Context()
         if self.include_pattern:
             include_pattern_relative_path = self.include_pattern_relative_path or shlex.quote(
                 str(git_base_path)
@@ -117,17 +119,31 @@ def package(self, path: Path, job_dir: str, name: str) -> str:
             # we first add git files into an uncompressed archive
             # then we add an extra files from pattern to that archive
             # finally we compress it (cannot compress right away, since adding files is not possible)
-            cmd = (
-                f"(cd {shlex.quote(str(git_base_path))} && git ls-files {git_sub_path} "
-                f"| tar -cf {output_file}.tmp -C {shlex.quote(str(git_base_path))} -T -) "
-                f"&& (cd {include_pattern_relative_path} && find {relative_include_pattern} -type f "
-                f"| tar -rf {output_file}.tmp -C {include_pattern_relative_path} -T -) "
-                f"&& gzip -c {output_file}.tmp > {output_file} && rm {output_file}.tmp"
+            git_archive_cmd = (
+                f"git archive --format=tar --output={output_file}.tmp {self.ref}:{git_sub_path}"
             )
+            include_pattern_cmd = f"find {relative_include_pattern} -type f | tar -cf {os.path.join(git_base_path, 'additional.tmp')} -T -"
+            tar_concatenate_cmd = f"tar -Af {output_file}.tmp additional.tmp"
+            gzip_cmd = f"gzip -c {output_file}.tmp > {output_file}"
+            rm_cmd = f"rm {output_file}.tmp additional.tmp"
+
+            with ctx.cd(git_base_path):
+                ctx.run(git_archive_cmd)
+
+            with ctx.cd(include_pattern_relative_path):
+                ctx.run(include_pattern_cmd)
+
+            with ctx.cd(git_base_path):
+                ctx.run(tar_concatenate_cmd)
+                ctx.run(gzip_cmd)
+                ctx.run(rm_cmd)
         else:
-            cmd = f"cd {shlex.quote(str(git_base_path))} && git archive --format=tar.gz --output={output_file} {self.ref}:{git_sub_path}"
-        ctx = Context()
-        ctx.run(cmd)
+            with ctx.cd(git_base_path):
+                git_archive_cmd = (
+                    f"git archive --format=tar.gz --output={output_file} {self.ref}:{git_sub_path}"
+                )
+                ctx.run(git_archive_cmd)
+
         return output_file
 
 

test/core/packaging/test_git.py

@@ -162,6 +162,50 @@ def test_package_with_include_pattern(packager, temp_repo):
         assert not cmp.diff_files
 
 
+@patch("nemo_run.core.packaging.git.Context", MockContext)
+def test_package_with_include_pattern_and_subpath(packager, temp_repo):
+    temp_repo = Path(temp_repo)
+    # Create extra files
+    (temp_repo / "extra").mkdir()
+    with open(temp_repo / "extra" / "extra_file1.txt", "w") as f:
+        f.write("Extra file 1")
+    with open(temp_repo / "extra" / "extra_file2.txt", "w") as f:
+        f.write("Extra file 2")
+
+    # Create extra files
+    (temp_repo / "extra2").mkdir()
+    with open(temp_repo / "extra2" / "extra2_file1.txt", "w") as f:
+        f.write("Extra file 1")
+    with open(temp_repo / "extra2" / "extra2_file2.txt", "w") as f:
+        f.write("Extra file 2")
+    subprocess.check_call(
+        [f"cd {temp_repo} && git add extra2 && git commit -m 'Extra2 commit'"], shell=True
+    )
+
+    packager = GitArchivePackager(ref="HEAD", include_pattern="extra", subpath="extra2")
+    with tempfile.TemporaryDirectory() as job_dir:
+        output_file = packager.package(Path(temp_repo), job_dir, "test_package")
+        assert os.path.exists(output_file)
+        subprocess.check_call(shlex.split(f"mkdir -p {os.path.join(job_dir, 'extracted_output')}"))
+        subprocess.check_call(
+            shlex.split(f"tar -xvzf {output_file} -C {os.path.join(job_dir, 'extracted_output')}"),
+        )
+        cmp = filecmp.dircmp(
+            os.path.join(temp_repo, "extra"),
+            os.path.join(job_dir, "extracted_output", "extra"),
+        )
+        assert cmp.left_list == cmp.right_list
+        assert not cmp.diff_files
+
+        cmp = filecmp.dircmp(
+            os.path.join(temp_repo, "extra2"),
+            os.path.join(job_dir, "extracted_output"),
+            ignore=["extra"],
+        )
+        assert cmp.left_list == cmp.right_list
+        assert not cmp.diff_files
+
+
 @patch("nemo_run.core.packaging.git.Context", MockContext)
 def test_package_with_include_pattern_multiple_directories(packager, temp_repo):
     temp_repo = Path(temp_repo)