Skip to content

Commit fbeb070

Browse files
hemildesaihemildesai
authored
Remove tar -Af to support Mac based systems (#113)
1 parent 0e42364 commit fbeb070

File tree

6 files changed

+41
-15
lines changed

6 files changed

+41
-15
lines changed

‎src/nemo_run/core/execution/docker.py‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -213,7 +213,9 @@ def package(self, packager: Packager, job_name: str):
213213
)
214214
ctx.run(f"mkdir -p {remote_nsys_extraction_path}")
215215
if local_pkg:
216-
ctx.run(f"tar -xvzf {local_pkg} -C {local_code_extraction_path}", hide=True)
216+
ctx.run(
217+
f"tar -xvzf {local_pkg} -C {local_code_extraction_path} --ignore-zeros", hide=True
218+
)
217219

218220
def cleanup(self, handle: str):
219221
_, _, app_id = parse_app_handle(handle)

‎src/nemo_run/core/execution/skypilot.py‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -306,7 +306,9 @@ def package(self, packager: Packager, job_name: str):
306306
)
307307
ctx.run(f"mkdir -p {remote_nsys_extraction_path}")
308308
if local_pkg:
309-
ctx.run(f"tar -xvzf {local_pkg} -C {local_code_extraction_path}", hide=True)
309+
ctx.run(
310+
f"tar -xvzf {local_pkg} -C {local_code_extraction_path} --ignore-zeros", hide=True
311+
)
310312

311313
def nnodes(self) -> int:
312314
return self.num_nodes

‎src/nemo_run/core/execution/slurm.py‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -569,7 +569,9 @@ def package(self, packager: Packager, job_name: str):
569569
# Touch hidden init file
570570
ctx.run(f"touch {remote_nsys_extraction_path}/.init")
571571
if local_pkg:
572-
ctx.run(f"tar -xvzf {local_pkg} -C {local_code_extraction_path}", hide=True)
572+
ctx.run(
573+
f"tar -xvzf {local_pkg} -C {local_code_extraction_path} --ignore-zeros", hide=True
574+
)
573575

574576
self.tunnel.packaging_jobs.add(job_name)
575577

‎src/nemo_run/core/packaging/git.py‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -120,7 +120,7 @@ def package(self, path: Path, job_dir: str, name: str) -> str:
120120
f"git archive --format=tar --output={output_file}.tmp {self.ref}:{git_sub_path}"
121121
)
122122
git_submodule_cmd = f"""git submodule foreach --recursive \
123-
'git archive --format=tar --prefix=$sm_path/ --output=$sha1.tmp HEAD && tar -Af {output_file}.tmp $sha1.tmp && rm $sha1.tmp'"""
123+
'git archive --format=tar --prefix=$sm_path/ --output=$sha1.tmp HEAD && cat $sha1.tmp >> {output_file}.tmp && rm $sha1.tmp'"""
124124
with ctx.cd(git_base_path):
125125
ctx.run(git_archive_cmd)
126126
if self.include_submodules:
@@ -134,7 +134,7 @@ def package(self, path: Path, job_dir: str, name: str) -> str:
134134
self.include_pattern, include_pattern_relative_path
135135
)
136136
include_pattern_cmd = f"find {relative_include_pattern} -type f | tar -cf {os.path.join(git_base_path, 'additional.tmp')} -T -"
137-
tar_concatenate_cmd = f"tar -Af {output_file}.tmp additional.tmp && rm additional.tmp"
137+
tar_concatenate_cmd = f"cat additional.tmp >> {output_file}.tmp && rm additional.tmp"
138138

139139
with ctx.cd(include_pattern_relative_path):
140140
ctx.run(include_pattern_cmd)

‎test/core/packaging/test_git.py‎

Lines changed: 27 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -63,7 +63,9 @@ def test_package(packager, temp_repo):
6363
assert os.path.exists(output_file)
6464
subprocess.check_call(shlex.split(f"mkdir -p {os.path.join(job_dir, 'extracted_output')}"))
6565
subprocess.check_call(
66-
shlex.split(f"tar -xvzf {output_file} -C {os.path.join(job_dir, 'extracted_output')}"),
66+
shlex.split(
67+
f"tar -xvzf {output_file} -C {os.path.join(job_dir, 'extracted_output')} --ignore-zeros"
68+
),
6769
)
6870
cmp = filecmp.dircmp(temp_repo, os.path.join(job_dir, "extracted_output"))
6971
assert cmp.left_list == cmp.right_list
@@ -84,7 +86,9 @@ def test_package_with_subpath(packager, temp_repo):
8486
assert os.path.exists(output_file)
8587
subprocess.check_call(shlex.split(f"mkdir -p {os.path.join(job_dir, 'extracted_output')}"))
8688
subprocess.check_call(
87-
shlex.split(f"tar -xvzf {output_file} -C {os.path.join(job_dir, 'extracted_output')}"),
89+
shlex.split(
90+
f"tar -xvzf {output_file} -C {os.path.join(job_dir, 'extracted_output')} --ignore-zeros"
91+
),
8892
)
8993
cmp = filecmp.dircmp(
9094
os.path.join(temp_repo, "subdir"), os.path.join(job_dir, "extracted_output")
@@ -110,7 +114,9 @@ def test_package_with_subpath_with_basepath(packager, temp_repo):
110114
assert os.path.exists(output_file)
111115
subprocess.check_call(shlex.split(f"mkdir -p {os.path.join(job_dir, 'extracted_output')}"))
112116
subprocess.check_call(
113-
shlex.split(f"tar -xvzf {output_file} -C {os.path.join(job_dir, 'extracted_output')}"),
117+
shlex.split(
118+
f"tar -xvzf {output_file} -C {os.path.join(job_dir, 'extracted_output')} --ignore-zeros"
119+
),
114120
)
115121
cmp = filecmp.dircmp(
116122
os.path.join(temp_repo, "subdir", "subdir2"),
@@ -152,7 +158,9 @@ def test_package_with_include_pattern(packager, temp_repo):
152158
assert os.path.exists(output_file)
153159
subprocess.check_call(shlex.split(f"mkdir -p {os.path.join(job_dir, 'extracted_output')}"))
154160
subprocess.check_call(
155-
shlex.split(f"tar -xvzf {output_file} -C {os.path.join(job_dir, 'extracted_output')}"),
161+
shlex.split(
162+
f"tar -xvzf {output_file} -C {os.path.join(job_dir, 'extracted_output')} --ignore-zeros"
163+
),
156164
)
157165
cmp = filecmp.dircmp(
158166
os.path.join(temp_repo, "extra"),
@@ -188,7 +196,9 @@ def test_package_with_include_pattern_and_subpath(packager, temp_repo):
188196
assert os.path.exists(output_file)
189197
subprocess.check_call(shlex.split(f"mkdir -p {os.path.join(job_dir, 'extracted_output')}"))
190198
subprocess.check_call(
191-
shlex.split(f"tar -xvzf {output_file} -C {os.path.join(job_dir, 'extracted_output')}"),
199+
shlex.split(
200+
f"tar -xvzf {output_file} -C {os.path.join(job_dir, 'extracted_output')} --ignore-zeros"
201+
),
192202
)
193203
cmp = filecmp.dircmp(
194204
os.path.join(temp_repo, "extra"),
@@ -228,7 +238,9 @@ def test_package_with_include_pattern_multiple_directories(packager, temp_repo):
228238
assert os.path.exists(output_file)
229239
subprocess.check_call(shlex.split(f"mkdir -p {os.path.join(job_dir, 'extracted_output')}"))
230240
subprocess.check_call(
231-
shlex.split(f"tar -xvzf {output_file} -C {os.path.join(job_dir, 'extracted_output')}"),
241+
shlex.split(
242+
f"tar -xvzf {output_file} -C {os.path.join(job_dir, 'extracted_output')} --ignore-zeros"
243+
),
232244
)
233245
cmp = filecmp.dircmp(
234246
os.path.join(temp_repo, "extra"),
@@ -263,7 +275,9 @@ def test_package_with_include_pattern_rel_path(packager, temp_repo, tmpdir):
263275
assert os.path.exists(output_file)
264276
subprocess.check_call(shlex.split(f"mkdir -p {os.path.join(job_dir, 'extracted_output')}"))
265277
subprocess.check_call(
266-
shlex.split(f"tar -xvzf {output_file} -C {os.path.join(job_dir, 'extracted_output')}"),
278+
shlex.split(
279+
f"tar -xvzf {output_file} -C {os.path.join(job_dir, 'extracted_output')} --ignore-zeros"
280+
),
267281
)
268282
cmp = filecmp.dircmp(
269283
os.path.join(tmpdir, "extra"),
@@ -313,7 +327,9 @@ def test_package_with_include_submodules(packager, temp_repo):
313327
assert os.path.exists(output_file)
314328
subprocess.check_call(shlex.split(f"mkdir -p {os.path.join(job_dir, 'extracted_output')}"))
315329
subprocess.check_call(
316-
shlex.split(f"tar -xvzf {output_file} -C {os.path.join(job_dir, 'extracted_output')}"),
330+
shlex.split(
331+
f"tar -xvzf {output_file} -C {os.path.join(job_dir, 'extracted_output')} --ignore-zeros"
332+
),
317333
)
318334
cmp = filecmp.dircmp(
319335
os.path.join(temp_repo, "submodule"),
@@ -344,6 +360,8 @@ def test_package_without_include_submodules(packager, temp_repo):
344360
assert os.path.exists(output_file)
345361
subprocess.check_call(shlex.split(f"mkdir -p {os.path.join(job_dir, 'extracted_output')}"))
346362
subprocess.check_call(
347-
shlex.split(f"tar -xvzf {output_file} -C {os.path.join(job_dir, 'extracted_output')}"),
363+
shlex.split(
364+
f"tar -xvzf {output_file} -C {os.path.join(job_dir, 'extracted_output')} --ignore-zeros"
365+
),
348366
)
349367
assert len(os.listdir(os.path.join(job_dir, "extracted_output", "submodule"))) == 0

‎test/core/packaging/test_pattern.py‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -40,7 +40,9 @@ def test_package_with_include_pattern_rel_path(tmpdir):
4040
assert os.path.exists(output_file)
4141
subprocess.check_call(shlex.split(f"mkdir -p {os.path.join(job_dir, 'extracted_output')}"))
4242
subprocess.check_call(
43-
shlex.split(f"tar -xvzf {output_file} -C {os.path.join(job_dir, 'extracted_output')}"),
43+
shlex.split(
44+
f"tar -xvzf {output_file} -C {os.path.join(job_dir, 'extracted_output')} --ignore-zeros"
45+
),
4446
)
4547
cmp = filecmp.dircmp(
4648
os.path.join(tmpdir, "extra"),

0 commit comments

Comments
 (0)