Skip to content

Commit 165f92f

Browse files
egalevmoelasmar
andauthored
Prevent already downloaded dependencies from hanging the download (#209)
* Prevent already downloaded dependencies from hanging the download process (it's waiting for user input) * Fixed test * Fixed tests * Formatted Co-authored-by: Mohamed Elasmar <[email protected]>
1 parent 2ad17ff commit 165f92f

File tree

3 files changed

+24
-21
lines changed

3 files changed

+24
-21
lines changed

aws_lambda_builders/workflows/python_pip/packager.py

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -583,7 +583,7 @@ def build_wheel(self, wheel, directory, compile_c=True):
583583

584584
def download_all_dependencies(self, requirements_filename, directory):
585585
"""Download all dependencies as sdist or wheel."""
586-
arguments = ["-r", requirements_filename, "--dest", directory]
586+
arguments = ["-r", requirements_filename, "--dest", directory, "--exists-action", "i"]
587587
rc, out, err = self._execute("download", arguments)
588588
# When downloading all dependencies we expect to get an rc of 0 back
589589
# since we are casting a wide net here letting pip have options about

tests/functional/workflows/python_pip/test_packager.py

Lines changed: 22 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -231,7 +231,7 @@ def test_can_get_whls_all_manylinux(self, tmpdir, pip_runner, osutils):
231231
appdir, builder = self._make_appdir_and_dependency_builder(reqs, tmpdir, runner)
232232
requirements_file = os.path.join(appdir, "requirements.txt")
233233
pip.packages_to_download(
234-
expected_args=["-r", requirements_file, "--dest", mock.ANY],
234+
expected_args=["-r", requirements_file, "--dest", mock.ANY, "--exists-action", "i"],
235235
packages=["foo-1.2-cp36-cp36m-manylinux1_x86_64.whl", "bar-1.2-cp36-cp36m-manylinux1_x86_64.whl"],
236236
)
237237

@@ -250,7 +250,7 @@ def test_can_use_abi3_whl_for_any_python3(self, tmpdir, pip_runner, osutils):
250250
appdir, builder = self._make_appdir_and_dependency_builder(reqs, tmpdir, runner)
251251
requirements_file = os.path.join(appdir, "requirements.txt")
252252
pip.packages_to_download(
253-
expected_args=["-r", requirements_file, "--dest", mock.ANY],
253+
expected_args=["-r", requirements_file, "--dest", mock.ANY, "--exists-action", "i"],
254254
packages=[
255255
"foo-1.2-cp33-abi3-manylinux1_x86_64.whl",
256256
"bar-1.2-cp34-abi3-manylinux1_x86_64.whl",
@@ -274,7 +274,7 @@ def test_can_expand_purelib_whl(self, tmpdir, pip_runner, osutils):
274274
appdir, builder = self._make_appdir_and_dependency_builder(reqs, tmpdir, runner)
275275
requirements_file = os.path.join(appdir, "requirements.txt")
276276
pip.packages_to_download(
277-
expected_args=["-r", requirements_file, "--dest", mock.ANY],
277+
expected_args=["-r", requirements_file, "--dest", mock.ANY, "--exists-action", "i"],
278278
packages=["foo-1.2-cp36-cp36m-manylinux1_x86_64.whl"],
279279
whl_contents=["foo-1.2.data/purelib/foo/"],
280280
)
@@ -294,7 +294,7 @@ def test_can_expand_platlib_whl(self, tmpdir, pip_runner, osutils):
294294
appdir, builder = self._make_appdir_and_dependency_builder(reqs, tmpdir, runner)
295295
requirements_file = os.path.join(appdir, "requirements.txt")
296296
pip.packages_to_download(
297-
expected_args=["-r", requirements_file, "--dest", mock.ANY],
297+
expected_args=["-r", requirements_file, "--dest", mock.ANY, "--exists-action", "i"],
298298
packages=["foo-1.2-cp36-cp36m-manylinux1_x86_64.whl"],
299299
whl_contents=["foo-1.2.data/platlib/foo/"],
300300
)
@@ -316,7 +316,7 @@ def test_can_expand_platlib_and_purelib(self, tmpdir, pip_runner, osutils):
316316
appdir, builder = self._make_appdir_and_dependency_builder(reqs, tmpdir, runner)
317317
requirements_file = os.path.join(appdir, "requirements.txt")
318318
pip.packages_to_download(
319-
expected_args=["-r", requirements_file, "--dest", mock.ANY],
319+
expected_args=["-r", requirements_file, "--dest", mock.ANY, "--exists-action", "i"],
320320
packages=["foo-1.2-cp36-cp36m-manylinux1_x86_64.whl"],
321321
whl_contents=["foo-1.2.data/platlib/foo/", "foo-1.2.data/purelib/bar/"],
322322
)
@@ -338,7 +338,7 @@ def test_does_ignore_data(self, tmpdir, pip_runner, osutils):
338338
appdir, builder = self._make_appdir_and_dependency_builder(reqs, tmpdir, runner)
339339
requirements_file = os.path.join(appdir, "requirements.txt")
340340
pip.packages_to_download(
341-
expected_args=["-r", requirements_file, "--dest", mock.ANY],
341+
expected_args=["-r", requirements_file, "--dest", mock.ANY, "--exists-action", "i"],
342342
packages=["foo-1.2-cp36-cp36m-manylinux1_x86_64.whl"],
343343
whl_contents=["foo/placeholder", "foo-1.2.data/data/bar/"],
344344
)
@@ -361,7 +361,7 @@ def test_does_ignore_include(self, tmpdir, pip_runner, osutils):
361361
appdir, builder = self._make_appdir_and_dependency_builder(reqs, tmpdir, runner)
362362
requirements_file = os.path.join(appdir, "requirements.txt")
363363
pip.packages_to_download(
364-
expected_args=["-r", requirements_file, "--dest", mock.ANY],
364+
expected_args=["-r", requirements_file, "--dest", mock.ANY, "--exists-action", "i"],
365365
packages=["foo-1.2-cp36-cp36m-manylinux1_x86_64.whl"],
366366
whl_contents=["foo/placeholder", "foo.1.2.data/includes/bar/"],
367367
)
@@ -384,7 +384,7 @@ def test_does_ignore_scripts(self, tmpdir, pip_runner, osutils):
384384
appdir, builder = self._make_appdir_and_dependency_builder(reqs, tmpdir, runner)
385385
requirements_file = os.path.join(appdir, "requirements.txt")
386386
pip.packages_to_download(
387-
expected_args=["-r", requirements_file, "--dest", mock.ANY],
387+
expected_args=["-r", requirements_file, "--dest", mock.ANY, "--exists-action", "i"],
388388
packages=["foo-1.2-cp36-cp36m-manylinux1_x86_64.whl"],
389389
whl_contents=["{package_name}/placeholder", "{data_dir}/scripts/bar/placeholder"],
390390
)
@@ -408,7 +408,7 @@ def test_can_expand_platlib_and_platlib_and_root(self, tmpdir, pip_runner, osuti
408408
appdir, builder = self._make_appdir_and_dependency_builder(reqs, tmpdir, runner)
409409
requirements_file = os.path.join(appdir, "requirements.txt")
410410
pip.packages_to_download(
411-
expected_args=["-r", requirements_file, "--dest", mock.ANY],
411+
expected_args=["-r", requirements_file, "--dest", mock.ANY, "--exists-action", "i"],
412412
packages=["foo-1.2-cp36-cp36m-manylinux1_x86_64.whl"],
413413
whl_contents=[
414414
"{package_name}/placeholder",
@@ -432,7 +432,7 @@ def test_can_get_whls_mixed_compat(self, tmpdir, osutils, pip_runner):
432432
appdir, builder = self._make_appdir_and_dependency_builder(reqs, tmpdir, runner)
433433
requirements_file = os.path.join(appdir, "requirements.txt")
434434
pip.packages_to_download(
435-
expected_args=["-r", requirements_file, "--dest", mock.ANY],
435+
expected_args=["-r", requirements_file, "--dest", mock.ANY, "--exists-action", "i"],
436436
packages=[
437437
"foo-1.0-cp36-none-any.whl",
438438
"bar-1.2-cp36-cp36m-manylinux1_x86_64.whl",
@@ -455,7 +455,7 @@ def test_can_get_py27_whls(self, tmpdir, osutils, pip_runner):
455455
appdir, builder = self._make_appdir_and_dependency_builder(reqs, tmpdir, runner)
456456
requirements_file = os.path.join(appdir, "requirements.txt")
457457
pip.packages_to_download(
458-
expected_args=["-r", requirements_file, "--dest", mock.ANY],
458+
expected_args=["-r", requirements_file, "--dest", mock.ANY, "--exists-action", "i"],
459459
packages=[
460460
"foo-1.0-cp27-none-any.whl",
461461
"bar-1.2-cp27-none-manylinux1_x86_64.whl",
@@ -501,7 +501,7 @@ def test_does_fail_on_narrow_py27_unicode(self, tmpdir, osutils, pip_runner):
501501
appdir, builder = self._make_appdir_and_dependency_builder(reqs, tmpdir, runner)
502502
requirements_file = os.path.join(appdir, "requirements.txt")
503503
pip.packages_to_download(
504-
expected_args=["-r", requirements_file, "--dest", mock.ANY],
504+
expected_args=["-r", requirements_file, "--dest", mock.ANY, "--exists-action", "i"],
505505
packages=["baz-1.5-cp27-cp27m-linux_x86_64.whl"],
506506
)
507507

@@ -523,7 +523,7 @@ def test_does_fail_on_python_1_whl(self, tmpdir, osutils, pip_runner):
523523
appdir, builder = self._make_appdir_and_dependency_builder(reqs, tmpdir, runner)
524524
requirements_file = os.path.join(appdir, "requirements.txt")
525525
pip.packages_to_download(
526-
expected_args=["-r", requirements_file, "--dest", mock.ANY],
526+
expected_args=["-r", requirements_file, "--dest", mock.ANY, "--exists-action", "i"],
527527
packages=["baz-1.5-cp14-cp14m-linux_x86_64.whl"],
528528
)
529529

@@ -545,7 +545,7 @@ def test_can_replace_incompat_whl(self, tmpdir, osutils, pip_runner):
545545
appdir, builder = self._make_appdir_and_dependency_builder(reqs, tmpdir, runner)
546546
requirements_file = os.path.join(appdir, "requirements.txt")
547547
pip.packages_to_download(
548-
expected_args=["-r", requirements_file, "--dest", mock.ANY],
548+
expected_args=["-r", requirements_file, "--dest", mock.ANY, "--exists-action", "i"],
549549
packages=["foo-1.0-cp36-none-any.whl", "bar-1.2-cp36-cp36m-macosx_10_6_intel.whl"],
550550
)
551551
# Once the initial download has 1 incompatible whl file. The second,
@@ -581,7 +581,7 @@ def test_allowlist_sqlalchemy(self, tmpdir, osutils, pip_runner):
581581
appdir, builder = self._make_appdir_and_dependency_builder(reqs, tmpdir, runner)
582582
requirements_file = os.path.join(appdir, "requirements.txt")
583583
pip.packages_to_download(
584-
expected_args=["-r", requirements_file, "--dest", mock.ANY],
584+
expected_args=["-r", requirements_file, "--dest", mock.ANY, "--exists-action", "i"],
585585
packages=["SQLAlchemy-1.1.18-cp36-cp36m-macosx_10_11_x86_64.whl"],
586586
)
587587
pip.packages_to_download(
@@ -614,7 +614,7 @@ def test_can_build_sdist(self, tmpdir, osutils, pip_runner):
614614
appdir, builder = self._make_appdir_and_dependency_builder(reqs, tmpdir, runner)
615615
requirements_file = os.path.join(appdir, "requirements.txt")
616616
pip.packages_to_download(
617-
expected_args=["-r", requirements_file, "--dest", mock.ANY],
617+
expected_args=["-r", requirements_file, "--dest", mock.ANY, "--exists-action", "i"],
618618
packages=["foo-1.2.zip", "bar-1.2-cp36-cp36m-manylinux1_x86_64.whl"],
619619
)
620620
# Foo is built from and is pure python so it yields a compatible
@@ -638,7 +638,7 @@ def test_build_sdist_makes_incompatible_whl(self, tmpdir, osutils, pip_runner):
638638
appdir, builder = self._make_appdir_and_dependency_builder(reqs, tmpdir, runner)
639639
requirements_file = os.path.join(appdir, "requirements.txt")
640640
pip.packages_to_download(
641-
expected_args=["-r", requirements_file, "--dest", mock.ANY],
641+
expected_args=["-r", requirements_file, "--dest", mock.ANY, "--exists-action", "i"],
642642
packages=["foo-1.2.zip", "bar-1.2-cp36-cp36m-manylinux1_x86_64.whl"],
643643
)
644644
# foo is compiled since downloading it failed to get any wheels. And
@@ -671,7 +671,10 @@ def test_can_build_package_with_optional_c_speedups_and_no_wheel(self, tmpdir, o
671671
# In this scenario we are downloading a package that has no wheel files
672672
# at all, and optional c speedups. The initial download will yield an
673673
# sdist since there were no wheels.
674-
pip.packages_to_download(expected_args=["-r", requirements_file, "--dest", mock.ANY], packages=["foo-1.2.zip"])
674+
pip.packages_to_download(
675+
expected_args=["-r", requirements_file, "--dest", mock.ANY, "--exists-action", "i"],
676+
packages=["foo-1.2.zip"],
677+
)
675678

676679
# Chalice should now try and build this into a wheel file. Since it has
677680
# optional c speedups it will build a platform dependent wheel file
@@ -713,7 +716,7 @@ def test_build_into_existing_dir_with_preinstalled_packages(self, tmpdir, osutil
713716
appdir, builder = self._make_appdir_and_dependency_builder(reqs, tmpdir, runner)
714717
requirements_file = os.path.join(appdir, "requirements.txt")
715718
pip.packages_to_download(
716-
expected_args=["-r", requirements_file, "--dest", mock.ANY],
719+
expected_args=["-r", requirements_file, "--dest", mock.ANY, "--exists-action", "i"],
717720
packages=["foo-1.2.zip", "bar-1.2-cp36-cp36m-manylinux1_x86_64.whl"],
718721
)
719722
pip.packages_to_download(

tests/unit/workflows/python_pip/test_packager.py

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -219,7 +219,7 @@ def test_download_all_deps(self, pip_factory):
219219

220220
assert len(pip.calls) == 1
221221
call = pip.calls[0]
222-
assert call.args == ["download", "-r", "requirements.txt", "--dest", "directory"]
222+
assert call.args == ["download", "-r", "requirements.txt", "--dest", "directory", "--exists-action", "i"]
223223
assert call.env_vars is None
224224
assert call.shim is None
225225

0 commit comments

Comments
 (0)