Skip to content

Commit 182ff82

Browse files
hawflauChris Rehn
andauthored
Support PEP 600 platform tags (#234)
* Support PEP 600 platform tags * fix test * Fix integration test in appveyor by setting GO111MODULE to auto * Fix GOPATH env var in appveyor script * Add note about setting GO111MODULE in appveyor script * Update python_pip integration test * Fix numpy version for py36 in integ test * Update aws_lambda_builders/workflows/python_pip/packager.py Co-authored-by: Chris Rehn <[email protected]>
1 parent b663326 commit 182ff82

File tree

5 files changed

+155
-28
lines changed

5 files changed

+155
-28
lines changed

.appveyor.yml

Lines changed: 11 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,6 @@ image:
44
- Ubuntu
55

66
environment:
7-
GOPATH: c:\gopath
87
GOVERSION: 1.11
98
GRADLE_OPTS: -Dorg.gradle.daemon=false
109
nodejs_version: "8.10.0"
@@ -45,6 +44,9 @@ for:
4544
only:
4645
- image: Visual Studio 2017
4746

47+
environment:
48+
GOPATH: c:\gopath
49+
4850
install:
4951
# To run Nodejs workflow integ tests
5052
- ps: Install-Product node 8.10
@@ -65,6 +67,10 @@ for:
6567
- "choco install dep"
6668
- setx PATH "C:\go\bin;C:\gopath\bin;C:\Program Files (x86)\Bazaar\;C:\Program Files\Mercurial;%PATH%;"
6769
- "go version"
70+
# set set GO111MODULE to auto to enable module-aware mode only when a go.mod file is present in the current directory or any parent directory
71+
# https://blog.golang.org/go116-module-changes#TOC_2.
72+
# This is required for the go dep integration tests
73+
- "go env -w GO111MODULE=auto"
6874
- "go env"
6975

7076
# setup Gradle
@@ -102,6 +108,10 @@ for:
102108
- sh: "wget https://services.gradle.org/distributions/gradle-5.5-bin.zip -P /tmp"
103109
- sh: "sudo unzip -d /opt/gradle /tmp/gradle-*.zip"
104110
- sh: "PATH=/opt/gradle/gradle-5.5/bin:$PATH"
111+
# set set GO111MODULE to auto to enable module-aware mode only when a go.mod file is present in the current directory or any parent directory
112+
# https://blog.golang.org/go116-module-changes#TOC_2.
113+
# This is required for the go dep integration tests
114+
- sh: "go env -w GO111MODULE=auto"
105115

106116
build_script:
107117
- "python -c \"import sys; print(sys.executable)\""

aws_lambda_builders/workflows/python_pip/packager.py

Lines changed: 67 additions & 25 deletions
Original file line numberDiff line numberDiff line change
@@ -150,13 +150,23 @@ class DependencyBuilder(object):
150150
packager.
151151
"""
152152

153-
_MANYLINUX_COMPATIBLE_PLATFORM = {
154-
"any",
155-
"linux_x86_64",
156-
"manylinux1_x86_64",
157-
"manylinux2010_x86_64",
158-
"manylinux2014_x86_64",
153+
_ADDITIONAL_COMPATIBLE_PLATFORM = {"any", "linux_x86_64"}
154+
_MANYLINUX_LEGACY_MAP = {
155+
"manylinux1_x86_64": "manylinux_2_5_x86_64",
156+
"manylinux2010_x86_64": "manylinux_2_12_x86_64",
157+
"manylinux2014_x86_64": "manylinux_2_17_x86_64",
159158
}
159+
# Mapping of abi to glibc version in Lambda runtime.
160+
_RUNTIME_GLIBC = {
161+
"cp27mu": (2, 17),
162+
"cp36m": (2, 17),
163+
"cp37m": (2, 17),
164+
"cp38": (2, 26),
165+
}
166+
# Fallback version if we're on an unknown python version
167+
# not in _RUNTIME_GLIBC.
168+
# Unlikely to hit this case.
169+
_DEFAULT_GLIBC = (2, 17)
160170
_COMPATIBLE_PACKAGE_ALLOWLIST = {"sqlalchemy"}
161171

162172
def __init__(self, osutils, runtime, pip_runner=None):
@@ -341,30 +351,62 @@ def _categorize_wheel_files(self, directory):
341351

342352
def _is_compatible_wheel_filename(self, filename):
343353
wheel = filename[:-4]
344-
implementation, abi, platform = wheel.split("-")[-3:]
345-
# Verify platform is compatible
346-
if platform not in self._MANYLINUX_COMPATIBLE_PLATFORM:
347-
return False
348-
349354
lambda_runtime_abi = get_lambda_abi(self.runtime)
355+
for implementation, abi, platform in self._iter_all_compatibility_tags(wheel):
356+
if not self._is_compatible_platform_tag(lambda_runtime_abi, platform):
357+
continue
358+
359+
# Verify that the ABI is compatible with lambda. Either none or the
360+
# correct type for the python version cp27mu for py27 and cp36m for
361+
# py36.
362+
if abi == "none":
363+
return True
364+
prefix_version = implementation[:3]
365+
if prefix_version == "cp3":
366+
# Deploying python 3 function which means we need cp36m abi
367+
# We can also accept abi3 which is the CPython 3 Stable ABI and
368+
# will work on any version of python 3.
369+
if abi == lambda_runtime_abi or abi == "abi3":
370+
return True
371+
elif prefix_version == "cp2":
372+
# Deploying to python 2 function which means we need cp27mu abi
373+
if abi == "cp27mu":
374+
return True
375+
# Don't know what we have but it didn't pass compatibility tests.
376+
return False
350377

351-
# Verify that the ABI is compatible with lambda. Either none or the
352-
# correct type for the python version cp27mu for py27 and cp36m for
353-
# py36.
354-
if abi == "none":
378+
def _is_compatible_platform_tag(self, expected_abi, platform):
379+
"""
380+
Verify if a platform tag is compatible based on PEP 600
381+
https://www.python.org/dev/peps/pep-0600/#specification
382+
383+
In addition to checking the tag pattern, we also need to verify the glibc version
384+
"""
385+
if platform in self._ADDITIONAL_COMPATIBLE_PLATFORM:
355386
return True
356-
prefix_version = implementation[:3]
357-
if prefix_version == "cp3":
358-
# Deploying python 3 function which means we need cp36m abi
359-
# We can also accept abi3 which is the CPython 3 Stable ABI and
360-
# will work on any version of python 3.
361-
return abi == lambda_runtime_abi or abi == "abi3"
362-
elif prefix_version == "cp2":
363-
# Deploying to python 2 function which means we need cp27mu abi
364-
return abi == "cp27mu"
365-
# Don't know what we have but it didn't pass compatibility tests.
387+
elif platform.startswith("manylinux"):
388+
perennial_tag = self._MANYLINUX_LEGACY_MAP.get(platform, platform)
389+
m = re.match("manylinux_([0-9]+)_([0-9]+)_(.*)", perennial_tag)
390+
if m is None:
391+
return False
392+
tag_major, tag_minor = [int(x) for x in m.groups()[:2]]
393+
runtime_major, runtime_minor = self._RUNTIME_GLIBC.get(expected_abi, self._DEFAULT_GLIBC)
394+
if (tag_major, tag_minor) <= (runtime_major, runtime_minor):
395+
# glibc version is compatible with Lambda Runtime
396+
return True
366397
return False
367398

399+
def _iter_all_compatibility_tags(self, wheel):
400+
"""
401+
Generates all possible combination of tag sets as described in PEP 425
402+
https://www.python.org/dev/peps/pep-0425/#compressed-tag-sets
403+
"""
404+
implementation_tag, abi_tag, platform_tag = wheel.split("-")[-3:]
405+
for implementation in implementation_tag.split("."):
406+
for abi in abi_tag.split("."):
407+
for platform in platform_tag.split("."):
408+
yield (implementation, abi, platform)
409+
368410
def _apply_wheel_allowlist(self, compatible_wheels, incompatible_wheels):
369411
compatible_wheels = set(compatible_wheels)
370412
actual_incompatible_wheels = set()

tests/functional/workflows/python_pip/test_packager.py

Lines changed: 72 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -449,6 +449,48 @@ def test_can_get_whls_mixed_compat(self, tmpdir, osutils, pip_runner):
449449
for req in reqs:
450450
assert req in installed_packages
451451

452+
def test_can_support_pep_600_tags(self, tmpdir, osutils, pip_runner):
453+
reqs = ["foo"]
454+
pip, runner = pip_runner
455+
appdir, builder = self._make_appdir_and_dependency_builder(reqs, tmpdir, runner)
456+
requirements_file = os.path.join(appdir, "requirements.txt")
457+
pip.packages_to_download(
458+
expected_args=["-r", requirements_file, "--dest", mock.ANY, "--exists-action", "i"],
459+
packages=[
460+
"foo-1.2-cp36-cp36m-manylinux_2_12_x86_64.whl",
461+
],
462+
)
463+
464+
site_packages = os.path.join(appdir, ".chalice.", "site-packages")
465+
with osutils.tempdir() as scratch_dir:
466+
builder.build_site_packages(requirements_file, site_packages, scratch_dir)
467+
installed_packages = os.listdir(site_packages)
468+
469+
pip.validate()
470+
for req in reqs:
471+
assert req in installed_packages
472+
473+
def test_can_support_compressed_tags(self, tmpdir, osutils, pip_runner):
474+
reqs = ["foo"]
475+
pip, runner = pip_runner
476+
appdir, builder = self._make_appdir_and_dependency_builder(reqs, tmpdir, runner)
477+
requirements_file = os.path.join(appdir, "requirements.txt")
478+
pip.packages_to_download(
479+
expected_args=["-r", requirements_file, "--dest", mock.ANY, "--exists-action", "i"],
480+
packages=[
481+
"foo-1.2-cp36-cp36m-manylinux_2_5_x86_64.manylinux1_x86_64.whl",
482+
],
483+
)
484+
485+
site_packages = os.path.join(appdir, ".chalice.", "site-packages")
486+
with osutils.tempdir() as scratch_dir:
487+
builder.build_site_packages(requirements_file, site_packages, scratch_dir)
488+
installed_packages = os.listdir(site_packages)
489+
490+
pip.validate()
491+
for req in reqs:
492+
assert req in installed_packages
493+
452494
def test_can_get_py27_whls(self, tmpdir, osutils, pip_runner):
453495
reqs = ["foo", "bar", "baz"]
454496
pip, runner = pip_runner
@@ -539,6 +581,36 @@ def test_does_fail_on_python_1_whl(self, tmpdir, osutils, pip_runner):
539581
assert missing_packages[0].identifier == "baz==1.5"
540582
assert len(installed_packages) == 0
541583

584+
def test_does_fail_on_pep_600_tag_with_unsupported_glibc_version(self, tmpdir, osutils, pip_runner):
585+
reqs = ["foo", "bar", "baz", "qux"]
586+
pip, runner = pip_runner
587+
appdir, builder = self._make_appdir_and_dependency_builder(reqs, tmpdir, runner)
588+
requirements_file = os.path.join(appdir, "requirements.txt")
589+
pip.packages_to_download(
590+
expected_args=["-r", requirements_file, "--dest", mock.ANY, "--exists-action", "i"],
591+
packages=[
592+
"foo-1.2-cp36-cp36m-manylinux_2_12_x86_64.whl",
593+
"bar-1.2-cp36-cp36m-manylinux_2_999_x86_64.whl",
594+
"baz-1.2-cp36-cp36m-manylinux_3_12_x86_64.whl",
595+
"qux-1.2-cp36-cp36m-manylinux_3_999_x86_64.whl",
596+
],
597+
)
598+
599+
site_packages = os.path.join(appdir, ".chalice.", "site-packages")
600+
with osutils.tempdir() as scratch_dir:
601+
with pytest.raises(MissingDependencyError) as e:
602+
builder.build_site_packages(requirements_file, site_packages, scratch_dir)
603+
installed_packages = os.listdir(site_packages)
604+
605+
missing_packages = list(e.value.missing)
606+
pip.validate()
607+
assert len(missing_packages) == 3
608+
missing_package_identifies = [package.identifier for package in missing_packages]
609+
assert "bar==1.2" in missing_package_identifies
610+
assert "baz==1.2" in missing_package_identifies
611+
assert "qux==1.2" in missing_package_identifies
612+
assert len(installed_packages) == 1
613+
542614
def test_can_replace_incompat_whl(self, tmpdir, osutils, pip_runner):
543615
reqs = ["foo", "bar"]
544616
pip, runner = pip_runner

tests/integration/workflows/python_pip/test_python_pip.py

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -47,8 +47,10 @@ def test_must_build_python_project(self):
4747

4848
if self.runtime == "python2.7":
4949
expected_files = self.test_data_files.union({"numpy", "numpy-1.15.4.data", "numpy-1.15.4.dist-info"})
50-
else:
50+
elif self.runtime == "python3.6":
5151
expected_files = self.test_data_files.union({"numpy", "numpy-1.17.4.dist-info"})
52+
else:
53+
expected_files = self.test_data_files.union({"numpy", "numpy-1.20.3.dist-info", "numpy.libs"})
5254
output_files = set(os.listdir(self.artifacts_dir))
5355
self.assertEqual(expected_files, output_files)
5456

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,2 +1,3 @@
11
numpy==1.15.4; python_version == '2.7'
2-
numpy==1.17.4; python_version >= '3.6'
2+
numpy==1.17.4; python_version == '3.6'
3+
numpy==1.20.3; python_version >= '3.7'

0 commit comments

Comments
 (0)