Skip to content

Commit c1e4026

Browse files
msimacekmsimacek
committed
[GR-43684][GR-43716] Make patched pip work with virtualenv
PullRequest: graalpython/2610
2 parents 105fc32 + 9591f8b commit c1e4026

File tree

13 files changed

+220
-67
lines changed

13 files changed

+220
-67
lines changed

‎graalpython/lib-graalpython/modules/graalpy_virtualenv/__init__.py renamed to ‎graalpy_virtualenv/graalpy_virtualenv/__init__.py

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
# Copyright (c) 2022, 2022, Oracle and/or its affiliates. All rights reserved.
1+
# Copyright (c) 2022, 2023, Oracle and/or its affiliates. All rights reserved.
22
# DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
33
#
44
# The Universal Permissive License (UPL), Version 1.0

‎graalpython/lib-graalpython/modules/graalpy_virtualenv/graalpy.py renamed to ‎graalpy_virtualenv/graalpy_virtualenv/graalpy.py

Lines changed: 53 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
# Copyright (c) 2022, 2022, Oracle and/or its affiliates. All rights reserved.
1+
# Copyright (c) 2022, 2023, Oracle and/or its affiliates. All rights reserved.
22
# DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
33
#
44
# The Universal Permissive License (UPL), Version 1.0
@@ -36,16 +36,16 @@
3636
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
3737
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
3838
# SOFTWARE.
39+
import sys
3940
from contextlib import contextmanager
41+
from functools import lru_cache
4042
from pathlib import Path
41-
from sys import executable as sys_executable
4243
from subprocess import check_output as subprocess_check_output
4344

44-
4545
from virtualenv.create.creator import Creator
4646
from virtualenv.create.describe import PosixSupports
4747
from virtualenv.seed.embed.pip_invoke import PipInvoke
48-
from virtualenv.seed.wheels import get_wheel, Version
48+
from virtualenv.seed.wheels import get_wheel
4949
from virtualenv.seed.wheels.bundle import from_dir
5050

5151

@@ -90,11 +90,13 @@ def can_describe(cls, interpreter):
9090
try:
9191
from virtualenv.run import SeederSelector
9292
_get_default_orig = SeederSelector._get_default
93+
9394
def _seeder_selector_get_default_override(self):
9495
if self.interpreter.implementation == "GraalVM":
9596
return "graalpy"
9697
else:
9798
return _get_default_orig(self)
99+
98100
SeederSelector._get_default = _seeder_selector_get_default_override
99101
except ImportError:
100102
pass
@@ -113,6 +115,42 @@ def create(self):
113115
"or environment variable VIRTUALENV_CREATOR=venv")
114116

115117

118+
@lru_cache()
119+
def get_ensurepip_path(exe):
120+
if exe.samefile(sys.executable):
121+
import ensurepip
122+
ensurepip_path = ensurepip.__path__[0]
123+
else:
124+
cmd = [exe, "-u", "-c", 'import ensurepip; print(ensurepip.__path__[0])']
125+
ensurepip_path = subprocess_check_output(cmd, universal_newlines=True).strip()
126+
return Path(ensurepip_path) / '_bundled'
127+
128+
129+
def pip_wheel_env_run(search_dirs, app_data, env, exe):
130+
env = env.copy()
131+
env.update({"PIP_USE_WHEEL": "1", "PIP_USER": "0", "PIP_NO_INPUT": "1"})
132+
ensurepip_path = get_ensurepip_path(exe)
133+
if ensurepip_path:
134+
wheel_paths = list(ensurepip_path.glob('pip-*.whl'))
135+
if wheel_paths:
136+
env["PYTHONPATH"] = str(wheel_paths[0])
137+
return env
138+
wheel = get_wheel(
139+
distribution="pip",
140+
version=None,
141+
for_py_version=f"{sys.version_info.major}.{sys.version_info.minor}",
142+
search_dirs=search_dirs,
143+
download=False,
144+
app_data=app_data,
145+
do_periodic_update=False,
146+
env=env,
147+
)
148+
if wheel is None:
149+
raise RuntimeError("could not find the embedded pip")
150+
env["PYTHONPATH"] = str(wheel.path)
151+
return env
152+
153+
116154
class GraalPySeeder(PipInvoke):
117155
"""
118156
Seeder for GraalPy that reuses the PipInvoke seeder to install setuptools
@@ -130,13 +168,21 @@ class GraalPySeeder(PipInvoke):
130168
regardless whether the seeder may support that flag or not (the latter case
131169
casing an error).
132170
"""
171+
172+
def run(self, creator):
173+
if not self.enabled:
174+
return
175+
for_py_version = creator.interpreter.version_release_str
176+
with self.get_pip_install_cmd(creator.exe, for_py_version) as cmd:
177+
env = pip_wheel_env_run(self.extra_search_dir, self.app_data, self.env, creator.exe)
178+
self._execute(cmd, env)
179+
133180
@contextmanager
134181
def get_pip_install_cmd(self, exe, for_py_version):
135182
cmd = [str(exe), "-m", "pip", "-q", "install", "--only-binary", ":all:", "--disable-pip-version-check"]
136183
if not self.download:
137184
cmd.append("--no-index")
138-
folders = set()
139-
ensurepip_path = self.get_ensurepip_path(exe)
185+
ensurepip_path = get_ensurepip_path(exe)
140186
for dist, version in self.distribution_to_versions().items():
141187
if ensurepip_path and version == 'bundle' and dist in ('pip', 'setuptools'):
142188
wheel = from_dir(dist, None, for_py_version, [ensurepip_path])
@@ -153,19 +199,5 @@ def get_pip_install_cmd(self, exe, for_py_version):
153199
)
154200
if wheel is None:
155201
raise RuntimeError("could not get wheel for distribution {}".format(dist))
156-
folders.add(str(wheel.path.parent))
157-
cmd.append(Version.as_pip_req(dist, wheel.version))
158-
for folder in sorted(folders):
159-
cmd.extend(["--find-links", str(folder)])
160-
print(f"{cmd=}")
202+
cmd.append(str(wheel.path))
161203
yield cmd
162-
163-
@staticmethod
164-
def get_ensurepip_path(exe):
165-
if exe.samefile(sys_executable):
166-
import ensurepip
167-
ensurepip_path = ensurepip.__path__[0]
168-
else:
169-
ensurepip_path = subprocess_check_output(
170-
(exe, "-u", "-c", 'import ensurepip; print(ensurepip.__path__[0])'), universal_newlines=True)
171-
return Path(ensurepip_path.strip()) / '_bundled'

‎graalpy_virtualenv/pyproject.toml

Lines changed: 67 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,67 @@
1+
# Copyright (c) 2023, Oracle and/or its affiliates. All rights reserved.
2+
# DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
3+
#
4+
# The Universal Permissive License (UPL), Version 1.0
5+
#
6+
# Subject to the condition set forth below, permission is hereby granted to any
7+
# person obtaining a copy of this software, associated documentation and/or
8+
# data (collectively the "Software"), free of charge and under any and all
9+
# copyright rights in the Software, and any and all patent rights owned or
10+
# freely licensable by each licensor hereunder covering either (i) the
11+
# unmodified Software as contributed to or provided by such licensor, or (ii)
12+
# the Larger Works (as defined below), to deal in both
13+
#
14+
# (a) the Software, and
15+
#
16+
# (b) any piece of software and/or hardware listed in the lrgrwrks.txt file if
17+
# one is included with the Software each a "Larger Work" to which the Software
18+
# is contributed by such licensors),
19+
#
20+
# without restriction, including without limitation the rights to copy, create
21+
# derivative works of, display, perform, and distribute the Software and make,
22+
# use, sell, offer for sale, import, export, have made, and have sold the
23+
# Software and the Larger Work(s), and to sublicense the foregoing rights on
24+
# either these or other terms.
25+
#
26+
# This license is subject to the following condition:
27+
#
28+
# The above copyright notice and either this complete permission notice or at a
29+
# minimum a reference to the UPL must be included in all copies or substantial
30+
# portions of the Software.
31+
#
32+
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
33+
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
34+
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
35+
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
36+
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
37+
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
38+
# SOFTWARE.
39+
40+
[build-system]
41+
requires = ["hatchling"]
42+
build-backend = "hatchling.build"
43+
44+
[project]
45+
name = "graalpy-virtualenv"
46+
version = "0.0.1"
47+
license = "UPL-1.0"
48+
authors = [
49+
{ name = "GraalPy Developers", email = "[email protected]" },
50+
]
51+
keywords = [
52+
"environments",
53+
"isolated",
54+
"virtual",
55+
]
56+
dependencies = [
57+
"virtualenv",
58+
]
59+
60+
[project.entry-points."virtualenv.create"]
61+
graalpy-posix = "graalpy_virtualenv.graalpy:GraalPyCreatorPosix"
62+
63+
[project.entry-points."virtualenv.seed"]
64+
graalpy = "graalpy_virtualenv:graalpy.GraalPySeeder"
65+
66+
[project.urls]
67+
Homepage = "https://graalvm.org/python"

‎graalpython/com.oracle.graal.python.test/src/tests/test_multiprocessing.py

Lines changed: 14 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
# Copyright (c) 2022, 2022, Oracle and/or its affiliates. All rights reserved.
1+
# Copyright (c) 2022, 2023, Oracle and/or its affiliates. All rights reserved.
22
# DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
33
#
44
# The Universal Permissive License (UPL), Version 1.0
@@ -37,6 +37,7 @@
3737
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
3838
# SOFTWARE.
3939
import multiprocessing
40+
import sys
4041
import time
4142
from multiprocessing.connection import wait
4243

@@ -71,3 +72,15 @@ def test_wait():
7172
assert set(res) == set([b, x])
7273
assert b.recv() == 1
7374
assert x.recv() == 2
75+
76+
77+
def test_array_read():
78+
# TODO multiprocessing.Array doesn't work on emulated backend
79+
if sys.implementation.name == 'graalpy' and __graalpython__.posix_module_backend() == 'java':
80+
return
81+
# This used to be buggy due to wrong usage of memoryview offsets when two objects were allocated in the same block
82+
# Don't remove the unused value on the next line
83+
# noinspection PyUnusedLocal
84+
num = multiprocessing.Value('d', 0.0)
85+
arr = multiprocessing.Array('i', range(10))
86+
assert arr[1] == 1

‎graalpython/com.oracle.graal.python.test/src/tests/unittest_tags/test_multiprocessing_main_handling.txt

Lines changed: 0 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -11,12 +11,3 @@
1111
*graalpython.lib-python.3.test.test_multiprocessing_main_handling.ForkServerCmdLineTest.test_script_compiled
1212
*graalpython.lib-python.3.test.test_multiprocessing_main_handling.ForkServerCmdLineTest.test_zipfile
1313
*graalpython.lib-python.3.test.test_multiprocessing_main_handling.ForkServerCmdLineTest.test_zipfile_compiled
14-
*graalpython.lib-python.3.test.test_multiprocessing_main_handling.SpawnCmdLineTest.test_basic_script
15-
*graalpython.lib-python.3.test.test_multiprocessing_main_handling.SpawnCmdLineTest.test_basic_script_no_suffix
16-
*graalpython.lib-python.3.test.test_multiprocessing_main_handling.SpawnCmdLineTest.test_directory
17-
*graalpython.lib-python.3.test.test_multiprocessing_main_handling.SpawnCmdLineTest.test_directory_compiled
18-
*graalpython.lib-python.3.test.test_multiprocessing_main_handling.SpawnCmdLineTest.test_ipython_workaround
19-
*graalpython.lib-python.3.test.test_multiprocessing_main_handling.SpawnCmdLineTest.test_module_in_package_in_zipfile
20-
*graalpython.lib-python.3.test.test_multiprocessing_main_handling.SpawnCmdLineTest.test_package
21-
*graalpython.lib-python.3.test.test_multiprocessing_main_handling.SpawnCmdLineTest.test_package_compiled
22-
*graalpython.lib-python.3.test.test_multiprocessing_main_handling.SpawnCmdLineTest.test_zipfile

‎graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/objects/dict/DictBuiltins.java

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
/*
2-
* Copyright (c) 2017, 2022, Oracle and/or its affiliates.
2+
* Copyright (c) 2017, 2023, Oracle and/or its affiliates.
33
* Copyright (c) 2013, Regents of the University of California
44
*
55
* All rights reserved.
@@ -218,7 +218,7 @@ public Object popWithoutDefault(VirtualFrame frame, PDict dict, Object key, @Sup
218218
if (retVal != null) {
219219
return retVal;
220220
}
221-
throw raise(PythonBuiltinClassType.KeyError, ErrorMessages.S, key);
221+
throw raise(KeyError, new Object[]{key});
222222
}
223223
}
224224

@@ -322,7 +322,7 @@ protected abstract static class DefaultMissingNode extends PNodeWithRaise {
322322

323323
@Specialization
324324
Object run(TruffleString key) {
325-
throw raise(KeyError, ErrorMessages.S, key);
325+
throw raise(KeyError, new Object[]{key});
326326
}
327327

328328
@Fallback
@@ -352,7 +352,7 @@ Object run(VirtualFrame frame, PDict self, Object key,
352352
if (found != null) {
353353
return PNone.NONE;
354354
}
355-
throw raise(KeyError, ErrorMessages.S, key);
355+
throw raise(KeyError, new Object[]{key});
356356
}
357357
}
358358

‎graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/objects/memoryview/PMemoryView.java

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
/*
2-
* Copyright (c) 2020, 2022, Oracle and/or its affiliates. All rights reserved.
2+
* Copyright (c) 2020, 2023, Oracle and/or its affiliates. All rights reserved.
33
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
44
*
55
* The Universal Permissive License (UPL), Version 1.0
@@ -381,7 +381,7 @@ void readIntoBuffer(int srcOffset, Object dest, int destOffset, int length, Pyth
381381
byte readByte(int byteOffset,
382382
@Shared("bufferLib") @CachedLibrary(limit = "3") PythonBufferAccessLibrary bufferLib) {
383383
assert isCContiguous() && !isReleased();
384-
return bufferLib.readByte(buffer, byteOffset);
384+
return bufferLib.readByte(buffer, offset + byteOffset);
385385
}
386386

387387
@ExportMessage

‎graalpython/lib-graalpython/patches/pip/whl/pip-22.2.2.patch

Lines changed: 16 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,16 @@
1+
diff --git a/pip/__init__.py b/pip/__init__.py
2+
index 3d4b45a..3aa11e8 100644
3+
--- a/pip/__init__.py
4+
+++ b/pip/__init__.py
5+
@@ -11,3 +11,6 @@ def main(args: Optional[List[str]] = None) -> int:
6+
from pip._internal.utils.entrypoints import _wrapper
7+
8+
return _wrapper(args)
9+
+
10+
+
11+
+__GRAALPY_PATCHED = True
112
diff --git a/pip/_internal/cli/cmdoptions.py b/pip/_internal/cli/cmdoptions.py
2-
index 47ed92779..1d182ded3 100644
13+
index 47ed927..1d182de 100644
314
--- a/pip/_internal/cli/cmdoptions.py
415
+++ b/pip/_internal/cli/cmdoptions.py
516
@@ -891,7 +891,7 @@ disable_pip_version_check: Callable[..., Option] = partial(
@@ -12,7 +23,7 @@ index 47ed92779..1d182ded3 100644
1223
"of pip is available for download. Implied with --no-index.",
1324
)
1425
diff --git a/pip/_internal/index/package_finder.py b/pip/_internal/index/package_finder.py
15-
index 9bf247f02..816734859 100644
26+
index 9bf247f..8167348 100644
1627
--- a/pip/_internal/index/package_finder.py
1728
+++ b/pip/_internal/index/package_finder.py
1829
@@ -38,6 +38,7 @@ from pip._internal.utils.logging import indent_log
@@ -32,7 +43,7 @@ index 9bf247f02..816734859 100644
3243
"""
3344
Function to pass as the `key` argument to a call to sorted() to sort
3445
diff --git a/pip/_internal/operations/install/wheel.py b/pip/_internal/operations/install/wheel.py
35-
index 1af8978d4..15fee7f35 100644
46+
index 1af8978..15fee7f 100644
3647
--- a/pip/_internal/operations/install/wheel.py
3748
+++ b/pip/_internal/operations/install/wheel.py
3849
@@ -587,6 +587,9 @@ def _install_wheel(
@@ -47,7 +58,7 @@ index 1af8978d4..15fee7f35 100644
4758
# file in .data maps to same location as file in wheel root).
4859
diff --git a/pip/_internal/utils/graalpy.py b/pip/_internal/utils/graalpy.py
4960
new file mode 100644
50-
index 000000000..d8d97eb4b
61+
index 0000000..d8d97eb
5162
--- /dev/null
5263
+++ b/pip/_internal/utils/graalpy.py
5364
@@ -0,0 +1,145 @@
@@ -197,7 +208,7 @@ index 000000000..d8d97eb4b
197208
+ patchfiles.append(tuple(version))
198209
+ return patchfiles
199210
diff --git a/pip/_internal/utils/unpacking.py b/pip/_internal/utils/unpacking.py
200-
index 78b5c13ce..18a184c4f 100644
211+
index 78b5c13..18a184c 100644
201212
--- a/pip/_internal/utils/unpacking.py
202213
+++ b/pip/_internal/utils/unpacking.py
203214
@@ -255,3 +255,5 @@ def unpack_file(

0 commit comments

Comments
 (0)