Skip to content

Commit b13bb1e

Browse files
ericsnowcurrentlyericsnowcurrently
authored
Further venv-related Cleanup (#154)
This also includes some small fixes.
1 parent 74a4b48 commit b13bb1e

File tree

8 files changed

+630
-471
lines changed

8 files changed

+630
-471
lines changed

‎pyperformance/_benchmark.py

Lines changed: 13 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -175,7 +175,7 @@ def run(self, python, runid=None, pyperf_opts=None, *,
175175
verbose=False,
176176
):
177177
if venv and python == sys.executable:
178-
python = venv.get_python_program()
178+
python = venv.python
179179

180180
if not runid:
181181
from ..run import get_run_id
@@ -218,8 +218,18 @@ def _run_perf_script(python, runscript, runid, *,
218218
else:
219219
opts, inherit_envvar = _resolve_restricted_opts(opts)
220220
argv, env = _prep_cmd(python, runscript, opts, runid, inherit_envvar)
221-
_utils.run_command(argv, env=env, hide_stderr=not verbose)
222-
221+
hide_stderr = not verbose
222+
ec, _, stderr = _utils.run_cmd(
223+
argv,
224+
env=env,
225+
capture='stderr' if hide_stderr else None,
226+
)
227+
if ec != 0:
228+
if hide_stderr:
229+
sys.stderr.flush()
230+
sys.stderr.write(stderr)
231+
sys.stderr.flush()
232+
raise RuntimeError("Benchmark died")
223233
return pyperf.BenchmarkSuite.load(tmp)
224234

225235

‎pyperformance/_pip.py

Lines changed: 141 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,141 @@
1+
import os
2+
import os.path
3+
import shlex
4+
import subprocess
5+
import sys
6+
7+
from . import _utils, _pythoninfo
8+
9+
10+
GET_PIP_URL = 'https://bootstrap.pypa.io/get-pip.py'
11+
# pip 6 is the first version supporting environment markers
12+
MIN_PIP = '6.0'
13+
OLD_PIP = '7.1.2'
14+
OLD_SETUPTOOLS = '18.5'
15+
16+
17+
def get_best_pip_version(python):
18+
"""Return the pip to install for the given Python executable."""
19+
if not python or isinstance(python, str):
20+
info = _pythoninfo.get_info(python)
21+
else:
22+
info = python
23+
# On Python: 3.5a0 <= version < 3.5.0 (final), install pip 7.1.2,
24+
# the last version working on Python 3.5a0:
25+
# https://sourceforge.net/p/pyparsing/bugs/100/
26+
if 0x30500a0 <= info.sys.hexversion < 0x30500f0:
27+
return OLD_PIP
28+
else:
29+
return None
30+
31+
32+
def run_pip(cmd, *args, **kwargs):
33+
"""Return the result of running pip with the given args."""
34+
return _utils.run_python('-m', 'pip', cmd, *args, **kwargs)
35+
36+
37+
def is_pip_installed(python, *, env=None):
38+
"""Return True if pip is installed on the given Python executable."""
39+
ec, _, _ = run_pip('--version', env=env, capture=True, verbose=False)
40+
return ec == 0
41+
42+
43+
def install_pip(python=sys.executable, *,
44+
info=None,
45+
downloaddir=None,
46+
env=None,
47+
**kwargs
48+
):
49+
"""Install pip on the given Python executable."""
50+
if not python:
51+
python = getattr(info, 'executable', None) or sys.executable
52+
53+
# python -m ensurepip
54+
res = _utils.run_python(
55+
'-m', 'ensurepip', '--verbose',
56+
python=python,
57+
**kwargs
58+
)
59+
ec, _, _ = res
60+
if ec == 0 and is_pip_installed(python, env=env):
61+
return res
62+
63+
##############################
64+
# Fall back to get-pip.py.
65+
66+
if not downloaddir:
67+
downloaddir = '.'
68+
os.makedirs(downloaddir, exist_ok=True)
69+
70+
# download get-pip.py
71+
filename = os.path.join(downloaddir, 'get-pip.py')
72+
if not os.path.exists(filename):
73+
print("Download %s into %s" % (GET_PIP_URL, filename))
74+
_utils.download(GET_PIP_URL, filename)
75+
76+
# python get-pip.py
77+
argv = [python, '-u', filename]
78+
version = get_best_pip_version(info or python)
79+
if version:
80+
argv.append(version)
81+
res = _utils.run_cmd(argv, env=env)
82+
ec, _, _ = res
83+
if ec != 0:
84+
# get-pip.py was maybe not properly downloaded: remove it to
85+
# download it again next time
86+
os.unlink(filename)
87+
return res
88+
89+
90+
def upgrade_pip(python=sys.executable, *,
91+
info=None,
92+
installer=False,
93+
**kwargs,
94+
):
95+
"""Upgrade pip on the given Python to the latest version."""
96+
if not python:
97+
python = getattr(info, 'executable', None) or sys.executable
98+
99+
version = get_best_pip_version(info or python)
100+
if version:
101+
reqs = [f'pip=={version}']
102+
if installer:
103+
reqs.append(f'setuptools=={OLD_SETUPTOOLS}')
104+
else:
105+
# pip 6 is the first version supporting environment markers
106+
reqs = [f'pip>={MIN_PIP}']
107+
res = install_requirements(*reqs, python=python, upgrade=True, **kwargs)
108+
ec, _, _ = res
109+
if ec != 0:
110+
return res
111+
112+
if installer:
113+
# Upgrade installer dependencies (setuptools, ...)
114+
reqs = [
115+
f'setuptools>={OLD_SETUPTOOLS}',
116+
# install wheel so pip can cache binary wheel packages locally,
117+
# and install prebuilt wheel packages from PyPI.
118+
'wheel',
119+
]
120+
res = install_requirements(*reqs, python=python, upgrade=True, **kwargs)
121+
return res
122+
123+
124+
def install_requirements(reqs, *extra,
125+
upgrade=True,
126+
**kwargs
127+
):
128+
"""Install the given packages from PyPI."""
129+
args = []
130+
if upgrade:
131+
args.append('-U') # --upgrade
132+
for reqs in [reqs, *extra]:
133+
if os.path.exists(reqs):
134+
args.append('-r') # --requirement
135+
args.append(reqs)
136+
return run_pip('install', *args, **kwargs)
137+
138+
139+
def install_editable(projectroot, **kwargs):
140+
"""Install the given project as an "editable" install."""
141+
return run_pip('install', '-e', projectroot, **kwargs)

‎pyperformance/_utils.py

Lines changed: 86 additions & 28 deletions
Original file line numberDiff line numberDiff line change
@@ -23,7 +23,12 @@
2323
import errno
2424
import os
2525
import os.path
26+
import shlex
27+
import shutil
28+
import subprocess
29+
import sys
2630
import tempfile
31+
import urllib.request
2732

2833

2934
@contextlib.contextmanager
@@ -66,6 +71,16 @@ def resolve_file(filename, relroot=None):
6671
return resolved
6772

6873

74+
def safe_rmtree(path):
75+
if not os.path.exists(path):
76+
return False
77+
78+
print("Remove directory %s" % path)
79+
# XXX Pass onerror to report on any files that could not be deleted?
80+
shutil.rmtree(path)
81+
return True
82+
83+
6984
#######################################
7085
# platform utils
7186

@@ -77,43 +92,86 @@ def resolve_file(filename, relroot=None):
7792
MS_WINDOWS = (sys.platform == 'win32')
7893

7994

80-
def run_command(command, env=None, *, hide_stderr=True):
81-
if hide_stderr:
82-
kw = {'stderr': subprocess.PIPE}
83-
else:
84-
kw = {}
85-
86-
logging.info("Running `%s`",
87-
" ".join(list(map(str, command))))
95+
def run_cmd(argv, *, env=None, capture=None, verbose=True):
96+
try:
97+
cmdstr = ' '.join(shlex.quote(a) for a in argv)
98+
except TypeError:
99+
print(argv)
100+
raise # re-raise
101+
102+
if capture is True:
103+
capture = 'both'
104+
kw = dict(
105+
env=env,
106+
)
107+
if capture == 'both':
108+
kw.update(dict(
109+
stdout=subprocess.PIPE,
110+
stderr=subprocess.PIPE,
111+
))
112+
elif capture == 'combined':
113+
kw.update(dict(
114+
stdout=subprocess.PIPE,
115+
stderr=subprocess.STDOUT,
116+
))
117+
elif capture == 'stdout':
118+
kw.update(dict(
119+
stdout=subprocess.PIPE,
120+
))
121+
elif capture == 'stderr':
122+
kw.update(dict(
123+
stderr=subprocess.PIPE,
124+
))
125+
elif capture:
126+
raise NotImplementedError(repr(capture))
127+
if capture:
128+
kw.update(dict(
129+
encoding='utf-8',
130+
))
131+
132+
# XXX Use a logger.
133+
if verbose:
134+
print('#', cmdstr)
88135

89136
# Explicitly flush standard streams, required if streams are buffered
90137
# (not TTY) to write lines in the expected order
91138
sys.stdout.flush()
92139
sys.stderr.flush()
93140

94-
proc = subprocess.Popen(command,
95-
universal_newlines=True,
96-
env=env,
97-
**kw)
98141
try:
99-
stderr = proc.communicate()[1]
100-
except: # noqa
101-
if proc.stderr:
102-
proc.stderr.close()
103-
try:
104-
proc.kill()
105-
except OSError:
106-
# process already exited
107-
pass
108-
proc.wait()
142+
proc = subprocess.run(argv, **kw)
143+
except OSError as exc:
144+
if exc.errno == errno.ENOENT:
145+
if verbose:
146+
print('command failed (not found)')
147+
return 127, None, None
109148
raise
149+
if proc.returncode != 0 and verbose:
150+
print(f'Command failed with exit code {proc.returncode}')
151+
return proc.returncode, proc.stdout, proc.stderr
152+
153+
154+
def run_python(*args, python=sys.executable, **kwargs):
155+
if not isinstance(python, str) and python is not None:
156+
try:
157+
# See _pythoninfo.get_info().
158+
python = python.sys.executable
159+
except AttributeError:
160+
raise TypeError(f'expected python str, got {python!r}')
161+
return run_cmd([python, *args], **kwargs)
162+
163+
164+
#######################################
165+
# network utils
166+
167+
def download(url, filename):
168+
response = urllib.request.urlopen(url)
169+
with response:
170+
content = response.read()
110171

111-
if proc.returncode != 0:
112-
if hide_stderr:
113-
sys.stderr.flush()
114-
sys.stderr.write(stderr)
115-
sys.stderr.flush()
116-
raise RuntimeError("Benchmark died")
172+
with open(filename, 'wb') as fp:
173+
fp.write(content)
174+
fp.flush()
117175

118176

119177
#######################################

0 commit comments

Comments
 (0)