Skip to content

Commit cfa05e8

Browse files
timfeltimfel
committed
[GR-17682] Make mx scripts Python 3 compatible
PullRequest: graalpython/633
2 parents 3b73d50 + 266f1b2 commit cfa05e8

File tree

8 files changed

+75
-24
lines changed

8 files changed

+75
-24
lines changed

‎ci.jsonnet

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -20,7 +20,7 @@ local builder = import 'ci_common/builder.libsonnet';
2020
//
2121
// ------------------------------------------------------------------------------------------------------
2222
local gates = [
23-
// unittests
23+
// unittests on JDK8
2424
builder.testGate(type="unittest", platform="linux"),
2525
builder.testGate(type="unittest", platform="darwin"),
2626
builder.testGateTime(type="tagged-unittest", platform="linux", timelimit=const.TIME_LIMIT["2h"]),
@@ -29,6 +29,10 @@ local builder = import 'ci_common/builder.libsonnet';
2929
builder.testGate(type="svm-unittest", platform="darwin"),
3030
builder.testGate(type="unittest-jython", platform="linux"),
3131

32+
// JDK11
33+
// builder.testGate11(type="unittest", platform="linux"),
34+
// builder.testGate11(type="svm-unittest", platform="linux"),
35+
3236
// junit
3337
builder.testGate(type="junit", platform="linux"),
3438
builder.testGate(type="junit", platform="darwin"),

‎ci_common/builder.libsonnet

Lines changed: 11 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,10 @@ local mixins = import 'mixins.libsonnet';
2525
local common = base + mixins.labsjdk8 + {
2626
dynamicImports:: "/compiler",
2727

28+
environment +: {
29+
MX_PYTHON_VERSION: "3",
30+
},
31+
2832
setup +: [
2933
["mx", "sforceimports"],
3034
["mx", "--dynamicimports", self.dynamicImports, "build"],
@@ -56,11 +60,15 @@ local mixins = import 'mixins.libsonnet';
5660
baseGraalGate:: baseGraalGate,
5761

5862
// specific gates
59-
testGate(type, platform)::
63+
local testGate = function(type, platform)
6064
baseGraalGate + {tags:: "python-"+type} + mixins.getPlatform(platform) + {name: "python-"+ type +"-"+platform},
65+
testGate:: testGate,
66+
67+
testGate11(type, platform)::
68+
testGate(type, platform) + mixins.labsjdk11 + {name: "python-"+ type +"-jdk11-"+platform},
6169

6270
testGateTime(type, platform, timelimit)::
63-
baseGraalGate + {tags:: "python-"+type} + mixins.getPlatform(platform) + {name: "python-"+ type +"-"+platform} + {timelimit: timelimit},
71+
testGate(type, platform) + {timelimit: timelimit},
6472

6573
local baseStyleGate = baseGraalGate + mixins.eclipse + mixins.linux + {
6674
tags:: "style",
@@ -115,4 +123,4 @@ local mixins = import 'mixins.libsonnet';
115123
name: "python-coverage"
116124
},
117125

118-
}
126+
}

‎ci_common/mixins.libsonnet

Lines changed: 17 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -38,6 +38,9 @@ local const = import 'constants.libsonnet';
3838
"pip:pylint": "==1.1.0",
3939
"llvm": "==4.0.1",
4040
},
41+
environment: {
42+
"LC_CTYPE": "en_US.UTF-8", // make sure we have the correct locale
43+
},
4144
},
4245
darwin:: darwin,
4346

@@ -72,18 +75,28 @@ local const = import 'constants.libsonnet';
7275
},
7376
eclipse:: eclipse,
7477

75-
local labsjdk8 = {
76-
downloads +: {
77-
JAVA_HOME: utils.download("oraclejdk", "8u212-jvmci-19.2-b01"),
78-
},
78+
local labsjdk = {
7979
environment +: {
8080
CI: "true",
8181
GRAALVM_CHECK_EXPERIMENTAL_OPTIONS: "true",
8282
PATH: "$JAVA_HOME/bin:$PATH",
8383
},
8484
},
85+
86+
local labsjdk8 = labsjdk + {
87+
downloads +: {
88+
JAVA_HOME: utils.download("oraclejdk", "8u221-jvmci-19.3-b01"),
89+
},
90+
},
8591
labsjdk8:: labsjdk8,
8692

93+
local labsjdk11 = labsjdk + {
94+
downloads +: {
95+
JAVA_HOME: utils.download("labsjdk", "11-20190830-095818"),
96+
},
97+
},
98+
labsjdk11:: labsjdk11,
99+
87100
local graal = labsjdk8 + {
88101
environment +: {
89102
HOST_VM: const.JVM.server,

‎graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/CodecsModuleBuiltins.java

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -106,6 +106,7 @@ public class CodecsModuleBuiltins extends PythonBuiltins {
106106
CHARSET_MAP.put("L1", StandardCharsets.ISO_8859_1);
107107

108108
// utf-8
109+
CHARSET_MAP.put("UTF-8", StandardCharsets.UTF_8);
109110
CHARSET_MAP.put("utf-8", StandardCharsets.UTF_8);
110111
CHARSET_MAP.put("utf_8", StandardCharsets.UTF_8);
111112
CHARSET_MAP.put("U8", StandardCharsets.UTF_8);

‎graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/PosixModuleBuiltins.java

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -746,7 +746,7 @@ private PException fileNoFound(String path) {
746746

747747
@TruffleBoundary(allowInlining = true, transferToInterpreterOnException = false)
748748
private static long strToLong(String name) throws NumberFormatException {
749-
return new Long(name).longValue();
749+
return Long.decode(name).longValue();
750750
}
751751

752752
@TruffleBoundary(allowInlining = true)

‎mx.graalpython/mx_graalpython.py

Lines changed: 11 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -50,6 +50,12 @@
5050
SUITE_SULONG = mx.suite("sulong")
5151

5252

53+
# compatibility between Python versions
54+
PY3 = sys.version_info[0] == 3
55+
if PY3:
56+
raw_input = input
57+
58+
5359
def _get_core_home():
5460
return os.path.join(SUITE.dir, "graalpython", "lib-graalpython")
5561

@@ -113,7 +119,7 @@ def do_run_python(args, extra_vm_args=None, env=None, jdk=None, **kwargs):
113119

114120
dists = ['GRAALPYTHON', 'TRUFFLE_NFI', 'SULONG']
115121
env["PYTHONUSERBASE"] = mx_subst.path_substitutions.substitute("<path:PYTHON_USERBASE>")
116-
122+
117123
vm_args, graalpython_args = mx.extract_VM_args(args, useDoubleDash=True, defaultAllVMArgs=False)
118124
graalpython_args, additional_dists = _extract_graalpython_internal_options(graalpython_args)
119125
dists += additional_dists
@@ -317,7 +323,7 @@ def _graalpytest_root():
317323

318324
def run_python_unittests(python_binary, args=None, paths=None, aot_compatible=True, exclude=None):
319325
args = args or []
320-
args = ["--experimental-options=true",
326+
args = ["--experimental-options=true",
321327
"--python.CatchAllExceptions=true",
322328
mx_subst.path_substitutions.substitute("--python.CAPI=<path:com.oracle.graal.python.cext>"),
323329
] + args
@@ -372,6 +378,7 @@ def graalpython_gate_runner(args, tasks):
372378
mx.log("Running tests with CPython")
373379
test_args = [_graalpytest_driver(), "-v", _graalpytest_root()]
374380
mx.run(["python3"] + test_args, nonZeroIsFatal=True)
381+
mx.run(["env"])
375382
run_python_unittests(python_gvm())
376383

377384
with Task('GraalPython sandboxed tests', tasks, tags=[GraalPythonTags.unittest_sandboxed]) as task:
@@ -498,7 +505,7 @@ def run_shared_lib_test(args=None):
498505
fd = name = progname = None
499506
try:
500507
fd, name = tempfile.mkstemp(suffix='.c')
501-
os.write(fd, """
508+
os.write(fd, b"""
502509
#include "stdio.h"
503510
#include "polyglot_api.h"
504511
@@ -605,7 +612,7 @@ def run_shared_lib_test(args=None):
605612
}
606613
return test_basic_python_function();
607614
}
608-
""" % ("1" if "sandboxed" in args else "0"))
615+
""" % (b"1" if "sandboxed" in args else b"0"))
609616
os.close(fd)
610617
progname = os.path.join(SUITE.dir, "graalpython-embedded-tool")
611618
mx.log("".join(["Running ", "'clang", "-I%s" % svm_lib_path, "-L%s" % svm_lib_path, name, "-o", progname, "-lpolyglot"]))

‎mx.graalpython/mx_graalpython_benchmark.py

Lines changed: 27 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -80,35 +80,54 @@ def is_sandboxed_configuration(conf):
8080
return conf == CONFIGURATION_SANDBOXED or conf == CONFIGURATION_SANDBOXED_MULTI
8181

8282

83+
# from six
84+
def add_metaclass(metaclass):
85+
"""Class decorator for creating a class with a metaclass."""
86+
def wrapper(cls):
87+
orig_vars = cls.__dict__.copy()
88+
slots = orig_vars.get('__slots__')
89+
if slots is not None:
90+
if isinstance(slots, str):
91+
slots = [slots]
92+
for slots_var in slots:
93+
orig_vars.pop(slots_var)
94+
orig_vars.pop('__dict__', None)
95+
orig_vars.pop('__weakref__', None)
96+
if hasattr(cls, '__qualname__'):
97+
orig_vars['__qualname__'] = cls.__qualname__
98+
return metaclass(cls.__name__, cls.__bases__, orig_vars)
99+
return wrapper
100+
101+
83102
@contextmanager
84103
def environ(env):
85-
def _handle_var((k, v)):
104+
def _handle_var(key_value):
105+
(k, v) = key_value
86106
if v is None:
87107
del os.environ[k]
88108
else:
89109
os.environ[k] = str(v)
90110

91111
if env:
92112
prev_env = {v: os.getenv(v) for v in env}
93-
map(_handle_var, env.items())
113+
list(map(_handle_var, list(env.items())))
94114
else:
95115
prev_env = None
96116

97117
try:
98118
yield
99119
finally:
100120
if prev_env:
101-
map(_handle_var, prev_env.items())
121+
list(map(_handle_var, list(prev_env.items())))
102122

103123

104124
# ----------------------------------------------------------------------------------------------------------------------
105125
#
106126
# the vm definitions
107127
#
108128
# ----------------------------------------------------------------------------------------------------------------------
129+
@add_metaclass(ABCMeta)
109130
class AbstractPythonVm(Vm):
110-
__metaclass__ = ABCMeta
111-
112131
def __init__(self, config_name, options=None, env=None):
113132
super(AbstractPythonVm, self).__init__()
114133
self._config_name = config_name
@@ -156,9 +175,8 @@ def run(self, cwd, args):
156175
return ret_code, out.data
157176

158177

178+
@add_metaclass(ABCMeta)
159179
class AbstractPythonIterationsControlVm(AbstractPythonVm):
160-
__metaclass__ = ABCMeta
161-
162180
def __init__(self, config_name, options=None, env=None, iterations=None):
163181
super(AbstractPythonIterationsControlVm, self).__init__(config_name, options=options, env=env)
164182
try:
@@ -424,7 +442,7 @@ def createVmCommandLineArgs(self, benchmarks, bmSuiteArgs):
424442
for pth in self._python_path:
425443
if hasattr(pth, '__call__'):
426444
pth = pth()
427-
assert isinstance(pth, (str, unicode))
445+
assert isinstance(pth, str)
428446
python_path.append(pth)
429447
cmd_args += ['-p', ",".join(python_path)]
430448

@@ -438,7 +456,7 @@ def createVmCommandLineArgs(self, benchmarks, bmSuiteArgs):
438456
return vm_options + vm_args + cmd_args
439457

440458
def benchmarkList(self, bm_suite_args):
441-
return self._benchmarks.keys()
459+
return list(self._benchmarks.keys())
442460

443461
def benchmarks(self):
444462
raise FutureWarning('the benchmarks method has been deprecated for VmBenchmarkSuite instances, '

‎mx.graalpython/suite.py

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -44,15 +44,15 @@
4444
},
4545
{
4646
"name": "sulong",
47-
"version": "432df38e6864205e10e52aa4058ce868aedab929",
47+
"version": "01bb0670bac1ab1e87322b0621bc4ff2b76e79bc",
4848
"subdir": True,
4949
"urls": [
5050
{"url": "https://github.com/oracle/graal", "kind": "git"},
5151
]
5252
},
5353
{
5454
"name": "regex",
55-
"version": "432df38e6864205e10e52aa4058ce868aedab929",
55+
"version": "01bb0670bac1ab1e87322b0621bc4ff2b76e79bc",
5656
"subdir": True,
5757
"urls": [
5858
{"url": "https://github.com/oracle/graal", "kind": "git"},

0 commit comments

Comments
 (0)