Skip to content

Commit a995c6d

Browse files
fangererfangerer
committed
Merge branch 'master' into topic/GR-13908
2 parents adf6312 + fca7296 commit a995c6d

File tree

28 files changed

+260
-108
lines changed

28 files changed

+260
-108
lines changed

‎CHANGELOG.md

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,15 @@
33
This changelog summarizes major changes between GraalVM versions of the Python
44
language runtime. The main focus is on user-observable behavior of the engine.
55

6+
## Version 1.0.0 RC13
7+
8+
* Support marshal.dumps and marshal.loads for code objects and some other built-in objects
9+
* Fix installation of NumPy in a venv
10+
* Initial support for module mmap
11+
* Support debugging with workspace files in the Chrome debugger
12+
* Support the PEP 553 breakpoint() message
13+
* Support running weak reference callbacks and signals on the main thread
14+
615
## Version 1.0.0 RC12
716

817
* Support the `__class__` variable in the class scope

‎ci.jsonnet

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
{
2-
overlay: "e08ca6c848943a04c5e46d54272ff927ff3d51f7",
2+
overlay: "d20cc2abdeb3cfb022e1a8035e40e350e5cfe5fc",
33

44
// ======================================================================================================
55
//

‎graalpython/com.oracle.graal.python.shell/src/com/oracle/graal/python/shell/GraalPythonMain.java

Lines changed: 0 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -563,12 +563,6 @@ protected void printHelp(OptionCategory maxCategory) {
563563
"GRAAL_PYTHON_OPTIONS: This environment variable can include default options that\n" +
564564
" are always passed to the launcher. These are not shell expanded and given to\n" +
565565
" the launcher as-is.");
566-
if (maxCategory.compareTo(OptionCategory.DEBUG) >= 0) {
567-
print("\nGraalPython performance debugging options:\n" +
568-
"-debug-perf : Enable tracing of Truffle compilations and its warnings\n" +
569-
"-dump : Enable dumping of compilation graphs to IGV\n" +
570-
"-compile-truffle-immediately : Start compiling on first invocation and throw compilation exceptions");
571-
}
572566
}
573567

574568
@Override

‎graalpython/com.oracle.graal.python.tck/src/com/oracle/graal/python/tck/PythonProvider.java

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -50,6 +50,7 @@
5050
import static org.graalvm.polyglot.tck.TypeDescriptor.NUMBER;
5151
import static org.graalvm.polyglot.tck.TypeDescriptor.OBJECT;
5252
import static org.graalvm.polyglot.tck.TypeDescriptor.STRING;
53+
import static org.graalvm.polyglot.tck.TypeDescriptor.INSTANTIABLE;
5354
import static org.graalvm.polyglot.tck.TypeDescriptor.array;
5455
import static org.graalvm.polyglot.tck.TypeDescriptor.executable;
5556
import static org.graalvm.polyglot.tck.TypeDescriptor.intersection;
@@ -81,6 +82,9 @@ public class PythonProvider implements LanguageProvider {
8182
private static final TypeDescriptor NUMBER_OBJECT = TypeDescriptor.union(NUMBER, BOOLEAN, OBJECT, array(ANY));
8283
private static final TypeDescriptor PSEQUENCE_OBJECT = TypeDescriptor.union(array(ANY), STRING);
8384

85+
// Python types are just objects
86+
private static final TypeDescriptor PYTHON_TYPE = TypeDescriptor.union(OBJECT, INSTANTIABLE);
87+
8488
public String getId() {
8589
return ID;
8690
}
@@ -268,7 +272,7 @@ public Collection<? extends Snippet> createStatements(Context context) {
268272
addStatementSnippet(context, snippets, "class", "class Custom0:\n" +
269273
" def __init__(self, val):\n" +
270274
" self.val = val\n" +
271-
"Custom0", OBJECT, ANY);
275+
"Custom0", PYTHON_TYPE, ANY);
272276
addStatementSnippet(context, snippets, "class", "class Custom1:\n" +
273277
" def __call__(self, val):\n" +
274278
" return val\n" +

‎graalpython/com.oracle.graal.python.test/src/com/oracle/graal/python/test/debug/PythonDebugTest.java

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -168,7 +168,7 @@ public void testSteppingAsExpected() throws Throwable {
168168
});
169169
expectSuspended((SuspendedEvent event) -> {
170170
DebugStackFrame frame = event.getTopStackFrame();
171-
assertEquals(9, frame.getSourceSection().getStartLine());
171+
assertEquals(8, frame.getSourceSection().getStartLine());
172172
event.prepareStepOut(1);
173173
});
174174
expectSuspended((SuspendedEvent event) -> {

‎graalpython/com.oracle.graal.python.test/src/com/oracle/graal/python/test/grammar/TestParserTranslator.java

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -86,6 +86,7 @@
8686
import com.oracle.graal.python.nodes.frame.WriteNode;
8787
import com.oracle.graal.python.nodes.function.FunctionDefinitionNode;
8888
import com.oracle.graal.python.nodes.function.GeneratorExpressionNode;
89+
import com.oracle.graal.python.nodes.function.InnerRootNode;
8990
import com.oracle.graal.python.nodes.generator.DictConcatNode;
9091
import com.oracle.graal.python.nodes.literal.BooleanLiteralNode;
9192
import com.oracle.graal.python.nodes.literal.ComplexLiteralNode;
@@ -155,6 +156,8 @@ private Node unpackModuleBodyWrappers(Node n) {
155156
if (((WriteLocalVariableNode) n).getIdentifier().equals(FrameSlotIDs.RETURN_SLOT_ID)) {
156157
actual = ((WriteLocalVariableNode) n).getRhs();
157158
}
159+
} else if (n instanceof InnerRootNode) {
160+
actual = n.getChildren().iterator().next();
158161
}
159162
if (actual == n) {
160163
return n;

‎graalpython/com.oracle.graal.python.test/src/python_unittests.py

Lines changed: 79 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
# Copyright (c) 2018, Oracle and/or its affiliates. All rights reserved.
1+
# Copyright (c) 2018, 2019, Oracle and/or its affiliates. All rights reserved.
22
# DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
33
#
44
# The Universal Permissive License (UPL), Version 1.0
@@ -74,6 +74,9 @@
7474
PTRN_JAVA_EXCEPTION = re.compile(r'^(?P<exception>com\.oracle\.[^:]*):(?P<message>.*)')
7575
PTRN_MODULE_NOT_FOUND = re.compile(r'.*ModuleNotFound: \'(?P<module>.*)\'\..*', re.DOTALL)
7676
PTRN_IMPORT_ERROR = re.compile(r".*cannot import name \'(?P<module>.*)\'.*", re.DOTALL)
77+
PTRN_REMOTE_HOST = re.compile(r"(?P<user>\w+)@(?P<host>[\w.]+):(?P<path>.+)")
78+
PTRN_VALID_CSV_NAME = re.compile(r"unittests-\d{4}-\d{2}-\d{2}.csv")
79+
7780

7881
# ----------------------------------------------------------------------------------------------------------------------
7982
#
@@ -192,6 +195,27 @@ def _sorter(iterable):
192195
return unittests[:limit] if limit else unittests
193196

194197

198+
def get_remote_host(scp_path):
199+
match = re.match(PTRN_REMOTE_HOST, scp_path)
200+
return match.group('user'), match.group('host'), match.group('path')
201+
202+
203+
def ssh_ls(scp_path):
204+
user, host, path = get_remote_host(scp_path)
205+
cmd = ['ssh', '{}@{}'.format(user, host), 'ls', '-l', path]
206+
return map(lambda l: l.split()[-1], _run_cmd(cmd)[1].splitlines())
207+
208+
209+
def read_csv(path):
210+
rows = []
211+
with open(path, "r") as CSV_FILE:
212+
reader = csv.reader(CSV_FILE)
213+
headers = next(reader)[1:]
214+
for row in reader:
215+
rows.append(row)
216+
return rows
217+
218+
195219
# ----------------------------------------------------------------------------------------------------------------------
196220
#
197221
# result (output processing)
@@ -597,7 +621,7 @@ def format_val(row, k):
597621
elif k == Col.NUM_TESTS:
598622
_class = "text-dark"
599623
else:
600-
_class = "text-danger" if value < 0 else "text-muted"
624+
_class = "text-danger" if value and value < 0 else "text-muted"
601625
return '<span class="{} h6"><b>{}</b></span>'.format(_class, value)
602626

603627
_tbody = '\n'.join([
@@ -694,6 +718,10 @@ def main(prog, args):
694718
parser.add_argument("-o", "--only_tests", help="Run only these unittests (comma sep values).", default=None)
695719
parser.add_argument("-s", "--skip_tests", help="Run all unittests except (comma sep values)."
696720
"the only_tets option takes precedence", default=None)
721+
parser.add_argument("-r", "--regression_running_tests", help="Regression threshold for running tests.", type=float,
722+
default=None)
723+
parser.add_argument("-g", "--gate", help="Run in gate mode (Skip cpython runs; Do not upload results; "
724+
"Detect regressions).", action="store_true")
697725
parser.add_argument("path", help="Path to store the csv output and logs to.", nargs='?', default=None)
698726

699727
global flags
@@ -708,6 +736,12 @@ def main(prog, args):
708736
else:
709737
log("[INFO] results will not be saved remotely")
710738

739+
if flags.gate:
740+
log("[INFO] running in gate mode")
741+
if not flags.regression_running_tests:
742+
log("[WARNING] --regression_running_tests not set while in gate mode. "
743+
"Regression detection will not be performed")
744+
711745
def _fmt(t):
712746
t = t.strip()
713747
return os.path.join(flags.tests_path, t if t.endswith(".py") else t + ".py")
@@ -720,15 +754,22 @@ def _fmt(t):
720754
unittests = get_unittests(flags.tests_path, limit=flags.limit, skip_tests=skip_tests)
721755

722756
# get cpython stats
723-
log(HR)
724-
log("[INFO] get cpython stats")
725-
cpy_results = run_unittests(unittests, 60 * 5, with_cpython=True)
726-
cpy_stats = process_output('\n'.join(cpy_results))[-1]
757+
if not flags.gate:
758+
log(HR)
759+
log("[INFO] get cpython stats")
760+
cpy_results = run_unittests(unittests, 60 * 5, with_cpython=True)
761+
cpy_stats = process_output('\n'.join(cpy_results))[-1]
762+
# handle the timeout
763+
timeout = flags.timeout if flags.timeout else None
764+
else:
765+
cpy_stats = None
766+
# handle the timeout
767+
timeout = flags.timeout if flags.timeout else 60 * 5 # 5 minutes if no value specified (in gate mode only)
727768

728769
# get graalpython stats
729770
log(HR)
730771
log("[INFO] get graalpython stats")
731-
results = run_unittests(unittests, flags.timeout, with_cpython=False)
772+
results = run_unittests(unittests, timeout, with_cpython=False)
732773
txt_report_path = file_name(TXT_RESULTS_NAME, current_date)
733774
output = save_as_txt(txt_report_path, results)
734775

@@ -749,15 +790,44 @@ def _fmt(t):
749790
log("[JAVA ISSUES] \n{}", pformat(dict(java_issues)))
750791

751792
html_report_path = file_name(HTML_RESULTS_NAME, current_date)
752-
save_as_html(html_report_path, rows, totals, missing_modules, cannot_import_modules, java_issues, current_date)
793+
if not flags.gate:
794+
save_as_html(html_report_path, rows, totals, missing_modules, cannot_import_modules, java_issues, current_date)
753795

754-
if flags.path:
796+
if not flags.gate and flags.path:
755797
log("[SAVE] saving results to {} ... ", flags.path)
756798
scp(txt_report_path, flags.path)
757799
scp(csv_report_path, flags.path)
758800
scp(html_report_path, flags.path)
759801

802+
gate_failed = False
803+
if flags.gate and flags.regression_running_tests:
804+
log("[REGRESSION] detecting regression, acceptance threshold = {}%".format(
805+
flags.regression_running_tests * 100))
806+
csv_files = list(filter(lambda entry: True if PTRN_VALID_CSV_NAME.match(entry) else False, ssh_ls(flags.path)))
807+
last_csv = csv_files[-1]
808+
# log('\n'.join(csv_files))
809+
# read the remote csv and extract stats
810+
log("[REGRESSION] comparing against: {}".format(last_csv))
811+
scp('{}/{}'.format(flags.path, last_csv), '.', destination_name=last_csv)
812+
rows = read_csv(last_csv)
813+
prev_totals = {
814+
Col.NUM_TESTS: int(rows[-1][1]),
815+
Col.NUM_FAILS: int(rows[-1][2]),
816+
Col.NUM_ERRORS: int(rows[-1][3]),
817+
Col.NUM_SKIPPED: int(rows[-1][4]),
818+
Col.NUM_PASSES: int(rows[-1][5]),
819+
}
820+
print(prev_totals)
821+
if float(totals[Col.NUM_TESTS]) < float(prev_totals[Col.NUM_TESTS]) * (1.0 - flags.regression_running_tests):
822+
log("[REGRESSION] REGRESSION DETECTED, passed {} tests vs {} from {}".format(
823+
totals[Col.NUM_TESTS], prev_totals[Col.NUM_TESTS], last_csv))
824+
gate_failed = True
825+
else:
826+
log("[REGRESSION] no regression detected")
827+
760828
log("[DONE]")
829+
if flags.gate and gate_failed:
830+
exit(1)
761831

762832

763833
if __name__ == "__main__":

‎graalpython/com.oracle.graal.python.test/src/tests/test_itertools.py

Lines changed: 26 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,26 @@
1+
# Copyright (c) 2019, Oracle and/or its affiliates.
2+
# Copyright (C) 1996-2017 Python Software Foundation
3+
#
4+
# Licensed under the PYTHON SOFTWARE FOUNDATION LICENSE VERSION 2
5+
6+
import unittest
7+
from itertools import *
8+
9+
class CombinationsTests(unittest.TestCase):
10+
11+
def test_combinations_with_replacement(self):
12+
cwr = combinations_with_replacement
13+
self.assertRaises(TypeError, cwr, 'abc') # missing r argument
14+
self.assertRaises(TypeError, cwr, 'abc', 2, 1) # too many arguments
15+
self.assertRaises(TypeError, cwr, None) # pool is not iterable
16+
self.assertRaises(ValueError, cwr, 'abc', -2) # r is negative
17+
18+
result = list()
19+
for a in cwr('ABC', 2):
20+
result += a
21+
correct = [('A','A'), ('A','B'), ('A','C'), ('B','B'), ('B','C'), ('C','C')]
22+
compare = list();
23+
for a in correct:
24+
compare += a;
25+
self.assertEqual(result,compare)
26+

‎graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/TruffleCextBuiltins.java

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1436,7 +1436,7 @@ Object doUnpackLong(long start, long stop, long step, long length) {
14361436
}
14371437

14381438
@Specialization(replaces = {"doUnpackLong", "doUnpack"})
1439-
Object doUnpackLongOvf(long start, long stop, long step, long length) {
1439+
Object doUnpackLongOvf0(long start, long stop, long step, long length) {
14401440
try {
14411441
PSlice tmpSlice = factory().createSlice(PInt.intValueExact(start), PInt.intValueExact(stop), PInt.intValueExact(step));
14421442
SliceInfo actualIndices = tmpSlice.computeIndices(length > Integer.MAX_VALUE ? Integer.MAX_VALUE : PInt.intValueExact(length));
@@ -1446,8 +1446,8 @@ Object doUnpackLongOvf(long start, long stop, long step, long length) {
14461446
}
14471447
}
14481448

1449-
@Specialization(replaces = {"doUnpackLongOvf"})
1450-
Object doUnpackLongOvf(Object start, Object stop, Object step, Object lengthObj,
1449+
@Specialization(replaces = {"doUnpackLongOvf0"})
1450+
Object doUnpackLongOvf1(Object start, Object stop, Object step, Object lengthObj,
14511451
@Cached("createOverflow()") CastToIndexNode castToIndexNode,
14521452
@Cached("create()") IsBuiltinClassProfile profile,
14531453
@Cached("create()") SliceLiteralNode sliceLiteralNode) {

‎graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/objects/code/PCode.java

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -304,7 +304,7 @@ private static String extractName(RootNode rootNode) {
304304
if (rootNode instanceof ModuleRootNode) {
305305
name = rootNode.getName();
306306
} else if (rootNode instanceof FunctionRootNode) {
307-
name = ((FunctionRootNode) rootNode).getFunctionName();
307+
name = ((FunctionRootNode) rootNode).getName();
308308
} else {
309309
name = rootNode.getName();
310310
}

0 commit comments

Comments
 (0)