Merge remote-tracking branch 'origin/master' into tim/GR-9957/dynamic-cast

Author: timfel

graalpython/com.oracle.graal.python.cext/src/capi.h

@@ -73,6 +73,7 @@ PyAPI_DATA(PyTypeObject) _PyExc_BaseException;
 
 extern void* to_java(PyObject* obj);
 extern void* to_java_type(PyTypeObject* cls);
+void* native_to_java(PyObject* obj);
 extern PyObject* to_sulong(void *o);
 #define as_char_pointer(obj) polyglot_invoke(PY_TRUFFLE_CEXT, "to_char_pointer", to_java(obj))
 #define as_long(obj) ((long)polyglot_as_i64(polyglot_invoke(PY_TRUFFLE_CEXT, "to_long", to_java(obj))))

graalpython/com.oracle.graal.python.cext/src/object.c

@@ -539,7 +539,6 @@ PyObject* PyObject_CallObject(PyObject* callable, PyObject* args) {
 PyObject* PyObject_CallFunction(PyObject* callable, const char* fmt, ...) {
     PyObject* args;
     CALL_WITH_VARARGS(args, Py_BuildValue, 2, fmt);
-    args = to_sulong(args);
     if (strlen(fmt) < 2) {
         PyObject* singleArg = args;
         args = PyTuple_New(strlen(fmt));

graalpython/com.oracle.graal.python.test/src/python_unittests.py

@@ -36,32 +36,38 @@
 # SOFTWARE.
 
 import csv
+import gzip
 import os
 import re
 import subprocess
+import sys
 from collections import defaultdict
 from json import dumps
 from multiprocessing import Pool
 from pprint import pformat
+from time import gmtime, strftime
 
 import argparse
-import sys
-from time import gmtime, strftime
 
 # global CLI flags
 flags = None
 
 # constants
 PATH_UNITTESTS = "graalpython/lib-python/3/test/"
 
-CSV_RESULTS_NAME = "unittests.csv"
-HTML_RESULTS_NAME = "unittests.html"
+_BASE_NAME = "unittests"
+TXT_RESULTS_NAME = "{}.txt.gz".format(_BASE_NAME)
+CSV_RESULTS_NAME = "{}.csv".format(_BASE_NAME)
+HTML_RESULTS_NAME = "{}.html".format(_BASE_NAME)
+
 
 PTRN_ERROR = re.compile(r'^(?P<error>[A-Z][a-z][a-zA-Z]+):(?P<message>.*)$')
 PTRN_UNITTEST = re.compile(r'^#### running: graalpython/lib-python/3/test/(?P<unittest>.*)$')
 PTRN_NUM_TESTS = re.compile(r'^Ran (?P<num_tests>\d+) test.*$')
 PTRN_NUM_ERRORS = re.compile(
     r'^FAILED \((failures=(?P<failures>\d+))?(, )?(errors=(?P<errors>\d+))?(, )?(skipped=(?P<skipped>\d+))?\)$')
+PTRN_JAVA_EXCEPTION = re.compile(r'^(?P<exception>com\.oracle\.[^:]*):(?P<message>.*)')
+PTRN_MODULE_NOT_FOUND = re.compile(r'.*ModuleNotFound: \'(?P<module>.*)\'\..*', re.DOTALL)
 
 
 # ----------------------------------------------------------------------------------------------------------------------
@@ -79,7 +85,7 @@ def debug(msg, *args, **kwargs):
 
 
 def file_name(name, current_date_time):
-    idx = name.rindex('.')
+    idx = name.index('.')
     if idx > 0:
         return '{}-{}{}'.format(name[:idx], current_date_time, name[idx:])
     return '{}-{}'.format(name, current_date_time)
@@ -117,7 +123,7 @@ def scp(results_file_path, destination_path, destination_name=None):
 
 
 def _run_unittest(test_path):
-    cmd = ["mx", "python3", "--python.CatchAllExceptions=true", test_path]
+    cmd = ["mx", "python3", "--python.CatchAllExceptions=true", test_path, "-v"]
     success, output = _run_cmd(cmd)
     output = '''
 ##############################################################
@@ -128,14 +134,17 @@ def _run_unittest(test_path):
 
 def run_unittests(unittests):
     assert isinstance(unittests, (list, tuple))
-    log("[EXEC] running {} unittests ... ".format(len(unittests)))
+    num_unittests = len(unittests)
+    log("[EXEC] running {} unittests ... ", num_unittests)
     results = []
+
     pool = Pool()
     for ut in unittests:
-        results.append(pool.apply_async(_run_unittest, args=(ut,)))
+        results.append(pool.apply_async(_run_unittest, args=(ut, )))
     pool.close()
     pool.join()
-    return [r.get()[1] for r in results]
+
+    return [res.get()[1] for res in results]
 
 
 def get_unittests(base_tests_path, limit=None, sort=True):
@@ -177,6 +186,7 @@ def process_output(output_lines):
 
     unittests = []
     error_messages = defaultdict(set)
+    java_exceptions = defaultdict(set)
     stats = defaultdict(StatEntry)
 
     for line in output_lines:
@@ -191,6 +201,12 @@ def process_output(output_lines):
             error_messages[unittests[-1]].add((match.group('error'), match.group('message')))
             continue
 
+        # extract java exceptions
+        match = re.match(PTRN_JAVA_EXCEPTION, line)
+        if match:
+            java_exceptions[unittests[-1]].add((match.group('exception'), match.group('message')))
+            continue
+
         # stats
         if line.strip() == 'OK':
             stats[unittests[-1]].all_ok()
@@ -213,27 +229,34 @@ def process_output(output_lines):
             stats[unittests[-1]].num_errors = int(errs) if errs else 0
             stats[unittests[-1]].num_skipped = int(skipped) if skipped else 0
 
-    return unittests, error_messages, stats
+    return unittests, error_messages, java_exceptions, stats
 
 
 # ----------------------------------------------------------------------------------------------------------------------
 #
 # python  error processing
 #
 # ----------------------------------------------------------------------------------------------------------------------
-def process_errors(unittests, error_messages, error, msg_processor):
+def process_errors(unittests, error_messages, error=None, msg_processor=None):
+    def _err_filter(item):
+        if not error:
+            return True
+        return item[0] == error
+
+    def _processor(msg):
+        if not msg_processor:
+            return msg
+        return msg_processor(msg)
+
     missing_modules = defaultdict(lambda: 0)
     for ut in unittests:
         errors = error_messages[ut]
-        for name in map(msg_processor, (msg for err, msg in errors if err == error)):
+        for name in map(_processor, (msg for err, msg in filter(_err_filter, errors))):
             missing_modules[name] = missing_modules[name] + 1
 
     return missing_modules
 
 
-PTRN_MODULE_NOT_FOUND = re.compile(r'.*ModuleNotFound: \'(?P<module>.*)\'\..*', re.DOTALL)
-
-
 def get_missing_module(msg):
     match = re.match(PTRN_MODULE_NOT_FOUND, msg)
     return match.group('module') if match else None
@@ -278,7 +301,14 @@ class Stat(object):
     TEST_PERCENT_PASS = "test_percent_pass"  # percentage of tests which pass from all running tests (all unittests)
 
 
-def save_as_csv(report_path, unittests, error_messages, stats, current_date):
+def save_as_txt(report_path, results):
+    with gzip.open(report_path, 'wb') as TXT:
+        output = '\n'.join(results)
+        TXT.write(bytes(output, 'utf-8'))
+        return output
+
+
+def save_as_csv(report_path, unittests, error_messages, java_exceptions, stats):
     rows = []
     with open(report_path, 'w') as CSV:
         totals = {
@@ -294,6 +324,9 @@ def save_as_csv(report_path, unittests, error_messages, stats, current_date):
         for unittest in unittests:
             unittest_stats = stats[unittest]
             unittest_errmsg = error_messages[unittest]
+            if not unittest_errmsg:
+                unittest_errmsg = java_exceptions[unittest]
+
             rows.append({
                 Col.UNITTEST: unittest,
                 Col.NUM_TESTS: unittest_stats.num_tests,
@@ -424,7 +457,7 @@ def save_as_csv(report_path, unittests, error_messages, stats, current_date):
 '''
 
 
-def save_as_html(report_name, rows, totals, missing_modules, current_date):
+def save_as_html(report_name, rows, totals, missing_modules, java_issues, current_date):
     def grid(*components):
         def _fmt(cmp):
             if isinstance(cmp, tuple):
@@ -522,10 +555,15 @@ def format_val(row, k):
         '''
 
     missing_modules_info = ul('missing modules', [
-        '<b>{}</b>&nbsp;<span class="text-muted">count: {}</span>'.format(name, cnt)
+        '<b>{}</b>&nbsp;<span class="text-muted">imported by {} unittests</span>'.format(name, cnt)
         for cnt, name in sorted(((cnt, name) for name, cnt in missing_modules.items()), reverse=True)
     ])
 
+    java_issues_info = ul('Java issues', [
+        '<b>{}</b>&nbsp;<span class="text-muted">caused by {} unittests</span>'.format(name, cnt)
+        for cnt, name in sorted(((cnt, name) for name, cnt in java_issues.items()), reverse=True)
+    ])
+
     total_stats_info = ul("<b>Summary</b>", [
         grid('<b># total</b> unittests: {}'.format(totals[Stat.UT_TOTAL])),
         grid((progress_bar(totals[Stat.UT_PERCENT_RUNS], color="info"), 3),
@@ -539,7 +577,7 @@ def format_val(row, k):
 
     table_stats = table('stats', CSV_HEADER, rows)
 
-    content = ' <br> '.join([total_stats_info, table_stats, missing_modules_info])
+    content = ' <br> '.join([total_stats_info, table_stats, missing_modules_info, java_issues_info])
 
     report = HTML_TEMPLATE.format(
         title='GraalPython Unittests Stats',
@@ -595,19 +633,27 @@ def _fmt(t):
         unittests = get_unittests(flags.tests_path, limit=flags.limit)
 
     results = run_unittests(unittests)
-    unittests, error_messages, stats = process_output('\n'.join(results))
+    txt_report_path = file_name(TXT_RESULTS_NAME, current_date)
+    output = save_as_txt(txt_report_path, results)
+
+    unittests, error_messages, java_exceptions, stats = process_output(output)
 
     csv_report_path = file_name(CSV_RESULTS_NAME, current_date)
-    rows, totals = save_as_csv(csv_report_path, unittests, error_messages, stats, current_date)
+    rows, totals = save_as_csv(csv_report_path, unittests, error_messages, java_exceptions, stats)
 
-    missing_modules = process_errors(unittests, error_messages, 'ModuleNotFoundError', get_missing_module)
+    missing_modules = process_errors(unittests, error_messages, error='ModuleNotFoundError',
+                                     msg_processor=get_missing_module)
     log("[MISSING MODULES] \n{}", pformat(dict(missing_modules)))
 
+    java_issues = process_errors(unittests, java_exceptions)
+    log("[JAVA ISSUES] \n{}", pformat(dict(java_issues)))
+
     html_report_path = file_name(HTML_RESULTS_NAME, current_date)
-    save_as_html(html_report_path, rows, totals, missing_modules, current_date)
+    save_as_html(html_report_path, rows, totals, missing_modules, java_issues, current_date)
 
     if flags.path:
         log("[SAVE] saving results to {} ... ", flags.path)
+        scp(txt_report_path, flags.path)
         scp(csv_report_path, flags.path)
         scp(html_report_path, flags.path)
 

graalpython/com.oracle.graal.python.test/src/tests/cpyext/__init__.py

@@ -69,7 +69,8 @@ def setUp(self):
 
 def ccompile(self, name):
     from distutils.core import setup, Extension
-    module = Extension(name, sources=['%s/%s.c' % (__dir__, name)])
+    source_file = '%s/%s.c' % (__dir__, name)
+    module = Extension(name, sources=[source_file])
     args = ['--quiet', 'build', 'install_lib', '-f', '--install-dir=%s' % __dir__]
     setup(
         script_name='setup',
@@ -79,6 +80,13 @@ def ccompile(self, name):
         description='',
         ext_modules=[module]
     )
+    # ensure file was really written
+    try:
+        stat_result = os.stat(source_file)
+        if stat_result[6] == 0:
+            raise SystemError("empty source file %s" % (source_file,))
+    except FileNotFoundError:
+        raise SystemError("source file %s not available" % (source_file,))
 
 
 c_template = """
@@ -377,6 +385,7 @@ def create_module(self, name=None):
 
 
 class UnseenFormatter(Formatter):
+
     def get_value(self, key, args, kwds):
         if isinstance(key, str):
             try:

graalpython/com.oracle.graal.python.test/src/tests/test_iterator.py

@@ -100,3 +100,16 @@ def test_iter():
 
 def test_zip_no_args():
     assert list(zip(*[])) == []
+
+
+def test_iter_try_except():
+    it = iter(range(3))
+    exit_via_break = False
+    while 1:
+        try:
+            next(it)
+        except StopIteration:
+            exit_via_break = True
+            break
+
+    assert exit_via_break