Merge branch 'master' into topic/GR-21836

Author: msimacek

ci.jsonnet

@@ -1 +1 @@
-{ "overlay": "d521478b3a031b5b55bb1b629650320cc9c9d425" }
+{ "overlay": "309437bc2ceea142484fb6568804b9d9e22949ac" }

graalpython/com.oracle.graal.python.benchmarks/python/harness.py

@@ -40,6 +40,7 @@
 import _io
 import os
 import sys
+import types
 from time import time
 
 
@@ -229,7 +230,7 @@ def get_bench_module(bench_file):
             return bench_module
 
         else:
-            bench_module = type(sys)(name, bench_file)
+            bench_module = types.ModuleType(name, bench_file)
             with _io.FileIO(bench_file, "r") as f:
                 bench_module.__file__ = bench_file
                 bench_module.ccompile = ccompile
@@ -256,7 +257,10 @@ def run(self):
         args = self._call_attr(ATTR_PROCESS_ARGS, *self.bench_args)
         if args is None:
             # default args processor considers all args as ints
-            args = list(map(int, self.bench_args))
+            if sys.version_info.major < 3:
+                args = list(map(lambda x: int(x.replace("_", "")), self.bench_args))
+            else:
+                args = list(map(int, self.bench_args))
 
         print("### args = ", args)
         print(_HRULE)

graalpython/com.oracle.graal.python.benchmarks/python/meso/meteor3.py

@@ -41,6 +41,7 @@
 #
 # contributed by Daniel Nanz, 2008-08-21
 # 2to3
+from __future__ import print_function
 from bisect import bisect
 import time
 
@@ -181,7 +182,7 @@ def main(n):
 
 def measure(num):
     main(num)
-    
+
 
 def __benchmark__(num=2098):
     measure(num)

graalpython/com.oracle.graal.python.benchmarks/python/meso/richards3.py

@@ -45,6 +45,8 @@
 #  Translation from C++, Mario Wolczko
 #  Outer loop added by Alex Jacoby
 
+from __future__ import print_function
+
 # Task IDs
 I_IDLE = 1
 I_WORK = 2

graalpython/com.oracle.graal.python.test/src/com/oracle/graal/python/test/interop/InteropLibraryTest.java

@@ -40,19 +40,24 @@
  */
 package com.oracle.graal.python.test.interop;
 
-import static org.junit.Assert.assertEquals;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
 import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertTrue;
 
 import org.graalvm.polyglot.Context;
 import org.graalvm.polyglot.Context.Builder;
 import org.graalvm.polyglot.Value;
-import org.junit.After;
-import org.junit.Before;
-import org.junit.Test;
+import org.graalvm.polyglot.proxy.ProxyArray;
 
 import com.oracle.graal.python.test.PythonTests;
 
+import java.util.Arrays;
+import java.util.Iterator;
+import java.util.List;
+
 public class InteropLibraryTest extends PythonTests {
     private Context context;
 
@@ -137,4 +142,62 @@ public void testPListRemovable() {
     public void testIsNull() {
         assertTrue(v("None").isNull());
     }
+
+    @Test
+    public void testForItemInLazyArray() {
+        // @formatter:off
+        Value collect = context.eval("python", ""
+                + "def iter (arr):\n"
+                + "\tcollect = []\n"
+                + "\tfor item in arr:\n"
+                + "\t\tcollect.append(item)\n"
+                + "\t\tcollect.append(item)\n"
+                + "\t\tcollect.append(item)\n"
+                + "\treturn collect\n"
+                + "iter\n"
+        );
+        // @formatter:on
+
+        final List<Integer> list = Arrays.asList(5, 7, 11, 13, 17, 23);
+
+        Value tripples = collect.execute(new LazyArray(list.iterator()));
+        assertTrue("Array returned", tripples.hasArrayElements());
+        assertEquals(list.size() * 3, tripples.getArraySize());
+    }
+
+    private static final class LazyArray implements ProxyArray {
+
+        private final Iterator<?> it;
+        private long at;
+
+        LazyArray(Iterator<?> it) {
+            this.it = it;
+            this.at = 0;
+        }
+
+        @Override
+        public Object get(long index) {
+            if (index == at) {
+                at++;
+                return it.next();
+            }
+            return null;
+        }
+
+        @Override
+        public void set(long index, Value value) {
+            throw new UnsupportedOperationException();
+        }
+
+        @Override
+        public boolean remove(long index) {
+            throw new UnsupportedOperationException();
+        }
+
+        @Override
+        public long getSize() {
+            return it.hasNext() ? at + 1 : at;
+        }
+    }
+
 }

graalpython/com.oracle.graal.python.test/src/python_unittests.py

@@ -40,14 +40,17 @@
 import csv
 import gzip
 import os
+import signal
 import re
 import html
+import time
 import subprocess
 from collections import defaultdict
 from json import dumps
 from multiprocessing import Pool, TimeoutError
 from pprint import pformat
 
+   
 import argparse
 import sys
 from time import gmtime, strftime
@@ -125,29 +128,44 @@ def file_name(name, current_date_time):
         return '{}-{}{}'.format(name[:idx], current_date_time, name[idx:])
     return '{}-{}'.format(name, current_date_time)
 
+def get_tail(output, count=15):
+    lines = output.split("\n")
+    start = max(0, len(lines) - count)
+    return '\n'.join(lines[start:])
+
+TIMEOUT = 60 * 20  # 20 mins per unittest wait time max ...
 
 # ----------------------------------------------------------------------------------------------------------------------
 #
 # exec utils
 #
 # ----------------------------------------------------------------------------------------------------------------------
-def _run_cmd(cmd, capture_on_failure=True):
+def _run_cmd(cmd, timeout=TIMEOUT, capture_on_failure=True):
     if isinstance(cmd, str):
         cmd = cmd.split(" ")
     assert isinstance(cmd, (list, tuple))
 
-    log("[EXEC] cmd: {} ...".format(' '.join(cmd)))
-    success = True
-    output = None
+    cmd_string = ' '.join(cmd)
+    log("[EXEC] starting '{}' ...".format(cmd_string))
 
+    start_time = time.monotonic()
+    # os.setsid is used to create a process group, to be able to call os.killpg upon timeout
+    proc = subprocess.Popen(cmd, preexec_fn=os.setsid, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
     try:
-        output = subprocess.check_output(cmd, stderr=subprocess.STDOUT)
-    except subprocess.CalledProcessError as e:
-        log("[ERR] Could not execute CMD. Reason: {}".format(e))
-        if capture_on_failure:
-            output = e.output
-
-    return success, output.decode("utf-8", "ignore")
+        output = proc.communicate(timeout=timeout)[0]
+    except subprocess.TimeoutExpired as e:
+        delta = time.monotonic() - start_time
+        os.killpg(proc.pid, signal.SIGKILL)
+        output = proc.communicate()[0]
+        msg = "TimeoutExpired: {:.3f}s".format(delta)
+        tail = get_tail(output.decode('utf-8', 'ignore'))
+        log("[ERR] timeout '{}' after {:.3f}s, killing process group {}, last lines of output:\n{}\n{}", cmd_string, delta, proc.pid, tail, HR)
+    else:
+        delta = time.monotonic() - start_time
+        log("[EXEC] finished '{}' with exit code {} in {:.3f}s", cmd_string, proc.returncode, delta)
+        msg = "Finished: {:.3f}s".format(delta)
+    
+    return proc.returncode == 0, output.decode("utf-8", "ignore") + "\n" + msg
 
 
 def scp(results_file_path, destination_path, destination_name=None):
@@ -157,54 +175,42 @@ def scp(results_file_path, destination_path, destination_name=None):
     return _run_cmd(cmd)[0]
 
 
-def _run_unittest(test_path, with_cpython=False):
+def _run_unittest(test_path, timeout, with_cpython=False):
     if with_cpython:
         cmd = ["python3", test_path, "-v"]
     else:
         cmd = ["mx", "python3", "--python.CatchAllExceptions=true", test_path, "-v"]
-    success, output = _run_cmd(cmd)
+    output = _run_cmd(cmd, timeout)[1]
     output = '''
 ##############################################################
 #### running: {} 
     '''.format(test_path) + output
-    return success, output
-
+    return output
 
-TIMEOUT = 60 * 20  # 20 mins per unittest wait time max ...
 
 
 def run_unittests(unittests, timeout, with_cpython=False):
     assert isinstance(unittests, (list, tuple))
     num_unittests = len(unittests)
     log("[EXEC] running {} unittests ... ", num_unittests)
     log("[EXEC] timeout per unittest: {} seconds", timeout)
-    results = []
 
-    pool = Pool()
+    start_time = time.monotonic()
+    pool = Pool(processes=(os.cpu_count() // 4) or 1) # to account for hyperthreading and some additional overhead
+    
+    out = []
+    def callback(result):
+        out.append(result)
+        log("[PROGRESS] {} / {}: \t {:.1f}%", len(out), num_unittests, len(out) * 100 / num_unittests)
+        
+    # schedule all unittest runs
     for ut in unittests:
-        results.append(pool.apply_async(_run_unittest, args=(ut, with_cpython)))
+        pool.apply_async(_run_unittest, args=(ut, timeout, with_cpython), callback=callback)
+        
     pool.close()
-
-    log("[INFO] collect results ... ")
-    out = []
-    timed_out = []
-    for i, res in enumerate(results):
-        try:
-            _, output = res.get(timeout)
-            out.append(output)
-        except TimeoutError:
-            log("[ERR] timeout while getting results for {}, skipping!", unittests[i])
-            timed_out.append(unittests[i])
-        log("[PROGRESS] {} / {}: \t {}%", i+1, num_unittests, int(((i+1) * 100.0) / num_unittests))
-
-    if timed_out:
-        log(HR)
-        for t in timed_out:
-            log("[TIMEOUT] skipped: {}", t)
-        log(HR)
-    log("[STATS] processed {} out of {} unittests", num_unittests - len(timed_out), num_unittests)
-    pool.terminate()
     pool.join()
+    pool.terminate()
+    log("[STATS] processed {} unittests in {:.3f}s", num_unittests, time.monotonic() - start_time)
     return out
 
 
@@ -346,11 +352,12 @@ def process_output(output_lines):
         match = re.match(PTRN_ERROR, line)
         if match:
             error_message = (match.group('error'), match.group('message'))
-            error_message_dict = error_messages[unittests[-1]]
-            d = error_message_dict.get(error_message)
-            if not d:
-                d = 0
-            error_message_dict[error_message] = d + 1
+            if not error_message[0] == 'Directory' and not error_message[0] == 'Components':
+                error_message_dict = error_messages[unittests[-1]]
+                d = error_message_dict.get(error_message)
+                if not d:
+                    d = 0
+                error_message_dict[error_message] = d + 1
             continue
 
         # extract java exceptions
@@ -521,6 +528,8 @@ def save_as_csv(report_path, unittests, error_messages, java_exceptions, stats,
             unittest_errmsg = error_messages[unittest]
             if not unittest_errmsg:
                 unittest_errmsg = java_exceptions[unittest]
+            if not unittest_errmsg:
+                unittest_errmsg = {}
 
             rows.append({
                 Col.UNITTEST: unittest,
@@ -867,7 +876,7 @@ def main(prog, args):
                         action="store_true")
     parser.add_argument("-l", "--limit", help="Limit the number of unittests to run.", default=None, type=int)
     parser.add_argument("-t", "--tests_path", help="Unittests path.", default=PATH_UNITTESTS)
-    parser.add_argument("-T", "--timeout", help="Timeout per unittest run.", default=TIMEOUT, type=int)
+    parser.add_argument("-T", "--timeout", help="Timeout per unittest run (seconds).", default=TIMEOUT, type=int)
     parser.add_argument("-o", "--only_tests", help="Run only these unittests (comma sep values).", default=None)
     parser.add_argument("-s", "--skip_tests", help="Run all unittests except (comma sep values)."
                                                    "the only_tets option takes precedence", default=None)