Merge branch 'master' into feature/GR-24297

Author: ppisl

graalpython/com.oracle.graal.python.cext/src/dictobject.c

@@ -101,7 +101,7 @@ int _PyDict_Next(PyObject *d, Py_ssize_t *ppos, PyObject **pkey, PyObject **pval
     	*pvalue = PyTuple_GetItem(tresult, 1);
     }
     if (phash != NULL) {
-    	*phash = PyTuple_GetItem(tresult, 2);
+    	*phash = PyLong_AsSsize_t(PyTuple_GetItem(tresult, 2));
     }
     return 1;
 

graalpython/com.oracle.graal.python.shell/src/com/oracle/graal/python/shell/GraalPythonMain.java

@@ -153,6 +153,15 @@ protected List<String> preprocessArguments(List<String> givenArgs, Map<String, S
                 case "-S":
                     noSite = true;
                     break;
+                case "-X":
+                    i++;
+                    if (i < arguments.size()) {
+                        // CPython ignores unknown/unsupported -X options, so we can do that too
+                    } else {
+                        print("Argument expected for the -X option");
+                        printShortHelp();
+                    }
+                    break;
                 case "-v":
                     verboseFlag = true;
                     break;
@@ -585,6 +594,7 @@ protected void printHelp(OptionCategory maxCategory) {
                         "         can be supplied multiple times to increase verbosity\n" +
                         "-V     : print the Python version number and exit (also --version)\n" +
                         "         when given twice, print more information about the build\n" +
+                        "-X opt : CPython implementation-specific options. Ignored on GraalPython\n" +
                         // "-W arg : warning control; arg is
                         // action:message:category:module:lineno\n" +
                         // " also PYTHONWARNINGS=arg\n" +

graalpython/com.oracle.graal.python.test/src/com/oracle/graal/python/test/interop/JavaInteropTest.java

@@ -52,15 +52,6 @@
 import java.util.Arrays;
 import java.util.List;
 
-import com.oracle.graal.python.runtime.interop.InteropArray;
-import com.oracle.graal.python.test.PythonTests;
-import com.oracle.truffle.api.interop.ArityException;
-import com.oracle.truffle.api.interop.InteropLibrary;
-import com.oracle.truffle.api.interop.TruffleObject;
-import com.oracle.truffle.api.interop.UnknownIdentifierException;
-import com.oracle.truffle.api.library.ExportLibrary;
-import com.oracle.truffle.api.library.ExportMessage;
-
 import org.graalvm.polyglot.Context;
 import org.graalvm.polyglot.Context.Builder;
 import org.graalvm.polyglot.Engine;
@@ -76,6 +67,15 @@
 import org.junit.runners.Parameterized.Parameter;
 import org.junit.runners.Parameterized.Parameters;
 
+import com.oracle.graal.python.runtime.interop.InteropArray;
+import com.oracle.graal.python.test.PythonTests;
+import com.oracle.truffle.api.interop.ArityException;
+import com.oracle.truffle.api.interop.InteropLibrary;
+import com.oracle.truffle.api.interop.TruffleObject;
+import com.oracle.truffle.api.interop.UnknownIdentifierException;
+import com.oracle.truffle.api.library.ExportLibrary;
+import com.oracle.truffle.api.library.ExportMessage;
+
 @RunWith(Enclosed.class)
 public class JavaInteropTest {
     public static class GeneralInterop extends PythonTests {
@@ -301,13 +301,14 @@ public void accessSuitePy() throws IOException {
                             "suite.py").build();
             Value suite = context.eval(suitePy);
 
+            Value listConverter = context.eval("python", "list");
             Value libraries = suite.getMember("libraries");
             assertNotNull("libraries found", libraries);
-            final List<Object> suiteKeys = Arrays.asList(suite.invokeMember("keys").as(Object[].class));
+            final List<Object> suiteKeys = Arrays.asList(listConverter.execute(suite.invokeMember("keys")).as(Object[].class));
             assertTrue("Libraries found among keys: " + suiteKeys, suiteKeys.contains("libraries"));
 
             Value dacapo = null;
-            for (Object k : libraries.invokeMember("keys").as(List.class)) {
+            for (Object k : listConverter.execute(libraries.invokeMember("keys")).as(List.class)) {
                 System.err.println("k " + k);
                 if ("DACAPO".equals(k)) {
                     dacapo = libraries.getMember((String) k);

graalpython/com.oracle.graal.python.test/src/tests/cpyext/test_dict.py

@@ -1,4 +1,4 @@
-# Copyright (c) 2018, 2019, Oracle and/or its affiliates. All rights reserved.
+# Copyright (c) 2018, 2020, Oracle and/or its affiliates. All rights reserved.
 # DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
 #
 # The Universal Permissive License (UPL), Version 1.0
@@ -255,6 +255,31 @@ def compile_module(self, name):
         cmpfunc=lambda x, y: type(x) == tuple and type(y) == tuple and len(x) == 3 and len(y) == 3 and (x[0] == 0 and y[0] == 0 or x == y)
     )
 
+    # _PyDict_SetItem_KnownHash
+    test__PyDict_SetItem_KnownHash = CPyExtFunction(
+        lambda args: {'a': "hello"},
+        lambda: (({'a': "hello"}, ),),
+        code='''PyObject* wrap__PyDict_SetItem_KnownHash(PyObject* dict) {
+            PyObject* result = PyDict_New();
+            
+            Py_ssize_t ppos = 0;
+            PyObject* key;
+            PyObject* value;
+            Py_hash_t phash;
+            
+            int res = 0;
+            
+           _PyDict_Next(dict, &ppos, &key, &value, &phash);
+            res = _PyDict_SetItem_KnownHash(result, key, value, phash);
+            return result;
+        }
+        ''',
+        resultspec="O",
+        argspec='O',
+        arguments=["PyObject* dict"],
+        callfunction="wrap__PyDict_SetItem_KnownHash",
+    )
+
     # PyDict_Size
     test_PyDict_Size = CPyExtFunction(
         lambda args: len(args[0]),

graalpython/com.oracle.graal.python.test/src/tests/run_cpython_test.py

@@ -0,0 +1,65 @@
+# Copyright (c) 2020, Oracle and/or its affiliates. All rights reserved.
+# DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+#
+# The Universal Permissive License (UPL), Version 1.0
+#
+# Subject to the condition set forth below, permission is hereby granted to any
+# person obtaining a copy of this software, associated documentation and/or
+# data (collectively the "Software"), free of charge and under any and all
+# copyright rights in the Software, and any and all patent rights owned or
+# freely licensable by each licensor hereunder covering either (i) the
+# unmodified Software as contributed to or provided by such licensor, or (ii)
+# the Larger Works (as defined below), to deal in both
+#
+# (a) the Software, and
+#
+# (b) any piece of software and/or hardware listed in the lrgrwrks.txt file if
+# one is included with the Software each a "Larger Work" to which the Software
+# is contributed by such licensors),
+#
+# without restriction, including without limitation the rights to copy, create
+# derivative works of, display, perform, and distribute the Software and make,
+# use, sell, offer for sale, import, export, have made, and have sold the
+# Software and the Larger Work(s), and to sublicense the foregoing rights on
+# either these or other terms.
+#
+# This license is subject to the following condition:
+#
+# The above copyright notice and either this complete permission notice or at a
+# minimum a reference to the UPL must be included in all copies or substantial
+# portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+# SOFTWARE.
+
+import os
+import sys
+import unittest
+
+sys.path.insert(0, os.getcwd())
+
+
+class TestLoader(unittest.TestLoader):
+    def loadTestsFromModule(self, module, pattern=None):
+        suite = super().loadTestsFromModule(module, pattern=pattern)
+        test_main = getattr(module, 'test_main', None)
+        if callable(test_main):
+            class TestMain(unittest.TestCase):
+                pass
+
+            TestMain.__module__ = test_main.__module__
+            TestMain.__qualname__ = TestMain.__name__
+            TestMain.test_main = staticmethod(test_main)
+            suite.addTests(self.loadTestsFromTestCase(TestMain))
+        return suite
+
+
+# We would normmally just pass the loader to the main, but there are tests for the framework itself (test_unittest)
+# that interact weirdly with non-default loaders
+unittest.defaultTestLoader = TestLoader()
+unittest.main(module=None, testLoader=unittest.defaultTestLoader)

graalpython/com.oracle.graal.python.test/src/tests/test_binary_arithmetic.py

@@ -299,6 +299,19 @@ def test_pow():
     # (0xffffffffffffffff >> 63) is used to produce a non-narrowed int
     assert 2**(0xffffffffffffffff >> 63) == 2
 
+    # test that two-argument pow functions work
+    class M:
+        def __pow__(self, power):
+            return (type(self), power)
+    assert pow(M(), 2) == (M, 2)
+    assert pow(M(), 2, None) == (M, 2)
+    try:
+        pow(M(), 2, 3)
+    except TypeError:
+        assert True
+    else:
+        assert False
+
     if sys.version_info.minor >= 8:
         # for some reason this hangs CPython on the CI even if it's just parsed
         from pow_tests import test_pow

graalpython/com.oracle.graal.python.test/src/tests/test_dict.py

@@ -165,6 +165,18 @@ def test_init3():
         assert False, "expected ValueError"
     except ValueError as e:
         assert "dictionary update sequence element #4 has length 1; 2 is required" == str(e), "invalid error message"
+        
+    try:
+        dict("5")
+        assert False, "expected ValueError"
+    except ValueError as e:
+        assert "dictionary update sequence element #0 has length 1; 2 is required" == str(e), "invalid error message"
+        
+    try:
+        dict([("a", 1), ("b", 2), ("c", 3), ("d", 4), "5"])
+        assert False, "expected ValueError"
+    except ValueError as e:
+        assert "dictionary update sequence element #4 has length 1; 2 is required" == str(e), "invalid error message"
 
 
 def test_init4():
@@ -198,7 +210,13 @@ def __len__(self):
     assert set(d.keys()) == key_set, "unexpected keys: %s" % str(d.keys())
     assert set(d.values()) == {97, 98, 99, 100}, "unexpected values: %s" % str(d.values())
 
-
+def test_init6():
+    try:
+        dict(1)
+        assert False, "expected TypeError"
+    except TypeError as e:
+        assert "'int' object is not iterable" == str(e), "invalid error message"
+        
 def test_init_kwargs():
     kwargs = {'ONE': 'one', 'TWO': 'two'}
     d = dict([(1, 11), (2, 22)], **kwargs)

graalpython/com.oracle.graal.python.test/src/tests/test_tagged_unittests.py

@@ -50,6 +50,9 @@
     TAGS_DIR = "null"
 
 
+RUNNER = os.path.join(os.path.dirname(__file__), "run_cpython_test.py")
+
+
 def working_selectors(tagfile):
     if os.path.exists(tagfile):
         with open(tagfile) as f:
@@ -84,7 +87,7 @@ def fun(self):
                 cmd.append("--inspect")
             if "-debug-java" in sys.argv:
                 cmd.append("-debug-java")
-            cmd += ["-S", "-m", "unittest"]
+            cmd += ["-S", RUNNER]
             for testpattern in working_test[1]:
                 cmd.extend(["-k", testpattern])
             testmod = working_test[0].rpartition(".")[2]
@@ -163,7 +166,13 @@ def parse_unittest_output(output):
             # entirely
             testfile_stem = os.path.splitext(os.path.basename(testfile))[0]
             testmod = "test." + testfile_stem
-            cmd = [timeout, "-s", "9", "120"] + executable + ["-S", "-m"]
+            cmd = [timeout, "-s", "9", "120"] + executable
+            if repeat == 0:
+                # Allow catching Java exceptions in the first iteration only, so that subsequent iterations
+                # (there will be one even if everything succeeds) filter out possible false-passes caused by
+                # the tests catching all exceptions somewhere
+                cmd += ['--experimental-options', '--python.CatchAllExceptions']
+            cmd += ["-S", RUNNER, "-v"]
             tagfile = os.path.join(TAGS_DIR, testfile_stem + ".txt")
             if retag and repeat == 0:
                 test_selectors = []
@@ -176,7 +185,6 @@ def parse_unittest_output(output):
                 continue
 
             print("[%d/%d, Try %d] Testing %s" %(idx + 1, len(testfiles), repeat + 1, testmod))
-            cmd += ["unittest", "-v"]
             for selector in test_selectors:
                 cmd += ["-k", selector]
             cmd.append(testfile)

graalpython/com.oracle.graal.python.test/src/tests/unittest_tags/test_ast.txt

@@ -2,3 +2,4 @@
 *graalpython.lib-python.3.test.test_ast.AST_Tests.test_field_attr_writable
 *graalpython.lib-python.3.test.test_ast.AST_Tests.test_issue31592
 *graalpython.lib-python.3.test.test_ast.AST_Tests.test_realtype
+*graalpython.lib-python.3.test.test_ast.ConstantTests.test_load_const

graalpython/com.oracle.graal.python.test/src/tests/unittest_tags/test_base64.txt

@@ -1 +1,12 @@
+*graalpython.lib-python.3.test.test_base64.BaseXYTestCase.test_ErrorHeritage
+*graalpython.lib-python.3.test.test_base64.BaseXYTestCase.test_a85_padding
+*graalpython.lib-python.3.test.test_base64.BaseXYTestCase.test_a85decode_errors
+*graalpython.lib-python.3.test.test_base64.BaseXYTestCase.test_b32decode_casefold
+*graalpython.lib-python.3.test.test_base64.BaseXYTestCase.test_b32decode_error
+*graalpython.lib-python.3.test.test_base64.BaseXYTestCase.test_b85_padding
+*graalpython.lib-python.3.test.test_base64.BaseXYTestCase.test_b85decode_errors
+*graalpython.lib-python.3.test.test_base64.BaseXYTestCase.test_decode_nonascii_str
 *graalpython.lib-python.3.test.test_base64.LegacyBase64TestCase.test_decode
+*graalpython.lib-python.3.test.test_base64.LegacyBase64TestCase.test_decodestring_warns
+*graalpython.lib-python.3.test.test_base64.LegacyBase64TestCase.test_encode
+*graalpython.lib-python.3.test.test_base64.LegacyBase64TestCase.test_encodestring_warns