[GR-21590] Update Python imports

timfel · timfel · commit c6fc7c03df46 · 2021-10-01T06:05:13.000Z
PullRequest: graalpython/1976
diff --git a/ci.jsonnet b/ci.jsonnet
@@ -1 +1 @@
-{ "overlay": "6072f5daa1c3f2c08824306d295f7b596d2f4fa2" }
+{ "overlay": "6bfa0859ae0b0712ef63ece4d69b6e87c0afa86c" }
diff --git a/graalpython/com.oracle.graal.python.cext/setup.py b/graalpython/com.oracle.graal.python.cext/setup.py
@@ -49,6 +49,7 @@
 __dir__ = __file__.rpartition("/")[0]
 cflags_warnings = [ "-Wno-int-to-pointer-cast"
                   , "-Wno-int-conversion"
+                  , "-Wno-void-pointer-to-int-cast"
                   , "-Wno-incompatible-pointer-types-discards-qualifiers"
                   , "-Wno-pointer-type-mismatch"
                   , "-Wno-braced-scalar-init"
@@ -269,7 +270,7 @@ def build(self, extracted_dir=None):
             with open(makefile_path, "w") as f:
                 f.write(content)
 
-        parallel_arg =  "-j" + str(os.cpu_count()) if threaded else ""
+        parallel_arg =  "-j" + str(min(4, os.cpu_count())) if threaded else ""
         system("make -C '%s' %s -f '%s' CC='%s'" % (lib_src_folder, parallel_arg, self.makefile, get_config_var("CC")), msg="Could not build libbz2")
         return lib_src_folder
 
@@ -309,11 +310,11 @@ class LZMADepedency(CAPIDependency):
     def build(self, extracted_dir=None):
         if not extracted_dir:
             extracted_dir = self.download()
-        
+
         xz_src_path = os.path.join(extracted_dir, self.package_name + "-" + self.version)
         lzma_support_path = os.path.join(__dir__, 'lzma')
-        parallel_arg =  "-j" + str(min(4, os.cpu_count())) if threaded else ""
-        make_args = ['make', parallel_arg, '-C', lzma_support_path]
+        # not using parallel build for xz
+        make_args = ['make', '-C', lzma_support_path]
         make_args += ["CC='%s'" % get_config_var("CC")]
         make_args += ["XZ_ROOT='%s'" % xz_src_path]
         make_args += ["CONFIG_H_DIR='%s'" % lzma_support_path]
@@ -327,7 +328,7 @@ def install(self, build_dir=None):
         if os.path.exists(lib_path):
             # library has been built earlier, so just return the install directory.
             return self.lib_install_dir
-        
+
         return self.build()
 
 
@@ -526,18 +527,18 @@ def build(capi_home):
     try:
         build_libhpy(capi_home)
         build_libposix(capi_home)
-        build_nativelibsupport(capi_home, 
-                                subdir="zlib", 
-                                libname="libzsupport", 
+        build_nativelibsupport(capi_home,
+                                subdir="zlib",
+                                libname="libzsupport",
                                 libs=['z'])
-        build_nativelibsupport(capi_home, 
-                                subdir="bz2", 
-                                libname="libbz2support", 
+        build_nativelibsupport(capi_home,
+                                subdir="bz2",
+                                libname="libbz2support",
                                 deps=[Bzip2Depedency("bz2", "bzip2==1.0.8", "BZIP2")],
                                 extra_link_args=["-Wl,-rpath,%s/lib/%s/" % (relative_rpath, SOABI)])
-        build_nativelibsupport(capi_home, 
-                                subdir="lzma", 
-                                libname="liblzmasupport", 
+        build_nativelibsupport(capi_home,
+                                subdir="lzma",
+                                libname="liblzmasupport",
                                 deps=[LZMADepedency("lzma", "xz==5.2.5", "XZ-5.2.5")],
                                 extra_link_args=["-Wl,-rpath,%s/lib/%s/" % (relative_rpath, SOABI)])
         build_libpython(capi_home)
diff --git a/graalpython/com.oracle.graal.python.test/src/tests/unittest_tags/test_ast.txt b/graalpython/com.oracle.graal.python.test/src/tests/unittest_tags/test_ast.txt
@@ -1,5 +1,6 @@
 *graalpython.lib-python.3.test.test_ast.ASTHelpers_Test.test_parse_in_error
 *graalpython.lib-python.3.test.test_ast.AST_Tests.test_AST_garbage_collection
+*graalpython.lib-python.3.test.test_ast.AST_Tests.test_base_classes
 *graalpython.lib-python.3.test.test_ast.AST_Tests.test_field_attr_writable
 *graalpython.lib-python.3.test.test_ast.AST_Tests.test_issue31592
 *graalpython.lib-python.3.test.test_ast.AST_Tests.test_realtype
diff --git a/graalpython/com.oracle.graal.python.test/src/tests/unittest_tags/test_logging.txt b/graalpython/com.oracle.graal.python.test/src/tests/unittest_tags/test_logging.txt
@@ -89,6 +89,7 @@
 *graalpython.lib-python.3.test.test_logging.FormatterTest.test_percent
 *graalpython.lib-python.3.test.test_logging.FormatterTest.test_time
 *graalpython.lib-python.3.test.test_logging.HTTPHandlerTest.test_output
+*graalpython.lib-python.3.test.test_logging.HandlerTest.test_builtin_handlers
 *graalpython.lib-python.3.test.test_logging.HandlerTest.test_name
 *graalpython.lib-python.3.test.test_logging.HandlerTest.test_path_objects
 *graalpython.lib-python.3.test.test_logging.HandlerTest.test_post_fork_child_no_deadlock
diff --git a/graalpython/com.oracle.graal.python.test/src/tests/unittest_tags/test_multiprocessing_main_handling.txt b/graalpython/com.oracle.graal.python.test/src/tests/unittest_tags/test_multiprocessing_main_handling.txt
@@ -17,7 +17,6 @@
 *graalpython.lib-python.3.test.test_multiprocessing_main_handling.SpawnCmdLineTest.test_directory_compiled
 *graalpython.lib-python.3.test.test_multiprocessing_main_handling.SpawnCmdLineTest.test_ipython_workaround
 *graalpython.lib-python.3.test.test_multiprocessing_main_handling.SpawnCmdLineTest.test_module_in_package_in_zipfile
-*graalpython.lib-python.3.test.test_multiprocessing_main_handling.SpawnCmdLineTest.test_module_in_subpackage_in_zipfile
 *graalpython.lib-python.3.test.test_multiprocessing_main_handling.SpawnCmdLineTest.test_package
 *graalpython.lib-python.3.test.test_multiprocessing_main_handling.SpawnCmdLineTest.test_package_compiled
 *graalpython.lib-python.3.test.test_multiprocessing_main_handling.SpawnCmdLineTest.test_zipfile
diff --git a/graalpython/com.oracle.graal.python.test/src/tests/unittest_tags/test_multiprocessing_spawn.txt b/graalpython/com.oracle.graal.python.test/src/tests/unittest_tags/test_multiprocessing_spawn.txt
@@ -127,7 +127,6 @@
 *graalpython.lib-python.3.test.test_multiprocessing_spawn.WithProcessesTestPoolWorkerLifetime.test_pool_worker_lifetime
 *graalpython.lib-python.3.test.test_multiprocessing_spawn.WithProcessesTestPoolWorkerLifetime.test_pool_worker_lifetime_early_close
 *graalpython.lib-python.3.test.test_multiprocessing_spawn.WithProcessesTestPoolWorkerLifetime.test_worker_finalization_via_atexit_handler_of_multiprocessing
-*graalpython.lib-python.3.test.test_multiprocessing_spawn.WithProcessesTestProcess.test_active_children
 *graalpython.lib-python.3.test.test_multiprocessing_spawn.WithProcessesTestProcess.test_child_fd_inflation
 *graalpython.lib-python.3.test.test_multiprocessing_spawn.WithProcessesTestProcess.test_close
 *graalpython.lib-python.3.test.test_multiprocessing_spawn.WithProcessesTestProcess.test_cpu_count
@@ -174,6 +173,7 @@
 *graalpython.lib-python.3.test.test_multiprocessing_spawn.WithProcessesTestSharedMemory.test_shared_memory_cleaned_after_process_termination
 *graalpython.lib-python.3.test.test_multiprocessing_spawn.WithProcessesTestSubclassingProcess.test_stderr_flush
 *graalpython.lib-python.3.test.test_multiprocessing_spawn.WithProcessesTestSubclassingProcess.test_subclassing
+*graalpython.lib-python.3.test.test_multiprocessing_spawn.WithProcessesTestSubclassingProcess.test_sys_exit
 *graalpython.lib-python.3.test.test_multiprocessing_spawn.WithProcessesTestValue.test_getobj_getlock
 *graalpython.lib-python.3.test.test_multiprocessing_spawn.WithProcessesTestValue.test_rawvalue
 *graalpython.lib-python.3.test.test_multiprocessing_spawn.WithProcessesTestValue.test_value
diff --git a/graalpython/com.oracle.graal.python.test/src/tests/unittest_tags/test_nntplib.txt b/graalpython/com.oracle.graal.python.test/src/tests/unittest_tags/test_nntplib.txt
@@ -1,5 +1,6 @@
 *graalpython.lib-python.3.test.test_nntplib.CapsAfterLoginNNTPv2Tests.test_caps_only_after_login
 *graalpython.lib-python.3.test.test_nntplib.LocalServerTests.test_starttls
+*graalpython.lib-python.3.test.test_nntplib.MiscTests.test_decode_header
 *graalpython.lib-python.3.test.test_nntplib.MiscTests.test_parse_datetime
 *graalpython.lib-python.3.test.test_nntplib.MiscTests.test_parse_overview
 *graalpython.lib-python.3.test.test_nntplib.MiscTests.test_parse_overview_fmt
diff --git a/graalpython/com.oracle.graal.python.test/src/tests/unittest_tags/test_secrets.txt b/graalpython/com.oracle.graal.python.test/src/tests/unittest_tags/test_secrets.txt
@@ -1,4 +1,7 @@
+*graalpython.lib-python.3.test.test_secrets.Compare_Digest_Tests.test_bad_types
 *graalpython.lib-python.3.test.test_secrets.Compare_Digest_Tests.test_bool
+*graalpython.lib-python.3.test.test_secrets.Compare_Digest_Tests.test_equal
+*graalpython.lib-python.3.test.test_secrets.Compare_Digest_Tests.test_unequal
 *graalpython.lib-python.3.test.test_secrets.Random_Tests.test_choice
 *graalpython.lib-python.3.test.test_secrets.Random_Tests.test_randbelow
 *graalpython.lib-python.3.test.test_secrets.Random_Tests.test_randbits
diff --git a/graalpython/com.oracle.graal.python.test/src/tests/unittest_tags/test_smtplib.txt b/graalpython/com.oracle.graal.python.test/src/tests/unittest_tags/test_smtplib.txt
@@ -25,4 +25,5 @@
 *graalpython.lib-python.3.test.test_smtplib.SMTPAUTHInitialResponseSimTests.testAUTH_PLAIN_initial_response_auth
 *graalpython.lib-python.3.test.test_smtplib.SMTPSimTests.testBasic
 *graalpython.lib-python.3.test.test_smtplib.SMTPSimTests.testEXPN
+*graalpython.lib-python.3.test.test_smtplib.SMTPSimTests.testVRFY
 *graalpython.lib-python.3.test.test_smtplib.TooLongLineTests.testLineTooLong
diff --git a/graalpython/com.oracle.graal.python.test/src/tests/unittest_tags/test_tarfile.txt b/graalpython/com.oracle.graal.python.test/src/tests/unittest_tags/test_tarfile.txt
@@ -101,6 +101,8 @@
 *graalpython.lib-python.3.test.test_tarfile.CommandLineTest.test_extract_command_verbose
 *graalpython.lib-python.3.test.test_tarfile.CommandLineTest.test_list_command
 *graalpython.lib-python.3.test.test_tarfile.CommandLineTest.test_list_command_invalid_file
+*graalpython.lib-python.3.test.test_tarfile.CommandLineTest.test_list_command_verbose
+*graalpython.lib-python.3.test.test_tarfile.CommandLineTest.test_test_command
 *graalpython.lib-python.3.test.test_tarfile.GzipListTest.test_list
 *graalpython.lib-python.3.test.test_tarfile.GzipListTest.test_list_members
 *graalpython.lib-python.3.test.test_tarfile.GzipUstarReadTest.test_fileobj_iter
diff --git a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/PythonBuiltinClassType.java b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/PythonBuiltinClassType.java
@@ -311,6 +311,7 @@ public enum PythonBuiltinClassType implements TruffleObject {
     SSLSyscallError("SSLSyscallError", "_ssl", Flags.EXCEPTION),
     SSLEOFError("SSLEOFError", "_ssl", Flags.EXCEPTION),
     SSLCertVerificationError("SSLCertVerificationError", "_ssl", Flags.EXCEPTION),
+    PForeignException("ForeignException", Flags.FOREIGN_EXCEPTION),
 
     // todo: all OS errors
 
@@ -352,6 +353,7 @@ public enum PythonBuiltinClassType implements TruffleObject {
     private static class Flags {
 
         static final Flags EXCEPTION = new Flags(true, true, true);
+        static final Flags FOREIGN_EXCEPTION = new Flags(false, false, true);
         static final Flags PRIVATE_DERIVED_WDICT = new Flags(false, false, true);
         static final Flags PRIVATE_BASE_WDICT = new Flags(false, true, true);
         static final Flags PRIVATE_BASE_WODICT = new Flags(false, true, false);
@@ -617,6 +619,8 @@ public final Shape getInstanceShape(PythonLanguage lang) {
         PicklingError.base = PickleError;
         UnpicklingError.base = PickleError;
 
+        PForeignException.base = PBaseException;
+
         // warnings
         Warning.base = Exception;
         BytesWarning.base = Warning;
diff --git a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/objects/cext/capi/ExternalFunctionNodes.java b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/objects/cext/capi/ExternalFunctionNodes.java
@@ -103,6 +103,7 @@
 import com.oracle.graal.python.nodes.truffle.PythonTypes;
 import com.oracle.graal.python.runtime.ExecutionContext.CalleeContext;
 import com.oracle.graal.python.runtime.ExecutionContext.IndirectCallContext;
+import com.oracle.graal.python.runtime.GilNode;
 import com.oracle.graal.python.runtime.PythonContext;
 import com.oracle.graal.python.runtime.PythonContext.GetThreadStateNode;
 import com.oracle.graal.python.runtime.PythonContext.PythonThreadState;
@@ -546,6 +547,7 @@ static final class ExternalFunctionInvokeNode extends PNodeWithContext implement
         @Child private InteropLibrary lib;
         @Child private PRaiseNode raiseNode;
         @Child private GetThreadStateNode getThreadStateNode = GetThreadStateNodeGen.create();
+        @Child private GilNode gilNode = GilNode.create();
 
         @CompilationFinal private Assumption nativeCodeDoesntNeedExceptionState = Truffle.getRuntime().createAssumption();
         @CompilationFinal private Assumption nativeCodeDoesntNeedMyFrame = Truffle.getRuntime().createAssumption();
@@ -612,8 +614,17 @@ public Object execute(VirtualFrame frame, String name, Object callable, Object[]
                 CompilerDirectives.transferToInterpreterAndInvalidate();
                 throw ensureRaiseNode().raise(PythonBuiltinClassType.TypeError, ErrorMessages.CALLING_NATIVE_FUNC_EXPECTED_ARGS, name, e.getExpectedMinArity(), e.getActualArity());
             } finally {
-                // special case after calling a C function: transfer caught exception back to frame
-                // to simulate the global state semantics
+                /*
+                 * Always re-acquire the GIL here. This is necessary because it could happen that C
+                 * extensions are releasing the GIL and if then an LLVM exception occurs, C code
+                 * wouldn't re-acquire it (unexpectedly).
+                 */
+                gilNode.acquire();
+
+                /*
+                 * Special case after calling a C function: transfer caught exception back to frame
+                 * to simulate the global state semantics.
+                 */
                 PArguments.setException(frame, threadState.getCaughtException());
                 IndirectCallContext.exit(frame, threadState, state);
             }
diff --git a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/nodes/object/GetClassNode.java b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/nodes/object/GetClassNode.java
@@ -58,13 +58,19 @@
 import com.oracle.graal.python.nodes.PGuards;
 import com.oracle.graal.python.nodes.PNodeWithContext;
 import com.oracle.graal.python.nodes.truffle.PythonTypes;
+import com.oracle.graal.python.runtime.exception.PException;
+import com.oracle.truffle.api.CompilerDirectives;
 import com.oracle.truffle.api.dsl.Bind;
 import com.oracle.truffle.api.dsl.Cached;
 import com.oracle.truffle.api.dsl.Fallback;
 import com.oracle.truffle.api.dsl.GenerateUncached;
 import com.oracle.truffle.api.dsl.ImportStatic;
 import com.oracle.truffle.api.dsl.Specialization;
 import com.oracle.truffle.api.dsl.TypeSystemReference;
+import com.oracle.truffle.api.exception.AbstractTruffleException;
+import com.oracle.truffle.api.interop.ExceptionType;
+import com.oracle.truffle.api.interop.InteropLibrary;
+import com.oracle.truffle.api.interop.UnsupportedMessageException;
 import com.oracle.truffle.api.library.CachedLibrary;
 import com.oracle.truffle.api.object.DynamicObjectLibrary;
 import com.oracle.truffle.api.object.Shape;
@@ -190,6 +196,18 @@ static Object getNativeVoidPtr(@SuppressWarnings("unused") PythonNativeVoidPtr o
         return PythonBuiltinClassType.PInt;
     }
 
+    @Specialization(guards = "isForeignRuntimeException(object, lib)")
+    static Object getTruffleException(@SuppressWarnings("unused") AbstractTruffleException object,
+                    @SuppressWarnings("unused") @CachedLibrary(limit = "3") InteropLibrary lib) {
+        /*
+         * Special case: if Python code asks for the class of a foreign exception, we return a
+         * Python type that inherits from BaseException. We do this because Python users usually
+         * expect that every exception inherits from BaseException.
+         */
+        assert !(object instanceof PException);
+        return PythonBuiltinClassType.PForeignException;
+    }
+
     @Fallback
     static Object getForeign(@SuppressWarnings("unused") Object object) {
         return PythonBuiltinClassType.ForeignObject;
@@ -198,4 +216,12 @@ static Object getForeign(@SuppressWarnings("unused") Object object) {
     protected static boolean hasInitialClass(Shape shape) {
         return (shape.getFlags() & PythonObject.CLASS_CHANGED_FLAG) == 0;
     }
+
+    protected static boolean isForeignRuntimeException(AbstractTruffleException e, InteropLibrary lib) {
+        try {
+            return lib.isException(e) && lib.getExceptionType(e) == ExceptionType.RUNTIME_ERROR;
+        } catch (UnsupportedMessageException ex) {
+            throw CompilerDirectives.shouldNotReachHere(ex);
+        }
+    }
 }
diff --git a/mx.graalpython/mx_graalpython.py b/mx.graalpython/mx_graalpython.py
@@ -26,6 +26,7 @@
 
 import contextlib
 import datetime
+import getpass
 import glob
 import itertools
 import json
@@ -400,6 +401,9 @@ def update_unittest_tags(args):
         ('test_faulthandler.txt', '*graalpython.lib-python.3.test.test_faulthandler.FaultHandlerTests.test_sigill'),
         # Disabled due to transient failure
         ('test_multiprocessing_main_handling.txt', '*graalpython.lib-python.3.test.test_multiprocessing_main_handling.SpawnCmdLineTest.test_module_in_package'),
+        ('test_multiprocessing_main_handling.txt', '*graalpython.lib-python.3.test.test_multiprocessing_main_handling.SpawnCmdLineTest.test_module_in_subpackage_in_zipfile'),
+        ('test_multiprocessing_spawn.txt', '*graalpython.lib-python.3.test.test_multiprocessing_spawn.TestNoForkBomb.test_noforkbomb'),
+        ('test_multiprocessing_spawn.txt', '*graalpython.lib-python.3.test.test_multiprocessing_spawn.WithProcessesTestProcess.test_active_children'),
     }
 
     result_tags = linux_tags & darwin_tags - tag_exclusions
@@ -1188,6 +1192,9 @@ def update_import_cmd(args):
     shutil.copy(
         join(mx.suite("truffle").dir, "..", "common.json"),
         join(overlaydir, "python", "graal-common.json"))
+    with open(join(overlaydir, "fastr", "common.libsonnet")) as fastrCommon:
+        with open(join(overlaydir, "python", "fastr-common.libsonnet"), "w") as fastrCopy:
+            fastrCopy.write(fastrCommon.read().replace("graal_common.json", "../python/graal-common.json"))
 
     # update the graal-enterprise revision in the overlay (used by benchmarks)
     with open(join(overlaydir, "python", "imported-constants.json"), 'w') as fp:
@@ -1201,7 +1208,8 @@ def update_import_cmd(args):
 
     with open(join(overlaydir, "python", "vm-tests.json"), 'r') as fp:
         d = json.load(fp)
-        d['downloads']['JAVA_HOME']['version'] = oraclejdk8_ver
+        for job in d:
+            job['downloads']['JAVA_HOME']['version'] = oraclejdk8_ver
 
     with open(join(overlaydir, "python", "vm-tests.json"), 'w') as fp:
         json.dump(d, fp, indent=2)
@@ -1237,12 +1245,31 @@ def update_import_cmd(args):
             repos_updated.append(repo)
 
     # push all repos
-    for repo in repos_updated:
-        try:
-            mx._opts.very_verbose = True
-            vc.git_command(repo, ["push", "-u", "origin", "HEAD:%s" % current_branch], abortOnError=True)
-        finally:
-            mx._opts.very_verbose = prev_verbosity
+    if "--no-push" not in args:
+        for repo in repos_updated:
+            try:
+                mx._opts.very_verbose = True
+                vc.git_command(repo, ["push", "-u", "origin", "HEAD:%s" % current_branch], abortOnError=True)
+            finally:
+                mx._opts.very_verbose = prev_verbosity
+
+        if repos_updated and input('Use automation tool to create PRs (Y/n)? ').lower() != "n":
+            username = input('Username: ')
+            password = getpass.getpass('Password: ')
+            cmds = []
+            for repo in repos_updated:
+                reponame = os.path.basename(repo)
+                cmd = [
+                    "ol-cli", "bitbucket", "--user='%s'" % username, "--password='${SSO_PASSWORD}'",
+                    "create-pr", "--project=G", "--repo=%s" % reponame,
+                    "--title='[GR-21590] Update Python imports'",
+                    "--from-branch='%s'" % current_branch, "--to-branch=master"
+                ]
+                cmds.append(cmd)
+                print(" ".join(cmd))
+            for cmd in cmds:
+                cmd[3].replace("${SSO_PASSWORD}", password)
+                mx.run(cmd, nonZeroIsFatal=False)
 
     if repos_updated:
         mx.log("\n  ".join(["These repos were updated:"] + repos_updated))
@@ -2263,7 +2290,7 @@ def run_leak_launcher(input_args, out=None):
     'python3': [python, '[Python args|@VM options]'],
     'deploy-binary-if-master': [deploy_binary_if_main, ''],
     'python-gate': [python_gate, '--tags [gates]'],
-    'python-update-import': [update_import_cmd, '[--no-pull] [import-name, default: truffle]'],
+    'python-update-import': [update_import_cmd, '[--no-pull] [--no-push] [import-name, default: truffle]'],
     'python-style': [python_style_checks, '[--fix] [--no-spotbugs]'],
     'python-svm': [python_svm, ''],
     'python-gvm': [python_gvm, ''],
diff --git a/mx.graalpython/suite.py b/mx.graalpython/suite.py

Original file line number	Diff line number	Diff line change
`@@ -1 +1 @@`
`1`		`-{ "overlay": "6072f5daa1c3f2c08824306d295f7b596d2f4fa2" }`
	`1`	`+{ "overlay": "6bfa0859ae0b0712ef63ece4d69b6e87c0afa86c" }`