[GR-23272] Implement unraisable exceptions

PullRequest: graalpython/1049

Author: msimacek

graalpython/com.oracle.graal.python.test/src/tests/cpyext/__init__.py

@@ -39,6 +39,8 @@
 
 import sys
 import os
+from io import StringIO
+
 from importlib import invalidate_caches
 from string import Formatter
 __dir__ = __file__.rpartition("/")[0]
@@ -289,7 +291,7 @@ def file_not_empty(path):
 
 class CPyExtFunction():
 
-    def __init__(self, pfunc, parameters, template=c_template, cmpfunc=None, **kwargs):
+    def __init__(self, pfunc, parameters, template=c_template, cmpfunc=None, stderr_validator=None, **kwargs):
         self.template = template
         self.pfunc = pfunc
         self.parameters = parameters
@@ -306,6 +308,7 @@ def __init__(self, pfunc, parameters, template=c_template, cmpfunc=None, **kwarg
         kwargs["resultspec"] = kwargs["resultspec"] if "resultspec" in kwargs else "O"
         self.formatargs = kwargs
         self.cmpfunc = cmpfunc or self.do_compare
+        self.stderr_validator = stderr_validator
 
     def do_compare(self, x, y):
         if isinstance(x, BaseException):
@@ -359,11 +362,18 @@ def test(self):
         cargs = self.parameters()
         pargs = self.parameters()
         for i in range(len(cargs)):
-            cresult = presult = None
+            real_stderr = sys.stderr
+            sys.stderr = StringIO()
             try:
                 cresult = ctest(cargs[i])
             except BaseException as e:
                 cresult = e
+            else:
+                if self.stderr_validator:
+                    s = sys.stderr.getvalue()
+                    assert self.stderr_validator(cargs[i], s), f"captured stderr didn't match expectations. Stderr: {s}"
+            finally:
+                sys.stderr = real_stderr
             try:
                 presult = self.pfunc(pargs[i])
             except BaseException as e:

graalpython/com.oracle.graal.python.test/src/tests/cpyext/test_err.py

@@ -213,6 +213,7 @@ def compile_module(self, name):
         argspec='i',
         arguments=["int n"],
         callfunction="wrap_PyErr_PrintEx",
+        stderr_validator=lambda args, stderr: 'unknown key whatsoever' in stderr and 'Traceback' not in stderr,
         cmpfunc=unhandled_error_compare
     )
 
@@ -339,11 +340,20 @@ def compile_module(self, name):
     test_PyErr_WriteUnraisable = CPyExtFunctionVoid(
         lambda args: None,
         lambda: (
+            (None,),
             ("hello",),
         ),
         resultspec="O",
         argspec='O',
         arguments=["PyObject* obj"],
+        code="""void wrap_PyErr_WriteUnraisable(PyObject* object) {
+            PyErr_SetString(PyExc_RuntimeError, "unraisable exception");
+            if (object == Py_None)
+                object = NULL;
+            PyErr_WriteUnraisable(object);
+        }""",
+        callfunction="wrap_PyErr_WriteUnraisable",
+        stderr_validator=lambda args, stderr: 'unraisable exception' in stderr,
         cmpfunc=unhandled_error_compare
     )
 

graalpython/com.oracle.graal.python.test/src/tests/unittest_tags/test_yield_from.txt

@@ -1,5 +1,6 @@
 *graalpython.lib-python.3.test.test_yield_from.TestPEP380Operation.test_attempted_yield_from_loop
 *graalpython.lib-python.3.test.test_yield_from.TestPEP380Operation.test_attempting_to_send_to_non_generator
+*graalpython.lib-python.3.test.test_yield_from.TestPEP380Operation.test_broken_getattr_handling
 *graalpython.lib-python.3.test.test_yield_from.TestPEP380Operation.test_catching_exception_from_subgen_and_returning
 *graalpython.lib-python.3.test.test_yield_from.TestPEP380Operation.test_close_with_cleared_frame
 *graalpython.lib-python.3.test.test_yield_from.TestPEP380Operation.test_conversion_of_sendNone_to_next

graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/PythonCextBuiltins.java

@@ -201,6 +201,7 @@
 import com.oracle.graal.python.nodes.PNodeWithContext;
 import com.oracle.graal.python.nodes.PRaiseNode;
 import com.oracle.graal.python.nodes.SpecialMethodNames;
+import com.oracle.graal.python.nodes.WriteUnraisableNode;
 import com.oracle.graal.python.nodes.argument.keywords.ExecuteKeywordStarargsNode.ExpandKeywordStarargsNode;
 import com.oracle.graal.python.nodes.argument.positional.ExecutePositionalStarargsNode;
 import com.oracle.graal.python.nodes.attributes.HasInheritedAttributeNode;
@@ -686,6 +687,18 @@ Object run(Object typ, PBaseException val, Object tb) {
         }
     }
 
+    @Builtin(name = "PyTruffle_WriteUnraisable", minNumOfPositionalArgs = 2)
+    @GenerateNodeFactory
+    abstract static class PyTruffleWriteUnraisable extends PythonBuiltinNode {
+
+        @Specialization
+        Object run(PBaseException exception, Object object,
+                        @Cached WriteUnraisableNode writeUnraisableNode) {
+            writeUnraisableNode.execute(null, exception, null, (object instanceof PNone) ? PNone.NONE : object);
+            return PNone.NO_VALUE;
+        }
+    }
+
     @Builtin(name = "PyUnicode_FromString", minNumOfPositionalArgs = 1)
     @GenerateNodeFactory
     abstract static class PyUnicodeFromStringNode extends PythonBuiltinNode {

graalpython/com.oracle.graal.python/src/com/oracle/graal/python/nodes/WriteUnraisableNode.java

@@ -0,0 +1,118 @@
+/*
+ * Copyright (c) 2020, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * The Universal Permissive License (UPL), Version 1.0
+ *
+ * Subject to the condition set forth below, permission is hereby granted to any
+ * person obtaining a copy of this software, associated documentation and/or
+ * data (collectively the "Software"), free of charge and under any and all
+ * copyright rights in the Software, and any and all patent rights owned or
+ * freely licensable by each licensor hereunder covering either (i) the
+ * unmodified Software as contributed to or provided by such licensor, or (ii)
+ * the Larger Works (as defined below), to deal in both
+ *
+ * (a) the Software, and
+ *
+ * (b) any piece of software and/or hardware listed in the lrgrwrks.txt file if
+ * one is included with the Software each a "Larger Work" to which the Software
+ * is contributed by such licensors),
+ *
+ * without restriction, including without limitation the rights to copy, create
+ * derivative works of, display, perform, and distribute the Software and make,
+ * use, sell, offer for sale, import, export, have made, and have sold the
+ * Software and the Larger Work(s), and to sublicense the foregoing rights on
+ * either these or other terms.
+ *
+ * This license is subject to the following condition:
+ *
+ * The above copyright notice and either this complete permission notice or at a
+ * minimum a reference to the UPL must be included in all copies or substantial
+ * portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+package com.oracle.graal.python.nodes;
+
+import com.oracle.graal.python.PythonLanguage;
+import com.oracle.graal.python.builtins.objects.PNone;
+import com.oracle.graal.python.builtins.objects.exception.GetExceptionTracebackNode;
+import com.oracle.graal.python.builtins.objects.exception.PBaseException;
+import com.oracle.graal.python.builtins.objects.module.PythonModule;
+import com.oracle.graal.python.builtins.objects.object.PythonObjectLibrary;
+import com.oracle.graal.python.nodes.attributes.GetAttributeNode;
+import com.oracle.graal.python.nodes.call.CallNode;
+import com.oracle.graal.python.runtime.PythonContext;
+import com.oracle.graal.python.runtime.exception.PException;
+import com.oracle.graal.python.runtime.object.PythonObjectFactory;
+import com.oracle.truffle.api.CompilerDirectives.TruffleBoundary;
+import com.oracle.truffle.api.TruffleLanguage.ContextReference;
+import com.oracle.truffle.api.dsl.Cached;
+import com.oracle.truffle.api.dsl.CachedContext;
+import com.oracle.truffle.api.dsl.ImportStatic;
+import com.oracle.truffle.api.dsl.Specialization;
+import com.oracle.truffle.api.frame.VirtualFrame;
+import com.oracle.truffle.api.library.CachedLibrary;
+import com.oracle.truffle.api.nodes.Node;
+
+@ImportStatic(BuiltinNames.class)
+public abstract class WriteUnraisableNode extends Node {
+    static final String UNRAISABLE_HOOK_ARGUMENTS_CLASS = "__UnraisableHookArgs";
+
+    public abstract void execute(VirtualFrame frame, PBaseException exception, String message, Object object);
+
+    @Specialization(limit = "1")
+    static void writeUnraisable(VirtualFrame frame, PBaseException exception, String message, Object object,
+                    @CachedContext(PythonLanguage.class) ContextReference<PythonContext> contextRef,
+                    @CachedLibrary("exception") PythonObjectLibrary lib,
+                    @Cached PythonObjectFactory factory,
+                    @Cached GetExceptionTracebackNode getExceptionTracebackNode,
+                    @Cached("create(UNRAISABLEHOOK)") GetAttributeNode getUnraisableHook,
+                    @Cached CallNode callUnraisableHook,
+                    @Cached("create(UNRAISABLE_HOOK_ARGUMENTS_CLASS)") GetAttributeNode getArgumentsFactory,
+                    @Cached CallNode callArgumentsFactory) {
+        try {
+            PythonModule sysModule = contextRef.get().getCore().lookupBuiltinModule("sys");
+            Object unraisablehook = getUnraisableHook.executeObject(frame, sysModule);
+            Object argumentsFactory = getArgumentsFactory.executeObject(frame, sysModule);
+            Object exceptionType = lib.getLazyPythonClass(exception);
+            Object traceback = getExceptionTracebackNode.execute(frame, exception);
+            if (traceback == null) {
+                traceback = PNone.NONE;
+            }
+            Object messageObj = PNone.NONE;
+            if (message != null) {
+                messageObj = formatMessage(message);
+            }
+            Object hookArguments = callArgumentsFactory.execute(frame, argumentsFactory,
+                            factory.createTuple(new Object[]{exceptionType, exception, traceback, messageObj, object != null ? object : PNone.NONE}));
+            callUnraisableHook.execute(frame, unraisablehook, hookArguments);
+        } catch (PException e) {
+            ignoreException(message);
+        }
+    }
+
+    @TruffleBoundary
+    private static void ignoreException(String message) {
+        if (message != null) {
+            System.err.println(formatMessage(message));
+        } else {
+            System.err.println("Exception ignored in sys.unraisablehook");
+        }
+    }
+
+    @TruffleBoundary
+    private static Object formatMessage(String message) {
+        return "Exception ignored " + message;
+    }
+
+    public static WriteUnraisableNode create() {
+        return WriteUnraisableNodeGen.create();
+    }
+}

graalpython/com.oracle.graal.python/src/com/oracle/graal/python/nodes/generator/YieldFromNode.java

@@ -45,6 +45,7 @@
 import com.oracle.graal.python.builtins.objects.exception.PBaseException;
 import com.oracle.graal.python.builtins.objects.function.PArguments;
 import com.oracle.graal.python.builtins.objects.traceback.GetTracebackNode;
+import com.oracle.graal.python.nodes.WriteUnraisableNode;
 import com.oracle.graal.python.nodes.attributes.GetAttributeNode;
 import com.oracle.graal.python.nodes.call.CallNode;
 import com.oracle.graal.python.nodes.control.GetIteratorExpressionNode.GetIteratorNode;
@@ -76,6 +77,7 @@ public class YieldFromNode extends AbstractYieldNode implements GeneratorControl
     @Child private CallNode callSendNode;
 
     @Child private GetTracebackNode getTracebackNode;
+    @Child private WriteUnraisableNode writeUnraisableNode;
 
     @Child private IsBuiltinClassProfile stopIterProfile1 = IsBuiltinClassProfile.create();
     @Child private IsBuiltinClassProfile stopIterProfile2 = IsBuiltinClassProfile.create();
@@ -143,8 +145,7 @@ public Object execute(VirtualFrame frame) {
                             close = getGetCloseNode().executeObject(frame, _i);
                         } catch (PException pe) {
                             if (!hasNoCloseProfile.profileException(pe, PythonBuiltinClassType.AttributeError)) {
-                                // TODO msimacek: CPython writes the exception (!=AttributeError) as
-                                // unraisable and discards it
+                                ensureWriteUnraisable().execute(frame, pe.setCatchingFrameAndGetEscapedException(frame), null, _i);
                             }
                         }
                         if (close != null) {
@@ -286,6 +287,14 @@ private GetTracebackNode ensureGetTracebackNode() {
         return getTracebackNode;
     }
 
+    private WriteUnraisableNode ensureWriteUnraisable() {
+        if (writeUnraisableNode == null) {
+            CompilerDirectives.transferToInterpreterAndInvalidate();
+            writeUnraisableNode = insert(WriteUnraisableNode.create());
+        }
+        return writeUnraisableNode;
+    }
+
     public void setIteratorSlot(int slot) {
         this.iteratorSlot = slot;
     }

graalpython/lib-graalpython/python_cext.py

@@ -1243,38 +1243,13 @@ def _handle_system_exit():
 def PyErr_WriteUnraisable(obj):
     fetched = PyErr_Fetch()
     typ, val, tb = fetched if fetched is not native_null else (None, None, None)
-    try:
-        if sys.stderr is None:
-            return
-
-        if obj:
-            obj_str_arg = None
-            try:
-                obj_str_arg = repr(obj)
-            except:
-                obj_str_arg = "<object repr() failed>"
-            sys.stderr.write("Exception ignored in: %s\n" % obj_str_arg)
-
-        try:
-            import tb
-            tb.print_tb(tb, file=sys.stderr)
-        except:
-            pass
-
-        if not typ:
-            return
-
-        if typ.__module__ is None or typ.__name__ is None:
-            sys.stderr.write("<unknown>")
-
-        str_exc = None
-        try:
-            str_exc = str(obj)
-        except:
-            str_exc = "<exception str() failed>"
-        sys.stderr.write("%s.%s: %s" % (typ.__module__, typ.__name__, str_exc))
-    except:
-        pass
+    if val is None:
+        # This means an invalid call, but this function is not supposed to raise exceptions
+        return
+    if tb is native_null:
+        tb = None
+    val.__traceback__ = tb
+    PyTruffle_WriteUnraisable(val, obj)
 
 
 def _is_exception_class(exc):
@@ -1534,4 +1509,4 @@ def PyEval_GetBuiltins():
 
 
 def sequence_clear():
-    return 0
+    return 0

graalpython/lib-graalpython/sys.py

@@ -135,6 +135,16 @@ def make_hash_info_class():
 del make_hash_info_class
 
 
+def make_unraisable_hook_args_class():
+    from _descriptor import make_named_tuple_class
+    return make_named_tuple_class(
+        "UnraisableHookArgs",
+        ["exc_type", "exc_value", "exc_traceback", "err_msg", "object"],
+    )
+__UnraisableHookArgs = make_unraisable_hook_args_class()
+del make_unraisable_hook_args_class
+
+
 meta_path = []
 path_hooks = []
 path_importer_cache = {}
@@ -205,8 +215,22 @@ def __print_traceback__(typ, value, tb):
 
 def make_unraisablehook():
     def __unraisablehook__(unraisable, /):
-        # We don't currently use it and there's no way to construct the parameter in python code (internal type)
-        pass
+        try:
+            if unraisable.object:
+                try:
+                    r = repr(unraisable.object)
+                except Exception:
+                    r = "<object repr() failed>"
+                if unraisable.err_msg:
+                    print(f"{unraisable.err_msg}: {r}", file=stderr)
+                else:
+                    print(f"Exception ignored in: {r}", file=stderr)
+            elif unraisable.err_msg:
+                print(f"{unraisable.err_msg}:", file=stderr)
+        except BaseException:
+            # let it fall through to the exception printer
+            pass
+        __excepthook__(unraisable.exc_type, unraisable.exc_value, unraisable.exc_traceback)
 
     return __unraisablehook__