gh-76785: Show the Traceback for Uncaught Subinterpreter Exceptions (gh-113034)

When an exception is uncaught in Interpreter.exec_sync(), it helps to show that exception's error display if uncaught in the calling interpreter.  We do so here by generating a TracebackException in the subinterpreter and passing it between interpreters using pickle.

Author: ericsnowcurrently

Include/internal/pycore_crossinterp.h

@@ -188,6 +188,8 @@ typedef struct _excinfo {
         const char *module;
     } type;
     const char *msg;
+    const char *pickled;
+    Py_ssize_t pickled_len;
 } _PyXI_excinfo;
 
 

Lib/test/support/interpreters/__init__.py

@@ -34,17 +34,36 @@ def __getattr__(name):
         raise AttributeError(name)
 
 
+_EXEC_FAILURE_STR = """
+{superstr}
+
+Uncaught in the interpreter:
+
+{formatted}
+""".strip()
+
 class ExecFailure(RuntimeError):
 
     def __init__(self, excinfo):
         msg = excinfo.formatted
         if not msg:
-            if excinfo.type and snapshot.msg:
-                msg = f'{snapshot.type.__name__}: {snapshot.msg}'
+            if excinfo.type and excinfo.msg:
+                msg = f'{excinfo.type.__name__}: {excinfo.msg}'
             else:
-                msg = snapshot.type.__name__ or snapshot.msg
+                msg = excinfo.type.__name__ or excinfo.msg
         super().__init__(msg)
-        self.snapshot = excinfo
+        self.excinfo = excinfo
+
+    def __str__(self):
+        try:
+            formatted = ''.join(self.excinfo.tbexc.format()).rstrip()
+        except Exception:
+            return super().__str__()
+        else:
+            return _EXEC_FAILURE_STR.format(
+                superstr=super().__str__(),
+                formatted=formatted,
+            )
 
 
 def create():

Lib/test/test_interpreters/test_api.py

@@ -525,6 +525,54 @@ def test_failure(self):
         with self.assertRaises(interpreters.ExecFailure):
             interp.exec_sync('raise Exception')
 
+    def test_display_preserved_exception(self):
+        tempdir = self.temp_dir()
+        modfile = self.make_module('spam', tempdir, text="""
+            def ham():
+                raise RuntimeError('uh-oh!')
+
+            def eggs():
+                ham()
+            """)
+        scriptfile = self.make_script('script.py', tempdir, text="""
+            from test.support import interpreters
+
+            def script():
+                import spam
+                spam.eggs()
+
+            interp = interpreters.create()
+            interp.exec_sync(script)
+            """)
+
+        stdout, stderr = self.assert_python_failure(scriptfile)
+        self.maxDiff = None
+        interpmod_line, = (l for l in stderr.splitlines() if ' exec_sync' in l)
+        #      File "{interpreters.__file__}", line 179, in exec_sync
+        self.assertEqual(stderr, dedent(f"""\
+            Traceback (most recent call last):
+              File "{scriptfile}", line 9, in <module>
+                interp.exec_sync(script)
+                ~~~~~~~~~~~~~~~~^^^^^^^^
+              {interpmod_line.strip()}
+                raise ExecFailure(excinfo)
+            test.support.interpreters.ExecFailure: RuntimeError: uh-oh!
+
+            Uncaught in the interpreter:
+
+            Traceback (most recent call last):
+              File "{scriptfile}", line 6, in script
+                spam.eggs()
+                ~~~~~~~~~^^
+              File "{modfile}", line 6, in eggs
+                ham()
+                ~~~^^
+              File "{modfile}", line 3, in ham
+                raise RuntimeError('uh-oh!')
+            RuntimeError: uh-oh!
+            """))
+        self.assertEqual(stdout, '')
+
     def test_in_thread(self):
         interp = interpreters.create()
         script, file = _captured_script('print("it worked!", end="")')

Lib/test/test_interpreters/utils.py

@@ -1,9 +1,16 @@
 import contextlib
 import os
+import os.path
+import subprocess
+import sys
+import tempfile
 import threading
 from textwrap import dedent
 import unittest
 
+from test import support
+from test.support import os_helper
+
 from test.support import interpreters
 
 
@@ -71,5 +78,70 @@ def ensure_closed(fd):
         self.addCleanup(lambda: ensure_closed(w))
         return r, w
 
+    def temp_dir(self):
+        tempdir = tempfile.mkdtemp()
+        tempdir = os.path.realpath(tempdir)
+        self.addCleanup(lambda: os_helper.rmtree(tempdir))
+        return tempdir
+
+    def make_script(self, filename, dirname=None, text=None):
+        if text:
+            text = dedent(text)
+        if dirname is None:
+            dirname = self.temp_dir()
+        filename = os.path.join(dirname, filename)
+
+        os.makedirs(os.path.dirname(filename), exist_ok=True)
+        with open(filename, 'w', encoding='utf-8') as outfile:
+            outfile.write(text or '')
+        return filename
+
+    def make_module(self, name, pathentry=None, text=None):
+        if text:
+            text = dedent(text)
+        if pathentry is None:
+            pathentry = self.temp_dir()
+        else:
+            os.makedirs(pathentry, exist_ok=True)
+        *subnames, basename = name.split('.')
+
+        dirname = pathentry
+        for subname in subnames:
+            dirname = os.path.join(dirname, subname)
+            if os.path.isdir(dirname):
+                pass
+            elif os.path.exists(dirname):
+                raise Exception(dirname)
+            else:
+                os.mkdir(dirname)
+            initfile = os.path.join(dirname, '__init__.py')
+            if not os.path.exists(initfile):
+                with open(initfile, 'w'):
+                    pass
+        filename = os.path.join(dirname, basename + '.py')
+
+        with open(filename, 'w', encoding='utf-8') as outfile:
+            outfile.write(text or '')
+        return filename
+
+    @support.requires_subprocess()
+    def run_python(self, *argv):
+        proc = subprocess.run(
+            [sys.executable, *argv],
+            capture_output=True,
+            text=True,
+        )
+        return proc.returncode, proc.stdout, proc.stderr
+
+    def assert_python_ok(self, *argv):
+        exitcode, stdout, stderr = self.run_python(*argv)
+        self.assertNotEqual(exitcode, 1)
+        return stdout, stderr
+
+    def assert_python_failure(self, *argv):
+        exitcode, stdout, stderr = self.run_python(*argv)
+        self.assertNotEqual(exitcode, 0)
+        return stdout, stderr
+
     def tearDown(self):
         clean_up_interpreters()