Merge branch 'main' into unhandledexc

Author: kumaraditya303

Doc/using/cmdline.rst

@@ -535,12 +535,13 @@ Miscellaneous options
      location indicators when the interpreter displays tracebacks. See also
      :envvar:`PYTHONNODEBUGRANGES`.
    * ``-X frozen_modules`` determines whether or not frozen modules are
-     ignored by the import machinery.  A value of "on" means they get
-     imported and "off" means they are ignored.  The default is "on"
+     ignored by the import machinery.  A value of ``on`` means they get
+     imported and ``off`` means they are ignored.  The default is ``on``
      if this is an installed Python (the normal case).  If it's under
-     development (running from the source tree) then the default is "off".
-     Note that the "importlib_bootstrap" and "importlib_bootstrap_external"
-     frozen modules are always used, even if this flag is set to "off".
+     development (running from the source tree) then the default is ``off``.
+     Note that the :mod:`!importlib_bootstrap` and
+     :mod:`!importlib_bootstrap_external` frozen modules are always used, even
+     if this flag is set to ``off``. See also :envvar:`PYTHON_FROZEN_MODULES`.
    * ``-X perf`` enables support for the Linux ``perf`` profiler.
      When this option is provided, the ``perf`` profiler will be able to
      report Python calls. This option is only available on some platforms and
@@ -1095,6 +1096,20 @@ conflict.
 
    .. versionadded:: 3.13
 
+.. envvar:: PYTHON_FROZEN_MODULES
+
+   If this variable is set to ``on`` or ``off``, it determines whether or not
+   frozen modules are ignored by the import machinery.  A value of ``on`` means
+   they get imported and ``off`` means they are ignored.  The default is ``on``
+   for non-debug builds (the normal case) and ``off`` for debug builds.
+   Note that the :mod:`!importlib_bootstrap` and
+   :mod:`!importlib_bootstrap_external` frozen modules are always used, even
+   if this flag is set to ``off``.
+
+   See also the :option:`-X frozen_modules <-X>` command-line option.
+
+   .. versionadded:: 3.13
+
 
 Debug-mode variables
 ~~~~~~~~~~~~~~~~~~~~

Doc/whatsnew/3.13.rst

@@ -120,6 +120,11 @@ Other Language Changes
   is rejected when the global is used in the :keyword:`else` block.
   (Contributed by Irit Katriel in :gh:`111123`.)
 
+* Added a new environment variable :envvar:`PYTHON_FROZEN_MODULES`. It
+  determines whether or not frozen modules are ignored by the import machinery,
+  equivalent of the :option:`-X frozen_modules <-X>` command-line option.
+  (Contributed by Yilei Yang in :gh:`111374`.)
+
 New Modules
 ===========
 
@@ -1035,7 +1040,8 @@ New Features
 * If Python is built in :ref:`debug mode <debug-build>` or :option:`with
   assertions <--with-assertions>`, :c:func:`PyTuple_SET_ITEM` and
   :c:func:`PyList_SET_ITEM` now check the index argument with an assertion.
-  If the assertion fails, make sure that the size is set before.
+  If the assertion fails in :c:func:`PyTuple_SET_ITEM`, make sure that the
+  tuple size is set before.
   (Contributed by Victor Stinner in :gh:`106168`.)
 
 * Add :c:func:`PyModule_Add` function: similar to

Include/internal/pycore_crossinterp.h

@@ -8,6 +8,8 @@ extern "C" {
 #  error "this header requires Py_BUILD_CORE define"
 #endif
 
+#include "pycore_pyerrors.h"
+
 
 /***************************/
 /* cross-interpreter calls */
@@ -124,6 +126,8 @@ struct _xidregitem {
 };
 
 struct _xidregistry {
+    int global;  /* builtin types or heap types */
+    int initialized;
     PyThread_type_lock mutex;
     struct _xidregitem *head;
 };
@@ -133,6 +137,130 @@ PyAPI_FUNC(int) _PyCrossInterpreterData_UnregisterClass(PyTypeObject *);
 PyAPI_FUNC(crossinterpdatafunc) _PyCrossInterpreterData_Lookup(PyObject *);
 
 
+/*****************************/
+/* runtime state & lifecycle */
+/*****************************/
+
+struct _xi_runtime_state {
+    // builtin types
+    // XXX Remove this field once we have a tp_* slot.
+    struct _xidregistry registry;
+};
+
+struct _xi_state {
+    // heap types
+    // XXX Remove this field once we have a tp_* slot.
+    struct _xidregistry registry;
+
+    // heap types
+    PyObject *PyExc_NotShareableError;
+};
+
+extern PyStatus _PyXI_Init(PyInterpreterState *interp);
+extern void _PyXI_Fini(PyInterpreterState *interp);
+
+
+/***************************/
+/* short-term data sharing */
+/***************************/
+
+typedef enum error_code {
+    _PyXI_ERR_NO_ERROR = 0,
+    _PyXI_ERR_UNCAUGHT_EXCEPTION = -1,
+    _PyXI_ERR_OTHER = -2,
+    _PyXI_ERR_NO_MEMORY = -3,
+    _PyXI_ERR_ALREADY_RUNNING = -4,
+    _PyXI_ERR_MAIN_NS_FAILURE = -5,
+    _PyXI_ERR_APPLY_NS_FAILURE = -6,
+    _PyXI_ERR_NOT_SHAREABLE = -7,
+} _PyXI_errcode;
+
+
+typedef struct _sharedexception {
+    // The originating interpreter.
+    PyInterpreterState *interp;
+    // The kind of error to propagate.
+    _PyXI_errcode code;
+    // The exception information to propagate, if applicable.
+    // This is populated only for _PyXI_ERR_UNCAUGHT_EXCEPTION.
+    _Py_excinfo uncaught;
+} _PyXI_exception_info;
+
+PyAPI_FUNC(void) _PyXI_ApplyExceptionInfo(
+    _PyXI_exception_info *info,
+    PyObject *exctype);
+
+typedef struct xi_session _PyXI_session;
+typedef struct _sharedns _PyXI_namespace;
+
+PyAPI_FUNC(void) _PyXI_FreeNamespace(_PyXI_namespace *ns);
+PyAPI_FUNC(_PyXI_namespace *) _PyXI_NamespaceFromNames(PyObject *names);
+PyAPI_FUNC(int) _PyXI_FillNamespaceFromDict(
+    _PyXI_namespace *ns,
+    PyObject *nsobj,
+    _PyXI_session *session);
+PyAPI_FUNC(int) _PyXI_ApplyNamespace(
+    _PyXI_namespace *ns,
+    PyObject *nsobj,
+    PyObject *dflt);
+
+
+// A cross-interpreter session involves entering an interpreter
+// (_PyXI_Enter()), doing some work with it, and finally exiting
+// that interpreter (_PyXI_Exit()).
+//
+// At the boundaries of the session, both entering and exiting,
+// data may be exchanged between the previous interpreter and the
+// target one in a thread-safe way that does not violate the
+// isolation between interpreters.  This includes setting objects
+// in the target's __main__ module on the way in, and capturing
+// uncaught exceptions on the way out.
+struct xi_session {
+    // Once a session has been entered, this is the tstate that was
+    // current before the session.  If it is different from cur_tstate
+    // then we must have switched interpreters.  Either way, this will
+    // be the current tstate once we exit the session.
+    PyThreadState *prev_tstate;
+    // Once a session has been entered, this is the current tstate.
+    // It must be current when the session exits.
+    PyThreadState *init_tstate;
+    // This is true if init_tstate needs cleanup during exit.
+    int own_init_tstate;
+
+    // This is true if, while entering the session, init_thread took
+    // "ownership" of the interpreter's __main__ module.  This means
+    // it is the only thread that is allowed to run code there.
+    // (Caveat: for now, users may still run exec() against the
+    // __main__ module's dict, though that isn't advisable.)
+    int running;
+    // This is a cached reference to the __dict__ of the entered
+    // interpreter's __main__ module.  It is looked up when at the
+    // beginning of the session as a convenience.
+    PyObject *main_ns;
+
+    // This is set if the interpreter is entered and raised an exception
+    // that needs to be handled in some special way during exit.
+    _PyXI_errcode *exc_override;
+    // This is set if exit captured an exception to propagate.
+    _PyXI_exception_info *exc;
+
+    // -- pre-allocated memory --
+    _PyXI_exception_info _exc;
+    _PyXI_errcode _exc_override;
+};
+
+PyAPI_FUNC(int) _PyXI_Enter(
+    _PyXI_session *session,
+    PyInterpreterState *interp,
+    PyObject *nsupdates);
+PyAPI_FUNC(void) _PyXI_Exit(_PyXI_session *session);
+
+PyAPI_FUNC(void) _PyXI_ApplyCapturedException(
+    _PyXI_session *session,
+    PyObject *excwrapper);
+PyAPI_FUNC(int) _PyXI_HasCapturedException(_PyXI_session *session);
+
+
 #ifdef __cplusplus
 }
 #endif

Include/internal/pycore_interp.h

@@ -153,8 +153,8 @@ struct _is {
     Py_ssize_t co_extra_user_count;
     freefunc co_extra_freefuncs[MAX_CO_EXTRA_USERS];
 
-    // XXX Remove this field once we have a tp_* slot.
-    struct _xidregistry xidregistry;
+    /* cross-interpreter data and utils */
+    struct _xi_state xi;
 
 #ifdef HAVE_FORK
     PyObject *before_forkers;

Include/internal/pycore_pyerrors.h

@@ -68,6 +68,30 @@ extern PyStatus _PyErr_InitTypes(PyInterpreterState *);
 extern void _PyErr_FiniTypes(PyInterpreterState *);
 
 
+/* exception snapshots */
+
+// Ultimately we'd like to preserve enough information about the
+// exception and traceback that we could re-constitute (or at least
+// simulate, a la traceback.TracebackException), and even chain, a copy
+// of the exception in the calling interpreter.
+
+typedef struct _excinfo {
+    const char *type;
+    const char *msg;
+} _Py_excinfo;
+
+extern void _Py_excinfo_Clear(_Py_excinfo *info);
+extern int _Py_excinfo_Copy(_Py_excinfo *dest, _Py_excinfo *src);
+extern const char * _Py_excinfo_InitFromException(
+    _Py_excinfo *info,
+    PyObject *exc);
+extern void _Py_excinfo_Apply(_Py_excinfo *info, PyObject *exctype);
+extern const char * _Py_excinfo_AsUTF8(
+    _Py_excinfo *info,
+    char *buf,
+    size_t bufsize);
+
+
 /* other API */
 
 static inline PyObject* _PyErr_Occurred(PyThreadState *tstate)

Include/internal/pycore_runtime.h

@@ -200,8 +200,8 @@ typedef struct pyruntimestate {
      possible to facilitate out-of-process observability
      tools. */
 
-    // XXX Remove this field once we have a tp_* slot.
-    struct _xidregistry xidregistry;
+    /* cross-interpreter data and utils */
+    struct _xi_runtime_state xi;
 
     struct _pymem_allocators allocators;
     struct _obmalloc_global_state obmalloc;

Include/internal/pycore_runtime_init.h

@@ -95,6 +95,11 @@ extern PyTypeObject _PyExc_MemoryError;
               until _PyInterpreterState_Enable() is called. */ \
             .next_id = -1, \
         }, \
+        .xi = { \
+            .registry = { \
+                .global = 1, \
+            }, \
+        }, \
         /* A TSS key must be initialized with Py_tss_NEEDS_INIT \
            in accordance with the specification. */ \
         .autoTSSkey = Py_tss_NEEDS_INIT, \

Lib/test/support/__init__.py

@@ -2355,8 +2355,15 @@ def adjust_int_max_str_digits(max_digits):
 #For recursion tests, easily exceeds default recursion limit
 EXCEEDS_RECURSION_LIMIT = 5000
 
-# The default C recursion limit (from Include/cpython/pystate.h).
-Py_C_RECURSION_LIMIT = 1500
+def _get_c_recursion_limit():
+    try:
+        import _testcapi
+        return _testcapi.Py_C_RECURSION_LIMIT
+    except (ImportError, AttributeError):
+        return 1500  #  (from Include/cpython/pystate.h)
+
+# The default C recursion limit.
+Py_C_RECURSION_LIMIT = _get_c_recursion_limit()
 
 #Windows doesn't have os.uname() but it doesn't support s390x.
 skip_on_s390x = unittest.skipIf(hasattr(os, 'uname') and os.uname().machine == 's390x',

Lib/test/support/interpreters.py

@@ -92,7 +92,7 @@ def close(self):
         return _interpreters.destroy(self._id)
 
     # XXX Rename "run" to "exec"?
-    def run(self, src_str, /, *, channels=None):
+    def run(self, src_str, /, channels=None):
         """Run the given source code in the interpreter.
 
         This is essentially the same as calling the builtin "exec"

Lib/test/test__xxsubinterpreters.py

@@ -102,6 +102,8 @@ def test_default_shareables(self):
                 'spam',
                 10,
                 -10,
+                True,
+                False,
                 100.0,
                 ]
         for obj in shareables:
@@ -121,8 +123,6 @@ class SubBytes(bytes):
 
         not_shareables = [
                 # singletons
-                True,
-                False,
                 NotImplemented,
                 ...,
                 # builtin types and objects
@@ -189,6 +189,9 @@ def test_non_shareable_int(self):
                 with self.assertRaises(OverflowError):
                     _testinternalcapi.get_crossinterp_data(i)
 
+    def test_bool(self):
+        self._assert_values([True, False])
+
     def test_float(self):
         self._assert_values([0.0, 1.1, -1.0, 0.12345678, -0.12345678])