bpo-34206: Improve docs and test coverage for pre-init functions

Doc/c-api/init.rst

@@ -19,6 +19,12 @@ a few functions and the :ref:`global configuration variables
 
 The following functions can be safely called before Python is initialized:
 
+* Functions that initialize the interpreter:
+
+  * :c:func:`Py_Initialize`
+  * :c:func:`Py_InitializeEx`
+  * :c:func:`Py_Main`
+
 * Configuration functions:
 
   * :c:func:`PyImport_AppendInittab`
@@ -44,6 +50,7 @@ The following functions can be safely called before Python is initialized:
   * :c:func:`Py_GetCopyright`
   * :c:func:`Py_GetPlatform`
   * :c:func:`Py_GetVersion`
+  * :c:func:`Py_IsInitialized`
 
 * Utilities:
 
@@ -307,6 +314,47 @@ Initializing and finalizing the interpreter
    disregards the return value.
 
 
+.. c:function:: int Py_Main(int argc, wchar_t **argv)
+
+   The main program for the standard interpreter, encapsulating a full
+   :c:func:`Py_Initialize`/:c:func:`Py_Finalize` cycle, as well as additional
+   behaviour to implement reading configurations settings from the environment
+   and command line, and then executing ``__main__`` in accordance with
+   :ref:`using-on-cmdline`.
+
+   This is made available for programs which wish to support the full CPython
+   command line interface, rather than just embedding a Python runtime in a
+   larger application.
+
+   The *argc* and *argv* parameters should be prepared exactly as those which
+   are passed to a C program's :c:func:`main` function (converted to ``wchar_t``
+   with :c:func:`Py_DecodeLocale`).  It is important to note that the argument
+   list may be modified (but the contents of the strings pointed to by the
+   argument list are not).
+
+   The return value will be ``0`` if the interpreter exits normally (i.e.,
+   without an exception), ``1`` if the interpreter exits due to an exception,
+   or ``2`` if the argument list does not represent a valid Python command
+   line.
+
+   Note that if an otherwise unhandled :exc:`SystemExit` is raised, this
+   function will not return ``1``, but exit the process, as long as
+   ``Py_InspectFlag`` is not set. If ``Py_InspectFlag`` is set, execution will
+   drop into interactive Python prompt, at which point a second otherwise
+   unhandled :exc:`SystemExit` will still exit the process, while any other
+   means of exiting will set the return value as described above.
+
+   In normal usage, an embedding application will call this function
+   *instead* of calling :c:func:`Py_Initialize` or :c:func:`Py_InitializeEx`
+   directly, and all settings will be applied as described elsewhere in this
+   documentation. If this function is instead called *after* a preceding
+   ``Py_Initialize`` or ``Py_InitializeEx`` call, then exactly which
+   environmental and command line configuration settings will be updated is
+   version dependent (as it depends on which settings correctly support
+   being modified after they have already been set once in the first call to
+   ``Py_Initialize`` or ``Py_InitializeEx``).
+
+
 Process-wide parameters
 =======================
 

Doc/c-api/veryhigh.rst

@@ -25,23 +25,6 @@ are only passed to these functions if it is certain that they were created by
 the same library that the Python runtime is using.
 
 
-.. c:function:: int Py_Main(int argc, wchar_t **argv)
-
-   The main program for the standard interpreter.  This is made available for
-   programs which embed Python.  The *argc* and *argv* parameters should be
-   prepared exactly as those which are passed to a C program's :c:func:`main`
-   function (converted to wchar_t according to the user's locale).  It is
-   important to note that the argument list may be modified (but the contents of
-   the strings pointed to by the argument list are not). The return value will
-   be ``0`` if the interpreter exits normally (i.e., without an exception),
-   ``1`` if the interpreter exits due to an exception, or ``2`` if the parameter
-   list does not represent a valid Python command line.
-
-   Note that if an otherwise unhandled :exc:`SystemExit` is raised, this
-   function will not return ``1``, but exit the process, as long as
-   ``Py_InspectFlag`` is not set.
-
-
 .. c:function:: int PyRun_AnyFile(FILE *fp, const char *filename)
 
    This is a simplified interface to :c:func:`PyRun_AnyFileExFlags` below, leaving

Lib/test/test_embed.py

@@ -96,7 +96,8 @@ def run_repeated_init_and_subinterpreters(self):
             # Parse the line from the loop.  The first line is the main
             # interpreter and the 3 afterward are subinterpreters.
             interp = Interp(*match.groups())
-            if support.verbose > 1:
+            if support.verbose > 2:
+                # 5 lines per pass is super-spammy, so limit that to -vvv
                 print(interp)
             self.assertTrue(interp.interp)
             self.assertTrue(interp.tstate)
@@ -195,6 +196,10 @@ def test_pre_initialization_api(self):
         """
         env = dict(os.environ, PYTHONPATH=os.pathsep.join(sys.path))
         out, err = self.run_embedded_interpreter("pre_initialization_api", env=env)
+        if support.verbose > 1:
+            print()
+            print(out)
+            print(err)
         if sys.platform == "win32":
             expected_path = self.test_exe
         else:
@@ -211,6 +216,10 @@ def test_pre_initialization_sys_options(self):
         env = dict(os.environ, PYTHONPATH=os.pathsep.join(sys.path))
         out, err = self.run_embedded_interpreter(
                         "pre_initialization_sys_options", env=env)
+        if support.verbose > 1:
+            print()
+            print(out)
+            print(err)
         expected_output = (
             "sys.warnoptions: ['once', 'module', 'default']\n"
             "sys._xoptions: {'not_an_option': '1', 'also_not_an_option': '2'}\n"

Misc/NEWS.d/next/Core and Builtins/2018-06-30-21-48-16.bpo-34008.2Wjtm0.rst

@@ -0,0 +1,8 @@
+Make it explicit that we expect ``Py_Main`` to typically be called instead
+of ``Py_Initialize`` rather than after it (since ``Py_Main`` makes its own
+call to ``Py_Initialize``). However, also document that calling both is
+supported, but it's version dependent as to exactly which settings
+will be applied correctly in that case.
+
+Also add Py_IsInitialized to the list of APIs that are safe to call before
+the interpreter is initialized, and update the embedding tests to cover it.

Misc/NEWS.d/next/Documentation/2018-07-04-20-35-25.bpo-34008.bqecIb.rst

@@ -0,0 +1,2 @@
+The Py_Main documentation moved from the Very High Level API section to the
+Initialization and Finalization section.

Programs/_testembed.c

@@ -149,14 +149,36 @@ static int test_pre_initialization_api(void)
     _Py_EMBED_PREINIT_CHECK("Checking Py_SetProgramName\n");
     Py_SetProgramName(program);
 
+    _Py_EMBED_PREINIT_CHECK("Checking !Py_IsInitialized pre-initialization\n");
+    if (Py_IsInitialized()) {
+        fprintf(stderr, "Fatal error: initialized before initialization!\n");
+        return 1;
+    }
+
     _Py_EMBED_PREINIT_CHECK("Initializing interpreter\n");
     Py_Initialize();
+
+    _Py_EMBED_PREINIT_CHECK("Checking Py_IsInitialized post-initialization\n");
+    if (!Py_IsInitialized()) {
+        fprintf(stderr, "Fatal error: not initialized after initialization!\n");
+        return 1;
+    }
+
     _Py_EMBED_PREINIT_CHECK("Check sys module contents\n");
-    PyRun_SimpleString("import sys; "
-                       "print('sys.executable:', sys.executable)");
+    PyRun_SimpleString(
+        "import sys; "
+        "print('sys.executable:', sys.executable); "
+        "sys.stdout.flush(); "
+    );
     _Py_EMBED_PREINIT_CHECK("Finalizing interpreter\n");
     Py_Finalize();
 
+    _Py_EMBED_PREINIT_CHECK("Checking !Py_IsInitialized post-finalization\n");
+    if (Py_IsInitialized()) {
+        fprintf(stderr, "Fatal error: still initialized after finalization!\n");
+        return 1;
+    }
+
     _Py_EMBED_PREINIT_CHECK("Freeing memory allocated by Py_DecodeLocale\n");
     PyMem_RawFree(program);
     return 0;
@@ -202,12 +224,15 @@ static int test_pre_initialization_sys_options(void)
     _Py_EMBED_PREINIT_CHECK("Initializing interpreter\n");
     _testembed_Py_Initialize();
     _Py_EMBED_PREINIT_CHECK("Check sys module contents\n");
-    PyRun_SimpleString("import sys; "
-                       "print('sys.warnoptions:', sys.warnoptions); "
-                       "print('sys._xoptions:', sys._xoptions); "
-                       "warnings = sys.modules['warnings']; "
-                       "latest_filters = [f[0] for f in warnings.filters[:3]]; "
-                       "print('warnings.filters[:3]:', latest_filters)");
+    PyRun_SimpleString(
+        "import sys; "
+        "print('sys.warnoptions:', sys.warnoptions); "
+        "print('sys._xoptions:', sys._xoptions); "
+        "warnings = sys.modules['warnings']; "
+        "latest_filters = [f[0] for f in warnings.filters[:3]]; "
+        "print('warnings.filters[:3]:', latest_filters); "
+        "sys.stdout.flush(); "
+    );
     _Py_EMBED_PREINIT_CHECK("Finalizing interpreter\n");
     Py_Finalize();