bpo-43445: Add frozen modules to sys.stdlib_module_names (GH-24798)

Add frozen modules to sys.stdlib_module_names. For example, add
"_frozen_importlib" and "_frozen_importlib_external" names.

Add "list_frozen" command to Programs/_testembed.

Author: vstinner

Makefile.pre.in

@@ -913,7 +913,7 @@ regen-keyword:
 	$(UPDATE_FILE) $(srcdir)/Lib/keyword.py $(srcdir)/Lib/keyword.py.new
 
 .PHONY: regen-stdlib-module-names
-regen-stdlib-module-names: build_all
+regen-stdlib-module-names: build_all Programs/_testembed
 	# Regenerate Python/stdlib_module_names.h
 	# using Tools/scripts/generate_stdlib_module_names.py
 	$(RUNSHARED) ./$(BUILDPYTHON) \

Misc/NEWS.d/next/Library/2021-03-09-11-36-19.bpo-43445.jnj-UB.rst

@@ -0,0 +1,2 @@
+Add frozen modules to :data:`sys.stdlib_module_names`. For example, add
+``"_frozen_importlib"`` and ``"_frozen_importlib_external"`` names.

Programs/_testembed.c

@@ -1721,6 +1721,20 @@ static int test_unicode_id_init(void)
 }
 
 
+// List frozen modules.
+// Command used by Tools/scripts/generate_stdlib_module_names.py script.
+static int list_frozen(void)
+{
+    const struct _frozen *p;
+    for (p = PyImport_FrozenModules; ; p++) {
+        if (p->name == NULL)
+            break;
+        printf("%s\n", p->name);
+    }
+    return 0;
+}
+
+
 
 /* *********************************************************
  * List of test cases and the function that implements it.
@@ -1792,6 +1806,8 @@ static struct TestCase TestCases[] = {
     {"test_audit_run_stdin", test_audit_run_stdin},
 
     {"test_unicode_id_init", test_unicode_id_init},
+
+    {"list_frozen", list_frozen},
     {NULL, NULL}
 };
 

Python/stdlib_module_names.h

@@ -32,6 +32,8 @@ static const char* _Py_stdlib_module_names[] = {
 "_dbm",
 "_decimal",
 "_elementtree",
+"_frozen_importlib",
+"_frozen_importlib_external",
 "_functools",
 "_gdbm",
 "_hashlib",

Tools/scripts/generate_stdlib_module_names.py

@@ -11,14 +11,16 @@
 STDLIB_PATH = os.path.join(SRC_DIR, 'Lib')
 MODULES_SETUP = os.path.join(SRC_DIR, 'Modules', 'Setup')
 SETUP_PY = os.path.join(SRC_DIR, 'setup.py')
+TEST_EMBED = os.path.join(SRC_DIR, 'Programs', '_testembed')
 
 IGNORE = {
     '__init__',
     '__pycache__',
     'site-packages',
 
-    # test modules
-    '__phello__.foo',
+    # Test modules and packages
+    '__hello__',
+    '__phello__',
     '_ctypes_test',
     '_testbuffer',
     '_testcapi',
@@ -103,13 +105,40 @@ def list_modules_setup_extensions(names):
             names.add(name)
 
 
+# List frozen modules of the PyImport_FrozenModules list (Python/frozen.c).
+# Use the "./Programs/_testembed list_frozen" command.
+def list_frozen(names):
+    args = [TEST_EMBED, 'list_frozen']
+    proc = subprocess.run(args, stdout=subprocess.PIPE, text=True)
+    exitcode = proc.returncode
+    if exitcode:
+        cmd = ' '.join(args)
+        print(f"{cmd} failed with exitcode {exitcode}")
+        sys.exit(exitcode)
+    for line in proc.stdout.splitlines():
+        name = line.strip()
+        names.add(name)
+
+
 def list_modules():
     names = set(sys.builtin_module_names) | set(WINDOWS_MODULES)
     list_modules_setup_extensions(names)
     list_setup_extensions(names)
     list_packages(names)
     list_python_modules(names)
-    names -= set(IGNORE)
+    list_frozen(names)
+
+    # Remove ignored packages and modules
+    for name in list(names):
+        package_name = name.split('.')[0]
+        # package_name can be equal to name
+        if package_name in IGNORE:
+            names.discard(name)
+
+    for name in names:
+        if "." in name:
+            raise Exception("sub-modules must not be listed")
+
     return names