pythongh-135906: Test the internal C API in test_cext (python#136247)

Remove duplicated definition: atexit_datacallbackfunc type
is already defined by Include/cpython/pylifecycle.h.

Author: vstinner

Include/internal/pycore_interp_structs.h

@@ -130,8 +130,6 @@ struct _atexit_runtime_state {
 //###################
 // interpreter atexit
 
-typedef void (*atexit_datacallbackfunc)(void *);
-
 typedef struct atexit_callback {
     atexit_datacallbackfunc func;
     void *data;

Lib/test/test_cext/__init__.py

@@ -28,29 +28,13 @@
 @support.requires_venv_with_pip()
 @support.requires_subprocess()
 @support.requires_resource('cpu')
-class TestExt(unittest.TestCase):
+class BaseTests:
+    TEST_INTERNAL_C_API = False
+
     # Default build with no options
     def test_build(self):
         self.check_build('_test_cext')
 
-    def test_build_c11(self):
-        self.check_build('_test_c11_cext', std='c11')
-
-    @unittest.skipIf(support.MS_WINDOWS, "MSVC doesn't support /std:c99")
-    def test_build_c99(self):
-        # In public docs, we say C API is compatible with C11. However,
-        # in practice we do maintain C99 compatibility in public headers.
-        # Please ask the C API WG before adding a new C11-only feature.
-        self.check_build('_test_c99_cext', std='c99')
-
-    @support.requires_gil_enabled('incompatible with Free Threading')
-    def test_build_limited(self):
-        self.check_build('_test_limited_cext', limited=True)
-
-    @support.requires_gil_enabled('broken for now with Free Threading')
-    def test_build_limited_c11(self):
-        self.check_build('_test_limited_c11_cext', limited=True, std='c11')
-
     def check_build(self, extension_name, std=None, limited=False):
         venv_dir = 'env'
         with support.setup_venv_with_pip_setuptools(venv_dir) as python_exe:
@@ -70,6 +54,7 @@ def run_cmd(operation, cmd):
             if limited:
                 env['CPYTHON_TEST_LIMITED'] = '1'
             env['CPYTHON_TEST_EXT_NAME'] = extension_name
+            env['TEST_INTERNAL_C_API'] = str(int(self.TEST_INTERNAL_C_API))
             if support.verbose:
                 print('Run:', ' '.join(map(shlex.quote, cmd)))
                 subprocess.run(cmd, check=True, env=env)
@@ -110,5 +95,29 @@ def run_cmd(operation, cmd):
         run_cmd('Import', cmd)
 
 
+class TestPublicCAPI(BaseTests, unittest.TestCase):
+    @support.requires_gil_enabled('incompatible with Free Threading')
+    def test_build_limited(self):
+        self.check_build('_test_limited_cext', limited=True)
+
+    @support.requires_gil_enabled('broken for now with Free Threading')
+    def test_build_limited_c11(self):
+        self.check_build('_test_limited_c11_cext', limited=True, std='c11')
+
+    def test_build_c11(self):
+        self.check_build('_test_c11_cext', std='c11')
+
+    @unittest.skipIf(support.MS_WINDOWS, "MSVC doesn't support /std:c99")
+    def test_build_c99(self):
+        # In public docs, we say C API is compatible with C11. However,
+        # in practice we do maintain C99 compatibility in public headers.
+        # Please ask the C API WG before adding a new C11-only feature.
+        self.check_build('_test_c99_cext', std='c99')
+
+
+class TestInteralCAPI(BaseTests, unittest.TestCase):
+    TEST_INTERNAL_C_API = True
+
+
 if __name__ == "__main__":
     unittest.main()

Lib/test/test_cext/extension.c

@@ -1,11 +1,31 @@
 // gh-116869: Basic C test extension to check that the Python C API
 // does not emit C compiler warnings.
+//
+// Test also the internal C API if the TEST_INTERNAL_C_API macro is defined.
 
 // Always enable assertions
 #undef NDEBUG
 
+#ifdef TEST_INTERNAL_C_API
+#  define Py_BUILD_CORE_MODULE 1
+#endif
+
 #include "Python.h"
 
+#ifdef TEST_INTERNAL_C_API
+   // gh-135906: Check for compiler warnings in the internal C API.
+   // - Cython uses pycore_frame.h.
+   // - greenlet uses pycore_frame.h, pycore_interpframe_structs.h and
+   //   pycore_interpframe.h.
+#  include "internal/pycore_frame.h"
+#  include "internal/pycore_gc.h"
+#  include "internal/pycore_interp.h"
+#  include "internal/pycore_interpframe.h"
+#  include "internal/pycore_interpframe_structs.h"
+#  include "internal/pycore_object.h"
+#  include "internal/pycore_pystate.h"
+#endif
+
 #ifndef MODULE_NAME
 #  error "MODULE_NAME macro must be defined"
 #endif

Lib/test/test_cext/setup.py

@@ -14,10 +14,15 @@
 
 if not support.MS_WINDOWS:
     # C compiler flags for GCC and clang
-    CFLAGS = [
+    BASE_CFLAGS = [
         # The purpose of test_cext extension is to check that building a C
         # extension using the Python C API does not emit C compiler warnings.
         '-Werror',
+    ]
+
+    # C compiler flags for GCC and clang
+    PUBLIC_CFLAGS = [
+        *BASE_CFLAGS,
 
         # gh-120593: Check the 'const' qualifier
         '-Wcast-qual',
@@ -26,27 +31,40 @@
         '-pedantic-errors',
     ]
     if not support.Py_GIL_DISABLED:
-        CFLAGS.append(
+        PUBLIC_CFLAGS.append(
             # gh-116869: The Python C API must be compatible with building
             # with the -Werror=declaration-after-statement compiler flag.
             '-Werror=declaration-after-statement',
         )
+    INTERNAL_CFLAGS = [*BASE_CFLAGS]
 else:
     # MSVC compiler flags
-    CFLAGS = [
-        # Display warnings level 1 to 4
-        '/W4',
+    BASE_CFLAGS = [
         # Treat all compiler warnings as compiler errors
         '/WX',
     ]
+    PUBLIC_CFLAGS = [
+        *BASE_CFLAGS,
+        # Display warnings level 1 to 4
+        '/W4',
+    ]
+    INTERNAL_CFLAGS = [
+        *BASE_CFLAGS,
+        # Display warnings level 1 to 3
+        '/W3',
+    ]
 
 
 def main():
     std = os.environ.get("CPYTHON_TEST_STD", "")
     module_name = os.environ["CPYTHON_TEST_EXT_NAME"]
     limited = bool(os.environ.get("CPYTHON_TEST_LIMITED", ""))
+    internal = bool(int(os.environ.get("TEST_INTERNAL_C_API", "0")))
 
-    cflags = list(CFLAGS)
+    if not internal:
+        cflags = list(PUBLIC_CFLAGS)
+    else:
+        cflags = list(INTERNAL_CFLAGS)
     cflags.append(f'-DMODULE_NAME={module_name}')
 
     # Add -std=STD or /std:STD (MSVC) compiler flag
@@ -75,6 +93,9 @@ def main():
         version = sys.hexversion
         cflags.append(f'-DPy_LIMITED_API={version:#x}')
 
+    if internal:
+        cflags.append('-DTEST_INTERNAL_C_API=1')
+
     # On Windows, add PCbuild\amd64\ to include and library directories
     include_dirs = []
     library_dirs = []

Lib/test/test_cppext/__init__.py

@@ -24,7 +24,7 @@
 @support.requires_venv_with_pip()
 @support.requires_subprocess()
 @support.requires_resource('cpu')
-class TestCPPExt(unittest.TestCase):
+class BaseTests:
     def test_build(self):
         self.check_build('_testcppext')
 
@@ -34,10 +34,6 @@ def test_build_cpp03(self):
         # Please ask the C API WG before adding a new C++11-only feature.
         self.check_build('_testcpp03ext', std='c++03')
 
-    @support.requires_gil_enabled('incompatible with Free Threading')
-    def test_build_limited_cpp03(self):
-        self.check_build('_test_limited_cpp03ext', std='c++03', limited=True)
-
     @unittest.skipIf(support.MS_WINDOWS, "MSVC doesn't support /std:c++11")
     def test_build_cpp11(self):
         self.check_build('_testcpp11ext', std='c++11')
@@ -48,10 +44,6 @@ def test_build_cpp11(self):
     def test_build_cpp14(self):
         self.check_build('_testcpp14ext', std='c++14')
 
-    @support.requires_gil_enabled('incompatible with Free Threading')
-    def test_build_limited(self):
-        self.check_build('_testcppext_limited', limited=True)
-
     def check_build(self, extension_name, std=None, limited=False):
         venv_dir = 'env'
         with support.setup_venv_with_pip_setuptools(venv_dir) as python_exe:
@@ -111,5 +103,19 @@ def run_cmd(operation, cmd):
         run_cmd('Import', cmd)
 
 
+class TestPublicCAPI(BaseTests, unittest.TestCase):
+    @support.requires_gil_enabled('incompatible with Free Threading')
+    def test_build_limited_cpp03(self):
+        self.check_build('_test_limited_cpp03ext', std='c++03', limited=True)
+
+    @support.requires_gil_enabled('incompatible with Free Threading')
+    def test_build_limited(self):
+        self.check_build('_testcppext_limited', limited=True)
+
+
+class TestInteralCAPI(BaseTests, unittest.TestCase):
+    TEST_INTERNAL_C_API = True
+
+
 if __name__ == "__main__":
     unittest.main()

Lib/test/test_cppext/extension.cpp

@@ -6,8 +6,17 @@
 // Always enable assertions
 #undef NDEBUG
 
+#ifdef TEST_INTERNAL_C_API
+#  define Py_BUILD_CORE 1
+#endif
+
 #include "Python.h"
 
+#ifdef TEST_INTERNAL_C_API
+   // gh-135906: Check for compiler warnings in the internal C API
+#  include "internal/pycore_frame.h"
+#endif
+
 #ifndef MODULE_NAME
 #  error "MODULE_NAME macro must be defined"
 #endif

Lib/test/test_cppext/setup.py

@@ -47,6 +47,7 @@ def main():
     std = os.environ.get("CPYTHON_TEST_CPP_STD", "")
     module_name = os.environ["CPYTHON_TEST_EXT_NAME"]
     limited = bool(os.environ.get("CPYTHON_TEST_LIMITED", ""))
+    internal = bool(int(os.environ.get("TEST_INTERNAL_C_API", "0")))
 
     cppflags = list(CPPFLAGS)
     cppflags.append(f'-DMODULE_NAME={module_name}')
@@ -82,6 +83,9 @@ def main():
         version = sys.hexversion
         cppflags.append(f'-DPy_LIMITED_API={version:#x}')
 
+    if internal:
+        cppflags.append('-DTEST_INTERNAL_C_API=1')
+
     # On Windows, add PCbuild\amd64\ to include and library directories
     include_dirs = []
     library_dirs = []