From 8a747474526a7c02eb390a911d4f6070fe76e8f4 Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Wed, 16 Oct 2024 08:52:18 -0600 Subject: [PATCH 01/11] Move _Py_DebugOffsets to its own header file. --- Include/internal/pycore_debugger_utils.h | 162 +++++++++++++++++++++++ Include/internal/pycore_runtime.h | 147 +------------------- Include/internal/pycore_runtime_init.h | 1 + Makefile.pre.in | 1 + Modules/_testexternalinspection.c | 4 +- PCbuild/pythoncore.vcxproj | 1 + PCbuild/pythoncore.vcxproj.filters | 3 + 7 files changed, 173 insertions(+), 146 deletions(-) create mode 100644 Include/internal/pycore_debugger_utils.h diff --git a/Include/internal/pycore_debugger_utils.h b/Include/internal/pycore_debugger_utils.h new file mode 100644 index 00000000000000..c5c07cbc0ac22d --- /dev/null +++ b/Include/internal/pycore_debugger_utils.h @@ -0,0 +1,162 @@ +#ifndef Py_INTERNAL_DEBUGGER_UTILS_H +#define Py_INTERNAL_DEBUGGER_UTILS_H +#ifdef __cplusplus +extern "C" { +#endif + +#ifndef Py_BUILD_CORE +# error "this header requires Py_BUILD_CORE define" +#endif + + +#define _Py_Debug_Cookie "xdebugpy" + +#ifdef Py_GIL_DISABLED +# define _Py_Debug_gilruntimestate_enabled offsetof(struct _gil_runtime_state, enabled) +# define _Py_Debug_Free_Threaded 1 +#else +# define _Py_Debug_gilruntimestate_enabled 0 +# define _Py_Debug_Free_Threaded 0 +#endif + + +typedef struct _Py_DebugOffsets { + char cookie[8]; + uint64_t version; + uint64_t free_threaded; + // Runtime state offset; + struct _runtime_state { + uint64_t size; + uint64_t finalizing; + uint64_t interpreters_head; + } runtime_state; + + // Interpreter state offset; + struct _interpreter_state { + uint64_t size; + uint64_t id; + uint64_t next; + uint64_t threads_head; + uint64_t gc; + uint64_t imports_modules; + uint64_t sysdict; + uint64_t builtins; + uint64_t ceval_gil; + uint64_t gil_runtime_state; + uint64_t gil_runtime_state_enabled; + uint64_t gil_runtime_state_locked; + uint64_t gil_runtime_state_holder; + } interpreter_state; + + // Thread state offset; + struct _thread_state{ + uint64_t size; + uint64_t prev; + uint64_t next; + uint64_t interp; + uint64_t current_frame; + uint64_t thread_id; + uint64_t native_thread_id; + uint64_t datastack_chunk; + uint64_t status; + } thread_state; + + // InterpreterFrame offset; + struct _interpreter_frame { + uint64_t size; + uint64_t previous; + uint64_t executable; + uint64_t instr_ptr; + uint64_t localsplus; + uint64_t owner; + } interpreter_frame; + + // Code object offset; + struct _code_object { + uint64_t size; + uint64_t filename; + uint64_t name; + uint64_t qualname; + uint64_t linetable; + uint64_t firstlineno; + uint64_t argcount; + uint64_t localsplusnames; + uint64_t localspluskinds; + uint64_t co_code_adaptive; + } code_object; + + // PyObject offset; + struct _pyobject { + uint64_t size; + uint64_t ob_type; + } pyobject; + + // PyTypeObject object offset; + struct _type_object { + uint64_t size; + uint64_t tp_name; + uint64_t tp_repr; + uint64_t tp_flags; + } type_object; + + // PyTuple object offset; + struct _tuple_object { + uint64_t size; + uint64_t ob_item; + uint64_t ob_size; + } tuple_object; + + // PyList object offset; + struct _list_object { + uint64_t size; + uint64_t ob_item; + uint64_t ob_size; + } list_object; + + // PyDict object offset; + struct _dict_object { + uint64_t size; + uint64_t ma_keys; + uint64_t ma_values; + } dict_object; + + // PyFloat object offset; + struct _float_object { + uint64_t size; + uint64_t ob_fval; + } float_object; + + // PyLong object offset; + struct _long_object { + uint64_t size; + uint64_t lv_tag; + uint64_t ob_digit; + } long_object; + + // PyBytes object offset; + struct _bytes_object { + uint64_t size; + uint64_t ob_size; + uint64_t ob_sval; + } bytes_object; + + // Unicode object offset; + struct _unicode_object { + uint64_t size; + uint64_t state; + uint64_t length; + uint64_t asciiobject_size; + } unicode_object; + + // GC runtime state offset; + struct _gc { + uint64_t size; + uint64_t collecting; + } gc; +} _Py_DebugOffsets; + + +#ifdef __cplusplus +} +#endif +#endif /* !Py_INTERNAL_DEBUGGER_UTILS_H */ diff --git a/Include/internal/pycore_runtime.h b/Include/internal/pycore_runtime.h index d4291b87261ae0..64c38c47e50916 100644 --- a/Include/internal/pycore_runtime.h +++ b/Include/internal/pycore_runtime.h @@ -10,7 +10,8 @@ extern "C" { #include "pycore_atexit.h" // struct _atexit_runtime_state #include "pycore_ceval_state.h" // struct _ceval_runtime_state -#include "pycore_crossinterp.h" // struct _xidregistry +#include "pycore_crossinterp.h" // struct _xidregistry +#include "pycore_debugger_utils.h" // _Py_DebugOffsets #include "pycore_faulthandler.h" // struct _faulthandler_runtime_state #include "pycore_floatobject.h" // struct _Py_float_runtime_state #include "pycore_import.h" // struct _import_runtime_state @@ -44,156 +45,12 @@ struct _gilstate_runtime_state { /* Runtime audit hook state */ -#define _Py_Debug_Cookie "xdebugpy" - -#ifdef Py_GIL_DISABLED -# define _Py_Debug_gilruntimestate_enabled offsetof(struct _gil_runtime_state, enabled) -# define _Py_Debug_Free_Threaded 1 -#else -# define _Py_Debug_gilruntimestate_enabled 0 -# define _Py_Debug_Free_Threaded 0 -#endif typedef struct _Py_AuditHookEntry { struct _Py_AuditHookEntry *next; Py_AuditHookFunction hookCFunction; void *userData; } _Py_AuditHookEntry; -typedef struct _Py_DebugOffsets { - char cookie[8]; - uint64_t version; - uint64_t free_threaded; - // Runtime state offset; - struct _runtime_state { - uint64_t size; - uint64_t finalizing; - uint64_t interpreters_head; - } runtime_state; - - // Interpreter state offset; - struct _interpreter_state { - uint64_t size; - uint64_t id; - uint64_t next; - uint64_t threads_head; - uint64_t gc; - uint64_t imports_modules; - uint64_t sysdict; - uint64_t builtins; - uint64_t ceval_gil; - uint64_t gil_runtime_state; - uint64_t gil_runtime_state_enabled; - uint64_t gil_runtime_state_locked; - uint64_t gil_runtime_state_holder; - } interpreter_state; - - // Thread state offset; - struct _thread_state{ - uint64_t size; - uint64_t prev; - uint64_t next; - uint64_t interp; - uint64_t current_frame; - uint64_t thread_id; - uint64_t native_thread_id; - uint64_t datastack_chunk; - uint64_t status; - } thread_state; - - // InterpreterFrame offset; - struct _interpreter_frame { - uint64_t size; - uint64_t previous; - uint64_t executable; - uint64_t instr_ptr; - uint64_t localsplus; - uint64_t owner; - } interpreter_frame; - - // Code object offset; - struct _code_object { - uint64_t size; - uint64_t filename; - uint64_t name; - uint64_t qualname; - uint64_t linetable; - uint64_t firstlineno; - uint64_t argcount; - uint64_t localsplusnames; - uint64_t localspluskinds; - uint64_t co_code_adaptive; - } code_object; - - // PyObject offset; - struct _pyobject { - uint64_t size; - uint64_t ob_type; - } pyobject; - - // PyTypeObject object offset; - struct _type_object { - uint64_t size; - uint64_t tp_name; - uint64_t tp_repr; - uint64_t tp_flags; - } type_object; - - // PyTuple object offset; - struct _tuple_object { - uint64_t size; - uint64_t ob_item; - uint64_t ob_size; - } tuple_object; - - // PyList object offset; - struct _list_object { - uint64_t size; - uint64_t ob_item; - uint64_t ob_size; - } list_object; - - // PyDict object offset; - struct _dict_object { - uint64_t size; - uint64_t ma_keys; - uint64_t ma_values; - } dict_object; - - // PyFloat object offset; - struct _float_object { - uint64_t size; - uint64_t ob_fval; - } float_object; - - // PyLong object offset; - struct _long_object { - uint64_t size; - uint64_t lv_tag; - uint64_t ob_digit; - } long_object; - - // PyBytes object offset; - struct _bytes_object { - uint64_t size; - uint64_t ob_size; - uint64_t ob_sval; - } bytes_object; - - // Unicode object offset; - struct _unicode_object { - uint64_t size; - uint64_t state; - uint64_t length; - uint64_t asciiobject_size; - } unicode_object; - - // GC runtime state offset; - struct _gc { - uint64_t size; - uint64_t collecting; - } gc; -} _Py_DebugOffsets; - /* Reference tracer state */ struct _reftracer_runtime_state { PyRefTracer tracer_func; diff --git a/Include/internal/pycore_runtime_init.h b/Include/internal/pycore_runtime_init.h index a17ba46966daa1..9dcf4bb18db708 100644 --- a/Include/internal/pycore_runtime_init.h +++ b/Include/internal/pycore_runtime_init.h @@ -9,6 +9,7 @@ extern "C" { #endif #include "pycore_ceval_state.h" // _PyEval_RUNTIME_PERF_INIT +#include "pycore_debugger_utils.h" // _Py_Debug_gilruntimestate_enabled #include "pycore_faulthandler.h" // _faulthandler_runtime_state_INIT #include "pycore_floatobject.h" // _py_float_format_unknown #include "pycore_function.h" diff --git a/Makefile.pre.in b/Makefile.pre.in index 07c8a4d20142db..9d8abf390545bd 100644 --- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -1193,6 +1193,7 @@ PYTHON_HEADERS= \ $(srcdir)/Include/internal/pycore_context.h \ $(srcdir)/Include/internal/pycore_critical_section.h \ $(srcdir)/Include/internal/pycore_crossinterp.h \ + $(srcdir)/Include/internal/pycore_debugger_utils.h \ $(srcdir)/Include/internal/pycore_descrobject.h \ $(srcdir)/Include/internal/pycore_dict.h \ $(srcdir)/Include/internal/pycore_dict_state.h \ diff --git a/Modules/_testexternalinspection.c b/Modules/_testexternalinspection.c index 2476346777c319..d6e2a6fad1672d 100644 --- a/Modules/_testexternalinspection.c +++ b/Modules/_testexternalinspection.c @@ -51,7 +51,9 @@ # define Py_BUILD_CORE_MODULE 1 #endif #include "Python.h" -#include +#include // _Py_DebugOffsets +#include // FRAME_OWNED_BY_CSTACK +#include // Py_TAG_BITS #ifndef HAVE_PROCESS_VM_READV # define HAVE_PROCESS_VM_READV 0 diff --git a/PCbuild/pythoncore.vcxproj b/PCbuild/pythoncore.vcxproj index 3b33c6bf6bb91d..192079d6c37083 100644 --- a/PCbuild/pythoncore.vcxproj +++ b/PCbuild/pythoncore.vcxproj @@ -227,6 +227,7 @@ + diff --git a/PCbuild/pythoncore.vcxproj.filters b/PCbuild/pythoncore.vcxproj.filters index ee2930b10439a9..2865fcae870cae 100644 --- a/PCbuild/pythoncore.vcxproj.filters +++ b/PCbuild/pythoncore.vcxproj.filters @@ -603,6 +603,9 @@ Include\internal + + Include\internal + Include\internal From 5fc5b7d9f985da85c67f1f19952f7816cfaa4dcc Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Wed, 16 Oct 2024 09:03:10 -0600 Subject: [PATCH 02/11] Factor out GENERATE_DEBUG_SECTION(). --- Include/internal/pycore_debugger_utils.h | 37 ++++++++++++++++++++++++ Python/pylifecycle.c | 24 ++------------- 2 files changed, 40 insertions(+), 21 deletions(-) diff --git a/Include/internal/pycore_debugger_utils.h b/Include/internal/pycore_debugger_utils.h index c5c07cbc0ac22d..bdfc0550fc839f 100644 --- a/Include/internal/pycore_debugger_utils.h +++ b/Include/internal/pycore_debugger_utils.h @@ -9,6 +9,43 @@ extern "C" { #endif +#if defined(__APPLE__) +# include +#endif + + +// Macros to burn global values in custom sections so out-of-process +// profilers can locate them easily + +#define GENERATE_DEBUG_SECTION(name, declaration) \ + _GENERATE_DEBUG_SECTION_WINDOWS(name) \ + _GENERATE_DEBUG_SECTION_APPLE(name) \ + declaration \ + _GENERATE_DEBUG_SECTION_LINUX(name) + +#if defined(MS_WINDOWS) +#define _GENERATE_DEBUG_SECTION_WINDOWS(name) \ + _Pragma(Py_STRINGIFY(section(Py_STRINGIFY(name), read, write))) \ + __declspec(allocate(Py_STRINGIFY(name))) +#else +#define _GENERATE_DEBUG_SECTION_WINDOWS(name) +#endif + +#if defined(__APPLE__) +#define _GENERATE_DEBUG_SECTION_APPLE(name) \ + __attribute__((section(SEG_DATA "," Py_STRINGIFY(name)))) +#else +#define _GENERATE_DEBUG_SECTION_APPLE(name) +#endif + +#if defined(__linux__) && (defined(__GNUC__) || defined(__clang__)) +#define _GENERATE_DEBUG_SECTION_LINUX(name) \ + __attribute__((section("." Py_STRINGIFY(name)))) +#else +#define _GENERATE_DEBUG_SECTION_LINUX(name) +#endif + + #define _Py_Debug_Cookie "xdebugpy" #ifdef Py_GIL_DISABLED diff --git a/Python/pylifecycle.c b/Python/pylifecycle.c index ebeee4f41d795d..80bae87f9aa4d3 100644 --- a/Python/pylifecycle.c +++ b/Python/pylifecycle.c @@ -6,6 +6,7 @@ #include "pycore_ceval.h" // _PyEval_FiniGIL() #include "pycore_codecs.h" // _PyCodec_Lookup() #include "pycore_context.h" // _PyContext_Init() +#include "pycore_debugger_utils.h" // GENERATE_DEBUG_SECTION #include "pycore_dict.h" // _PyDict_Fini() #include "pycore_exceptions.h" // _PyExc_InitTypes() #include "pycore_fileutils.h" // _Py_ResetForceASCII() @@ -43,10 +44,6 @@ # include // isatty() #endif -#if defined(__APPLE__) -# include -#endif - #ifdef HAVE_SIGNAL_H # include // SIG_IGN #endif @@ -86,24 +83,9 @@ static void call_ll_exitfuncs(_PyRuntimeState *runtime); /* Suppress deprecation warning for PyBytesObject.ob_shash */ _Py_COMP_DIAG_PUSH _Py_COMP_DIAG_IGNORE_DEPR_DECLS - -#if defined(MS_WINDOWS) - -#pragma section("PyRuntime", read, write) -__declspec(allocate("PyRuntime")) - -#elif defined(__APPLE__) - -__attribute__(( - section(SEG_DATA ",PyRuntime") -)) - -#endif - +GENERATE_DEBUG_SECTION(PyRuntime, _PyRuntimeState _PyRuntime -#if defined(__linux__) && (defined(__GNUC__) || defined(__clang__)) -__attribute__ ((section (".PyRuntime"))) -#endif +) = _PyRuntimeState_INIT(_PyRuntime, _Py_Debug_Cookie); _Py_COMP_DIAG_POP From c4df901f0f453ad0d392558d0b3c7a5d3ab531e8 Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Wed, 16 Oct 2024 09:07:11 -0600 Subject: [PATCH 03/11] Update a comment. --- Include/internal/pycore_runtime.h | 3 ++- Python/pylifecycle.c | 2 ++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/Include/internal/pycore_runtime.h b/Include/internal/pycore_runtime.h index 64c38c47e50916..74fc90f251dc17 100644 --- a/Include/internal/pycore_runtime.h +++ b/Include/internal/pycore_runtime.h @@ -60,7 +60,8 @@ struct _reftracer_runtime_state { /* Full Python runtime state */ /* _PyRuntimeState holds the global state for the CPython runtime. - That data is exposed in the internal API as a static variable (_PyRuntime). + That data is exported by the internal API as a global variable + (_PyRuntime, defined near the top of pylifecycle.c). */ typedef struct pyruntimestate { /* This field must be first to facilitate locating it by out of process diff --git a/Python/pylifecycle.c b/Python/pylifecycle.c index 80bae87f9aa4d3..913e08690efc8a 100644 --- a/Python/pylifecycle.c +++ b/Python/pylifecycle.c @@ -75,6 +75,7 @@ static void wait_for_thread_shutdown(PyThreadState *tstate); static void finalize_subinterpreters(void); static void call_ll_exitfuncs(_PyRuntimeState *runtime); + /* The following places the `_PyRuntime` structure in a location that can be * found without any external information. This is meant to ease access to the * interpreter state for various runtime debugging tools, but is *not* an @@ -89,6 +90,7 @@ _PyRuntimeState _PyRuntime = _PyRuntimeState_INIT(_PyRuntime, _Py_Debug_Cookie); _Py_COMP_DIAG_POP + static int runtime_initialized = 0; PyStatus From 0766de700851999ef71d46fb351270642d21b236 Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Wed, 16 Oct 2024 09:17:10 -0600 Subject: [PATCH 04/11] Inline struct _gilstate_runtime_state. --- Include/internal/pycore_runtime.h | 22 ++++++++++------------ 1 file changed, 10 insertions(+), 12 deletions(-) diff --git a/Include/internal/pycore_runtime.h b/Include/internal/pycore_runtime.h index 74fc90f251dc17..fda21f87ea9dce 100644 --- a/Include/internal/pycore_runtime.h +++ b/Include/internal/pycore_runtime.h @@ -32,17 +32,6 @@ struct _getargs_runtime_state { /* GIL state */ -struct _gilstate_runtime_state { - /* bpo-26558: Flag to disable PyGILState_Check(). - If set to non-zero, PyGILState_Check() always return 1. */ - int check_enabled; - /* The single PyInterpreterState used by this process' - GILState implementation - */ - /* TODO: Given interp_main, it may be possible to kill this ref */ - PyInterpreterState *autoInterpreterState; -}; - /* Runtime audit hook state */ typedef struct _Py_AuditHookEntry { @@ -157,7 +146,16 @@ typedef struct pyruntimestate { struct _import_runtime_state imports; struct _ceval_runtime_state ceval; - struct _gilstate_runtime_state gilstate; + struct _gilstate_runtime_state { + /* bpo-26558: Flag to disable PyGILState_Check(). + If set to non-zero, PyGILState_Check() always return 1. */ + int check_enabled; + /* The single PyInterpreterState used by this process' + GILState implementation + */ + /* TODO: Given interp_main, it may be possible to kill this ref */ + PyInterpreterState *autoInterpreterState; + } gilstate; struct _getargs_runtime_state getargs; struct _fileutils_state fileutils; struct _faulthandler_runtime_state faulthandler; From 35b9aeb16e16a983f0b68e7fa8c1fd1288291bbc Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Wed, 16 Oct 2024 09:22:41 -0600 Subject: [PATCH 05/11] Move struct _reftracer_runtime_state to pycore_object_state.h. --- Include/internal/pycore_object_state.h | 8 ++++++++ Include/internal/pycore_runtime.h | 8 -------- Objects/object.c | 1 + 3 files changed, 9 insertions(+), 8 deletions(-) diff --git a/Include/internal/pycore_object_state.h b/Include/internal/pycore_object_state.h index e7fa7c1f10d6d1..8a47a6d9e6eb0d 100644 --- a/Include/internal/pycore_object_state.h +++ b/Include/internal/pycore_object_state.h @@ -11,6 +11,14 @@ extern "C" { #include "pycore_freelist_state.h" // _Py_freelists #include "pycore_hashtable.h" // _Py_hashtable_t + +/* Reference tracer state */ +struct _reftracer_runtime_state { + PyRefTracer tracer_func; + void* tracer_data; +}; + + struct _py_object_runtime_state { #ifdef Py_REF_DEBUG Py_ssize_t interpreter_leaks; diff --git a/Include/internal/pycore_runtime.h b/Include/internal/pycore_runtime.h index fda21f87ea9dce..f8c1207e9cd7e6 100644 --- a/Include/internal/pycore_runtime.h +++ b/Include/internal/pycore_runtime.h @@ -30,8 +30,6 @@ struct _getargs_runtime_state { struct _PyArg_Parser *static_parsers; }; -/* GIL state */ - /* Runtime audit hook state */ typedef struct _Py_AuditHookEntry { @@ -40,12 +38,6 @@ typedef struct _Py_AuditHookEntry { void *userData; } _Py_AuditHookEntry; -/* Reference tracer state */ -struct _reftracer_runtime_state { - PyRefTracer tracer_func; - void* tracer_data; -}; - /* Full Python runtime state */ /* _PyRuntimeState holds the global state for the CPython runtime. diff --git a/Objects/object.c b/Objects/object.c index 4a4c5bf7d7f08a..1a15b70d3dc63f 100644 --- a/Objects/object.c +++ b/Objects/object.c @@ -17,6 +17,7 @@ #include "pycore_memoryobject.h" // _PyManagedBuffer_Type #include "pycore_namespace.h" // _PyNamespace_Type #include "pycore_object.h" // PyAPI_DATA() _Py_SwappedOp definition +#include "pycore_object_state.h" // struct _reftracer_runtime_state #include "pycore_long.h" // _PyLong_GetZero() #include "pycore_optimizer.h" // _PyUOpExecutor_Type, _PyUOpOptimizer_Type, ... #include "pycore_pyerrors.h" // _PyErr_Occurred() From 943b3dab095c2b69bcf1e31fe83cc9f5dfe0bb6b Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Wed, 16 Oct 2024 09:24:31 -0600 Subject: [PATCH 06/11] Inline struct _getargs_runtime_state. --- Include/internal/pycore_runtime.h | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/Include/internal/pycore_runtime.h b/Include/internal/pycore_runtime.h index f8c1207e9cd7e6..5d9afd5539d069 100644 --- a/Include/internal/pycore_runtime.h +++ b/Include/internal/pycore_runtime.h @@ -26,9 +26,6 @@ extern "C" { #include "pycore_typeobject.h" // struct _types_runtime_state #include "pycore_unicodeobject.h" // struct _Py_unicode_runtime_state -struct _getargs_runtime_state { - struct _PyArg_Parser *static_parsers; -}; /* Runtime audit hook state */ @@ -148,7 +145,9 @@ typedef struct pyruntimestate { /* TODO: Given interp_main, it may be possible to kill this ref */ PyInterpreterState *autoInterpreterState; } gilstate; - struct _getargs_runtime_state getargs; + struct _getargs_runtime_state { + struct _PyArg_Parser *static_parsers; + } getargs; struct _fileutils_state fileutils; struct _faulthandler_runtime_state faulthandler; struct _tracemalloc_runtime_state tracemalloc; @@ -252,6 +251,7 @@ _PyRuntimeState_SetFinalizing(_PyRuntimeState *runtime, PyThreadState *tstate) { } } + #ifdef __cplusplus } #endif From 35c2ab4f879cd978a48b4824f2f4c4ada1211029 Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Wed, 16 Oct 2024 09:42:46 -0600 Subject: [PATCH 07/11] Move the audit API to its own header files. --- Include/Python.h | 1 + Include/audit.h | 30 +++++++++++++++++++++++++ Include/cpython/audit.h | 8 +++++++ Include/cpython/sysmodule.h | 4 ---- Include/internal/pycore_audit.h | 35 +++++++++++++++++++++++++++++ Include/internal/pycore_runtime.h | 9 +------- Include/internal/pycore_sysmodule.h | 10 --------- Include/sysmodule.h | 11 --------- Makefile.pre.in | 3 +++ PCbuild/pythoncore.vcxproj | 3 +++ PCbuild/pythoncore.vcxproj.filters | 9 ++++++++ Python/bytecodes.c | 2 +- Python/ceval.c | 2 +- Python/errors.c | 3 ++- Python/import.c | 3 ++- Python/legacy_tracing.c | 2 +- Python/pylifecycle.c | 3 ++- Python/pystate.c | 2 +- Python/pythonrun.c | 3 ++- Python/sysmodule.c | 1 + 20 files changed, 103 insertions(+), 41 deletions(-) create mode 100644 Include/audit.h create mode 100644 Include/cpython/audit.h create mode 100644 Include/internal/pycore_audit.h diff --git a/Include/Python.h b/Include/Python.h index e1abdd16f031fb..717e27feab62db 100644 --- a/Include/Python.h +++ b/Include/Python.h @@ -124,6 +124,7 @@ #include "pylifecycle.h" #include "ceval.h" #include "sysmodule.h" +#include "audit.h" #include "osmodule.h" #include "intrcheck.h" #include "import.h" diff --git a/Include/audit.h b/Include/audit.h new file mode 100644 index 00000000000000..793b7077e1027b --- /dev/null +++ b/Include/audit.h @@ -0,0 +1,30 @@ +#ifndef Py_AUDIT_H +#define Py_AUDIT_H +#ifdef __cplusplus +extern "C" { +#endif + + +#if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 >= 0x030d0000 +PyAPI_FUNC(int) PySys_Audit( + const char *event, + const char *argFormat, + ...); + +PyAPI_FUNC(int) PySys_AuditTuple( + const char *event, + PyObject *args); +#endif + + +#ifndef Py_LIMITED_API +# define Py_CPYTHON_AUDIT_H +# include "cpython/audit.h" +# undef Py_CPYTHON_AUDIT_H +#endif + + +#ifdef __cplusplus +} +#endif +#endif /* !Py_AUDIT_H */ diff --git a/Include/cpython/audit.h b/Include/cpython/audit.h new file mode 100644 index 00000000000000..3c5c7a8c06091d --- /dev/null +++ b/Include/cpython/audit.h @@ -0,0 +1,8 @@ +#ifndef Py_CPYTHON_AUDIT_H +# error "this header file must not be included directly" +#endif + + +typedef int(*Py_AuditHookFunction)(const char *, PyObject *, void *); + +PyAPI_FUNC(int) PySys_AddAuditHook(Py_AuditHookFunction, void*); diff --git a/Include/cpython/sysmodule.h b/Include/cpython/sysmodule.h index a3ac07f538a94f..a4685768b9d181 100644 --- a/Include/cpython/sysmodule.h +++ b/Include/cpython/sysmodule.h @@ -2,10 +2,6 @@ # error "this header file must not be included directly" #endif -typedef int(*Py_AuditHookFunction)(const char *, PyObject *, void *); - -PyAPI_FUNC(int) PySys_AddAuditHook(Py_AuditHookFunction, void*); - typedef struct { FILE* perf_map; PyThread_type_lock map_lock; diff --git a/Include/internal/pycore_audit.h b/Include/internal/pycore_audit.h new file mode 100644 index 00000000000000..2811aaa6236123 --- /dev/null +++ b/Include/internal/pycore_audit.h @@ -0,0 +1,35 @@ +#ifndef Py_INTERNAL_AUDIT_H +#define Py_INTERNAL_AUDIT_H +#ifdef __cplusplus +extern "C" { +#endif + +#ifndef Py_BUILD_CORE +# error "this header requires Py_BUILD_CORE define" +#endif + + +/* Runtime audit hook state */ + +typedef struct _Py_AuditHookEntry { + struct _Py_AuditHookEntry *next; + Py_AuditHookFunction hookCFunction; + void *userData; +} _Py_AuditHookEntry; + + +extern int _PySys_Audit( + PyThreadState *tstate, + const char *event, + const char *argFormat, + ...); + +// _PySys_ClearAuditHooks() must not be exported: use extern rather than +// PyAPI_FUNC(). We want minimal exposure of this function. +extern void _PySys_ClearAuditHooks(PyThreadState *tstate); + + +#ifdef __cplusplus +} +#endif +#endif /* !Py_INTERNAL_AUDIT_H */ diff --git a/Include/internal/pycore_runtime.h b/Include/internal/pycore_runtime.h index 5d9afd5539d069..b00ca39a05fa4d 100644 --- a/Include/internal/pycore_runtime.h +++ b/Include/internal/pycore_runtime.h @@ -9,6 +9,7 @@ extern "C" { #endif #include "pycore_atexit.h" // struct _atexit_runtime_state +#include "pycore_audit.h" // _Py_AuditHookEntry #include "pycore_ceval_state.h" // struct _ceval_runtime_state #include "pycore_crossinterp.h" // struct _xidregistry #include "pycore_debugger_utils.h" // _Py_DebugOffsets @@ -27,14 +28,6 @@ extern "C" { #include "pycore_unicodeobject.h" // struct _Py_unicode_runtime_state -/* Runtime audit hook state */ - -typedef struct _Py_AuditHookEntry { - struct _Py_AuditHookEntry *next; - Py_AuditHookFunction hookCFunction; - void *userData; -} _Py_AuditHookEntry; - /* Full Python runtime state */ /* _PyRuntimeState holds the global state for the CPython runtime. diff --git a/Include/internal/pycore_sysmodule.h b/Include/internal/pycore_sysmodule.h index a1d795e284f6ac..99968df54a45f6 100644 --- a/Include/internal/pycore_sysmodule.h +++ b/Include/internal/pycore_sysmodule.h @@ -14,16 +14,6 @@ PyAPI_FUNC(PyObject*) _PySys_GetAttr(PyThreadState *tstate, PyObject *name); // Export for '_pickle' shared extension PyAPI_FUNC(size_t) _PySys_GetSizeOf(PyObject *); -extern int _PySys_Audit( - PyThreadState *tstate, - const char *event, - const char *argFormat, - ...); - -// _PySys_ClearAuditHooks() must not be exported: use extern rather than -// PyAPI_FUNC(). We want minimal exposure of this function. -extern void _PySys_ClearAuditHooks(PyThreadState *tstate); - extern int _PySys_SetAttr(PyObject *, PyObject *); extern int _PySys_ClearAttrString(PyInterpreterState *interp, diff --git a/Include/sysmodule.h b/Include/sysmodule.h index 5a0af2e1578eb7..f839a66e70fd9d 100644 --- a/Include/sysmodule.h +++ b/Include/sysmodule.h @@ -21,17 +21,6 @@ Py_DEPRECATED(3.13) PyAPI_FUNC(void) PySys_ResetWarnOptions(void); PyAPI_FUNC(PyObject *) PySys_GetXOptions(void); -#if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 >= 0x030d0000 -PyAPI_FUNC(int) PySys_Audit( - const char *event, - const char *argFormat, - ...); - -PyAPI_FUNC(int) PySys_AuditTuple( - const char *event, - PyObject *args); -#endif - #ifndef Py_LIMITED_API # define Py_CPYTHON_SYSMODULE_H # include "cpython/sysmodule.h" diff --git a/Makefile.pre.in b/Makefile.pre.in index 9d8abf390545bd..6f611c941445b9 100644 --- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -1032,6 +1032,7 @@ python.worker.js: $(srcdir)/Tools/wasm/python.worker.js PYTHON_HEADERS= \ $(srcdir)/Include/Python.h \ $(srcdir)/Include/abstract.h \ + $(srcdir)/Include/audit.h \ $(srcdir)/Include/bltinmodule.h \ $(srcdir)/Include/boolobject.h \ $(srcdir)/Include/bytearrayobject.h \ @@ -1110,6 +1111,7 @@ PYTHON_HEADERS= \ $(PARSER_HEADERS) \ \ $(srcdir)/Include/cpython/abstract.h \ + $(srcdir)/Include/cpython/audit.h \ $(srcdir)/Include/cpython/bytearrayobject.h \ $(srcdir)/Include/cpython/bytesobject.h \ $(srcdir)/Include/cpython/cellobject.h \ @@ -1174,6 +1176,7 @@ PYTHON_HEADERS= \ $(srcdir)/Include/internal/pycore_ast.h \ $(srcdir)/Include/internal/pycore_ast_state.h \ $(srcdir)/Include/internal/pycore_atexit.h \ + $(srcdir)/Include/internal/pycore_audit.h \ $(srcdir)/Include/internal/pycore_backoff.h \ $(srcdir)/Include/internal/pycore_bitutils.h \ $(srcdir)/Include/internal/pycore_blocks_output_buffer.h \ diff --git a/PCbuild/pythoncore.vcxproj b/PCbuild/pythoncore.vcxproj index 192079d6c37083..1f91c951227063 100644 --- a/PCbuild/pythoncore.vcxproj +++ b/PCbuild/pythoncore.vcxproj @@ -128,6 +128,7 @@ + @@ -137,6 +138,7 @@ + @@ -208,6 +210,7 @@ + diff --git a/PCbuild/pythoncore.vcxproj.filters b/PCbuild/pythoncore.vcxproj.filters index 2865fcae870cae..044f8ffffdc77f 100644 --- a/PCbuild/pythoncore.vcxproj.filters +++ b/PCbuild/pythoncore.vcxproj.filters @@ -48,6 +48,9 @@ Include + + Include + Include @@ -354,6 +357,9 @@ Include\cpython + + Include\cpython + Include\cpython @@ -552,6 +558,9 @@ Include\internal + + Include\internal + Include\internal diff --git a/Python/bytecodes.c b/Python/bytecodes.c index b22916aeaa248b..b2482b6a0754c9 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -8,6 +8,7 @@ #include "Python.h" #include "pycore_abstract.h" // _PyIndex_Check() +#include "pycore_audit.h" // _PySys_Audit() #include "pycore_backoff.h" #include "pycore_cell.h" // PyCell_GetRef() #include "pycore_ceval.h" @@ -27,7 +28,6 @@ #include "pycore_range.h" // _PyRangeIterObject #include "pycore_setobject.h" // _PySet_NextEntry() #include "pycore_sliceobject.h" // _PyBuildSlice_ConsumeRefs -#include "pycore_sysmodule.h" // _PySys_Audit() #include "pycore_tuple.h" // _PyTuple_ITEMS() #include "pycore_typeobject.h" // _PySuper_Lookup() diff --git a/Python/ceval.c b/Python/ceval.c index f4e0add3034707..65dd6113d6e255 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -4,6 +4,7 @@ #include "Python.h" #include "pycore_abstract.h" // _PyIndex_Check() +#include "pycore_audit.h" // _PySys_Audit() #include "pycore_backoff.h" #include "pycore_call.h" // _PyObject_CallNoArgs() #include "pycore_cell.h" // PyCell_GetRef() @@ -26,7 +27,6 @@ #include "pycore_range.h" // _PyRangeIterObject #include "pycore_setobject.h" // _PySet_Update() #include "pycore_sliceobject.h" // _PyBuildSlice_ConsumeRefs -#include "pycore_sysmodule.h" // _PySys_Audit() #include "pycore_tuple.h" // _PyTuple_ITEMS() #include "pycore_typeobject.h" // _PySuper_Lookup() #include "pycore_uop_ids.h" // Uops diff --git a/Python/errors.c b/Python/errors.c index 9e2a3ce062a6fe..7f3b4aabc432d7 100644 --- a/Python/errors.c +++ b/Python/errors.c @@ -2,12 +2,13 @@ /* Error handling */ #include "Python.h" +#include "pycore_audit.h" // _PySys_Audit() #include "pycore_call.h" // _PyObject_CallNoArgs() #include "pycore_initconfig.h" // _PyStatus_ERR() #include "pycore_pyerrors.h" // _PyErr_Format() #include "pycore_pystate.h" // _PyThreadState_GET() #include "pycore_structseq.h" // _PyStructSequence_FiniBuiltin() -#include "pycore_sysmodule.h" // _PySys_Audit() +#include "pycore_sysmodule.h" // _PySys_GetAttr() #include "pycore_traceback.h" // _PyTraceBack_FromFrame() #ifdef MS_WINDOWS diff --git a/Python/import.c b/Python/import.c index acf849f14562b9..d8ad37b2422795 100644 --- a/Python/import.c +++ b/Python/import.c @@ -1,6 +1,7 @@ /* Module definition and import implementation */ #include "Python.h" +#include "pycore_audit.h" // _PySys_Audit() #include "pycore_ceval.h" #include "pycore_hashtable.h" // _Py_hashtable_new_full() #include "pycore_import.h" // _PyImport_BootstrapImp() @@ -14,7 +15,7 @@ #include "pycore_pylifecycle.h" #include "pycore_pymem.h" // _PyMem_SetDefaultAllocator() #include "pycore_pystate.h" // _PyInterpreterState_GET() -#include "pycore_sysmodule.h" // _PySys_Audit() +#include "pycore_sysmodule.h" // _PySys_ClearAttrString() #include "pycore_time.h" // _PyTime_AsMicroseconds() #include "pycore_weakref.h" // _PyWeakref_GET_REF() diff --git a/Python/legacy_tracing.c b/Python/legacy_tracing.c index 1436921a19b768..45af275f1f6dce 100644 --- a/Python/legacy_tracing.c +++ b/Python/legacy_tracing.c @@ -3,9 +3,9 @@ */ #include "Python.h" +#include "pycore_audit.h" // _PySys_Audit() #include "pycore_ceval.h" // export _PyEval_SetProfile() #include "pycore_object.h" -#include "pycore_sysmodule.h" // _PySys_Audit() #include "opcode.h" #include diff --git a/Python/pylifecycle.c b/Python/pylifecycle.c index 913e08690efc8a..91dfb8d4ec2f86 100644 --- a/Python/pylifecycle.c +++ b/Python/pylifecycle.c @@ -2,6 +2,7 @@ #include "Python.h" +#include "pycore_audit.h" // _PySys_ClearAuditHooks() #include "pycore_call.h" // _PyObject_CallMethod() #include "pycore_ceval.h" // _PyEval_FiniGIL() #include "pycore_codecs.h" // _PyCodec_Lookup() @@ -27,7 +28,7 @@ #include "pycore_runtime_init.h" // _PyRuntimeState_INIT #include "pycore_setobject.h" // _PySet_NextEntry() #include "pycore_sliceobject.h" // _PySlice_Fini() -#include "pycore_sysmodule.h" // _PySys_ClearAuditHooks() +#include "pycore_sysmodule.h" // _PySys_GetAttr() #include "pycore_traceback.h" // _Py_DumpTracebackThreads() #include "pycore_uniqueid.h" // _PyType_FinalizeIdPool() #include "pycore_typeobject.h" // _PyTypes_InitTypes() diff --git a/Python/pystate.c b/Python/pystate.c index 5d94b7714bd607..43dee172308a06 100644 --- a/Python/pystate.c +++ b/Python/pystate.c @@ -3,6 +3,7 @@ #include "Python.h" #include "pycore_abstract.h" // _PyIndex_Check() +#include "pycore_audit.h" // _Py_AuditHookEntry #include "pycore_ceval.h" #include "pycore_code.h" // stats #include "pycore_critical_section.h" // _PyCriticalSection_Resume() @@ -18,7 +19,6 @@ #include "pycore_pymem.h" // _PyMem_SetDefaultAllocator() #include "pycore_pystate.h" #include "pycore_runtime_init.h" // _PyRuntimeState_INIT -#include "pycore_sysmodule.h" // _PySys_Audit() #include "pycore_obmalloc.h" // _PyMem_obmalloc_state_on_heap() #include "pycore_uniqueid.h" // _PyType_FinalizeThreadLocalRefcounts() diff --git a/Python/pythonrun.c b/Python/pythonrun.c index b67597113ead45..fc0f11bc4e8af4 100644 --- a/Python/pythonrun.c +++ b/Python/pythonrun.c @@ -13,6 +13,7 @@ #include "Python.h" #include "pycore_ast.h" // PyAST_mod2obj() +#include "pycore_audit.h" // _PySys_Audit() #include "pycore_ceval.h" // _Py_EnterRecursiveCall() #include "pycore_compile.h" // _PyAST_Compile() #include "pycore_interp.h" // PyInterpreterState.importlib @@ -22,7 +23,7 @@ #include "pycore_pylifecycle.h" // _Py_FdIsInteractive() #include "pycore_pystate.h" // _PyInterpreterState_GET() #include "pycore_pythonrun.h" // export _PyRun_InteractiveLoopObject() -#include "pycore_sysmodule.h" // _PySys_Audit() +#include "pycore_sysmodule.h" // _PySys_GetAttr() #include "pycore_traceback.h" // _PyTraceBack_Print() #include "errcode.h" // E_EOF diff --git a/Python/sysmodule.c b/Python/sysmodule.c index ac343a8048e008..8b9209324002ce 100644 --- a/Python/sysmodule.c +++ b/Python/sysmodule.c @@ -15,6 +15,7 @@ Data members: */ #include "Python.h" +#include "pycore_audit.h" // _Py_AuditHookEntry #include "pycore_call.h" // _PyObject_CallNoArgs() #include "pycore_ceval.h" // _PyEval_SetAsyncGenFinalizer() #include "pycore_dict.h" // _PyDict_GetItemWithError() From a1b0897791f62ffed720ff2178cf064dee3d4f54 Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Wed, 16 Oct 2024 10:19:46 -0600 Subject: [PATCH 08/11] Move the perf trampoline API to the right header file. --- Include/cpython/ceval.h | 18 ++++++++++++++++++ Include/cpython/sysmodule.h | 18 ------------------ Include/sysmodule.h | 6 ------ Makefile.pre.in | 1 - PCbuild/pythoncore.vcxproj | 1 - PCbuild/pythoncore.vcxproj.filters | 3 --- 6 files changed, 18 insertions(+), 29 deletions(-) delete mode 100644 Include/cpython/sysmodule.h diff --git a/Include/cpython/ceval.h b/Include/cpython/ceval.h index 78f7405661662f..ca8109e3248a8d 100644 --- a/Include/cpython/ceval.h +++ b/Include/cpython/ceval.h @@ -23,3 +23,21 @@ _PyEval_RequestCodeExtraIndex(freefunc f) { PyAPI_FUNC(int) _PyEval_SliceIndex(PyObject *, Py_ssize_t *); PyAPI_FUNC(int) _PyEval_SliceIndexNotNone(PyObject *, Py_ssize_t *); + + +// Trampoline API + +typedef struct { + FILE* perf_map; + PyThread_type_lock map_lock; +} PerfMapState; + +PyAPI_FUNC(int) PyUnstable_PerfMapState_Init(void); +PyAPI_FUNC(int) PyUnstable_WritePerfMapEntry( + const void *code_addr, + unsigned int code_size, + const char *entry_name); +PyAPI_FUNC(void) PyUnstable_PerfMapState_Fini(void); +PyAPI_FUNC(int) PyUnstable_CopyPerfMapFile(const char* parent_filename); +PyAPI_FUNC(int) PyUnstable_PerfTrampoline_CompileCode(PyCodeObject *); +PyAPI_FUNC(int) PyUnstable_PerfTrampoline_SetPersistAfterFork(int enable); diff --git a/Include/cpython/sysmodule.h b/Include/cpython/sysmodule.h deleted file mode 100644 index a4685768b9d181..00000000000000 --- a/Include/cpython/sysmodule.h +++ /dev/null @@ -1,18 +0,0 @@ -#ifndef Py_CPYTHON_SYSMODULE_H -# error "this header file must not be included directly" -#endif - -typedef struct { - FILE* perf_map; - PyThread_type_lock map_lock; -} PerfMapState; - -PyAPI_FUNC(int) PyUnstable_PerfMapState_Init(void); -PyAPI_FUNC(int) PyUnstable_WritePerfMapEntry( - const void *code_addr, - unsigned int code_size, - const char *entry_name); -PyAPI_FUNC(void) PyUnstable_PerfMapState_Fini(void); -PyAPI_FUNC(int) PyUnstable_CopyPerfMapFile(const char* parent_filename); -PyAPI_FUNC(int) PyUnstable_PerfTrampoline_CompileCode(PyCodeObject *); -PyAPI_FUNC(int) PyUnstable_PerfTrampoline_SetPersistAfterFork(int enable); diff --git a/Include/sysmodule.h b/Include/sysmodule.h index f839a66e70fd9d..c1d5f610fe08a5 100644 --- a/Include/sysmodule.h +++ b/Include/sysmodule.h @@ -21,12 +21,6 @@ Py_DEPRECATED(3.13) PyAPI_FUNC(void) PySys_ResetWarnOptions(void); PyAPI_FUNC(PyObject *) PySys_GetXOptions(void); -#ifndef Py_LIMITED_API -# define Py_CPYTHON_SYSMODULE_H -# include "cpython/sysmodule.h" -# undef Py_CPYTHON_SYSMODULE_H -#endif - #ifdef __cplusplus } #endif diff --git a/Makefile.pre.in b/Makefile.pre.in index 6f611c941445b9..d044e81e30495a 100644 --- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -1161,7 +1161,6 @@ PYTHON_HEADERS= \ $(srcdir)/Include/cpython/pythonrun.h \ $(srcdir)/Include/cpython/pythread.h \ $(srcdir)/Include/cpython/setobject.h \ - $(srcdir)/Include/cpython/sysmodule.h \ $(srcdir)/Include/cpython/traceback.h \ $(srcdir)/Include/cpython/tracemalloc.h \ $(srcdir)/Include/cpython/tupleobject.h \ diff --git a/PCbuild/pythoncore.vcxproj b/PCbuild/pythoncore.vcxproj index 1f91c951227063..ad2d4a581189d1 100644 --- a/PCbuild/pythoncore.vcxproj +++ b/PCbuild/pythoncore.vcxproj @@ -187,7 +187,6 @@ - diff --git a/PCbuild/pythoncore.vcxproj.filters b/PCbuild/pythoncore.vcxproj.filters index 044f8ffffdc77f..739dda7528c53a 100644 --- a/PCbuild/pythoncore.vcxproj.filters +++ b/PCbuild/pythoncore.vcxproj.filters @@ -519,9 +519,6 @@ Include\cpython - - Include\cpython - Include\cpython From d4373a20a295347ab2aafaf0c1a333cb00a1e193 Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Wed, 16 Oct 2024 10:54:59 -0600 Subject: [PATCH 09/11] Revert "Factor out GENERATE_DEBUG_SECTION()." This reverts commit 5fc5b7d9f985da85c67f1f19952f7816cfaa4dcc. --- Include/internal/pycore_debugger_utils.h | 37 ------------------------ Python/pylifecycle.c | 24 +++++++++++++-- 2 files changed, 21 insertions(+), 40 deletions(-) diff --git a/Include/internal/pycore_debugger_utils.h b/Include/internal/pycore_debugger_utils.h index bdfc0550fc839f..c5c07cbc0ac22d 100644 --- a/Include/internal/pycore_debugger_utils.h +++ b/Include/internal/pycore_debugger_utils.h @@ -9,43 +9,6 @@ extern "C" { #endif -#if defined(__APPLE__) -# include -#endif - - -// Macros to burn global values in custom sections so out-of-process -// profilers can locate them easily - -#define GENERATE_DEBUG_SECTION(name, declaration) \ - _GENERATE_DEBUG_SECTION_WINDOWS(name) \ - _GENERATE_DEBUG_SECTION_APPLE(name) \ - declaration \ - _GENERATE_DEBUG_SECTION_LINUX(name) - -#if defined(MS_WINDOWS) -#define _GENERATE_DEBUG_SECTION_WINDOWS(name) \ - _Pragma(Py_STRINGIFY(section(Py_STRINGIFY(name), read, write))) \ - __declspec(allocate(Py_STRINGIFY(name))) -#else -#define _GENERATE_DEBUG_SECTION_WINDOWS(name) -#endif - -#if defined(__APPLE__) -#define _GENERATE_DEBUG_SECTION_APPLE(name) \ - __attribute__((section(SEG_DATA "," Py_STRINGIFY(name)))) -#else -#define _GENERATE_DEBUG_SECTION_APPLE(name) -#endif - -#if defined(__linux__) && (defined(__GNUC__) || defined(__clang__)) -#define _GENERATE_DEBUG_SECTION_LINUX(name) \ - __attribute__((section("." Py_STRINGIFY(name)))) -#else -#define _GENERATE_DEBUG_SECTION_LINUX(name) -#endif - - #define _Py_Debug_Cookie "xdebugpy" #ifdef Py_GIL_DISABLED diff --git a/Python/pylifecycle.c b/Python/pylifecycle.c index 91dfb8d4ec2f86..6d7a57695c0bdf 100644 --- a/Python/pylifecycle.c +++ b/Python/pylifecycle.c @@ -7,7 +7,6 @@ #include "pycore_ceval.h" // _PyEval_FiniGIL() #include "pycore_codecs.h" // _PyCodec_Lookup() #include "pycore_context.h" // _PyContext_Init() -#include "pycore_debugger_utils.h" // GENERATE_DEBUG_SECTION #include "pycore_dict.h" // _PyDict_Fini() #include "pycore_exceptions.h" // _PyExc_InitTypes() #include "pycore_fileutils.h" // _Py_ResetForceASCII() @@ -45,6 +44,10 @@ # include // isatty() #endif +#if defined(__APPLE__) +# include +#endif + #ifdef HAVE_SIGNAL_H # include // SIG_IGN #endif @@ -85,9 +88,24 @@ static void call_ll_exitfuncs(_PyRuntimeState *runtime); /* Suppress deprecation warning for PyBytesObject.ob_shash */ _Py_COMP_DIAG_PUSH _Py_COMP_DIAG_IGNORE_DEPR_DECLS -GENERATE_DEBUG_SECTION(PyRuntime, + +#if defined(MS_WINDOWS) + +#pragma section("PyRuntime", read, write) +__declspec(allocate("PyRuntime")) + +#elif defined(__APPLE__) + +__attribute__(( + section(SEG_DATA ",PyRuntime") +)) + +#endif + _PyRuntimeState _PyRuntime -) +#if defined(__linux__) && (defined(__GNUC__) || defined(__clang__)) +__attribute__ ((section (".PyRuntime"))) +#endif = _PyRuntimeState_INIT(_PyRuntime, _Py_Debug_Cookie); _Py_COMP_DIAG_POP From 99e7e069dd0619ff26201ef003aef9081fca7ede Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Wed, 16 Oct 2024 11:35:44 -0600 Subject: [PATCH 10/11] Factor out _Py_DebugOffsets_INIT(). --- Include/internal/pycore_debugger_utils.h | 107 ++++++++++++++++++++++ Include/internal/pycore_runtime_init.h | 108 +---------------------- 2 files changed, 109 insertions(+), 106 deletions(-) diff --git a/Include/internal/pycore_debugger_utils.h b/Include/internal/pycore_debugger_utils.h index c5c07cbc0ac22d..b87b3975a621ec 100644 --- a/Include/internal/pycore_debugger_utils.h +++ b/Include/internal/pycore_debugger_utils.h @@ -156,6 +156,113 @@ typedef struct _Py_DebugOffsets { } _Py_DebugOffsets; +#define _Py_DebugOffsets_INIT(debug_cookie) { \ + .cookie = debug_cookie, \ + .version = PY_VERSION_HEX, \ + .free_threaded = _Py_Debug_Free_Threaded, \ + .runtime_state = { \ + .size = sizeof(_PyRuntimeState), \ + .finalizing = offsetof(_PyRuntimeState, _finalizing), \ + .interpreters_head = offsetof(_PyRuntimeState, interpreters.head), \ + }, \ + .interpreter_state = { \ + .size = sizeof(PyInterpreterState), \ + .id = offsetof(PyInterpreterState, id), \ + .next = offsetof(PyInterpreterState, next), \ + .threads_head = offsetof(PyInterpreterState, threads.head), \ + .gc = offsetof(PyInterpreterState, gc), \ + .imports_modules = offsetof(PyInterpreterState, imports.modules), \ + .sysdict = offsetof(PyInterpreterState, sysdict), \ + .builtins = offsetof(PyInterpreterState, builtins), \ + .ceval_gil = offsetof(PyInterpreterState, ceval.gil), \ + .gil_runtime_state = offsetof(PyInterpreterState, _gil), \ + .gil_runtime_state_enabled = _Py_Debug_gilruntimestate_enabled, \ + .gil_runtime_state_locked = offsetof(PyInterpreterState, _gil.locked), \ + .gil_runtime_state_holder = offsetof(PyInterpreterState, _gil.last_holder), \ + }, \ + .thread_state = { \ + .size = sizeof(PyThreadState), \ + .prev = offsetof(PyThreadState, prev), \ + .next = offsetof(PyThreadState, next), \ + .interp = offsetof(PyThreadState, interp), \ + .current_frame = offsetof(PyThreadState, current_frame), \ + .thread_id = offsetof(PyThreadState, thread_id), \ + .native_thread_id = offsetof(PyThreadState, native_thread_id), \ + .datastack_chunk = offsetof(PyThreadState, datastack_chunk), \ + .status = offsetof(PyThreadState, _status), \ + }, \ + .interpreter_frame = { \ + .size = sizeof(_PyInterpreterFrame), \ + .previous = offsetof(_PyInterpreterFrame, previous), \ + .executable = offsetof(_PyInterpreterFrame, f_executable), \ + .instr_ptr = offsetof(_PyInterpreterFrame, instr_ptr), \ + .localsplus = offsetof(_PyInterpreterFrame, localsplus), \ + .owner = offsetof(_PyInterpreterFrame, owner), \ + }, \ + .code_object = { \ + .size = sizeof(PyCodeObject), \ + .filename = offsetof(PyCodeObject, co_filename), \ + .name = offsetof(PyCodeObject, co_name), \ + .qualname = offsetof(PyCodeObject, co_qualname), \ + .linetable = offsetof(PyCodeObject, co_linetable), \ + .firstlineno = offsetof(PyCodeObject, co_firstlineno), \ + .argcount = offsetof(PyCodeObject, co_argcount), \ + .localsplusnames = offsetof(PyCodeObject, co_localsplusnames), \ + .localspluskinds = offsetof(PyCodeObject, co_localspluskinds), \ + .co_code_adaptive = offsetof(PyCodeObject, co_code_adaptive), \ + }, \ + .pyobject = { \ + .size = sizeof(PyObject), \ + .ob_type = offsetof(PyObject, ob_type), \ + }, \ + .type_object = { \ + .size = sizeof(PyTypeObject), \ + .tp_name = offsetof(PyTypeObject, tp_name), \ + .tp_repr = offsetof(PyTypeObject, tp_repr), \ + .tp_flags = offsetof(PyTypeObject, tp_flags), \ + }, \ + .tuple_object = { \ + .size = sizeof(PyTupleObject), \ + .ob_item = offsetof(PyTupleObject, ob_item), \ + .ob_size = offsetof(PyTupleObject, ob_base.ob_size), \ + }, \ + .list_object = { \ + .size = sizeof(PyListObject), \ + .ob_item = offsetof(PyListObject, ob_item), \ + .ob_size = offsetof(PyListObject, ob_base.ob_size), \ + }, \ + .dict_object = { \ + .size = sizeof(PyDictObject), \ + .ma_keys = offsetof(PyDictObject, ma_keys), \ + .ma_values = offsetof(PyDictObject, ma_values), \ + }, \ + .float_object = { \ + .size = sizeof(PyFloatObject), \ + .ob_fval = offsetof(PyFloatObject, ob_fval), \ + }, \ + .long_object = { \ + .size = sizeof(PyLongObject), \ + .lv_tag = offsetof(PyLongObject, long_value.lv_tag), \ + .ob_digit = offsetof(PyLongObject, long_value.ob_digit), \ + }, \ + .bytes_object = { \ + .size = sizeof(PyBytesObject), \ + .ob_size = offsetof(PyBytesObject, ob_base.ob_size), \ + .ob_sval = offsetof(PyBytesObject, ob_sval), \ + }, \ + .unicode_object = { \ + .size = sizeof(PyUnicodeObject), \ + .state = offsetof(PyUnicodeObject, _base._base.state), \ + .length = offsetof(PyUnicodeObject, _base._base.length), \ + .asciiobject_size = sizeof(PyASCIIObject), \ + }, \ + .gc = { \ + .size = sizeof(struct _gc_runtime_state), \ + .collecting = offsetof(struct _gc_runtime_state, collecting), \ + }, \ +} + + #ifdef __cplusplus } #endif diff --git a/Include/internal/pycore_runtime_init.h b/Include/internal/pycore_runtime_init.h index 9dcf4bb18db708..66ab8aaa79a91b 100644 --- a/Include/internal/pycore_runtime_init.h +++ b/Include/internal/pycore_runtime_init.h @@ -9,7 +9,7 @@ extern "C" { #endif #include "pycore_ceval_state.h" // _PyEval_RUNTIME_PERF_INIT -#include "pycore_debugger_utils.h" // _Py_Debug_gilruntimestate_enabled +#include "pycore_debugger_utils.h" // _Py_DebugOffsets_INIT() #include "pycore_faulthandler.h" // _faulthandler_runtime_state_INIT #include "pycore_floatobject.h" // _py_float_format_unknown #include "pycore_function.h" @@ -33,111 +33,7 @@ extern PyTypeObject _PyExc_MemoryError; #define _PyRuntimeState_INIT(runtime, debug_cookie) \ { \ - .debug_offsets = { \ - .cookie = debug_cookie, \ - .version = PY_VERSION_HEX, \ - .free_threaded = _Py_Debug_Free_Threaded, \ - .runtime_state = { \ - .size = sizeof(_PyRuntimeState), \ - .finalizing = offsetof(_PyRuntimeState, _finalizing), \ - .interpreters_head = offsetof(_PyRuntimeState, interpreters.head), \ - }, \ - .interpreter_state = { \ - .size = sizeof(PyInterpreterState), \ - .id = offsetof(PyInterpreterState, id), \ - .next = offsetof(PyInterpreterState, next), \ - .threads_head = offsetof(PyInterpreterState, threads.head), \ - .gc = offsetof(PyInterpreterState, gc), \ - .imports_modules = offsetof(PyInterpreterState, imports.modules), \ - .sysdict = offsetof(PyInterpreterState, sysdict), \ - .builtins = offsetof(PyInterpreterState, builtins), \ - .ceval_gil = offsetof(PyInterpreterState, ceval.gil), \ - .gil_runtime_state = offsetof(PyInterpreterState, _gil), \ - .gil_runtime_state_enabled = _Py_Debug_gilruntimestate_enabled, \ - .gil_runtime_state_locked = offsetof(PyInterpreterState, _gil.locked), \ - .gil_runtime_state_holder = offsetof(PyInterpreterState, _gil.last_holder), \ - }, \ - .thread_state = { \ - .size = sizeof(PyThreadState), \ - .prev = offsetof(PyThreadState, prev), \ - .next = offsetof(PyThreadState, next), \ - .interp = offsetof(PyThreadState, interp), \ - .current_frame = offsetof(PyThreadState, current_frame), \ - .thread_id = offsetof(PyThreadState, thread_id), \ - .native_thread_id = offsetof(PyThreadState, native_thread_id), \ - .datastack_chunk = offsetof(PyThreadState, datastack_chunk), \ - .status = offsetof(PyThreadState, _status), \ - }, \ - .interpreter_frame = { \ - .size = sizeof(_PyInterpreterFrame), \ - .previous = offsetof(_PyInterpreterFrame, previous), \ - .executable = offsetof(_PyInterpreterFrame, f_executable), \ - .instr_ptr = offsetof(_PyInterpreterFrame, instr_ptr), \ - .localsplus = offsetof(_PyInterpreterFrame, localsplus), \ - .owner = offsetof(_PyInterpreterFrame, owner), \ - }, \ - .code_object = { \ - .size = sizeof(PyCodeObject), \ - .filename = offsetof(PyCodeObject, co_filename), \ - .name = offsetof(PyCodeObject, co_name), \ - .qualname = offsetof(PyCodeObject, co_qualname), \ - .linetable = offsetof(PyCodeObject, co_linetable), \ - .firstlineno = offsetof(PyCodeObject, co_firstlineno), \ - .argcount = offsetof(PyCodeObject, co_argcount), \ - .localsplusnames = offsetof(PyCodeObject, co_localsplusnames), \ - .localspluskinds = offsetof(PyCodeObject, co_localspluskinds), \ - .co_code_adaptive = offsetof(PyCodeObject, co_code_adaptive), \ - }, \ - .pyobject = { \ - .size = sizeof(PyObject), \ - .ob_type = offsetof(PyObject, ob_type), \ - }, \ - .type_object = { \ - .size = sizeof(PyTypeObject), \ - .tp_name = offsetof(PyTypeObject, tp_name), \ - .tp_repr = offsetof(PyTypeObject, tp_repr), \ - .tp_flags = offsetof(PyTypeObject, tp_flags), \ - }, \ - .tuple_object = { \ - .size = sizeof(PyTupleObject), \ - .ob_item = offsetof(PyTupleObject, ob_item), \ - .ob_size = offsetof(PyTupleObject, ob_base.ob_size), \ - }, \ - .list_object = { \ - .size = sizeof(PyListObject), \ - .ob_item = offsetof(PyListObject, ob_item), \ - .ob_size = offsetof(PyListObject, ob_base.ob_size), \ - }, \ - .dict_object = { \ - .size = sizeof(PyDictObject), \ - .ma_keys = offsetof(PyDictObject, ma_keys), \ - .ma_values = offsetof(PyDictObject, ma_values), \ - }, \ - .float_object = { \ - .size = sizeof(PyFloatObject), \ - .ob_fval = offsetof(PyFloatObject, ob_fval), \ - }, \ - .long_object = { \ - .size = sizeof(PyLongObject), \ - .lv_tag = offsetof(PyLongObject, long_value.lv_tag), \ - .ob_digit = offsetof(PyLongObject, long_value.ob_digit), \ - }, \ - .bytes_object = { \ - .size = sizeof(PyBytesObject), \ - .ob_size = offsetof(PyBytesObject, ob_base.ob_size), \ - .ob_sval = offsetof(PyBytesObject, ob_sval), \ - }, \ - .unicode_object = { \ - .size = sizeof(PyUnicodeObject), \ - .state = offsetof(PyUnicodeObject, _base._base.state), \ - .length = offsetof(PyUnicodeObject, _base._base.length), \ - .asciiobject_size = sizeof(PyASCIIObject), \ - }, \ - .gc = { \ - .size = sizeof(struct _gc_runtime_state), \ - .collecting = offsetof(struct _gc_runtime_state, collecting), \ - }, \ - }, \ + .debug_offsets = _Py_DebugOffsets_INIT(debug_cookie), \ .allocators = { \ .standard = _pymem_allocators_standard_INIT(runtime), \ .debug = _pymem_allocators_debug_INIT, \ From 438b723b9b3e02104c6214c11a4952f061cb1d42 Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Thu, 17 Oct 2024 11:41:24 -0600 Subject: [PATCH 11/11] pycore_debugger_utils.h -> pycore_debug_offsets.h --- .../{pycore_debugger_utils.h => pycore_debug_offsets.h} | 6 +++--- Include/internal/pycore_runtime.h | 2 +- Include/internal/pycore_runtime_init.h | 2 +- Makefile.pre.in | 2 +- Modules/_testexternalinspection.c | 2 +- PCbuild/pythoncore.vcxproj | 2 +- PCbuild/pythoncore.vcxproj.filters | 2 +- 7 files changed, 9 insertions(+), 9 deletions(-) rename Include/internal/{pycore_debugger_utils.h => pycore_debug_offsets.h} (98%) diff --git a/Include/internal/pycore_debugger_utils.h b/Include/internal/pycore_debug_offsets.h similarity index 98% rename from Include/internal/pycore_debugger_utils.h rename to Include/internal/pycore_debug_offsets.h index b87b3975a621ec..184f4b9360b6d3 100644 --- a/Include/internal/pycore_debugger_utils.h +++ b/Include/internal/pycore_debug_offsets.h @@ -1,5 +1,5 @@ -#ifndef Py_INTERNAL_DEBUGGER_UTILS_H -#define Py_INTERNAL_DEBUGGER_UTILS_H +#ifndef Py_INTERNAL_DEBUG_OFFSETS_H +#define Py_INTERNAL_DEBUG_OFFSETS_H #ifdef __cplusplus extern "C" { #endif @@ -266,4 +266,4 @@ typedef struct _Py_DebugOffsets { #ifdef __cplusplus } #endif -#endif /* !Py_INTERNAL_DEBUGGER_UTILS_H */ +#endif /* !Py_INTERNAL_DEBUG_OFFSETS_H */ diff --git a/Include/internal/pycore_runtime.h b/Include/internal/pycore_runtime.h index b00ca39a05fa4d..7f592aa6cf9f05 100644 --- a/Include/internal/pycore_runtime.h +++ b/Include/internal/pycore_runtime.h @@ -12,7 +12,7 @@ extern "C" { #include "pycore_audit.h" // _Py_AuditHookEntry #include "pycore_ceval_state.h" // struct _ceval_runtime_state #include "pycore_crossinterp.h" // struct _xidregistry -#include "pycore_debugger_utils.h" // _Py_DebugOffsets +#include "pycore_debug_offsets.h" // _Py_DebugOffsets #include "pycore_faulthandler.h" // struct _faulthandler_runtime_state #include "pycore_floatobject.h" // struct _Py_float_runtime_state #include "pycore_import.h" // struct _import_runtime_state diff --git a/Include/internal/pycore_runtime_init.h b/Include/internal/pycore_runtime_init.h index 66ab8aaa79a91b..e99febab2f3d57 100644 --- a/Include/internal/pycore_runtime_init.h +++ b/Include/internal/pycore_runtime_init.h @@ -9,7 +9,7 @@ extern "C" { #endif #include "pycore_ceval_state.h" // _PyEval_RUNTIME_PERF_INIT -#include "pycore_debugger_utils.h" // _Py_DebugOffsets_INIT() +#include "pycore_debug_offsets.h" // _Py_DebugOffsets_INIT() #include "pycore_faulthandler.h" // _faulthandler_runtime_state_INIT #include "pycore_floatobject.h" // _py_float_format_unknown #include "pycore_function.h" diff --git a/Makefile.pre.in b/Makefile.pre.in index d044e81e30495a..fb6f22d57397db 100644 --- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -1195,7 +1195,7 @@ PYTHON_HEADERS= \ $(srcdir)/Include/internal/pycore_context.h \ $(srcdir)/Include/internal/pycore_critical_section.h \ $(srcdir)/Include/internal/pycore_crossinterp.h \ - $(srcdir)/Include/internal/pycore_debugger_utils.h \ + $(srcdir)/Include/internal/pycore_debug_offsets.h \ $(srcdir)/Include/internal/pycore_descrobject.h \ $(srcdir)/Include/internal/pycore_dict.h \ $(srcdir)/Include/internal/pycore_dict_state.h \ diff --git a/Modules/_testexternalinspection.c b/Modules/_testexternalinspection.c index d6e2a6fad1672d..0807d1e47b6736 100644 --- a/Modules/_testexternalinspection.c +++ b/Modules/_testexternalinspection.c @@ -51,7 +51,7 @@ # define Py_BUILD_CORE_MODULE 1 #endif #include "Python.h" -#include // _Py_DebugOffsets +#include // _Py_DebugOffsets #include // FRAME_OWNED_BY_CSTACK #include // Py_TAG_BITS diff --git a/PCbuild/pythoncore.vcxproj b/PCbuild/pythoncore.vcxproj index ad2d4a581189d1..a4881e9256e4dd 100644 --- a/PCbuild/pythoncore.vcxproj +++ b/PCbuild/pythoncore.vcxproj @@ -229,7 +229,7 @@ - + diff --git a/PCbuild/pythoncore.vcxproj.filters b/PCbuild/pythoncore.vcxproj.filters index 739dda7528c53a..6b294683320a73 100644 --- a/PCbuild/pythoncore.vcxproj.filters +++ b/PCbuild/pythoncore.vcxproj.filters @@ -609,7 +609,7 @@ Include\internal - + Include\internal