pythonGH-131238: Core header refactor (pythonGH-131250)

* Moves most structs in pycore_ header files into pycore_structs.h and pycore_runtime_structs.h

* Removes many cross-header dependencies

Author: markshannon

Include/cpython/code.h

@@ -8,70 +8,19 @@
 extern "C" {
 #endif
 
-/* Total tool ids available */
-#define  _PY_MONITORING_TOOL_IDS 8
-/* Count of all local monitoring events */
-#define  _PY_MONITORING_LOCAL_EVENTS 11
-/* Count of all "real" monitoring events (not derived from other events) */
-#define _PY_MONITORING_UNGROUPED_EVENTS 16
-/* Count of all  monitoring events */
-#define _PY_MONITORING_EVENTS 19
-
-/* Tables of which tools are active for each monitored event. */
-typedef struct _Py_LocalMonitors {
-    uint8_t tools[_PY_MONITORING_LOCAL_EVENTS];
-} _Py_LocalMonitors;
-
-typedef struct _Py_GlobalMonitors {
-    uint8_t tools[_PY_MONITORING_UNGROUPED_EVENTS];
-} _Py_GlobalMonitors;
-
-
 typedef struct {
     PyObject *_co_code;
     PyObject *_co_varnames;
     PyObject *_co_cellvars;
     PyObject *_co_freevars;
 } _PyCoCached;
 
-/* Ancillary data structure used for instrumentation.
-   Line instrumentation creates this with sufficient
-   space for one entry per code unit. The total size
-   of the data will be `bytes_per_entry * Py_SIZE(code)` */
-typedef struct {
-    uint8_t bytes_per_entry;
-    uint8_t data[1];
-} _PyCoLineInstrumentationData;
-
-
 typedef struct {
     int size;
     int capacity;
     struct _PyExecutorObject *executors[1];
 } _PyExecutorArray;
 
-/* Main data structure used for instrumentation.
- * This is allocated when needed for instrumentation
- */
-typedef struct {
-    /* Monitoring specific to this code object */
-    _Py_LocalMonitors local_monitors;
-    /* Monitoring that is active on this code object */
-    _Py_LocalMonitors active_monitors;
-    /* The tools that are to be notified for events for the matching code unit */
-    uint8_t *tools;
-    /* The version of tools when they instrument the code */
-    uintptr_t tool_versions[_PY_MONITORING_TOOL_IDS];
-    /* Information to support line events */
-    _PyCoLineInstrumentationData *lines;
-    /* The tools that are to be notified for line events for the matching code unit */
-    uint8_t *line_tools;
-    /* Information to support instruction events */
-    /* The underlying instructions, which can themselves be instrumented */
-    uint8_t *per_instruction_opcodes;
-    /* The tools that are to be notified for instruction events for the matching code unit */
-    uint8_t *per_instruction_tools;
-} _PyCoMonitoringData;
 
 #ifdef Py_GIL_DISABLED
 
@@ -151,7 +100,7 @@ typedef struct {
     _PyExecutorArray *co_executors;      /* executors from optimizer */        \
     _PyCoCached *_co_cached;      /* cached co_* attributes */                 \
     uintptr_t _co_instrumentation_version; /* current instrumentation version */ \
-    _PyCoMonitoringData *_co_monitoring; /* Monitoring data */                 \
+    struct _PyCoMonitoringData *_co_monitoring; /* Monitoring data */          \
     Py_ssize_t _co_unique_id;     /* ID used for per-thread refcounting */   \
     int _co_firsttraceable;       /* index of first traceable instruction */   \
     /* Scratch space for extra data relating to the code object.               \

Include/cpython/object.h

@@ -512,7 +512,6 @@ PyAPI_FUNC(int) PyObject_VisitManagedDict(PyObject *obj, visitproc visit, void *
 PyAPI_FUNC(int) _PyObject_SetManagedDict(PyObject *obj, PyObject *new_dict);
 PyAPI_FUNC(void) PyObject_ClearManagedDict(PyObject *obj);
 
-#define TYPE_MAX_WATCHERS 8
 
 typedef int(*PyType_WatchCallback)(PyTypeObject *);
 PyAPI_FUNC(int) PyType_AddWatcher(PyType_WatchCallback callback);

Include/internal/pycore_atexit.h

@@ -12,44 +12,6 @@ extern "C" {
 #endif
 
 
-//###############
-// runtime atexit
-
-typedef void (*atexit_callbackfunc)(void);
-
-struct _atexit_runtime_state {
-    PyMutex mutex;
-#define NEXITFUNCS 32
-    atexit_callbackfunc callbacks[NEXITFUNCS];
-    int ncallbacks;
-};
-
-
-//###################
-// interpreter atexit
-
-typedef void (*atexit_datacallbackfunc)(void *);
-
-typedef struct atexit_callback {
-    atexit_datacallbackfunc func;
-    void *data;
-    struct atexit_callback *next;
-} atexit_callback;
-
-struct atexit_state {
-#ifdef Py_GIL_DISABLED
-    PyMutex ll_callbacks_lock;
-#endif
-    atexit_callback *ll_callbacks;
-
-    // XXX The rest of the state could be moved to the atexit module state
-    // and a low-level callback added for it during module exec.
-    // For the moment we leave it here.
-
-    // List containing tuples with callback information.
-    // e.g. [(func, args, kwargs), ...]
-    PyObject *callbacks;
-};
 
 #ifdef Py_GIL_DISABLED
 #  define _PyAtExit_LockCallbacks(state) PyMutex_Lock(&state->ll_callbacks_lock);

Include/internal/pycore_backoff.h

@@ -10,14 +10,7 @@ extern "C" {
 #endif
 
 #include <assert.h>
-#include <stdbool.h>
-#include <stdint.h>
-
-
-typedef struct {
-    uint16_t value_and_backoff;
-} _Py_BackoffCounter;
-
+#include "pycore_structs.h" // _Py_BackoffCounter
 
 /* 16-bit countdown counters using exponential backoff.
 

Include/internal/pycore_ceval_state.h

@@ -12,16 +12,6 @@ extern "C" {
 #include "pycore_gil.h"             // struct _gil_runtime_state
 
 
-typedef int (*_Py_pending_call_func)(void *);
-
-struct _pending_call {
-    _Py_pending_call_func func;
-    void *arg;
-    int flags;
-};
-
-#define PENDINGCALLSARRAYSIZE 300
-
 #define MAXPENDINGCALLS PENDINGCALLSARRAYSIZE
 /* For interpreter-level pending calls, we want to avoid spending too
    much time on pending calls in any one thread, so we apply a limit. */
@@ -40,69 +30,6 @@ struct _pending_call {
    pending calls for the main thread. */
 #define MAXPENDINGCALLSLOOP_MAIN 0
 
-struct _pending_calls {
-    PyThreadState *handling_thread;
-    PyMutex mutex;
-    /* Request for running pending calls. */
-    int32_t npending;
-    /* The maximum allowed number of pending calls.
-       If the queue fills up to this point then _PyEval_AddPendingCall()
-       will return _Py_ADD_PENDING_FULL. */
-    int32_t max;
-    /* We don't want a flood of pending calls to interrupt any one thread
-       for too long, so we keep a limit on the number handled per pass.
-       A value of 0 means there is no limit (other than the maximum
-       size of the list of pending calls). */
-    int32_t maxloop;
-    struct _pending_call calls[PENDINGCALLSARRAYSIZE];
-    int first;
-    int next;
-};
-
-
-typedef enum {
-    PERF_STATUS_FAILED = -1,  // Perf trampoline is in an invalid state
-    PERF_STATUS_NO_INIT = 0,  // Perf trampoline is not initialized
-    PERF_STATUS_OK = 1,       // Perf trampoline is ready to be executed
-} perf_status_t;
-
-#ifdef PY_HAVE_PERF_TRAMPOLINE
-struct code_arena_st;
-
-struct trampoline_api_st {
-    void* (*init_state)(void);
-    void (*write_state)(void* state, const void *code_addr,
-                        unsigned int code_size, PyCodeObject* code);
-    int (*free_state)(void* state);
-    void *state;
-    Py_ssize_t code_padding;
-};
-#endif
-
-
-struct _ceval_runtime_state {
-    struct {
-#ifdef PY_HAVE_PERF_TRAMPOLINE
-        perf_status_t status;
-        int perf_trampoline_type;
-        Py_ssize_t extra_code_index;
-        struct code_arena_st *code_arena;
-        struct trampoline_api_st trampoline_api;
-        FILE *map_file;
-        Py_ssize_t persist_after_fork;
-#else
-        int _not_used;
-#endif
-    } perf;
-    /* Pending calls to be made only on the main thread. */
-    // The signal machinery falls back on this
-    // so it must be especially stable and efficient.
-    // For example, we use a preallocated array
-    // for the list of pending calls.
-    struct _pending_calls pending_mainthread;
-    PyMutex sys_trace_profile_mutex;
-};
-
 
 #ifdef PY_HAVE_PERF_TRAMPOLINE
 # define _PyEval_RUNTIME_PERF_INIT \
@@ -116,18 +43,6 @@ struct _ceval_runtime_state {
 #endif
 
 
-struct _ceval_state {
-    /* This variable holds the global instrumentation version. When a thread is
-       running, this value is overlaid onto PyThreadState.eval_breaker so that
-       changes in the instrumentation version will trigger the eval breaker. */
-    uintptr_t instrumentation_version;
-    int recursion_limit;
-    struct _gil_runtime_state *gil;
-    int own_gil;
-    struct _pending_calls pending;
-};
-
-
 #ifdef __cplusplus
 }
 #endif

Include/internal/pycore_code.h

@@ -8,30 +8,13 @@ extern "C" {
 #  error "this header requires Py_BUILD_CORE define"
 #endif
 
+#include "pycore_structs.h"     // _Py_CODEUNIT
 #include "pycore_stackref.h"    // _PyStackRef
 #include "pycore_lock.h"        // PyMutex
 #include "pycore_backoff.h"     // _Py_BackoffCounter
 #include "pycore_tstate.h"      // _PyThreadStateImpl
 
 
-/* Each instruction in a code object is a fixed-width value,
- * currently 2 bytes: 1-byte opcode + 1-byte oparg.  The EXTENDED_ARG
- * opcode allows for larger values but the current limit is 3 uses
- * of EXTENDED_ARG (see Python/compile.c), for a maximum
- * 32-bit value.  This aligns with the note in Python/compile.c
- * (compiler_addop_i_line) indicating that the max oparg value is
- * 2**32 - 1, rather than INT_MAX.
- */
-
-typedef union {
-    uint16_t cache;
-    struct {
-        uint8_t code;
-        uint8_t arg;
-    } op;
-    _Py_BackoffCounter counter;  // First cache entry of specializable op
-} _Py_CODEUNIT;
-
 #define _PyCode_CODE(CO) _Py_RVALUE((_Py_CODEUNIT *)(CO)->co_code_adaptive)
 #define _PyCode_NBYTES(CO) (Py_SIZE(CO) * (Py_ssize_t)sizeof(_Py_CODEUNIT))
 
@@ -67,16 +50,10 @@ _py_set_opcode(_Py_CODEUNIT *word, uint8_t opcode)
 #define _PyCode_HAS_INSTRUMENTATION(CODE) \
     (CODE->_co_instrumentation_version > 0)
 
-struct _py_code_state {
-    PyMutex mutex;
-    // Interned constants from code objects. Used by the free-threaded build.
-    struct _Py_hashtable_t *constants;
-};
 
 extern PyStatus _PyCode_Init(PyInterpreterState *interp);
 extern void _PyCode_Fini(PyInterpreterState *interp);
 
-#define CODE_MAX_WATCHERS 8
 
 /* PEP 659
  * Specialization and quickening structs and helper functions
@@ -185,14 +162,6 @@ typedef struct {
 
 #define INLINE_CACHE_ENTRIES_CONTAINS_OP CACHE_ENTRIES(_PyContainsOpCache)
 
-// Borrowed references to common callables:
-struct callable_cache {
-    PyObject *isinstance;
-    PyObject *len;
-    PyObject *list_append;
-    PyObject *object__getattribute__;
-};
-
 /* "Locals plus" for a code object is the set of locals + cell vars +
  * free vars.  This relates to variable names as well as offsets into
  * the "fast locals" storage array of execution frames.  The compiler

Include/internal/pycore_codecs.h

@@ -9,6 +9,7 @@ extern "C" {
 #endif
 
 #include "pycore_lock.h"          // PyMutex
+#include "pycore_runtime_structs.h"    // struct codecs_state
 
 /* Initialize codecs-related state for the given interpreter, including
    registering the first codec search function. Must be called before any other
@@ -70,27 +71,6 @@ extern PyObject* _PyCodecInfo_GetIncrementalEncoder(
    PyObject *codec_info,
    const char *errors);
 
-// Per-interpreter state used by codecs.c.
-struct codecs_state {
-    // A list of callable objects used to search for codecs.
-    PyObject *search_path;
-
-    // A dict mapping codec names to codecs returned from a callable in
-    // search_path.
-    PyObject *search_cache;
-
-    // A dict mapping error handling strategies to functions to implement them.
-    PyObject *error_registry;
-
-#ifdef Py_GIL_DISABLED
-    // Used to safely delete a specific item from search_path.
-    PyMutex search_path_mutex;
-#endif
-
-    // Whether or not the rest of the state is initialized.
-    int initialized;
-};
-
 #ifdef __cplusplus
 }
 #endif

Include/internal/pycore_context.h

@@ -5,9 +5,7 @@
 #  error "this header requires Py_BUILD_CORE define"
 #endif
 
-#include "pycore_hamt.h"          // PyHamtObject
-
-#define CONTEXT_MAX_WATCHERS 8
+#include "pycore_structs.h"
 
 extern PyTypeObject _PyContextTokenMissing_Type;