remove private types from pyo3-ffi

newsfragments/5064.changed.md

@@ -0,0 +1 @@
+The FFI definition `PyCodeObject` is now an opaque struct on all Python versions.

newsfragments/5064.removed.md

@@ -0,0 +1 @@
+Remove private types from `pyo3-ffi` (i.e. starting with `_Py`) which are not referenced by public APIs: `_PyLocalMonitors`, `_Py_GlobalMonitors`, `_PyCoCached`, `_PyCoLineInstrumentationData`, `_PyCoMonitoringData`, `_PyCompilerSrcLocation`, `_PyErr_StackItem`.

pyo3-ffi/src/bytearrayobject.rs

@@ -17,7 +17,7 @@ pub struct PyByteArrayObject {
 }
 
 #[cfg(any(PyPy, GraalPy, Py_LIMITED_API))]
-opaque_struct!(PyByteArrayObject);
+opaque_struct!(pub PyByteArrayObject);
 
 #[cfg_attr(windows, link(name = "pythonXY"))]
 extern "C" {

pyo3-ffi/src/code.rs

@@ -1,4 +1,4 @@
 // This header doesn't exist in CPython, but Include/cpython/code.h does. We add
 // this here so that PyCodeObject has a definition under the limited API.
 
-opaque_struct!(PyCodeObject);
+opaque_struct!(pub PyCodeObject);

pyo3-ffi/src/cpython/bytesobject.rs

@@ -17,7 +17,7 @@ pub struct PyBytesObject {
 }
 
 #[cfg(any(PyPy, GraalPy, Py_LIMITED_API))]
-opaque_struct!(PyBytesObject);
+opaque_struct!(pub PyBytesObject);
 
 extern "C" {
     #[cfg_attr(PyPy, link_name = "_PyPyBytes_Resize")]

pyo3-ffi/src/cpython/code.rs

@@ -1,209 +1,43 @@
 use crate::object::*;
 use crate::pyport::Py_ssize_t;
 
-#[allow(unused_imports)]
-use std::os::raw::{c_char, c_int, c_short, c_uchar, c_void};
+#[cfg(not(GraalPy))]
+use std::os::raw::c_char;
+use std::os::raw::{c_int, c_void};
 #[cfg(not(any(PyPy, GraalPy)))]
 use std::ptr::addr_of_mut;
 
-#[cfg(all(Py_3_8, not(any(PyPy, GraalPy)), not(Py_3_11)))]
-opaque_struct!(_PyOpcache);
+// skipped private _PY_MONITORING_LOCAL_EVENTS
+// skipped private _PY_MONITORING_UNGROUPED_EVENTS
+// skipped private _PY_MONITORING_EVENTS
 
-#[cfg(Py_3_12)]
-pub const _PY_MONITORING_LOCAL_EVENTS: usize = 10;
-#[cfg(Py_3_12)]
-pub const _PY_MONITORING_UNGROUPED_EVENTS: usize = 15;
-#[cfg(Py_3_12)]
-pub const _PY_MONITORING_EVENTS: usize = 17;
+// skipped private _PyLocalMonitors
+// skipped private _Py_GlobalMonitors
 
-#[cfg(Py_3_12)]
-#[repr(C)]
-#[derive(Clone, Copy)]
-pub struct _Py_LocalMonitors {
-    pub tools: [u8; if cfg!(Py_3_13) {
-        _PY_MONITORING_LOCAL_EVENTS
-    } else {
-        _PY_MONITORING_UNGROUPED_EVENTS
-    }],
-}
-
-#[cfg(Py_3_12)]
-#[repr(C)]
-#[derive(Clone, Copy)]
-pub struct _Py_GlobalMonitors {
-    pub tools: [u8; _PY_MONITORING_UNGROUPED_EVENTS],
-}
-
-// skipped _Py_CODEUNIT
-
-// skipped _Py_OPCODE
-// skipped _Py_OPARG
-
-// skipped _py_make_codeunit
-
-// skipped _py_set_opcode
-
-// skipped _Py_MAKE_CODEUNIT
-// skipped _Py_SET_OPCODE
-
-#[cfg(Py_3_12)]
-#[repr(C)]
-#[derive(Copy, Clone)]
-pub struct _PyCoCached {
-    pub _co_code: *mut PyObject,
-    pub _co_varnames: *mut PyObject,
-    pub _co_cellvars: *mut PyObject,
-    pub _co_freevars: *mut PyObject,
-}
+// skipped private _Py_CODEUNIT
 
-#[cfg(Py_3_12)]
-#[repr(C)]
-#[derive(Copy, Clone)]
-pub struct _PyCoLineInstrumentationData {
-    pub original_opcode: u8,
-    pub line_delta: i8,
-}
-
-#[cfg(Py_3_12)]
-#[repr(C)]
-#[derive(Copy, Clone)]
-pub struct _PyCoMonitoringData {
-    pub local_monitors: _Py_LocalMonitors,
-    pub active_monitors: _Py_LocalMonitors,
-    pub tools: *mut u8,
-    pub lines: *mut _PyCoLineInstrumentationData,
-    pub line_tools: *mut u8,
-    pub per_instruction_opcodes: *mut u8,
-    pub per_instruction_tools: *mut u8,
-}
-
-#[cfg(all(not(any(PyPy, GraalPy)), not(Py_3_7)))]
-opaque_struct!(PyCodeObject);
-
-#[cfg(all(not(any(PyPy, GraalPy)), Py_3_7, not(Py_3_8)))]
-#[repr(C)]
-pub struct PyCodeObject {
-    pub ob_base: PyObject,
-    pub co_argcount: c_int,
-    pub co_kwonlyargcount: c_int,
-    pub co_nlocals: c_int,
-    pub co_stacksize: c_int,
-    pub co_flags: c_int,
-    pub co_firstlineno: c_int,
-    pub co_code: *mut PyObject,
-    pub co_consts: *mut PyObject,
-    pub co_names: *mut PyObject,
-    pub co_varnames: *mut PyObject,
-    pub co_freevars: *mut PyObject,
-    pub co_cellvars: *mut PyObject,
-    pub co_cell2arg: *mut Py_ssize_t,
-    pub co_filename: *mut PyObject,
-    pub co_name: *mut PyObject,
-    pub co_lnotab: *mut PyObject,
-    pub co_zombieframe: *mut c_void,
-    pub co_weakreflist: *mut PyObject,
-    pub co_extra: *mut c_void,
-}
+// skipped private _Py_OPCODE
+// skipped private _Py_OPARG
 
-#[cfg(Py_3_13)]
-opaque_struct!(_PyExecutorArray);
+// skipped private _py_make_codeunit
 
-#[cfg(all(not(any(PyPy, GraalPy)), Py_3_8, not(Py_3_11)))]
-#[repr(C)]
-pub struct PyCodeObject {
-    pub ob_base: PyObject,
-    pub co_argcount: c_int,
-    pub co_posonlyargcount: c_int,
-    pub co_kwonlyargcount: c_int,
-    pub co_nlocals: c_int,
-    pub co_stacksize: c_int,
-    pub co_flags: c_int,
-    pub co_firstlineno: c_int,
-    pub co_code: *mut PyObject,
-    pub co_consts: *mut PyObject,
-    pub co_names: *mut PyObject,
-    pub co_varnames: *mut PyObject,
-    pub co_freevars: *mut PyObject,
-    pub co_cellvars: *mut PyObject,
-    pub co_cell2arg: *mut Py_ssize_t,
-    pub co_filename: *mut PyObject,
-    pub co_name: *mut PyObject,
-    #[cfg(not(Py_3_10))]
-    pub co_lnotab: *mut PyObject,
-    #[cfg(Py_3_10)]
-    pub co_linetable: *mut PyObject,
-    pub co_zombieframe: *mut c_void,
-    pub co_weakreflist: *mut PyObject,
-    pub co_extra: *mut c_void,
-    pub co_opcache_map: *mut c_uchar,
-    pub co_opcache: *mut _PyOpcache,
-    pub co_opcache_flag: c_int,
-    pub co_opcache_size: c_uchar,
-}
+// skipped private _py_set_opcode
 
-#[cfg(all(not(any(PyPy, GraalPy)), Py_3_11))]
-#[repr(C)]
-pub struct PyCodeObject {
-    pub ob_base: PyVarObject,
-    pub co_consts: *mut PyObject,
-    pub co_names: *mut PyObject,
-    pub co_exceptiontable: *mut PyObject,
-    pub co_flags: c_int,
-    #[cfg(not(Py_3_12))]
-    pub co_warmup: c_int,
+// skipped private _Py_MAKE_CODEUNIT
+// skipped private _Py_SET_OPCODE
 
-    pub co_argcount: c_int,
-    pub co_posonlyargcount: c_int,
-    pub co_kwonlyargcount: c_int,
-    pub co_stacksize: c_int,
-    pub co_firstlineno: c_int,
+// skipped private _PyCoCached
+// skipped private _PyCoLineInstrumentationData
+// skipped private _PyCoMontoringData
 
-    pub co_nlocalsplus: c_int,
-    #[cfg(Py_3_12)]
-    pub co_framesize: c_int,
-    pub co_nlocals: c_int,
-    #[cfg(not(Py_3_12))]
-    pub co_nplaincellvars: c_int,
-    pub co_ncellvars: c_int,
-    pub co_nfreevars: c_int,
-    #[cfg(Py_3_12)]
-    pub co_version: u32,
+// skipped private _PyExecutorArray
 
-    pub co_localsplusnames: *mut PyObject,
-    pub co_localspluskinds: *mut PyObject,
-    pub co_filename: *mut PyObject,
-    pub co_name: *mut PyObject,
-    pub co_qualname: *mut PyObject,
-    pub co_linetable: *mut PyObject,
-    pub co_weakreflist: *mut PyObject,
-    #[cfg(not(Py_3_12))]
-    pub _co_code: *mut PyObject,
-    #[cfg(not(Py_3_12))]
-    pub _co_linearray: *mut c_char,
-    #[cfg(Py_3_13)]
-    pub co_executors: *mut _PyExecutorArray,
-    #[cfg(Py_3_12)]
-    pub _co_cached: *mut _PyCoCached,
-    #[cfg(all(Py_3_12, not(Py_3_13)))]
-    pub _co_instrumentation_version: u64,
-    #[cfg(Py_3_13)]
-    pub _co_instrumentation_version: libc::uintptr_t,
-    #[cfg(Py_3_12)]
-    pub _co_monitoring: *mut _PyCoMonitoringData,
-    pub _co_firsttraceable: c_int,
-    pub co_extra: *mut c_void,
-    pub co_code_adaptive: [c_char; 1],
-}
-
-#[cfg(PyPy)]
-#[repr(C)]
-pub struct PyCodeObject {
-    pub ob_base: PyObject,
-    pub co_name: *mut PyObject,
-    pub co_filename: *mut PyObject,
-    pub co_argcount: c_int,
-    pub co_flags: c_int,
-}
+opaque_struct!(
+    #[doc = "A Python code object.\n"]
+    #[doc = "\n"]
+    #[doc = "`pyo3-ffi` does not expose the contents of this struct, as it has no stability guarantees."]
+    pub PyCodeObject
+);
 
 /* Masks for co_flags */
 pub const CO_OPTIMIZED: c_int = 0x0001;

pyo3-ffi/src/cpython/compile.rs

@@ -33,7 +33,7 @@ pub struct PyCompilerFlags {
 #[cfg(all(Py_3_12, not(any(Py_3_13, PyPy, GraalPy))))]
 #[repr(C)]
 #[derive(Copy, Clone)]
-pub struct _PyCompilerSrcLocation {
+struct _PyCompilerSrcLocation {
     pub lineno: c_int,
     pub end_lineno: c_int,
     pub col_offset: c_int,

pyo3-ffi/src/cpython/critical_section.rs

@@ -17,10 +17,10 @@ pub struct PyCriticalSection2 {
 }
 
 #[cfg(not(Py_GIL_DISABLED))]
-opaque_struct!(PyCriticalSection);
+opaque_struct!(pub PyCriticalSection);
 
 #[cfg(not(Py_GIL_DISABLED))]
-opaque_struct!(PyCriticalSection2);
+opaque_struct!(pub PyCriticalSection2);
 
 extern "C" {
     pub fn PyCriticalSection_Begin(c: *mut PyCriticalSection, op: *mut PyObject);

pyo3-ffi/src/cpython/dictobject.rs

@@ -2,10 +2,10 @@ use crate::object::*;
 use crate::pyport::Py_ssize_t;
 use std::os::raw::c_int;
 
-opaque_struct!(PyDictKeysObject);
+opaque_struct!(pub PyDictKeysObject);
 
 #[cfg(Py_3_11)]
-opaque_struct!(PyDictValues);
+opaque_struct!(pub PyDictValues);
 
 #[cfg(not(GraalPy))]
 #[repr(C)]

pyo3-ffi/src/cpython/frameobject.rs

@@ -53,7 +53,7 @@ pub struct PyFrameObject {
 }
 
 #[cfg(any(PyPy, GraalPy, Py_3_11))]
-opaque_struct!(PyFrameObject);
+opaque_struct!(pub PyFrameObject);
 
 // skipped _PyFrame_IsRunnable
 // skipped _PyFrame_IsExecuting

pyo3-ffi/src/cpython/genobject.rs

@@ -1,7 +1,5 @@
 use crate::object::*;
 use crate::PyFrameObject;
-#[cfg(not(any(PyPy, GraalPy)))]
-use crate::_PyErr_StackItem;
 #[cfg(all(Py_3_11, not(any(PyPy, GraalPy))))]
 use std::os::raw::c_char;
 use std::os::raw::c_int;
@@ -20,7 +18,7 @@ pub struct PyGenObject {
     pub gi_weakreflist: *mut PyObject,
     pub gi_name: *mut PyObject,
     pub gi_qualname: *mut PyObject,
-    pub gi_exc_state: _PyErr_StackItem,
+    pub gi_exc_state: crate::cpython::pystate::_PyErr_StackItem,
     #[cfg(Py_3_11)]
     pub gi_origin_or_finalizer: *mut PyObject,
     #[cfg(Py_3_11)]

pyo3-ffi/src/cpython/pyframe.rs

@@ -1,2 +1,3 @@
+// NB used in `_PyEval_EvalFrameDefault`, maybe we remove this too.
 #[cfg(all(Py_3_11, not(PyPy)))]
-opaque_struct!(_PyInterpreterFrame);
+opaque_struct!(pub _PyInterpreterFrame);

pyo3-ffi/src/cpython/pystate.rs

@@ -27,16 +27,18 @@ pub const PyTrace_OPCODE: c_int = 7;
 // skipped PyTraceInfo
 // skipped CFrame
 
+/// Private structure used inline in `PyGenObject`
 #[cfg(not(PyPy))]
 #[repr(C)]
 #[derive(Clone, Copy)]
+#[doc(hidden)] // TODO should be able to make pub(crate) after MSRV 1.74
 pub struct _PyErr_StackItem {
     #[cfg(not(Py_3_11))]
-    pub exc_type: *mut PyObject,
-    pub exc_value: *mut PyObject,
+    exc_type: *mut PyObject,
+    exc_value: *mut PyObject,
     #[cfg(not(Py_3_11))]
-    pub exc_traceback: *mut PyObject,
-    pub previous_item: *mut _PyErr_StackItem,
+    exc_traceback: *mut PyObject,
+    previous_item: *mut _PyErr_StackItem,
 }
 
 // skipped _PyStackChunk

pyo3-ffi/src/cpython/weakrefobject.rs

@@ -1,3 +1,4 @@
+// NB publicly re-exported in `src/weakrefobject.rs`
 #[cfg(not(any(PyPy, GraalPy)))]
 pub struct _PyWeakReference {
     pub ob_base: crate::PyObject,

pyo3-ffi/src/dictobject.rs

@@ -118,4 +118,4 @@ extern "C" {
 
 #[cfg(any(PyPy, GraalPy, Py_LIMITED_API))]
 // TODO: remove (see https://github.com/PyO3/pyo3/pull/1341#issuecomment-751515985)
-opaque_struct!(PyDictObject);
+opaque_struct!(pub PyDictObject);

pyo3-ffi/src/floatobject.rs

@@ -4,7 +4,7 @@ use std::ptr::addr_of_mut;
 
 #[cfg(Py_LIMITED_API)]
 // TODO: remove (see https://github.com/PyO3/pyo3/pull/1341#issuecomment-751515985)
-opaque_struct!(PyFloatObject);
+opaque_struct!(pub PyFloatObject);
 
 #[cfg_attr(windows, link(name = "pythonXY"))]
 extern "C" {

pyo3-ffi/src/lib.rs

@@ -341,9 +341,10 @@
 // model opaque types:
 // https://doc.rust-lang.org/nomicon/ffi.html#representing-opaque-structs
 macro_rules! opaque_struct {
-    ($name:ident) => {
+    ($(#[$attrs:meta])* $pub:vis $name:ident) => {
+        $(#[$attrs])*
         #[repr(C)]
-        pub struct $name([u8; 0]);
+        $pub struct $name([u8; 0]);
     };
 }
 

pyo3-ffi/src/longobject.rs

@@ -4,7 +4,7 @@ use libc::size_t;
 use std::os::raw::{c_char, c_double, c_int, c_long, c_longlong, c_ulong, c_ulonglong, c_void};
 use std::ptr::addr_of_mut;
 
-opaque_struct!(PyLongObject);
+opaque_struct!(pub PyLongObject);
 
 #[inline]
 pub unsafe fn PyLong_Check(op: *mut PyObject) -> c_int {