Embed C API into Python extension (Qiskit#13934)

* Embed C API into Python extension

This commit adds the C extension crate into the Python extension build.
This is done by adding the qiskit-cext crate as a dependency for
qiskit-pyext. Then the contents of cext are re-exported from pyext. This
enables linking against the _accelerate shared library object using the
C API. This facilitates writing compiled Python extensions that
interface with Qiskit directly via C. For example, building a
SparseObservable object in a compiled language via the C API and then
returning that to Python for use with the rest of Qiskit.

Co-authored-by: Matthew Treinish <[email protected]>

* Remove python space function

The path is trivial to get if you need it and we shouldn't commit to
this in the public api yet.

* Add feature gate for python function

* Add optional python header and and fix rename overrides prefixing

* Correct dependencies in Makefile

* Fix cbindgen config

* Add cfg feature guard for python dependent functionality

* Adjust python gil usage and document it

* Update crates/pyext/src/lib.rs

* Update crates/cext/src/sparse_observable.rs

Co-authored-by: Jake Lishman <[email protected]>

* Manual object rename

We should find a better solution,
maybe the Rust interface of cbindgen
allows for more options here.

* pyo3 as optional dependency in cext

but implicitly it still depends on it via qiskit-accelerate

---------

Co-authored-by: Julien Gacon <[email protected]>
Co-authored-by: Jake Lishman <[email protected]>
Co-authored-by: Jake Lishman <[email protected]>

Author: 3 people

Cargo.lock

@@ -115,16 +115,18 @@ cformat:
 fix_cformat:
 	bash tools/run_clang_format.sh apply
 
-# The header file is managed by a different build tool - pretend it's always dirty.
-.PHONY: $(C_QISKIT_H)
-$(C_QISKIT_H):
-	cargo build --release --no-default-features --features cbinding
+# The library file is managed by a different build tool - pretend it's always dirty.
+.PHONY: $(C_LIB_CARGO_PATH)
+$(C_LIB_CARGO_PATH):
+	cargo build --release --no-default-features --features cbinding -p qiskit-cext
+
+$(C_QISKIT_H): $(C_LIB_CARGO_PATH)
 	cbindgen --crate qiskit-cext --output $(C_DIR_INCLUDE)/qiskit.h --lang C
 
 $(C_DIR_LIB):
 	mkdir -p $(C_DIR_LIB)
 
-$(C_LIBQISKIT): $(C_DIR_LIB)
+$(C_LIBQISKIT): $(C_DIR_LIB) $(C_LIB_CARGO_PATH)
 	cp $(C_LIB_CARGO_PATH) $(C_DIR_LIB)/$(subst _cext,,$(C_LIB_CARGO_FILENAME))
 
 .PHONY: cheader clib c

Makefile

@@ -17,10 +17,12 @@ workspace = true
 thiserror.workspace = true
 num-complex.workspace = true
 qiskit-accelerate.workspace = true
+pyo3 = { workspace = true, optional = true }
 
 [build-dependencies]
 cbindgen = "0.28"
 
 [features]
 default = ["cbinding"]
+python_binding = ["dep:pyo3"]
 cbinding = []

crates/cext/Cargo.toml

@@ -1,19 +1,24 @@
 include_version = true
-style = "type" 
+style = "type"
 sys_includes = ["complex.h"]
 after_includes = """
+#ifdef QISKIT_C_PYTHON_INTERFACE
+    #include <Python.h>
+#endif
+
 // Complex number typedefs -- note these are memory aligned but
 // not calling convention compatible.
-typedef float complex QkComplex32; 
+typedef float complex QkComplex32;
 typedef double complex QkComplex64;
 
-// Always expose [cfg(feature = "cbinding")] -- workaround for 
+// Always expose [cfg(feature = "cbinding")] -- workaround for
 // https://github.com/mozilla/cbindgen/issues/995
 #define QISKIT_WITH_CBINDINGS
 """
 
 [defines]
 "feature = cbinding" = "QISKIT_WITH_CBINDINGS"
+"feature = python_binding" = "QISKIT_C_PYTHON_INTERFACE"
 
 [parse]
 parse_deps = true
@@ -22,10 +27,8 @@ include = ["qiskit-accelerate"]
 [enum]
 prefix_with_name = true
 
-[export]
-prefix = "Qk"
-renaming_overrides_prefixing = false
-
 [export.rename]
-"SparseObservable" = "Obs"
-"CSparseTerm" = "ObsTerm"
+"SparseObservable" = "QkObs"
+"CSparseTerm" = "QkObsTerm"
+"BitTerm" = "QkBitTerm"
+"Complex64" = "QkComplex64"

crates/cext/cbindgen.toml

@@ -14,8 +14,16 @@ use std::ffi::{c_char, CString};
 
 use crate::exit_codes::{CInputError, ExitCode};
 use num_complex::Complex64;
+
 use qiskit_accelerate::sparse_observable::{BitTerm, SparseObservable, SparseTermView};
 
+#[cfg(feature = "python_binding")]
+use pyo3::ffi::PyObject;
+#[cfg(feature = "python_binding")]
+use pyo3::{Py, Python};
+#[cfg(feature = "python_binding")]
+use qiskit_accelerate::sparse_observable::PySparseObservable;
+
 /// @ingroup QkObsTerm
 /// A term in a [SparseObservable].
 ///
@@ -924,3 +932,34 @@ pub extern "C" fn qk_bitterm_label(bit_term: BitTerm) -> u8 {
         .next()
         .expect("Label has exactly one character") as u8
 }
+
+/// @ingroup SparseObservable
+/// Convert to a Python-space [PySparseObservable].
+///
+/// @param obs The C-space [SparseObservable] pointer.
+///
+/// @return A Python object representing the [PySparseObservable].
+///
+/// # Safety
+///
+/// Behavior is undefined if ``obs`` is not a valid, non-null pointer to a ``QkObs``.
+///
+/// It is assumed that the thread currently executing this function holds the
+/// Python GIL this is required to create the Python object returned by this
+/// function.
+#[no_mangle]
+#[cfg(feature = "python_binding")]
+#[cfg(feature = "cbinding")]
+pub unsafe extern "C" fn qk_obs_to_python(obs: *const SparseObservable) -> *mut PyObject {
+    // SAFETY: Per documentation, the pointer is non-null and aligned.
+    let obs = unsafe { const_ptr_as_ref(obs) };
+    let py_obs: PySparseObservable = obs.clone().into();
+
+    // SAFETY: the C caller is required to hold the GIL.
+    unsafe {
+        let py = Python::assume_gil_acquired();
+        Py::new(py, py_obs)
+            .expect("Unable to create a Python object")
+            .into_ptr()
+    }
+}

crates/cext/src/sparse_observable.rs

@@ -28,3 +28,4 @@ qiskit-accelerate.workspace = true
 qiskit-circuit.workspace = true
 qiskit-qasm2.workspace = true
 qiskit-qasm3.workspace = true
+qiskit-cext = { workspace = true, features = ["python_binding"] }

crates/pyext/Cargo.toml

@@ -11,6 +11,7 @@
 // that they have been altered from the originals.
 
 use pyo3::prelude::*;
+pub use qiskit_cext::*;
 
 #[inline(always)]
 #[doc(hidden)]

crates/pyext/src/lib.rs

@@ -131,6 +131,7 @@
 from qiskit.transpiler.preset_passmanagers import generate_preset_pass_manager
 from .version import __version__
 
+
 __all__ = [
     "AncillaRegister",
     "ClassicalRegister",