Use nanobind for python module

.github/workflows/linux.yml

@@ -56,6 +56,7 @@ jobs:
         run: cmake --build . --target run_sparrow_tests_direct
 
       - name: Install
+        if: matrix.build_shared == 'ON'
         working-directory: build
         run: cmake --install .
 
@@ -69,6 +70,14 @@ jobs:
       - name: Checkout repository
         uses: actions/checkout@v4
 
+      - name: Set up Python
+        uses: actions/setup-python@v6
+        with:
+          python-version: '3.12'
+
+      - name: Install Python test dependencies
+        run: pip install pytest polars pyarrow
+
       - name: Configure using cmake
         run: |
           cmake -G Ninja \
@@ -90,6 +99,12 @@ jobs:
         working-directory: build
         run: cmake --build . --target run_tests_with_junit_report
 
+      - name: Run sparrow integration tests
+        if: matrix.build_shared == 'ON'
+        working-directory: build
+        run: cmake --build . --target run_sparrow_tests_direct
+
       - name: Install
+        if: matrix.build_shared == 'ON'
         working-directory: build
         run: sudo cmake --install .

.github/workflows/osx.yml

@@ -55,12 +55,13 @@ jobs:
         working-directory: build
         run: cmake --build . --target run_tests_with_junit_report
 
-      # - name: Run Sparrow integration tests
-      #   if: matrix.build_shared == 'ON'
-      #   working-directory: build
-      #   run: cmake --build . --target run_sparrow_tests_direct
+      - name: Run Sparrow integration tests
+        if: matrix.build_shared == 'ON'
+        working-directory: build
+        run: cmake --build . --target run_sparrow_tests_direct
 
       - name: Install
+        if: matrix.build_shared == 'ON'
         working-directory: build
         run: cmake --install .
 
@@ -79,14 +80,23 @@ jobs:
           sudo xcode-select --switch /Applications/Xcode_16.4.app/Contents/Developer
           xcodebuild -version
 
+      - name: Set up Python
+        uses: actions/setup-python@v6
+        with:
+          python-version: '3.12'
+
+      - name: Install Python test dependencies
+        run: pip install pytest polars pyarrow
+
       - name: Configure using cmake
         run: |
           cmake -G Ninja \
           -Bbuild \
           -DCMAKE_BUILD_TYPE=${{ matrix.build_type }} \
           -DSPARROW_PYCAPSULE_BUILD_SHARED=${{ matrix.build_shared }} \
           -DSPARROW_PYCAPSULE_BUILD_TESTS=ON \
-          -DFETCH_DEPENDENCIES_WITH_CMAKE=MISSING
+          -DFETCH_DEPENDENCIES_WITH_CMAKE=MISSING \
+          -DPython_EXECUTABLE=$(which python3)
 
       - name: Build sparrow-pycapsule
         working-directory: build
@@ -100,6 +110,12 @@ jobs:
         working-directory: build
         run: cmake --build . --target run_tests_with_junit_report
 
+      - name: Run sparrow integration tests
+        if: matrix.build_shared == 'ON'
+        working-directory: build
+        run: cmake --build . --target run_sparrow_tests_direct
+
       - name: Install
+        if: matrix.build_shared == 'ON'
         working-directory: build
         run: sudo cmake --install .

.github/workflows/windows.yml

@@ -19,7 +19,7 @@ jobs:
         build_shared: [ON, OFF]
     steps:
       - name: Checkout repository
-        uses: actions/checkout@v4
+        uses: actions/checkout@v5
 
       - name: Create build environment
         uses: mamba-org/setup-micromamba@v2
@@ -61,6 +61,7 @@ jobs:
         run: cmake --build . --config ${{ matrix.build_type }} --target run_sparrow_tests_direct
 
       - name: Install
+        if: matrix.build_shared == 'ON'
         working-directory: build
         run: cmake --install . --config ${{ matrix.build_type }}
 
@@ -72,7 +73,7 @@ jobs:
         build_shared: [ON, OFF]
     steps:
       - name: Checkout repository
-        uses: actions/checkout@v4
+        uses: actions/checkout@v5
 
       - name: Configure using cmake
         run: |
@@ -89,12 +90,17 @@ jobs:
         working-directory: build
         run: cmake --build . --config ${{ matrix.build_type }} --target test_sparrow_pycapsule_lib
 
-      # TODO: This crashes in the CI
-      #- name: Run tests
-      #  working-directory: build
-      #  run: cmake --build . --config ${{ matrix.build_type }} --target run_tests_with_junit_report
+      - name: Run tests
+        working-directory: build
+        run: cmake --build . --config ${{ matrix.build_type }} --target run_tests_with_junit_report
+
+      - name: Run Sparrow integration tests
+        if: matrix.build_shared == 'ON'
+        working-directory: build
+        run: cmake --build . --config ${{ matrix.build_type }} --target run_sparrow_tests_direct
 
       - name: Install
+        if: matrix.build_shared == 'ON'
         working-directory: build
         run: cmake --install . --config ${{ matrix.build_type }}
 

CMakeLists.txt

@@ -174,8 +174,8 @@ add_library(sparrow-pycapsule ${SPARROW_PYCAPSULE_LIBRARY_TYPE} ${SPARROW_PYCAPS
 
 target_link_libraries(sparrow-pycapsule
     PUBLIC 
-        Python::Python
-        sparrow::sparrow)
+        sparrow::sparrow
+        Python::Python)
 
 target_compile_definitions(sparrow-pycapsule PUBLIC ${SPARROW_PYCAPSULE_COMPILE_DEFINITIONS})
 
@@ -246,6 +246,43 @@ if(BUILD_DOCS)
     add_subdirectory(docs)
 endif()
 
+# Python module
+# =============
+option(SPARROW_PYCAPSULE_BUILD_PYTHON_MODULE "Build the sparrow Python module" ON)
+message(STATUS "🔧 Build Python module: ${SPARROW_PYCAPSULE_BUILD_PYTHON_MODULE}")
+
+if(SPARROW_PYCAPSULE_BUILD_PYTHON_MODULE)
+    nanobind_add_module(sparrow_rockfinch src/sparrow_module.cpp)
+    target_link_libraries(sparrow_rockfinch PRIVATE sparrow-pycapsule sparrow::sparrow)
+    target_compile_features(sparrow_rockfinch PRIVATE cxx_std_20)
+    # Define the module name macro to match the output filename (includes debug suffix)
+    target_compile_definitions(sparrow_rockfinch PRIVATE
+        SPARROW_MODULE_NAME=$<TARGET_FILE_BASE_NAME:sparrow_rockfinch>
+    )
+    set_target_properties(sparrow_rockfinch PROPERTIES
+        LIBRARY_OUTPUT_DIRECTORY ${BINARY_BUILD_DIR}
+        # Set RPATH so the module can find libsparrow-pycapsule at runtime
+        BUILD_RPATH "$<TARGET_FILE_DIR:sparrow-pycapsule>"
+        BUILD_RPATH_USE_ORIGIN ON
+        MACOSX_RPATH ON
+        INSTALL_RPATH "@loader_path"
+    )
+
+    # Install Python module (for pip install / wheel building)
+    install(TARGETS sparrow_rockfinch
+        LIBRARY DESTINATION sparrow_rockfinch
+        RUNTIME DESTINATION sparrow_rockfinch
+    )
+    # Install __init__.py for the package
+    install(FILES ${CMAKE_CURRENT_SOURCE_DIR}/python/sparrow_rockfinch/__init__.py
+        DESTINATION sparrow_rockfinch
+    )
+    # Install py.typed marker for type hints
+    install(FILES ${CMAKE_CURRENT_SOURCE_DIR}/python/sparrow_rockfinch/py.typed
+        DESTINATION sparrow_rockfinch
+    )
+endif()
+
 # Tests
 # =====
 if(SPARROW_PYCAPSULE_BUILD_TESTS)

cmake/external_dependencies.cmake

@@ -64,4 +64,18 @@ if(SPARROW_PYCAPSULE_BUILD_TESTS)
     )
 endif()
 
-find_package(Python REQUIRED COMPONENTS Development)
+find_package(Python REQUIRED COMPONENTS Interpreter Development.Module Development.Embed)
+
+execute_process(
+    COMMAND "${Python_EXECUTABLE}" -m nanobind --cmake_dir
+    OUTPUT_STRIP_TRAILING_WHITESPACE OUTPUT_VARIABLE nanobind_ROOT)
+
+find_package_or_fetch(
+    PACKAGE_NAME nanobind
+    GIT_REPOSITORY https://github.com/wjakob/nanobind.git
+    TAG v2.9.2
+)
+
+if(TARGET nanobind)
+    nanobind_build_library(nanobind-static)
+endif()

environment-dev.yml

@@ -8,6 +8,8 @@ dependencies:
   # Dependencies
   - sparrow-devel
   - python
+  - nanobind
+  - nanobind-abi
   # Tests
   - doctest
   - polars

include/sparrow-pycapsule/pycapsule.hpp

@@ -2,6 +2,7 @@
 
 #include <utility>
 
+#define PY_SSIZE_T_CLEAN
 #include <Python.h>
 #include <sparrow-pycapsule/config/config.hpp>
 
@@ -16,64 +17,6 @@ struct ArrowArray;
 
 namespace sparrow::pycapsule
 {
-    /**
-     * @brief Capsule destructor for ArrowSchema PyCapsules.
-     *
-     * Calls the schema's release callback if not null, then frees the schema.
-     * This is used as the PyCapsule destructor to ensure proper cleanup.
-     *
-     * @param capsule The PyCapsule containing an ArrowSchema pointer
-     */
-    SPARROW_PYCAPSULE_API void release_arrow_schema_pycapsule(PyObject* capsule);
-
-    /**
-     * @brief Exports a sparrow array's schema to a PyCapsule.
-     *
-     * Creates a new ArrowSchema on the heap and transfers ownership from the array.
-     * The array is moved from and becomes invalid after this call.
-     *
-     * @param arr The sparrow array to export (will be moved from)
-     * @return A new PyCapsule containing the ArrowSchema, or nullptr on error
-     */
-    SPARROW_PYCAPSULE_API PyObject* export_arrow_schema_pycapsule(array& arr);
-
-    /**
-     * @brief Retrieves the ArrowSchema pointer from a PyCapsule.
-     *
-     * @param capsule The PyCapsule to extract the schema from
-     * @return Pointer to the ArrowSchema, or nullptr if the capsule is invalid (sets Python exception)
-     */
-    SPARROW_PYCAPSULE_API ArrowSchema* get_arrow_schema_pycapsule(PyObject* capsule);
-
-    /**
-     * @brief Capsule destructor for ArrowArray PyCapsules.
-     *
-     * Calls the array's release callback if not null, then frees the array.
-     * This is used as the PyCapsule destructor to ensure proper cleanup.
-     *
-     * @param capsule The PyCapsule containing an ArrowArray pointer
-     */
-    SPARROW_PYCAPSULE_API void release_arrow_array_pycapsule(PyObject* capsule);
-
-    /**
-     * @brief Exports a sparrow array's data to a PyCapsule.
-     *
-     * Creates a new ArrowArray on the heap and transfers ownership from the array.
-     * The array is moved from and becomes invalid after this call.
-     *
-     * @param arr The sparrow array to export (will be moved from)
-     * @return A new PyCapsule containing the ArrowArray, or nullptr on error
-     */
-    SPARROW_PYCAPSULE_API PyObject* export_arrow_array_pycapsule(array& arr);
-
-    /**
-     * @brief Retrieves the ArrowArray pointer from a PyCapsule.
-     *
-     * @param capsule The PyCapsule to extract the array from
-     * @return Pointer to the ArrowArray, or nullptr if the capsule is invalid (sets Python exception)
-     */
-    SPARROW_PYCAPSULE_API ArrowArray* get_arrow_array_pycapsule(PyObject* capsule);
-
     /**
      * @brief Imports a sparrow array from schema and array PyCapsules.
      *