wip

Author: Alex-PLACET

test/CMakeLists.txt

@@ -85,15 +85,17 @@ add_custom_target(run_tests_with_junit_report
 
 set_target_properties(run_tests_with_junit_report PROPERTIES FOLDER "Tests utilities")
 
-# Polars integration test helper library
-# =======================================
-add_library(test_polars_helper SHARED test_polars_helper.cpp test_polars_helper.hpp)
+# Polars integration test helper - Native Python Extension Module
+# ===============================================================
+# This builds a proper Python extension module (.cpython-*.so) that shares
+# the same Python runtime as the interpreter, avoiding dual-runtime issues.
+
+Python_add_library(test_polars_helper MODULE test_polars_helper_module.cpp)
 
 target_link_libraries(test_polars_helper
-    PUBLIC
+    PRIVATE
         sparrow-pycapsule
         sparrow::sparrow
-        Python::Python
 )
 
 target_compile_features(test_polars_helper PRIVATE cxx_std_20)
@@ -107,6 +109,8 @@ endif()
 set_target_properties(test_polars_helper PROPERTIES
     FOLDER tests
     LIBRARY_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/bin/${CMAKE_BUILD_TYPE}
+    # Python modules must not have a debug suffix - Python won't find them
+    DEBUG_POSTFIX ""
 )
 
 # Python integration test

test/test_library_load.py

@@ -21,7 +21,6 @@ def main():
     print("="*60)
 
     try:
-        # Step 1: Check environment variables
         step(1, "Checking environment variables")
         helper_path = os.environ.get('TEST_POLARS_HELPER_LIB_PATH')
         main_path = os.environ.get('SPARROW_PYCAPSULE_LIB_PATH')
@@ -37,7 +36,6 @@ def main():
             print("ERROR: SPARROW_PYCAPSULE_LIB_PATH not set")
             return 1
 
-        # Step 2: Check files exist
         step(2, "Checking library files exist")
         helper_file = Path(helper_path)
         main_file = Path(main_path)
@@ -53,21 +51,18 @@ def main():
             print(f"ERROR: Main library not found at {main_file}")
             return 1
 
-        # Step 3: Test ctypes import
         step(3, "Testing ctypes module")
         print("ctypes imported successfully")
         print(f"ctypes.CDLL: {ctypes.CDLL}")
 
-        # Step 4: Try loading main library
         step(4, "Loading sparrow-pycapsule library")
         try:
             main_lib = ctypes.CDLL(str(main_file))
             print("✓ Main library loaded successfully")
         except Exception as e:
             print(f"✗ Failed to load main library: {e}")
             return 1
-        
-        # Step 5: Try loading helper library
+
         step(5, "Loading test_polars_helper library")
         try:
             helper_lib = ctypes.CDLL(str(helper_file))
@@ -76,7 +71,6 @@ def main():
             print(f"✗ Failed to load helper library: {e}")
             return 1
 
-        # Step 6: Check if init_python exists
         step(6, "Checking for init_python function")
         try:
             if hasattr(helper_lib, 'init_python'):
@@ -89,24 +83,6 @@ def main():
             print(f"✗ Error checking for init_python: {e}")
             return 1
 
-        # Step 7: Call init_python
-        step(7, "Calling init_python()")
-        print("About to call init_python()...")
-        sys.stdout.flush()
-        
-        try:
-            helper_lib.init_python()
-            print("✓ init_python() called successfully")
-        except Exception as e:
-            print(f"✗ init_python() failed: {e}")
-            return 1
-        
-        # Step 8: Check Python state
-        step(8, "Checking Python interpreter state")
-        import sys as sys2
-        print(f"Python version: {sys2.version}")
-        print(f"Python initialized: {sys2.version_info}")
-        
         print("\n" + "="*60)
         print("✓ ALL STEPS COMPLETED SUCCESSFULLY")
         print("="*60)

test/test_polars_helper.cpp

@@ -3,7 +3,8 @@
  * @brief C++ helper library for Polars integration tests.
  *
  * This library provides C functions that can be called from Python via ctypes
- * to test the bidirectional data exchange between Polars and sparrow.
+ * to test the bidirectional data exchange between Polars and sparrow using
+ * the sparrow::pycapsule interface.
  */
 
 #include "test_polars_helper.hpp"
@@ -12,52 +13,18 @@
 #include <iostream>
 #include <vector>
 
-#include <Python.h>
-#include <sparrow-pycapsule/config/config.hpp>
-#include <sparrow-pycapsule/pycapsule.hpp>
-
 #include <sparrow/array.hpp>
 #include <sparrow/primitive_array.hpp>
 #include <sparrow/utils/nullable.hpp>
 
-// Export C API functions for ctypes
+#include <sparrow-pycapsule/pycapsule.hpp>
+
 extern "C"
 {
-    /**
-     * @brief Initialize Python interpreter if not already initialized.
-     *
-     * Note: When called from Python (via ctypes), Python is already initialized.
-     * This function only initializes if called from pure C++ context.
-     */
-    void init_python()
-    {
-        // When called from Python via ctypes, Python is already initialized
-        // So this check should always be true, and we do nothing
-        if (Py_IsInitialized())
-        {
-            // Python already initialized - this is the normal case when called from Python
-            return;
-        }
-
-        // Only initialize if we're being called from C++ without Python
-        Py_Initialize();
-    }
-
-    /**
-     * @brief Create a simple test array and return raw Arrow C pointers.
-     *
-     * Instead of creating PyCapsules in C++, we return raw pointers that Python
-     * will wrap in PyCapsules. This avoids Python C API calls from ctypes libraries.
-     *
-     * @param schema_ptr_out Output parameter for ArrowSchema pointer
-     * @param array_ptr_out Output parameter for ArrowArray pointer
-     * @return 0 on success, -1 on error
-     */
-    int create_test_array_as_pointers(void** schema_ptr_out, void** array_ptr_out)
+    int create_test_array_capsules(PyObject** schema_capsule_out, PyObject** array_capsule_out)
     {
         try
         {
-            // Create a test array with nullable integers
             std::vector<sparrow::nullable<int32_t>> values = {
                 sparrow::make_nullable<int32_t>(10, true),
                 sparrow::make_nullable<int32_t>(20, true),
@@ -69,120 +36,101 @@ extern "C"
             sparrow::primitive_array<int32_t> prim_array(std::move(values));
             sparrow::array arr(std::move(prim_array));
 
-            // Extract Arrow C structures
-            auto [arrow_array, arrow_schema] = sparrow::extract_arrow_structures(std::move(arr));
+            auto [schema_capsule, array_capsule] = sparrow::pycapsule::export_array_to_capsules(arr);
 
-            // Allocate on heap and transfer ownership to Python
-            ArrowSchema* schema_ptr = new ArrowSchema(std::move(arrow_schema));
-            ArrowArray* array_ptr = new ArrowArray(std::move(arrow_array));
+            if (schema_capsule == nullptr || array_capsule == nullptr)
+            {
+                std::cerr << "Failed to create PyCapsules\n";
+                Py_XDECREF(schema_capsule);
+                Py_XDECREF(array_capsule);
+                return -1;
+            }
 
-            *schema_ptr_out = schema_ptr;
-            *array_ptr_out = array_ptr;
+            *schema_capsule_out = schema_capsule;
+            *array_capsule_out = array_capsule;
 
             return 0;
         }
         catch (const std::exception& e)
         {
-            std::cerr << "Exception in create_test_array_as_pointers: " << e.what() << std::endl;
+            std::cerr << "Exception in create_test_array_capsules: " << e.what() << '\n';
             return -1;
         }
     }
 
-    /**
-     * @brief Import array from raw Arrow C pointers and return new pointers.
-     *
-     * @param schema_ptr_in Input ArrowSchema pointer
-     * @param array_ptr_in Input ArrowArray pointer
-     * @param schema_ptr_out Output ArrowSchema pointer
-     * @param array_ptr_out Output ArrowArray pointer
-     * @return 0 on success, -1 on error
-     */
-    int
-    roundtrip_array_pointers(void* schema_ptr_in, void* array_ptr_in, void** schema_ptr_out, void** array_ptr_out)
+    int roundtrip_array_capsules(
+        PyObject* schema_capsule_in,
+        PyObject* array_capsule_in,
+        PyObject** schema_capsule_out,
+        PyObject** array_capsule_out
+    )
     {
         try
         {
-            if (schema_ptr_in == nullptr || array_ptr_in == nullptr)
+            if (schema_capsule_in == nullptr || array_capsule_in == nullptr)
             {
-                std::cerr << "Null input pointers" << std::endl;
+                std::cerr << "Null input capsules\n";
                 return -1;
             }
 
-            ArrowSchema* schema_in = static_cast<ArrowSchema*>(schema_ptr_in);
-            ArrowArray* array_in = static_cast<ArrowArray*>(array_ptr_in);
-
-            // Move the data (mark originals as released to prevent double-free)
-            ArrowSchema schema_moved = *schema_in;
-            ArrowArray array_moved = *array_in;
-            schema_in->release = nullptr;
-            array_in->release = nullptr;
-
-            // Import into sparrow
-            sparrow::array arr(std::move(array_moved), std::move(schema_moved));
+            sparrow::array arr = sparrow::pycapsule::import_array_from_capsules(
+                schema_capsule_in,
+                array_capsule_in
+            );
 
-            std::cout << "Roundtrip array size: " << arr.size() << std::endl;
+            std::cout << "Roundtrip array size: " << arr.size() << '\n';
 
-            // Export back out
-            auto [arrow_array_out, arrow_schema_out] = sparrow::extract_arrow_structures(std::move(arr));
+            auto [schema_capsule, array_capsule] = sparrow::pycapsule::export_array_to_capsules(arr);
 
-            ArrowSchema* schema_out = new ArrowSchema(std::move(arrow_schema_out));
-            ArrowArray* array_out = new ArrowArray(std::move(arrow_array_out));
+            if (schema_capsule == nullptr || array_capsule == nullptr)
+            {
+                std::cerr << "Failed to create output PyCapsules\n";
+                Py_XDECREF(schema_capsule);
+                Py_XDECREF(array_capsule);
+                return -1;
+            }
 
-            *schema_ptr_out = schema_out;
-            *array_ptr_out = array_out;
+            *schema_capsule_out = schema_capsule;
+            *array_capsule_out = array_capsule;
 
             return 0;
         }
         catch (const std::exception& e)
         {
-            std::cerr << "Exception in roundtrip_array_pointers: " << e.what() << std::endl;
+            std::cerr << "Exception in roundtrip_array_capsules: " << e.what() << '\n';
             return -1;
         }
     }
 
-    /**
-     * @brief Verify that Arrow C structures have the expected size.
-     *
-     * @param schema_ptr ArrowSchema pointer
-     * @param array_ptr ArrowArray pointer
-     * @param expected_size Expected array size
-     * @return 0 if size matches, -1 otherwise
-     */
-    int verify_array_size_from_pointers(void* schema_ptr, void* array_ptr, size_t expected_size)
+    int verify_array_size_from_capsules(PyObject* schema_capsule, PyObject* array_capsule, size_t expected_size)
     {
         try
         {
-            if (schema_ptr == nullptr || array_ptr == nullptr)
+            if (schema_capsule == nullptr || array_capsule == nullptr)
             {
-                std::cerr << "Null pointers provided" << std::endl;
+                std::cerr << "Null capsules provided\n";
                 return -1;
             }
 
-            ArrowSchema* schema = static_cast<ArrowSchema*>(schema_ptr);
-            ArrowArray* array = static_cast<ArrowArray*>(array_ptr);
-
-            // Move the data (mark originals as released)
-            ArrowSchema schema_moved = *schema;
-            ArrowArray array_moved = *array;
-            schema->release = nullptr;
-            array->release = nullptr;
-
-            sparrow::array arr(std::move(array_moved), std::move(schema_moved));
+            sparrow::array arr = sparrow::pycapsule::import_array_from_capsules(
+                schema_capsule,
+                array_capsule
+            );
 
             if (arr.size() == expected_size)
             {
-                std::cout << "Array size verified: " << arr.size() << std::endl;
+                std::cout << "Array size verified: " << arr.size() << '\n';
                 return 0;
             }
             else
             {
-                std::cerr << "Size mismatch: expected " << expected_size << ", got " << arr.size() << std::endl;
+                std::cerr << "Size mismatch: expected " << expected_size << ", got " << arr.size() << '\n';
                 return -1;
             }
         }
         catch (const std::exception& e)
         {
-            std::cerr << "Exception in verify_array_size_from_pointers: " << e.what() << std::endl;
+            std::cerr << "Exception in verify_array_size_from_capsules: " << e.what() << '\n';
             return -1;
         }
     }