wip

Author: Alex-PLACET

.github/workflows/linux.yml

@@ -50,10 +50,10 @@ jobs:
         working-directory: build
         run: cmake --build . --target run_tests_with_junit_report
 
-      - name: Run Polars integration tests
+      - name: Run sparrow integration tests
         if: matrix.build_shared == 'ON'
         working-directory: build
-        run: cmake --build . --target run_polars_tests_direct
+        run: cmake --build . --target run_sparrow_tests_direct
 
       - name: Install
         working-directory: build

.github/workflows/osx.yml

@@ -55,10 +55,10 @@ jobs:
         working-directory: build
         run: cmake --build . --target run_tests_with_junit_report
 
-      - name: Run Polars integration tests
+      - name: Run Sparrow integration tests
         if: matrix.build_shared == 'ON'
         working-directory: build
-        run: cmake --build . --target run_polars_tests_direct
+        run: cmake --build . --target run_sparrow_tests_direct
 
       - name: Install
         working-directory: build

.github/workflows/windows.yml

@@ -55,10 +55,10 @@ jobs:
         run: |
           cmake --build . --config ${{ matrix.build_type }} --target run_tests_with_junit_report
 
-      - name: Run Polars integration tests
+      - name: Run Sparrow integration tests
         if: matrix.build_shared == 'ON'
         working-directory: build
-        run: cmake --build . --config ${{ matrix.build_type }} --target run_polars_tests_direct
+        run: cmake --build . --config ${{ matrix.build_type }} --target run_sparrow_tests_direct
 
       - name: Install
         working-directory: build

CMakeLists.txt

@@ -151,10 +151,12 @@ set(CMAKE_ARCHIVE_OUTPUT_DIRECTORY_RELEASE "${BINARY_BUILD_DIR}")
 set(SPARROW_PYCAPSULE_HEADERS
     ${SPARROW_PYCAPSULE_INCLUDE_DIR}/sparrow-pycapsule/config/sparrow_pycapsule_version.hpp
     ${SPARROW_PYCAPSULE_INCLUDE_DIR}/sparrow-pycapsule/pycapsule.hpp
+    ${SPARROW_PYCAPSULE_INCLUDE_DIR}/sparrow-pycapsule/sparrow_array_python_class.hpp
 )
 
 set(SPARROW_PYCAPSULE_SOURCES
     src/pycapsule.cpp
+    src/sparrow_array_python_class.cpp
 )
 
 option(SPARROW_PYCAPSULE_BUILD_SHARED "Build sparrow pycapsule as a shared library" ON)

include/sparrow-pycapsule/SparrowPythonClass.hpp renamed to include/sparrow-pycapsule/sparrow_array_python_class.hpp

@@ -1,4 +1,4 @@
-#include "sparrow-pycapsule/SparrowPythonClass.hpp"
+#include "sparrow-pycapsule/sparrow_array_python_class.hpp"
 
 #include <new>
 #include <utility>
@@ -67,6 +67,81 @@ namespace sparrow::pycapsule
         return PyLong_FromSize_t(self->arr->size());
     }
 
+    /**
+     * @brief Constructor for SparrowArray.
+     *
+     * Accepts an object implementing __arrow_c_array__ and imports it.
+     */
+    static PyObject* SparrowArray_new(PyTypeObject* type, PyObject* args, PyObject* kwargs)
+    {
+        static const char* kwlist[] = {"arrow_array", nullptr};
+        PyObject* arrow_array = nullptr;
+
+        if (!PyArg_ParseTupleAndKeywords(args, kwargs, "O", const_cast<char**>(kwlist), &arrow_array))
+        {
+            return nullptr;
+        }
+
+        // Get __arrow_c_array__ method from the input object
+        PyObject* arrow_c_array_method = PyObject_GetAttrString(arrow_array, "__arrow_c_array__");
+        if (arrow_c_array_method == nullptr)
+        {
+            PyErr_SetString(
+                PyExc_TypeError,
+                "Input object must implement __arrow_c_array__ (ArrowArrayExportable protocol)"
+            );
+            return nullptr;
+        }
+
+        // Call __arrow_c_array__() to get the capsules
+        PyObject* capsules = PyObject_CallNoArgs(arrow_c_array_method);
+        Py_DECREF(arrow_c_array_method);
+
+        if (capsules == nullptr)
+        {
+            return nullptr;
+        }
+
+        // Unpack the tuple (schema_capsule, array_capsule)
+        if (!PyTuple_Check(capsules) || PyTuple_Size(capsules) != 2)
+        {
+            Py_DECREF(capsules);
+            PyErr_SetString(PyExc_TypeError, "__arrow_c_array__ must return a tuple of 2 elements");
+            return nullptr;
+        }
+
+        PyObject* schema_capsule = PyTuple_GetItem(capsules, 0);
+        PyObject* array_capsule = PyTuple_GetItem(capsules, 1);
+
+        try
+        {
+            sparrow::array arr = import_array_from_capsules(schema_capsule, array_capsule);
+            Py_DECREF(capsules);
+
+            // Allocate the object
+            SparrowArrayObject* self = reinterpret_cast<SparrowArrayObject*>(type->tp_alloc(type, 0));
+            if (self == nullptr)
+            {
+                return nullptr;
+            }
+
+            self->arr = new sparrow::array(std::move(arr));
+            return reinterpret_cast<PyObject*>(self);
+        }
+        catch (const std::bad_alloc&)
+        {
+            Py_DECREF(capsules);
+            PyErr_NoMemory();
+            return nullptr;
+        }
+        catch (const std::exception& e)
+        {
+            Py_DECREF(capsules);
+            PyErr_SetString(PyExc_RuntimeError, e.what());
+            return nullptr;
+        }
+    }
+
     static PyMethodDef SparrowArray_methods[] = {
         {"__arrow_c_array__",
          reinterpret_cast<PyCFunction>(SparrowArray_arrow_c_array),
@@ -99,12 +174,17 @@ namespace sparrow::pycapsule
         .tp_dealloc = reinterpret_cast<destructor>(SparrowArray_dealloc),
         .tp_flags = Py_TPFLAGS_DEFAULT,
         .tp_doc = PyDoc_STR(
-            "SparrowArray - Arrow array wrapper implementing __arrow_c_array__.\n\n"
+            "SparrowArray(arrow_array) - Arrow array wrapper implementing __arrow_c_array__.\n\n"
             "This class wraps a sparrow array and implements the Arrow PyCapsule\n"
             "Interface (ArrowArrayExportable protocol), allowing it to be passed\n"
-            "directly to libraries like Polars via pl.from_arrow()."
+            "directly to libraries like Polars via pl.from_arrow().\n\n"
+            "Parameters\n"
+            "----------\n"
+            "arrow_array : ArrowArrayExportable\n"
+            "    An object implementing __arrow_c_array__ (e.g., PyArrow array)."
         ),
         .tp_methods = SparrowArray_methods,
+        .tp_new = SparrowArray_new,
     };
 
     static bool type_initialized = false;

src/SparrowPythonClass.cpp renamed to src/sparrow_array_python_class.cpp

@@ -85,28 +85,23 @@ add_custom_target(run_tests_with_junit_report
 
 set_target_properties(run_tests_with_junit_report PROPERTIES FOLDER "Tests utilities")
 
-# Polars integration test helper - Native Python Extension Module
-# ===============================================================
-# This builds a proper Python extension module (.cpython-*.so) that shares
-# the same Python runtime as the interpreter, avoiding dual-runtime issues.
+Python_add_library(test_sparrow_helper MODULE test_sparrow_helper_module.cpp)
 
-Python_add_library(test_polars_helper MODULE test_polars_helper_module.cpp)
-
-target_link_libraries(test_polars_helper
+target_link_libraries(test_sparrow_helper
     PRIVATE
         sparrow-pycapsule
         sparrow::sparrow
 )
 
-target_compile_features(test_polars_helper PRIVATE cxx_std_20)
+target_compile_features(test_sparrow_helper PRIVATE cxx_std_20)
 
 if(MSVC)
-    target_compile_options(test_polars_helper PRIVATE /W4)
+    target_compile_options(test_sparrow_helper PRIVATE /W4)
 else()
-    target_compile_options(test_polars_helper PRIVATE -Wall -Wextra -Wpedantic)
+    target_compile_options(test_sparrow_helper PRIVATE -Wall -Wextra -Wpedantic)
 endif()
 
-set_target_properties(test_polars_helper PROPERTIES
+set_target_properties(test_sparrow_helper PROPERTIES
     FOLDER tests
     LIBRARY_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/bin/${CMAKE_BUILD_TYPE}
     # Python modules must not have a debug suffix - Python won't find them
@@ -120,77 +115,76 @@ find_package(Python COMPONENTS Interpreter QUIET)
 if(Python_Interpreter_FOUND)
     # Add a test that runs the Python integration script
     add_test(
-        NAME test_polars_integration
-        COMMAND ${Python_EXECUTABLE} ${CMAKE_CURRENT_SOURCE_DIR}/test_polars_integration.py
+        NAME test_sparrow_integration
+        COMMAND ${Python_EXECUTABLE} ${CMAKE_CURRENT_SOURCE_DIR}/test_sparrow_integration.py
         WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}
     )
 
     # Set environment variables so Python can find the libraries
     # Use generator expressions to get the actual library paths from targets
-    set_tests_properties(test_polars_integration PROPERTIES
-        ENVIRONMENT "TEST_POLARS_HELPER_LIB_PATH=$<TARGET_FILE:test_polars_helper>;SPARROW_PYCAPSULE_LIB_PATH=$<TARGET_FILE:sparrow-pycapsule>"
+    set_tests_properties(test_sparrow_integration PROPERTIES
+        ENVIRONMENT "TEST_SPARROW_HELPER_LIB_PATH=$<TARGET_FILE:test_sparrow_helper>;SPARROW_PYCAPSULE_LIB_PATH=$<TARGET_FILE:sparrow-pycapsule>"
         TIMEOUT 300
-        DEPENDS test_polars_helper
+        DEPENDS test_sparrow_helper
     )
 
-    message(STATUS "Added Polars integration test (Python ${Python_VERSION})")
+    message(STATUS "Added sparrow integration test (Python ${Python_VERSION})")
 else()
-    message(WARNING "Python interpreter not found, skipping Polars integration test")
+    message(WARNING "Python interpreter not found, skipping sparrow integration test")
 endif()
 
-# Custom target to run Polars tests directly (with better output)
 if(Python_Interpreter_FOUND)
-    add_custom_target(run_polars_tests_direct
+    add_custom_target(run_sparrow_tests_direct
         COMMAND ${CMAKE_COMMAND} -E echo "=================================="
-        COMMAND ${CMAKE_COMMAND} -E echo "Polars Integration Test Runner"
+        COMMAND ${CMAKE_COMMAND} -E echo "Sparrow Integration Test Runner"
         COMMAND ${CMAKE_COMMAND} -E echo "=================================="
         COMMAND ${CMAKE_COMMAND} -E echo ""
         COMMAND ${CMAKE_COMMAND} -E echo "Checking Python dependencies..."
         COMMAND ${Python_EXECUTABLE} -c "import polars" || ${CMAKE_COMMAND} -E cmake_echo_color --red "ERROR: polars not installed. Install with: pip install polars"
         COMMAND ${Python_EXECUTABLE} -c "import pyarrow" || ${CMAKE_COMMAND} -E cmake_echo_color --red "ERROR: pyarrow not installed. Install with: pip install pyarrow"
         COMMAND ${CMAKE_COMMAND} -E echo ""
         COMMAND ${CMAKE_COMMAND} -E echo "Library paths:"
-        COMMAND ${CMAKE_COMMAND} -E echo "  TEST_POLARS_HELPER_LIB_PATH=$<TARGET_FILE:test_polars_helper>"
+        COMMAND ${CMAKE_COMMAND} -E echo "  TEST_SPARROW_HELPER_LIB_PATH=$<TARGET_FILE:test_sparrow_helper>"
         COMMAND ${CMAKE_COMMAND} -E echo "  SPARROW_PYCAPSULE_LIB_PATH=$<TARGET_FILE:sparrow-pycapsule>"
         COMMAND ${CMAKE_COMMAND} -E echo ""
         COMMAND ${CMAKE_COMMAND} -E echo "Running tests..."
         COMMAND ${CMAKE_COMMAND} -E echo ""
         COMMAND ${CMAKE_COMMAND} -E env 
-            "TEST_POLARS_HELPER_LIB_PATH=$<TARGET_FILE:test_polars_helper>"
+            "TEST_SPARROW_HELPER_LIB_PATH=$<TARGET_FILE:test_sparrow_helper>"
             "SPARROW_PYCAPSULE_LIB_PATH=$<TARGET_FILE:sparrow-pycapsule>"
-            ${Python_EXECUTABLE} ${CMAKE_CURRENT_SOURCE_DIR}/test_polars_integration.py
+            ${Python_EXECUTABLE} ${CMAKE_CURRENT_SOURCE_DIR}/test_sparrow_integration.py
         COMMAND ${CMAKE_COMMAND} -E echo ""
-        DEPENDS test_polars_helper sparrow-pycapsule
+        DEPENDS test_sparrow_helper sparrow-pycapsule
         WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}
-        COMMENT "Running Polars integration tests directly"
+        COMMENT "Running Sparrow integration tests directly"
         USES_TERMINAL
     )
 
-    set_target_properties(run_polars_tests_direct PROPERTIES FOLDER "Tests utilities")
+    set_target_properties(run_sparrow_tests_direct PROPERTIES FOLDER "Tests utilities")
 
-    # Custom target to check Polars dependencies
-    add_custom_target(check_polars_deps
-        COMMAND ${CMAKE_COMMAND} -E echo "Checking Polars integration test dependencies..."
+    # Custom target to check Sparrow dependencies
+    add_custom_target(check_sparrow_deps
+        COMMAND ${CMAKE_COMMAND} -E echo "Checking Sparrow integration test dependencies..."
         COMMAND ${Python_EXECUTABLE} ${CMAKE_CURRENT_SOURCE_DIR}/check_deps.py
         COMMAND ${CMAKE_COMMAND} -E echo "Environment variables that will be set:"
-        COMMAND ${CMAKE_COMMAND} -E echo "  TEST_POLARS_HELPER_LIB_PATH=$<TARGET_FILE:test_polars_helper>"
+        COMMAND ${CMAKE_COMMAND} -E echo "  TEST_SPARROW_HELPER_LIB_PATH=$<TARGET_FILE:test_sparrow_helper>"
         COMMAND ${CMAKE_COMMAND} -E echo "  SPARROW_PYCAPSULE_LIB_PATH=$<TARGET_FILE:sparrow-pycapsule>"
         WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}
-        COMMENT "Checking Polars test dependencies"
+        COMMENT "Checking Sparrow test dependencies"
         USES_TERMINAL
     )
 
-    set_target_properties(check_polars_deps PROPERTIES FOLDER "Tests utilities")
+    set_target_properties(check_sparrow_deps PROPERTIES FOLDER "Tests utilities")
 
     # Minimal library loading test for debugging segfaults
     add_custom_target(test_library_load
         COMMAND ${CMAKE_COMMAND} -E env 
-            "TEST_POLARS_HELPER_LIB_PATH=$<TARGET_FILE:test_polars_helper>"
+            "TEST_SPARROW_HELPER_LIB_PATH=$<TARGET_FILE:test_sparrow_helper>"
             "SPARROW_PYCAPSULE_LIB_PATH=$<TARGET_FILE:sparrow-pycapsule>"
             "PYTHONUNBUFFERED=1"
             ${Python_EXECUTABLE} -u ${CMAKE_CURRENT_SOURCE_DIR}/test_library_load.py
         DEPENDS 
-            test_polars_helper
+            test_sparrow_helper
             sparrow-pycapsule
         WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}
         COMMENT "Testing library loading step-by-step"

test/CMakeLists.txt

@@ -0,0 +1,72 @@
+"""
+Python helper module for sparrow-pycapsule integration tests.
+
+This module provides helper functions that wrap the C++ SparrowArray class,
+making it easy to create test arrays and perform roundtrip operations.
+"""
+
+from __future__ import annotations
+
+import sys
+import os
+from pathlib import Path
+from typing import Any, Protocol, Tuple
+
+class ArrowArrayExportable(Protocol):
+    """Protocol for objects implementing the Arrow PyCapsule Interface."""
+    
+    def __arrow_c_array__(
+        self, requested_schema: Any = None
+    ) -> Tuple[Any, Any]:
+        """Export the array as Arrow PyCapsules.
+        
+        Returns
+        -------
+        Tuple[Any, Any]
+            A tuple of (schema_capsule, array_capsule).
+        """
+        ...
+
+
+class SparrowArrayType(ArrowArrayExportable, Protocol):
+    """Type definition for SparrowArray from C++ extension."""
+    
+    def size(self) -> int:
+        """Get the number of elements in the array."""
+        ...
+
+
+def _setup_module_path() -> None:
+    """Add the build directory to Python path so we can import test_sparrow_helper."""
+    # Check for environment variable first
+    helper_path = os.environ.get('TEST_SPARROW_HELPER_PATH')
+    if helper_path:
+        module_dir = Path(helper_path).parent
+        if module_dir.exists():
+            sys.path.insert(0, str(module_dir))
+            return
+
+    # Try to find in build directory
+    test_dir = Path(__file__).parent
+    build_dirs = [
+        test_dir.parent / "build" / "bin" / "Debug",
+        test_dir.parent / "build" / "bin" / "Release",
+        test_dir.parent / "build" / "bin",
+    ]
+
+    for build_dir in build_dirs:
+        if build_dir.exists():
+            sys.path.insert(0, str(build_dir))
+            return
+
+    raise ImportError(
+        "Could not find test_sparrow_helper module. "
+        "Build the project first or set TEST_SPARROW_HELPER_PATH."
+    )
+
+
+# Set up module path and import the C++ module
+_setup_module_path()
+
+# Import the native Python extension module that provides SparrowArray
+from test_sparrow_helper import SparrowArray  # noqa: E402