feat: support pyarrow 19.0 (zen-xu#198)

* build: upgrade pyarrow min version to 19.0

* feat: support pyarrow 19.0

* omit mypy bool8 override error

Author: zen-xu

pixi.lock

@@ -89,6 +89,8 @@ from pyarrow.lib import (
     Time64Type,
     DurationType,
     FixedSizeBinaryType,
+    Decimal32Type,
+    Decimal64Type,
     Decimal128Type,
     Decimal256Type,
     BaseExtensionType,
@@ -403,6 +405,8 @@ __all__ = [
     "Time64Type",
     "DurationType",
     "FixedSizeBinaryType",
+    "Decimal32Type",
+    "Decimal64Type",
     "Decimal128Type",
     "Decimal256Type",
     "BaseExtensionType",

pyarrow-stubs/__init__.pyi

@@ -38,7 +38,15 @@ from pyarrow._stubs_typing import (
     SupportArrowArray,
     SupportArrowDeviceArray,
 )
-from pyarrow.lib import Buffer, MemoryPool, MonthDayNano, Tensor, _Weakrefable
+from pyarrow.lib import (
+    Buffer,
+    Device,
+    MemoryManager,
+    MemoryPool,
+    MonthDayNano,
+    Tensor,
+    _Weakrefable,
+)
 
 from . import scalar, types
 from .device import DeviceAllocationType
@@ -673,6 +681,14 @@ def nulls(
     size: int, types: types.Float64Type, memory_pool: MemoryPool | None = None
 ) -> DoubleArray: ...
 @overload
+def nulls(
+    size: int, types: types.Decimal32Type, memory_pool: MemoryPool | None = None
+) -> Decimal128Array: ...
+@overload
+def nulls(
+    size: int, types: types.Decimal64Type, memory_pool: MemoryPool | None = None
+) -> Decimal128Array: ...
+@overload
 def nulls(
     size: int, types: types.Decimal128Type, memory_pool: MemoryPool | None = None
 ) -> Decimal128Array: ...
@@ -1165,6 +1181,7 @@ class Array(_PandasConvertible[pd.Series], Generic[_Scalar_CoT]):
     @property
     def offset(self) -> int: ...
     def buffers(self) -> list[Buffer | None]: ...
+    def copy_to(self, destination: MemoryManager | Device) -> Self: ...
     def _export_to_c(self, out_ptr: int, out_schema_ptr: int = 0) -> None: ...
     @classmethod
     def _import_from_c(cls, in_ptr: int, type: int | DataType) -> Self: ...
@@ -1214,6 +1231,8 @@ class HalfFloatArray(FloatingPointArray[scalar.HalfFloatScalar]): ...
 class FloatArray(FloatingPointArray[scalar.FloatScalar]): ...
 class DoubleArray(FloatingPointArray[scalar.DoubleScalar]): ...
 class FixedSizeBinaryArray(Array[scalar.FixedSizeBinaryScalar]): ...
+class Decimal32Array(FixedSizeBinaryArray): ...
+class Decimal64Array(FixedSizeBinaryArray): ...
 class Decimal128Array(FixedSizeBinaryArray): ...
 class Decimal256Array(FixedSizeBinaryArray): ...
 
@@ -1536,6 +1555,7 @@ class StructArray(Array[scalar.StructScalar]):
         fields: list[Field] | None = None,
         mask=None,
         memory_pool: MemoryPool | None = None,
+        type: types.StructType | None = None,
     ) -> StructArray: ...
     def sort(self, order: Order = "ascending", by: str | None = None, **kwargs) -> StructArray: ...
 
@@ -1587,11 +1607,23 @@ class ExtensionArray(Array[scalar.ExtensionScalar], Generic[_ArrayT]):
         typ: types.BaseExtensionType, storage: _ArrayT
     ) -> ExtensionArray[_ArrayT]: ...
 
+class JsonArray(ExtensionArray[_ArrayT]): ...
+class UuidArray(ExtensionArray[_ArrayT]): ...
+
 class FixedShapeTensorArray(ExtensionArray[_ArrayT]):
     def to_numpy_ndarray(self) -> np.ndarray: ...
     def to_tensor(self) -> Tensor: ...
-    @staticmethod
-    def from_numpy_ndarray(obj: np.ndarray) -> FixedShapeTensorArray: ...
+    @classmethod
+    def from_numpy_ndarray(cls, obj: np.ndarray) -> Self: ...
+
+class OpaqueArray(ExtensionArray[_ArrayT]): ...
+
+class Bool8Array(ExtensionArray):
+    def to_numpy(self, zero_copy_only: bool = ..., writable: bool = ...) -> np.ndarray: ...
+    @classmethod
+    def from_storage(cls, storage: Int8Array) -> Self: ...  # type: ignore[override]
+    @classmethod
+    def from_numpy(cls, obj: np.ndarray) -> Self: ...
 
 def concat_arrays(arrays: Iterable[_ArrayT], memory_pool: MemoryPool | None = None) -> _ArrayT: ...
 def _empty_array(type: _DataTypeT) -> Array[scalar.Scalar[_DataTypeT]]: ...

pyarrow-stubs/__lib_pxi/array.pyi

@@ -120,6 +120,8 @@ class Int64Scalar(Scalar[types.Int64Type]): ...
 class HalfFloatScalar(Scalar[types.Float16Type]): ...
 class FloatScalar(Scalar[types.Float32Type]): ...
 class DoubleScalar(Scalar[types.Float64Type]): ...
+class Decimal32Scalar(Scalar[types.Decimal32Type]): ...
+class Decimal64Scalar(Scalar[types.Decimal64Type]): ...
 class Decimal128Scalar(Scalar[types.Decimal128Type]): ...
 class Decimal256Scalar(Scalar[types.Decimal256Type]): ...
 class Date32Scalar(Scalar[types.Date32Type]): ...
@@ -352,6 +354,20 @@ def scalar(
     memory_pool: MemoryPool | None = None,
 ) -> ListScalar[types.ListType[types.Float64Type]]: ...
 @overload
+def scalar(
+    value: CollectionValue[Decimal],
+    *,
+    from_pandas: bool | None = None,
+    memory_pool: MemoryPool | None = None,
+) -> ListScalar[types.ListType[types.Decimal32Type]]: ...
+@overload
+def scalar(
+    value: CollectionValue[Decimal],
+    *,
+    from_pandas: bool | None = None,
+    memory_pool: MemoryPool | None = None,
+) -> ListScalar[types.ListType[types.Decimal64Type]]: ...
+@overload
 def scalar(
     value: CollectionValue[Decimal],
     *,

pyarrow-stubs/__lib_pxi/scalar.pyi

@@ -44,7 +44,7 @@ from pyarrow._stubs_typing import (
 )
 from pyarrow.compute import Expression
 from pyarrow.interchange.dataframe import _PyArrowDataFrame
-from pyarrow.lib import Field, MemoryPool, MonthDayNano, Schema
+from pyarrow.lib import Device, Field, MemoryManager, MemoryPool, MonthDayNano, Schema
 
 from . import scalar
 from .array import Array, StructArray, _CastAs, _PandasConvertible
@@ -145,6 +145,8 @@ class ChunkedArray(_PandasConvertible[pd.Series], Generic[_Scalar_CoT]):
     def __arrow_c_stream__(self, requested_schema=None) -> Any: ...
     @classmethod
     def _import_from_c_capsule(cls, stream) -> Self: ...
+    @property
+    def is_cpu(self) -> bool: ...
 
 @overload
 def chunked_array(
@@ -507,6 +509,7 @@ class RecordBatch(_Tabular[Array]):
     def device_type(self) -> DeviceAllocationType: ...
     @property
     def is_cpu(self) -> bool: ...
+    def copy_to(self, destination: MemoryManager | Device) -> Self: ...
 
 def table_to_blocks(options, table: Table, categories, extension_columns): ...
 
@@ -594,6 +597,8 @@ class Table(_Tabular[ChunkedArray]):
         right_by: str | list[str] | None = None,
     ) -> Self: ...
     def __arrow_c_stream__(self, requested_schema=None): ...
+    @property
+    def is_cpu(self) -> bool: ...
 
 def record_batch(
     data: dict[str, list | Array]
@@ -641,6 +646,10 @@ class TableGroupBy:
     @property
     def _use_threads(self) -> bool: ...
 
+def concat_batches(
+    recordbatches: Iterable[RecordBatch], memory_pool: MemoryPool | None = None
+) -> RecordBatch: ...
+
 __all__ = [
     "ChunkedArray",
     "chunked_array",
@@ -652,4 +661,5 @@ __all__ = [
     "table",
     "concat_tables",
     "TableGroupBy",
+    "concat_batches",
 ]