fix: Handle duckdb breaking change in arrow (#3119)

Author: MarcoGorelli

docs/generating_sql.md

@@ -24,7 +24,7 @@ There are several ways to find out.
 
 ## Via DuckDB
 
-You can also generate SQL directly from DuckDB.
+You can generate SQL directly from DuckDB.
 
 ```python exec="1" source="above" session="generating-sql" result="sql"
 import duckdb

narwhals/_duckdb/dataframe.py

@@ -27,6 +27,7 @@
     not_implemented,
     parse_columns_to_drop,
     requires,
+    to_pyarrow_table,
     zip_strict,
 )
 from narwhals.dependencies import get_duckdb
@@ -138,7 +139,7 @@ def collect(
             from narwhals._arrow.dataframe import ArrowDataFrame
 
             return ArrowDataFrame(
-                self.native.arrow(),
+                to_pyarrow_table(self.native.arrow()),
                 validate_backend_version=True,
                 version=self._version,
                 validate_column_names=True,
@@ -255,7 +256,7 @@ def to_pandas(self) -> pd.DataFrame:
 
     def to_arrow(self) -> pa.Table:
         # only if version is v1, keep around for backcompat
-        return self.native.arrow()
+        return self.lazy().collect(Implementation.PYARROW).native  # type: ignore[no-any-return]
 
     def _with_version(self, version: Version) -> Self:
         return self.__class__(self.native, version=version)

narwhals/_ibis/dataframe.py

@@ -15,6 +15,7 @@
     Version,
     not_implemented,
     parse_columns_to_drop,
+    to_pyarrow_table,
     zip_strict,
 )
 from narwhals.exceptions import ColumnNotFoundError, InvalidOperationError
@@ -109,7 +110,7 @@ def collect(
             from narwhals._arrow.dataframe import ArrowDataFrame
 
             return ArrowDataFrame(
-                self.native.to_pyarrow(),
+                to_pyarrow_table(self.native.to_pyarrow()),
                 validate_backend_version=True,
                 version=self._version,
                 validate_column_names=True,

narwhals/_spark_like/dataframe.py

@@ -22,6 +22,7 @@
     generate_temporary_column_name,
     not_implemented,
     parse_columns_to_drop,
+    to_pyarrow_table,
     zip_strict,
 )
 from narwhals.exceptions import InvalidOperationError
@@ -184,7 +185,7 @@ def _collect_to_arrow(self) -> pa.Table:
             pa_schema = self._to_arrow_schema()
             return pa.Table.from_pandas(self.native.toPandas(), schema=pa_schema)
         else:
-            return self.native.toArrow()
+            return to_pyarrow_table(self.native.toArrow())
 
     def _iter_columns(self) -> Iterator[Column]:
         for col in self.columns:

narwhals/_utils.py

@@ -2148,3 +2148,11 @@ def __get__(
     def __get__(self, instance: LazyFrame[Any], owner: Any) -> _LazyAllowedImpl: ...
     def __get__(self, instance: Narwhals[Any] | None, owner: Any) -> Any:
         return self if instance is None else instance._compliant._implementation
+
+
+def to_pyarrow_table(tbl: pa.Table | pa.RecordBatchReader) -> pa.Table:
+    import pyarrow as pa  # ignore-banned-import
+
+    if isinstance(tbl, pa.RecordBatchReader):  # pragma: no cover
+        return pa.Table.from_batches(tbl)
+    return tbl

tests/expr_and_series/reduction_test.py

@@ -91,9 +91,15 @@ def test_empty_scalar_reduction_select(constructor: Constructor) -> None:
     assert_equal_data(result, expected)
 
 
-def test_empty_scalar_reduction_with_columns(constructor: Constructor) -> None:
+def test_empty_scalar_reduction_with_columns(
+    constructor: Constructor, request: pytest.FixtureRequest
+) -> None:
     if "duckdb" in str(constructor) and DUCKDB_VERSION < (1, 3):
         pytest.skip()
+    if any(
+        x in str(constructor) for x in ("duckdb", "sqlframe", "ibis")
+    ) and DUCKDB_VERSION >= (1, 4):
+        request.applymarker(pytest.mark.xfail)
     from itertools import chain
 
     data = {

tests/frame/collect_test.py

@@ -7,7 +7,7 @@
 import narwhals as nw
 from narwhals._utils import Implementation
 from narwhals.dependencies import get_cudf, get_modin, get_polars
-from tests.utils import POLARS_VERSION, Constructor, assert_equal_data
+from tests.utils import DUCKDB_VERSION, POLARS_VERSION, Constructor, assert_equal_data
 
 if TYPE_CHECKING:
     from narwhals._typing import Arrow, Dask, IntoBackend, Modin, Pandas, Polars
@@ -163,7 +163,11 @@ def test_collect_with_kwargs(constructor: Constructor) -> None:
     assert_equal_data(result, expected)
 
 
-def test_collect_empty(constructor: Constructor) -> None:
+def test_collect_empty(constructor: Constructor, request: pytest.FixtureRequest) -> None:
+    if any(
+        x in str(constructor) for x in ("duckdb", "sqlframe", "ibis")
+    ) and DUCKDB_VERSION >= (1, 4):
+        request.applymarker(pytest.mark.xfail)
     df = nw.from_native(constructor({"a": [1, 2, 3]}))
     lf = df.filter(nw.col("a").is_null()).with_columns(b=nw.lit(None)).lazy()
     result = lf.collect()

tests/utils.py

@@ -88,7 +88,7 @@ def assert_equal_data(result: Any, expected: Mapping[str, Any]) -> None:
         and result._compliant_frame._implementation.is_spark_like()
     )
     if is_duckdb:
-        result = from_native(result.to_native().arrow())
+        result = from_native(result.collect("pyarrow"))
     if is_ibis:
         result = from_native(result.to_native().to_pyarrow())
     if hasattr(result, "collect"):