chore: pandas-nightly and duckdb-nightly fixes (#3158)

Author: MarcoGorelli

.github/workflows/extremes.yml

@@ -29,7 +29,7 @@ jobs:
           cache-suffix: min-versions-${{ matrix.python-version }}
           cache-dependency-glob: "pyproject.toml"
       - name: install-minimum-versions
-        run: uv pip install pipdeptree tox virtualenv setuptools pandas==1.1.3 polars==0.20.4 numpy==1.19.3 pyarrow==13.0.0 "pyarrow-stubs<17" scipy==1.6.0 scikit-learn==1.1.0 duckdb==1.0 tzdata --system
+        run: uv pip install pipdeptree tox virtualenv setuptools pandas==1.1.3 polars==0.20.4 numpy==1.19.3 pyarrow==13.0.0 "pyarrow-stubs<17" scipy==1.6.0 scikit-learn==1.1.0 duckdb==1.1 tzdata --system
       - name: install-reqs
         run: |
           uv pip install -e . --group tests --system
@@ -44,9 +44,9 @@ jobs:
           echo "$DEPS" | grep 'pyarrow==13.0.0'
           echo "$DEPS" | grep 'scipy==1.6.0'
           echo "$DEPS" | grep 'scikit-learn==1.1.0'
-          echo "$DEPS" | grep 'duckdb==1.0'
+          echo "$DEPS" | grep 'duckdb==1.1'
       - name: Run pytest
-        run: pytest tests --cov=narwhals --cov=tests --cov-fail-under=50 --runslow --constructors=pandas,pyarrow,polars[eager],polars[lazy]
+        run: pytest tests --cov=narwhals --cov=tests --cov-fail-under=50 --runslow --constructors=pandas,pyarrow,polars[eager],polars[lazy],duckdb
 
   pretty_old_versions:
     strategy:
@@ -66,7 +66,7 @@ jobs:
           cache-suffix: pretty-old-versions-${{ matrix.python-version }}
           cache-dependency-glob: "pyproject.toml"
       - name: install-pretty-old-versions
-        run: uv pip install pipdeptree tox virtualenv setuptools pandas==1.1.5 polars==0.20.4 numpy==1.19.3 pyarrow==14.0.0 "pyarrow-stubs<17" scipy==1.6.0 scikit-learn==1.1.0 duckdb==1.0 tzdata --system
+        run: uv pip install pipdeptree tox virtualenv setuptools pandas==1.1.5 polars==0.20.4 numpy==1.19.3 pyarrow==14.0.0 "pyarrow-stubs<17" scipy==1.6.0 scikit-learn==1.1.0 duckdb==1.2 tzdata --system
       - name: install-reqs
         run: uv pip install -e . --group tests --system
       - name: show-deps
@@ -82,9 +82,9 @@ jobs:
           echo "$DEPS" | grep 'pyarrow==14.0.0'
           echo "$DEPS" | grep 'scipy==1.6.0'
           echo "$DEPS" | grep 'scikit-learn==1.1.0'
-          echo "$DEPS" | grep 'duckdb==1.0'
+          echo "$DEPS" | grep 'duckdb==1.2'
       - name: Run pytest
-        run: pytest tests --cov=narwhals --cov=tests --cov-fail-under=50 --runslow --constructors=pandas,pyarrow,polars[eager],polars[lazy]
+        run: pytest tests --cov=narwhals --cov=tests --cov-fail-under=50 --runslow --constructors=pandas,pyarrow,polars[eager],polars[lazy],duckdb
 
   not_so_old_versions:
     strategy:
@@ -104,7 +104,7 @@ jobs:
           cache-suffix: not-so-old-versions-${{ matrix.python-version }}
           cache-dependency-glob: "pyproject.toml"
       - name: install-not-so-old-versions
-        run: uv pip install tox virtualenv setuptools pandas==2.0.3 polars==0.20.8 numpy==1.24.4 pyarrow==15.0.0 "pyarrow-stubs<17" scipy==1.8.0 scikit-learn==1.3.0 duckdb==1.0 dask[dataframe]==2024.10 tzdata --system
+        run: uv pip install tox virtualenv setuptools pandas==2.0.3 polars==0.20.8 numpy==1.24.4 pyarrow==15.0.0 "pyarrow-stubs<17" scipy==1.8.0 scikit-learn==1.3.0 duckdb==1.3 dask[dataframe]==2024.10 tzdata --system
       - name: install-reqs
         run: uv pip install -e . --group tests --system
       - name: show-deps
@@ -119,9 +119,9 @@ jobs:
           echo "$DEPS" | grep 'scipy==1.8.0'
           echo "$DEPS" | grep 'scikit-learn==1.3.0'
           echo "$DEPS" | grep 'dask==2024.10'
-          echo "$DEPS" | grep 'duckdb==1.0'
+          echo "$DEPS" | grep 'duckdb==1.3'
       - name: Run pytest
-        run: pytest tests --cov=narwhals --cov=tests --cov-fail-under=50 --runslow --constructors=pandas,pyarrow,polars[eager],polars[lazy],dask
+        run: pytest tests --cov=narwhals --cov=tests --cov-fail-under=50 --runslow --constructors=pandas,pyarrow,polars[eager],polars[lazy],dask,duckdb
 
   nightlies:
     strategy:

narwhals/_duckdb/dataframe.py

@@ -41,14 +41,14 @@
     import pandas as pd
     import pyarrow as pa
     from duckdb import Expression
-    from duckdb.typing import DuckDBPyType
     from typing_extensions import Self, TypeIs
 
     from narwhals._compliant.typing import CompliantDataFrameAny
     from narwhals._duckdb.expr import DuckDBExpr
     from narwhals._duckdb.group_by import DuckDBGroupBy
     from narwhals._duckdb.namespace import DuckDBNamespace
     from narwhals._duckdb.series import DuckDBInterchangeSeries
+    from narwhals._duckdb.utils import duckdb_dtypes
     from narwhals._typing import _EagerAllowedImpl
     from narwhals._utils import _LimitedContext
     from narwhals.dataframe import LazyFrame
@@ -76,7 +76,7 @@ def __init__(
     ) -> None:
         self._native_frame: duckdb.DuckDBPyRelation = df
         self._version = version
-        self._cached_native_schema: dict[str, DuckDBPyType] | None = None
+        self._cached_native_schema: dict[str, duckdb_dtypes.DuckDBPyType] | None = None
         self._cached_columns: list[str] | None = None
         if validate_backend_version:
             self._validate_backend_version()

narwhals/_duckdb/expr_list.py

@@ -5,6 +5,7 @@
 from narwhals._compliant import LazyExprNamespace
 from narwhals._compliant.any_namespace import ListNamespace
 from narwhals._duckdb.utils import F, lit, when
+from narwhals._utils import requires
 
 if TYPE_CHECKING:
     from duckdb import Expression
@@ -19,6 +20,7 @@ class DuckDBExprListNamespace(
     def len(self) -> DuckDBExpr:
         return self.compliant._with_elementwise(lambda expr: F("len", expr))
 
+    @requires.backend_version((1, 3))  # bugged before 1.3
     def unique(self) -> DuckDBExpr:
         def func(expr: Expression) -> Expression:
             expr_distinct = F("list_distinct", expr)

narwhals/_duckdb/namespace.py

@@ -7,7 +7,6 @@
 
 import duckdb
 from duckdb import CoalesceOperator, Expression
-from duckdb.typing import BIGINT, VARCHAR
 
 from narwhals._duckdb.dataframe import DuckDBLazyFrame
 from narwhals._duckdb.expr import DuckDBExpr
@@ -16,6 +15,7 @@
     DeferredTimeZone,
     F,
     concat_str,
+    duckdb_dtypes,
     function,
     lit,
     narwhals_to_native_dtype,
@@ -108,9 +108,9 @@ def func(df: DuckDBLazyFrame) -> list[Expression]:
                 cols_separated = [
                     y
                     for x in [
-                        (col.cast(VARCHAR),)
+                        (col.cast(duckdb_dtypes.VARCHAR),)
                         if i == len(cols) - 1
-                        else (col.cast(VARCHAR), lit(separator))
+                        else (col.cast(duckdb_dtypes.VARCHAR), lit(separator))
                         for i, col in enumerate(cols)
                     ]
                     for y in x
@@ -130,7 +130,9 @@ def func(cols: Iterable[Expression]) -> Expression:
             cols = list(cols)
             return reduce(
                 operator.add, (CoalesceOperator(col, lit(0)) for col in cols)
-            ) / reduce(operator.add, (col.isnotnull().cast(BIGINT) for col in cols))
+            ) / reduce(
+                operator.add, (col.isnotnull().cast(duckdb_dtypes.BIGINT) for col in cols)
+            )
 
         return self._expr._from_elementwise_horizontal_op(func, *exprs)
 

narwhals/_duckdb/utils.py

@@ -4,9 +4,13 @@
 from typing import TYPE_CHECKING
 
 import duckdb
-import duckdb.typing as duckdb_dtypes
 from duckdb import Expression
-from duckdb.typing import DuckDBPyType
+
+try:
+    import duckdb.sqltypes as duckdb_dtypes
+except ModuleNotFoundError:
+    # DuckDB pre 1.3
+    import duckdb.typing as duckdb_dtypes
 
 from narwhals._utils import Version, isinstance_or_issubclass, zip_strict
 from narwhals.exceptions import ColumnNotFoundError
@@ -131,7 +135,9 @@ def time_zone(self) -> str:
 
 
 def native_to_narwhals_dtype(
-    duckdb_dtype: DuckDBPyType, version: Version, deferred_time_zone: DeferredTimeZone
+    duckdb_dtype: duckdb_dtypes.DuckDBPyType,
+    version: Version,
+    deferred_time_zone: DeferredTimeZone,
 ) -> DType:
     duckdb_dtype_id = duckdb_dtype.id
     dtypes = version.dtypes
@@ -216,7 +222,7 @@ def _non_nested_native_to_narwhals_dtype(duckdb_dtype_id: str, version: Version)
 
 
 dtypes = Version.MAIN.dtypes
-NW_TO_DUCKDB_DTYPES: Mapping[type[DType], DuckDBPyType] = {
+NW_TO_DUCKDB_DTYPES: Mapping[type[DType], duckdb_dtypes.DuckDBPyType] = {
     dtypes.Float64: duckdb_dtypes.DOUBLE,
     dtypes.Float32: duckdb_dtypes.FLOAT,
     dtypes.Binary: duckdb_dtypes.BLOB,
@@ -228,14 +234,14 @@ def _non_nested_native_to_narwhals_dtype(duckdb_dtype_id: str, version: Version)
     dtypes.Int16: duckdb_dtypes.SMALLINT,
     dtypes.Int32: duckdb_dtypes.INTEGER,
     dtypes.Int64: duckdb_dtypes.BIGINT,
-    dtypes.Int128: DuckDBPyType("INT128"),
+    dtypes.Int128: duckdb_dtypes.HUGEINT,
     dtypes.UInt8: duckdb_dtypes.UTINYINT,
     dtypes.UInt16: duckdb_dtypes.USMALLINT,
     dtypes.UInt32: duckdb_dtypes.UINTEGER,
     dtypes.UInt64: duckdb_dtypes.UBIGINT,
-    dtypes.UInt128: DuckDBPyType("UINT128"),
+    dtypes.UInt128: duckdb_dtypes.UHUGEINT,
 }
-TIME_UNIT_TO_TIMESTAMP: Mapping[TimeUnit, DuckDBPyType] = {
+TIME_UNIT_TO_TIMESTAMP: Mapping[TimeUnit, duckdb_dtypes.DuckDBPyType] = {
     "s": duckdb_dtypes.TIMESTAMP_S,
     "ms": duckdb_dtypes.TIMESTAMP_MS,
     "us": duckdb_dtypes.TIMESTAMP,
@@ -246,7 +252,7 @@ def _non_nested_native_to_narwhals_dtype(duckdb_dtype_id: str, version: Version)
 
 def narwhals_to_native_dtype(  # noqa: PLR0912, C901
     dtype: IntoDType, version: Version, deferred_time_zone: DeferredTimeZone
-) -> DuckDBPyType:
+) -> duckdb_dtypes.DuckDBPyType:
     dtypes = version.dtypes
     base_type = dtype.base_type()
     if duckdb_type := NW_TO_DUCKDB_DTYPES.get(base_type):
@@ -256,7 +262,7 @@ def narwhals_to_native_dtype(  # noqa: PLR0912, C901
             msg = "Converting to Enum is not supported in narwhals.stable.v1"
             raise NotImplementedError(msg)
         if isinstance(dtype, dtypes.Enum):
-            return DuckDBPyType(f"ENUM{dtype.categories!r}")
+            return duckdb_dtypes.DuckDBPyType(f"ENUM{dtype.categories!r}")
         msg = "Can not cast / initialize Enum without categories present"
         raise ValueError(msg)
     if isinstance_or_issubclass(dtype, dtypes.Datetime):
@@ -291,7 +297,7 @@ def narwhals_to_native_dtype(  # noqa: PLR0912, C901
             nw_inner = nw_inner.inner
         duckdb_inner = narwhals_to_native_dtype(nw_inner, version, deferred_time_zone)
         duckdb_shape_fmt = "".join(f"[{item}]" for item in dtype.shape)
-        return DuckDBPyType(f"{duckdb_inner}{duckdb_shape_fmt}")
+        return duckdb_dtypes.DuckDBPyType(f"{duckdb_inner}{duckdb_shape_fmt}")
     if issubclass(base_type, UNSUPPORTED_DTYPES):
         msg = f"Converting to {base_type.__name__} dtype is not supported for DuckDB."
         raise NotImplementedError(msg)
@@ -378,19 +384,39 @@ def function(name: str, *args: Expression) -> Expression:
     if name == "isnull":
         return args[0].isnull()
     if name == "count_distinct":
-        try:
-            from duckdb import SQLExpression
-        except ModuleNotFoundError as exc:  # pragma: no cover
-            msg = f"DuckDB>=1.3.0 is required for this operation. Found: DuckDB {duckdb.__version__}"
-            raise NotImplementedError(msg) from exc
-        return SQLExpression(f"count(distinct {args[0]})")
+        return sql_expression(f"count(distinct {args[0]})")
     return F(name, *args)
 
 
 def sql_expression(expr: str) -> Expression:
     try:
         from duckdb import SQLExpression
-    except ModuleNotFoundError as exc:  # pragma: no cover
+    except ImportError as exc:  # pragma: no cover
         msg = f"DuckDB>=1.3.0 is required for this operation. Found: DuckDB {duckdb.__version__}"
         raise NotImplementedError(msg) from exc
     return SQLExpression(expr)
+
+
+__all__ = [
+    "UNITS_DICT",
+    "DeferredTimeZone",
+    "F",
+    "catch_duckdb_exception",
+    "col",
+    "concat_str",
+    "duckdb_dtypes",
+    "evaluate_exprs",
+    "fetch_rel_time_zone",
+    "function",
+    "generate_order_by_sql",
+    "generate_partition_by_sql",
+    "join_column_names",
+    "lambda_expr",
+    "lit",
+    "narwhals_to_native_dtype",
+    "native_to_narwhals_dtype",
+    "parse_into_expression",
+    "sql_expression",
+    "when",
+    "window_expression",
+]

narwhals/_utils.py

@@ -604,7 +604,7 @@ def _backend_version(self) -> tuple[int, ...]:
     Implementation.PYSPARK_CONNECT: (3, 5),
     Implementation.POLARS: (0, 20, 4),
     Implementation.DASK: (2024, 8),
-    Implementation.DUCKDB: (1,),
+    Implementation.DUCKDB: (1, 1),
     Implementation.IBIS: (6,),
     Implementation.SQLFRAME: (3, 22, 0),
 }

pyproject.toml

@@ -42,7 +42,7 @@ pyspark = ["pyspark>=3.5.0"]
 pyspark-connect = ["pyspark[connect]>=3.5.0"]
 polars = ["polars>=0.20.4"]
 dask = ["dask[dataframe]>=2024.8"]
-duckdb = ["duckdb>=1.0"]
+duckdb = ["duckdb>=1.1"]
 ibis = ["ibis-framework>=6.0.0", "rich", "packaging", "pyarrow_hotfix"]
 sqlframe = ["sqlframe>=3.22.0,!=3.39.3"]
 

tests/dtypes_test.py

@@ -517,6 +517,7 @@ def test_datetime_w_tz_duckdb() -> None:
     assert result["b"] == nw.List(nw.List(nw.Datetime("us", "Asia/Kathmandu")))
 
 
+@pytest.mark.slow
 def test_datetime_w_tz_pyspark() -> None:  # pragma: no cover
     pytest.importorskip("pyspark")
     session = pyspark_session()

tests/expr_and_series/dt/convert_time_zone_test.py

@@ -154,6 +154,7 @@ def test_convert_time_zone_to_connection_tz_duckdb() -> None:
         )
 
 
+@pytest.mark.slow
 def test_convert_time_zone_to_connection_tz_pyspark() -> None:  # pragma: no cover
     pytest.importorskip("pyspark")
 

tests/expr_and_series/dt/replace_time_zone_test.py

@@ -142,6 +142,7 @@ def test_replace_time_zone_to_connection_tz_duckdb() -> None:
         )
 
 
+@pytest.mark.slow
 def test_replace_time_zone_to_connection_tz_pyspark() -> None:  # pragma: no cover
     pytest.importorskip("pyspark")