Merge branch 'main' into create_fromnative_daft

Author: ym-pett

.github/workflows/extremes.yml

@@ -152,8 +152,10 @@ jobs:
           uv pip install pandas --pre --index https://pypi.anaconda.org/scientific-python-nightly-wheels/simple --system -U
       - name: install pyarrow nightly
         run: |
-          uv pip uninstall pyarrow --system
-          uv pip install pyarrow --pre --index https://pypi.fury.io/arrow-nightlies/ --system -U
+          # commented out nightly whilst it fails to install
+          uv pip install -U pyarrow --system
+          # uv pip uninstall pyarrow --system
+          # uv pip install pyarrow --pre --index https://pypi.fury.io/arrow-nightlies/ --system -U
       - name: install numpy nightly
         run: |
           uv pip uninstall numpy --system
@@ -172,7 +174,7 @@ jobs:
         run: |
           DEPS=$(uv pip freeze)
           echo "$DEPS" | grep 'pandas.*dev'
-          echo "$DEPS" | grep 'pyarrow.*dev'
+          # echo "$DEPS" | grep 'pyarrow.*dev'
           echo "$DEPS" | grep 'numpy.*dev'
           echo "$DEPS" | grep 'dask.*@'
       - name: Run pytest

README.md

@@ -181,6 +181,7 @@ Join the party!
 - [plotly](https://plotly.com)
 - [pointblank](https://github.com/posit-dev/pointblank)
 - [pymarginaleffects](https://github.com/vincentarelbundock/pymarginaleffects)
+- [pyreadstat](https://github.com/Roche/pyreadstat)
 - [py-shiny](https://github.com/posit-dev/py-shiny)
 - [rio](https://github.com/rio-labs/rio)
 - [scikit-lego](https://github.com/koaning/scikit-lego)

docs/ecosystem.md

@@ -18,6 +18,7 @@ for their dataframe interoperability needs:
 * [plotly](https://github.com/plotly/plotly.py)
 * [pointblank](https://github.com/posit-dev/pointblank)
 * [pymarginaleffects](https://github.com/vincentarelbundock/pymarginaleffects)
+* [pyreadstat](https://github.com/Roche/pyreadstat)
 * [py-shiny](https://github.com/posit-dev/py-shiny)
 * [rio](https://github.com/rio-labs/rio)
 * [scikit-lego](https://github.com/koaning/scikit-lego)

narwhals/_dask/expr.py

@@ -186,105 +186,92 @@ def _with_alias_output_names(self, func: AliasNames | None, /) -> Self:
             scalar_kwargs=self._scalar_kwargs,
         )
 
-    def __add__(self, other: Any) -> Self:
-        return self._with_callable(
-            lambda expr, other: expr.__add__(other), "__add__", other=other
+    def _with_binary(
+        self,
+        call: Callable[[dx.Series, Any], dx.Series],
+        name: str,
+        other: Any,
+        *,
+        reverse: bool = False,
+    ) -> Self:
+        result = self._with_callable(
+            lambda expr, other: call(expr, other), name, other=other
         )
+        if reverse:
+            result = result.alias("literal")
+        return result
 
-    def __sub__(self, other: Any) -> Self:
-        return self._with_callable(
-            lambda expr, other: expr.__sub__(other), "__sub__", other=other
+    def _binary_op(self, op_name: str, other: Any) -> Self:
+        return self._with_binary(
+            lambda expr, other: getattr(expr, op_name)(other), op_name, other
         )
 
-    def __rsub__(self, other: Any) -> Self:
-        return self._with_callable(
-            lambda expr, other: other - expr, "__rsub__", other=other
-        ).alias("literal")
+    def _reverse_binary_op(
+        self, op_name: str, operator_func: Callable[..., dx.Series], other: Any
+    ) -> Self:
+        return self._with_binary(
+            lambda expr, other: operator_func(other, expr), op_name, other, reverse=True
+        )
+
+    def __add__(self, other: Any) -> Self:
+        return self._binary_op("__add__", other)
+
+    def __sub__(self, other: Any) -> Self:
+        return self._binary_op("__sub__", other)
 
     def __mul__(self, other: Any) -> Self:
-        return self._with_callable(
-            lambda expr, other: expr.__mul__(other), "__mul__", other=other
-        )
+        return self._binary_op("__mul__", other)
 
     def __truediv__(self, other: Any) -> Self:
-        return self._with_callable(
-            lambda expr, other: expr.__truediv__(other), "__truediv__", other=other
-        )
-
-    def __rtruediv__(self, other: Any) -> Self:
-        return self._with_callable(
-            lambda expr, other: other / expr, "__rtruediv__", other=other
-        ).alias("literal")
+        return self._binary_op("__truediv__", other)
 
     def __floordiv__(self, other: Any) -> Self:
-        return self._with_callable(
-            lambda expr, other: expr.__floordiv__(other), "__floordiv__", other=other
-        )
-
-    def __rfloordiv__(self, other: Any) -> Self:
-        return self._with_callable(
-            lambda expr, other: other // expr, "__rfloordiv__", other=other
-        ).alias("literal")
+        return self._binary_op("__floordiv__", other)
 
     def __pow__(self, other: Any) -> Self:
-        return self._with_callable(
-            lambda expr, other: expr.__pow__(other), "__pow__", other=other
-        )
-
-    def __rpow__(self, other: Any) -> Self:
-        return self._with_callable(
-            lambda expr, other: other**expr, "__rpow__", other=other
-        ).alias("literal")
+        return self._binary_op("__pow__", other)
 
     def __mod__(self, other: Any) -> Self:
-        return self._with_callable(
-            lambda expr, other: expr.__mod__(other), "__mod__", other=other
-        )
+        return self._binary_op("__mod__", other)
 
-    def __rmod__(self, other: Any) -> Self:
-        return self._with_callable(
-            lambda expr, other: other % expr, "__rmod__", other=other
-        ).alias("literal")
+    def __eq__(self, other: object) -> Self:  # type: ignore[override]
+        return self._binary_op("__eq__", other)
 
-    def __eq__(self, other: DaskExpr) -> Self:  # type: ignore[override]
-        return self._with_callable(
-            lambda expr, other: expr.__eq__(other), "__eq__", other=other
-        )
+    def __ne__(self, other: object) -> Self:  # type: ignore[override]
+        return self._binary_op("__ne__", other)
 
-    def __ne__(self, other: DaskExpr) -> Self:  # type: ignore[override]
-        return self._with_callable(
-            lambda expr, other: expr.__ne__(other), "__ne__", other=other
-        )
+    def __ge__(self, other: Any) -> Self:
+        return self._binary_op("__ge__", other)
 
-    def __ge__(self, other: DaskExpr | Any) -> Self:
-        return self._with_callable(
-            lambda expr, other: expr.__ge__(other), "__ge__", other=other
-        )
+    def __gt__(self, other: Any) -> Self:
+        return self._binary_op("__gt__", other)
 
-    def __gt__(self, other: DaskExpr) -> Self:
-        return self._with_callable(
-            lambda expr, other: expr.__gt__(other), "__gt__", other=other
-        )
+    def __le__(self, other: Any) -> Self:
+        return self._binary_op("__le__", other)
 
-    def __le__(self, other: DaskExpr) -> Self:
-        return self._with_callable(
-            lambda expr, other: expr.__le__(other), "__le__", other=other
-        )
+    def __lt__(self, other: Any) -> Self:
+        return self._binary_op("__lt__", other)
 
-    def __lt__(self, other: DaskExpr) -> Self:
-        return self._with_callable(
-            lambda expr, other: expr.__lt__(other), "__lt__", other=other
-        )
+    def __and__(self, other: Any) -> Self:
+        return self._binary_op("__and__", other)
 
-    def __and__(self, other: DaskExpr | Any) -> Self:
-        return self._with_callable(
-            lambda expr, other: expr.__and__(other), "__and__", other=other
-        )
+    def __or__(self, other: Any) -> Self:
+        return self._binary_op("__or__", other)
 
-    def __or__(self, other: DaskExpr) -> Self:
-        return self._with_callable(
-            lambda expr, other: expr.__or__(other), "__or__", other=other
-        )
+    def __rsub__(self, other: Any) -> Self:
+        return self._reverse_binary_op("__rsub__", lambda a, b: a - b, other)
+
+    def __rtruediv__(self, other: Any) -> Self:
+        return self._reverse_binary_op("__rtruediv__", lambda a, b: a / b, other)
+
+    def __rfloordiv__(self, other: Any) -> Self:
+        return self._reverse_binary_op("__rfloordiv__", lambda a, b: a // b, other)
+
+    def __rpow__(self, other: Any) -> Self:
+        return self._reverse_binary_op("__rpow__", lambda a, b: a**b, other)
+
+    def __rmod__(self, other: Any) -> Self:
+        return self._reverse_binary_op("__rmod__", lambda a, b: a % b, other)
 
     def __invert__(self) -> Self:
         return self._with_callable(lambda expr: expr.__invert__(), "__invert__")

narwhals/_ibis/dataframe.py

@@ -337,12 +337,12 @@ def sort(self, *by: str, descending: bool | Sequence[bool], nulls_last: bool) ->
         if isinstance(descending, bool):
             descending = [descending for _ in range(len(by))]
 
-        sort_cols = []
+        sort_cols: list[Any] = []
 
         for i in range(len(by)):
             direction_fn = ibis.desc if descending[i] else ibis.asc
             col = direction_fn(by[i], nulls_first=not nulls_last)
-            sort_cols.append(cast("ir.Column", col))
+            sort_cols.append(col)
 
         return self._with_native(self.native.order_by(*sort_cols))
 

narwhals/_ibis/expr.py

@@ -227,7 +227,7 @@ def _std(expr: ir.NumericColumn, ddof: int) -> ir.Value:
                 return expr.std(how="sample")
             n_samples = expr.count()
             std_pop = expr.std(how="pop")
-            ddof_lit = cast("ir.IntegerScalar", ibis.literal(ddof))
+            ddof_lit = lit(ddof)
             return std_pop * n_samples.sqrt() / (n_samples - ddof_lit).sqrt()
 
         return self._with_callable(lambda expr: _std(expr, ddof))
@@ -240,7 +240,7 @@ def _var(expr: ir.NumericColumn, ddof: int) -> ir.Value:
                 return expr.var(how="sample")
             n_samples = expr.count()
             var_pop = expr.var(how="pop")
-            ddof_lit = cast("ir.IntegerScalar", ibis.literal(ddof))
+            ddof_lit = lit(ddof)
             return var_pop * n_samples / (n_samples - ddof_lit)
 
         return self._with_callable(lambda expr: _var(expr, ddof))
@@ -290,35 +290,33 @@ def is_unique(self) -> Self:
         )
 
     def rank(self, method: RankMethod, *, descending: bool) -> Self:
-        def _rank(expr: ir.Column) -> ir.Column:
+        def _rank(expr: ir.Column) -> ir.Value:
             order_by = next(self._sort(expr, descending=[descending], nulls_last=[True]))
             window = ibis.window(order_by=order_by)
 
             if method == "dense":
                 rank_ = order_by.dense_rank()
             elif method == "ordinal":
-                rank_ = cast("ir.IntegerColumn", ibis.row_number().over(window))
+                rank_ = ibis.row_number().over(window)
             else:
                 rank_ = order_by.rank()
 
             # Ibis uses 0-based ranking. Add 1 to match polars 1-based rank.
-            rank_ = rank_ + cast("ir.IntegerValue", lit(1))
+            rank_ = rank_ + lit(1)
 
             # For "max" and "average", adjust using the count of rows in the partition.
             if method == "max":
                 # Define a window partitioned by expr (i.e. each distinct value)
                 partition = ibis.window(group_by=[expr])
-                cnt = cast("ir.IntegerValue", expr.count().over(partition))
-                rank_ = rank_ + cnt - cast("ir.IntegerValue", lit(1))
+                cnt = expr.count().over(partition)
+                rank_ = rank_ + cnt - lit(1)
             elif method == "average":
                 partition = ibis.window(group_by=[expr])
-                cnt = cast("ir.IntegerValue", expr.count().over(partition))
-                avg = cast(
-                    "ir.NumericValue", (cnt - cast("ir.IntegerScalar", lit(1))) / lit(2.0)
-                )
+                cnt = expr.count().over(partition)
+                avg = cast("ir.NumericValue", (cnt - lit(1)) / lit(2.0))
                 rank_ = rank_ + avg
 
-            return cast("ir.Column", ibis.cases((expr.notnull(), rank_)))
+            return ibis.cases((expr.notnull(), rank_))
 
         return self._with_callable(_rank)
 

narwhals/_ibis/namespace.py

@@ -3,7 +3,7 @@
 import operator
 from functools import reduce
 from itertools import chain
-from typing import TYPE_CHECKING, Any, cast
+from typing import TYPE_CHECKING, Any
 
 import ibis
 import ibis.expr.types as ir
@@ -88,8 +88,7 @@ def func(df: IbisLazyFrame) -> list[ir.Value]:
                 for col in cols_casted[1:]:
                     result = result + separator + col
             else:
-                sep = cast("ir.StringValue", lit(separator))
-                result = sep.join(cols_casted)
+                result = lit(separator).join(cols_casted)
 
             return [result]
 

narwhals/_ibis/utils.py

@@ -1,7 +1,7 @@
 from __future__ import annotations
 
 from functools import lru_cache
-from typing import TYPE_CHECKING, Any, Literal, cast
+from typing import TYPE_CHECKING, Any, Literal, cast, overload
 
 import ibis
 import ibis.expr.datatypes as ibis_dtypes
@@ -23,8 +23,27 @@
     from narwhals.dtypes import DType
     from narwhals.typing import IntoDType, PythonLiteral
 
-lit = ibis.literal
-"""Alias for `ibis.literal`."""
+Incomplete: TypeAlias = Any
+"""Marker for upstream issues."""
+
+
+@overload
+def lit(value: bool, dtype: None = ...) -> ir.BooleanScalar: ...  # noqa: FBT001
+@overload
+def lit(value: int, dtype: None = ...) -> ir.IntegerScalar: ...
+@overload
+def lit(value: float, dtype: None = ...) -> ir.FloatingScalar: ...
+@overload
+def lit(value: str, dtype: None = ...) -> ir.StringScalar: ...
+@overload
+def lit(value: PythonLiteral | ir.Value, dtype: None = ...) -> ir.Scalar: ...
+@overload
+def lit(value: Any, dtype: Any) -> Incomplete: ...
+def lit(value: Any, dtype: Any | None = None) -> Incomplete:
+    """Alias for `ibis.literal`."""
+    literal: Incomplete = ibis.literal
+    return literal(value, dtype)
+
 
 BucketUnit: TypeAlias = Literal[
     "years",
@@ -231,11 +250,11 @@ def timedelta_to_ibis_interval(td: timedelta) -> ibis.expr.types.temporal.Interv
 def function(name: str, *args: ir.Value | PythonLiteral) -> ir.Value:
     # Workaround SQL vs Ibis differences.
     if name == "row_number":
-        return ibis.row_number() + 1  # pyright: ignore[reportOperatorIssue]
+        return ibis.row_number() + lit(1)
     if name == "least":
-        return ibis.least(*args)  # pyright: ignore[reportOperatorIssue]
+        return ibis.least(*args)
     if name == "greatest":
-        return ibis.greatest(*args)  # pyright: ignore[reportOperatorIssue]
+        return ibis.greatest(*args)
     expr = args[0]
     if name == "var_pop":
         return cast("ir.NumericColumn", expr).var(how="pop")

narwhals/_utils.py

@@ -603,11 +603,7 @@ def backend_version(implementation: Implementation, /) -> tuple[int, ...]:
     impl = implementation
     module_name = _IMPLEMENTATION_TO_MODULE_NAME.get(impl, impl.value)
     native_namespace = _import_native_namespace(module_name)
-    if impl.is_polars():
-        from importlib import metadata
-
-        into_version = metadata.version("polars")
-    elif impl.is_sqlframe():
+    if impl.is_sqlframe():
         import sqlframe._version
 
         into_version = sqlframe._version

tests/expr_and_series/map_batches_test.py

@@ -1,5 +1,7 @@
 from __future__ import annotations
 
+import pytest
+
 import narwhals as nw
 from tests.utils import ConstructorEager, assert_equal_data
 
@@ -12,7 +14,12 @@ def test_map_batches_expr(constructor_eager: ConstructorEager) -> None:
     assert_equal_data(expected, {"a": [2, 3, 4], "b": [5, 6, 7]})
 
 
-def test_map_batches_expr_numpy(constructor_eager: ConstructorEager) -> None:
+def test_map_batches_expr_numpy(
+    constructor_eager: ConstructorEager, request: pytest.FixtureRequest
+) -> None:
+    if "polars" in str(constructor_eager):
+        # https://github.com/narwhals-dev/narwhals/issues/2995
+        request.applymarker(pytest.mark.xfail(strict=False))
     df = nw.from_native(constructor_eager(data))
     expected = df.select(
         nw.col("a")