fix: Always accept {Expr,Series}.is_in(other: Iterable)

narwhals/_utils.py

@@ -47,6 +47,7 @@
     is_numpy_array_1d_int,
     is_pandas_like_dataframe,
     is_pandas_like_series,
+    is_polars_series,
 )
 from narwhals.exceptions import ColumnNotFoundError, DuplicateError, InvalidOperationError
 
@@ -124,6 +125,7 @@
         CompliantLazyFrame,
         CompliantSeries,
         DTypes,
+        EagerAllowed,
         FileSource,
         IntoSeriesT,
         MultiIndexSelector,
@@ -2120,3 +2122,58 @@ def extend_bool(
     Stolen from https://github.com/pola-rs/polars/blob/b8bfb07a4a37a8d449d6d1841e345817431142df/py-polars/polars/_utils/various.py#L580-L594
     """
     return (value,) * n_match if isinstance(value, bool) else tuple(value)
+
+
+class _CanTo_List(Protocol):  # noqa: N801
+    def to_list(self, *args: Any, **kwds: Any) -> list[Any]: ...
+
+
+class _CanToList(Protocol):
+    def tolist(self, *args: Any, **kwds: Any) -> list[Any]: ...
+
+
+class _CanTo_PyList(Protocol):  # noqa: N801
+    def to_pylist(self, *args: Any, **kwds: Any) -> list[Any]: ...
+
+
+def can_to_list(obj: Any) -> TypeIs[_CanTo_List]:
+    return (
+        is_narwhals_series(obj)
+        or is_polars_series(obj)
+        or _hasattr_static(obj, "to_list")
+    )
+
+
+def can_tolist(obj: Any) -> TypeIs[_CanToList]:
+    return is_numpy_array_1d(obj) or _hasattr_static(obj, "tolist")
+
+
+def can_to_pylist(obj: Any) -> TypeIs[_CanTo_PyList]:
+    return (
+        (pa := get_pyarrow()) and isinstance(obj, (pa.Array, pa.ChunkedArray))
+    ) or _hasattr_static(obj, "to_pylist")
+
+
+# TODO @dangotbanned: Use in `{Expr,Series}.is_in`
+# TODO @dangotbanned: Add (brief) doc
+def iterable_to_sequence(
+    iterable: Iterable[Any], /, *, backend: EagerAllowed | None = None
+) -> Sequence[Any]:
+    result: Sequence[Any]
+    if backend is not None:
+        from narwhals.series import Series
+
+        result = Series.from_iterable("", iterable, backend=backend).to_list()
+    elif isinstance(iterable, (tuple, list)):
+        result = iterable
+    elif isinstance(iterable, (Iterator, Sequence)):
+        result = tuple(iterable)
+    elif can_to_list(iterable):
+        result = iterable.to_list()
+    elif can_tolist(iterable):
+        result = iterable.tolist()
+    elif can_to_pylist(iterable):
+        result = iterable.to_pylist()
+    else:
+        result = tuple(iterable)
+    return result

narwhals/expr.py

@@ -9,8 +9,14 @@
     ExprMetadata,
     apply_n_ary_operation,
     combine_metadata,
+    is_series,
+)
+from narwhals._utils import (
+    _validate_rolling_arguments,
+    ensure_type,
+    flatten,
+    iterable_to_sequence,
 )
-from narwhals._utils import _validate_rolling_arguments, ensure_type, flatten
 from narwhals.dtypes import _validate_dtype
 from narwhals.exceptions import ComputeError, InvalidOperationError
 from narwhals.expr_cat import ExprCatNamespace
@@ -19,7 +25,6 @@
 from narwhals.expr_name import ExprNameNamespace
 from narwhals.expr_str import ExprStringNamespace
 from narwhals.expr_struct import ExprStructNamespace
-from narwhals.translate import to_native
 
 if TYPE_CHECKING:
     from typing import NoReturn, TypeVar
@@ -991,10 +996,9 @@ def is_in(self, other: Any) -> Self:
             └──────────────────┘
         """
         if isinstance(other, Iterable) and not isinstance(other, (str, bytes)):
+            other = other.to_native() if is_series(other) else iterable_to_sequence(other)
             return self._with_elementwise(
-                lambda plx: self._to_compliant_expr(plx).is_in(
-                    to_native(other, pass_through=True)
-                )
+                lambda plx: self._to_compliant_expr(plx).is_in(other)
             )
         msg = "Narwhals `is_in` doesn't accept expressions as an argument, as opposed to Polars. You should provide an iterable instead."
         raise NotImplementedError(msg)

narwhals/series.py

@@ -14,6 +14,7 @@
     is_compliant_series,
     is_eager_allowed,
     is_index_selector,
+    iterable_to_sequence,
     qualified_type_name,
     supports_arrow_c_stream,
 )
@@ -25,7 +26,6 @@
 from narwhals.series_list import SeriesListNamespace
 from narwhals.series_str import SeriesStringNamespace
 from narwhals.series_struct import SeriesStructNamespace
-from narwhals.translate import to_native
 from narwhals.typing import IntoSeriesT
 
 if TYPE_CHECKING:
@@ -948,9 +948,12 @@ def is_in(self, other: Any) -> Self:
               ]
             ]
         """
-        return self._with_compliant(
-            self._compliant_series.is_in(to_native(other, pass_through=True))
+        other = (
+            other.to_native()
+            if isinstance(other, Series)
+            else iterable_to_sequence(other, backend=self.implementation)
         )
+        return self._with_compliant(self._compliant_series.is_in(other))
 
     def arg_true(self) -> Self:
         """Find elements where boolean Series is True.

tests/conftest.py

@@ -2,18 +2,30 @@
 
 import os
 import uuid
+from collections import deque
 from copy import deepcopy
 from functools import lru_cache
 from importlib.util import find_spec
 from typing import TYPE_CHECKING, Any, Callable, cast
 
 import pytest
 
-from narwhals._utils import Implementation, generate_temporary_column_name
+from narwhals._utils import (
+    Implementation,
+    generate_temporary_column_name,
+    qualified_type_name,
+)
 from tests.utils import ID_PANDAS_LIKE, PANDAS_VERSION, pyspark_session, sqlframe_session
 
 if TYPE_CHECKING:
-    from collections.abc import Sequence
+    from collections.abc import (
+        Generator,
+        Iterable,
+        Iterator,
+        KeysView,
+        Sequence,
+        ValuesView,
+    )
 
     import duckdb
     import ibis
@@ -27,7 +39,7 @@
     from narwhals._spark_like.dataframe import SQLFrameDataFrame
     from narwhals._typing import EagerAllowed
     from narwhals.typing import NativeDataFrame, NativeLazyFrame
-    from tests.utils import Constructor, ConstructorEager, ConstructorLazy
+    from tests.utils import Constructor, ConstructorEager, ConstructorLazy, IntoIterable
 
     Data: TypeAlias = "dict[str, list[Any]]"
 
@@ -317,7 +329,109 @@ def eager_backend(request: pytest.FixtureRequest) -> EagerAllowed:
     return request.param  # type: ignore[no-any-return]
 
 
-@pytest.fixture(params=[el for el in TEST_EAGER_BACKENDS if not isinstance(el, str)])
+@pytest.fixture(
+    params=[el for el in TEST_EAGER_BACKENDS if not isinstance(el, str)], scope="session"
+)
 def eager_implementation(request: pytest.FixtureRequest) -> EagerAllowed:
     """Use if a test is heavily parametric, skips `str` backend."""
     return request.param  # type: ignore[no-any-return]
+
+
+class UserDefinedIterable:
+    def __init__(self, iterable: Iterable[Any]) -> None:
+        self.iterable: Iterable[Any] = iterable
+
+    def __iter__(self) -> Iterator[Any]:
+        yield from self.iterable
+
+
+def generator_function(iterable: Iterable[Any]) -> Generator[Any, Any, None]:
+    yield from iterable
+
+
+def generator_expression(iterable: Iterable[Any]) -> Generator[Any, None, None]:
+    return (element for element in iterable)
+
+
+def dict_keys(iterable: Iterable[Any]) -> KeysView[Any]:
+    return dict.fromkeys(iterable).keys()
+
+
+def dict_values(iterable: Iterable[Any]) -> ValuesView[Any]:
+    return dict(enumerate(iterable)).values()
+
+
+def chunked_array(iterable: Any) -> Iterable[Any]:
+    import pyarrow as pa
+
+    return pa.chunked_array([iterable])
+
+
+def _ids_into_iter(obj: Any) -> str:
+    module: str = ""
+    if (obj_module := obj.__module__) and obj_module != __name__:
+        module = obj.__module__
+    name = qualified_type_name(obj)
+    if name in {"function", "builtin_function_or_method"} or "_cython" in name:
+        return f"{module}.{obj.__qualname__}" if module else obj.__qualname__
+    return name.removeprefix(__name__).strip(".")
+
+
+def _build_into_iter() -> Iterator[IntoIterable]:  # pragma: no cover
+    yield from (
+        # 1-4: should cover `Iterable`, `Sequence`, `Iterator`
+        list,
+        tuple,
+        iter,
+        deque,
+        # 5-6: cover `Generator`
+        generator_function,
+        generator_expression,
+        # 7-8: `Iterable`, but quite commonly cause issues upstream as they are `Sized` but not `Sequence`
+        dict_keys,
+        dict_values,
+        # 9: duck typing
+        UserDefinedIterable,
+    )
+    # 10: 1D numpy
+    if find_spec("numpy"):
+        import numpy as np
+
+        yield np.array
+    # 11-13: 1D pandas
+    if find_spec("pandas"):
+        import pandas as pd
+
+        yield from (pd.Index, pd.array, pd.Series)
+    # 14: 1D polars
+    if find_spec("polars"):
+        import polars as pl
+
+        yield pl.Series
+    # 15-16: 1D pyarrow
+    if find_spec("pyarrow"):
+        import pyarrow as pa
+
+        yield from (pa.array, chunked_array)
+
+
+def _into_iter_selector() -> Callable[[int], Iterator[IntoIterable]]:
+    callables = tuple(_build_into_iter())
+
+    def pick(n: int, /) -> Iterator[IntoIterable]:
+        yield from callables[:n]
+
+    return pick
+
+
+_into_iter: Callable[[int], Iterator[IntoIterable]] = _into_iter_selector()
+"""`into_iter` fixtures use the suffix `_<n>` to denote the maximum number of constructors.
+
+Anything greater than **10** may return less depending on available dependencies.
+"""
+
+
+@pytest.fixture(params=_into_iter(16), scope="session", ids=_ids_into_iter)
+def into_iter_16(request: pytest.FixtureRequest) -> IntoIterable:
+    function: IntoIterable = request.param
+    return function

tests/expr_and_series/is_in_test.py

@@ -5,7 +5,13 @@
 import pytest
 
 import narwhals as nw
-from tests.utils import Constructor, ConstructorEager, assert_equal_data
+from tests.utils import (
+    Constructor,
+    ConstructorEager,
+    IntoIterable,
+    assert_equal_data,
+    assert_equal_series,
+)
 
 data = {"a": [1, 4, 2, 5]}
 
@@ -50,3 +56,40 @@ def test_filter_is_in_with_series(constructor_eager: ConstructorEager) -> None:
     result = df.filter(nw.col("a").is_in(df["b"]))
     expected = {"a": [1, 2], "b": [1, 2]}
     assert_equal_data(result, expected)
+
+
+@pytest.mark.slow
+def test_expr_is_in_iterable(
+    constructor: Constructor, into_iter_16: IntoIterable
+) -> None:
+    df = nw.from_native(constructor(data))
+    expected = {"a": [False, True, True, False]}
+    iterable = into_iter_16((4, 2))
+    expr = nw.col("a").is_in(iterable)
+    result = df.select(expr)
+    assert_equal_data(result, expected)
+    # NOTE: For an `Iterator`, this will fail if we haven't collected it first
+    repeated = df.select(expr)
+    assert_equal_data(repeated, expected)
+
+
+@pytest.mark.slow
+def test_ser_is_in_iterable(
+    constructor_eager: ConstructorEager,
+    into_iter_16: IntoIterable,
+    request: pytest.FixtureRequest,
+) -> None:
+    test_name = request.node.name
+    # NOTE: This *could* be supported by using `ExtensionArray.tolist` (same path as numpy)
+    request.applymarker(
+        pytest.mark.xfail(
+            ("polars" in test_name and "pandas" in test_name and "array" in test_name),
+            raises=TypeError,
+            reason="Polars doesn't support `pd.array`.\nhttps://github.com/pola-rs/polars/issues/22757",
+        )
+    )
+    iterable = into_iter_16((4, 2))
+    ser = nw.from_native(constructor_eager(data)).get_column("a")
+    result = ser.is_in(iterable)
+    expected = [False, True, True, False]
+    assert_equal_series(result, expected, "a")