chore: Add Compliant*.from_native (#2315)

* feat(typing): Add `FromNative` protocol

* chore(typing): Add `FromNative` to `CompliantSeries`

- Adding `._is_native` made `TypeVar` invariant
- Realistically, it always was, but underspecified

* feat: Implement for `(Arrow|PandasLike)Series`

* feat: Implement for `PolarsSeries`

+ get some coverage

* chore: `ArrowSeries` coverage

* chore: `PandasLikeSeries` partial coverage

* ignore coverage for now ...

* feat(typing): Add `CompliantDataFrame.from_native`

* feat: Implement for `ArrowDataFrame`

Also coverage for `ArrowSeries`

* feat: Implement for `PandasLikeDataFrame`

Loads of coverage for both `PandasLike`

* feat: Implement for `PolarsDataFrame`

* refactor: Found one more

* chore(typing): Fix missing `SQLExpression` ignore

Was missed in (#2310)

* feat: Implement `EagerNamespace.from_native`

- `Polars*` will also need to handle `LazyFrame`
- `Lazy*` has other constraints

* feat: Add `Polars(Namespace|LazyFrame).from_native`

Probably need to add a `LazyNamespace` protocol for `LazyOnly`

* chore: Ignore coverage `PandasLikeDataFrame._is_native`

Nowhere to use it yet, current stuff uses the more precise `self.native.__class__`

* feat: Add all `CompliantLazyFrame.from_native`

* feat: Add `LazyNamespace.from_native`

* refactor: Get some lazy coverage

https://github.com/narwhals-dev/narwhals/actions/runs/14157697512/job/39659084059?pr=2315

* refactor: More `polars` coverage

https://github.com/narwhals-dev/narwhals/actions/runs/14158424987/job/39660662342?pr=2315

* refactor: reuse `is_spark_like_dataframe`

* Update narwhals/_compliant/namespace.py

Co-authored-by: Dan Redding <[email protected]>

---------

Co-authored-by: Marco Edward Gorelli <[email protected]>

Author: dangotbanned

narwhals/_arrow/dataframe.py

@@ -45,11 +45,11 @@
     import polars as pl
     from typing_extensions import Self
     from typing_extensions import TypeAlias
+    from typing_extensions import TypeIs
 
     from narwhals._arrow.expr import ArrowExpr
     from narwhals._arrow.group_by import ArrowGroupBy
     from narwhals._arrow.namespace import ArrowNamespace
-    from narwhals._arrow.series import ArrowSeries
     from narwhals._arrow.typing import ArrowChunkedArray
     from narwhals._arrow.typing import Indices  # type: ignore[attr-defined]
     from narwhals._arrow.typing import Mask  # type: ignore[attr-defined]
@@ -99,7 +99,7 @@ def __init__(
     @classmethod
     def from_arrow(cls, data: IntoArrowTable, /, *, context: _FullContext) -> Self:
         backend_version = context._backend_version
-        if isinstance(data, pa.Table):
+        if cls._is_native(data):
             native = data
         elif backend_version >= (14,) or isinstance(data, Collection):
             native = pa.table(data)
@@ -109,12 +109,7 @@ def from_arrow(cls, data: IntoArrowTable, /, *, context: _FullContext) -> Self:
         else:  # pragma: no cover
             msg = f"`from_arrow` is not supported for object of type {type(data).__name__!r}."
             raise TypeError(msg)
-        return cls(
-            native,
-            backend_version=backend_version,
-            version=context._version,
-            validate_column_names=True,
-        )
+        return cls.from_native(native, context=context)
 
     @classmethod
     def from_dict(
@@ -129,8 +124,16 @@ def from_dict(
 
         pa_schema = Schema(schema).to_arrow() if schema is not None else schema
         native = pa.Table.from_pydict(data, schema=pa_schema)
+        return cls.from_native(native, context=context)
+
+    @staticmethod
+    def _is_native(obj: pa.Table | Any) -> TypeIs[pa.Table]:
+        return isinstance(obj, pa.Table)
+
+    @classmethod
+    def from_native(cls, data: pa.Table, /, *, context: _FullContext) -> Self:
         return cls(
-            native,
+            data,
             backend_version=context._backend_version,
             version=context._version,
             validate_column_names=True,
@@ -152,12 +155,7 @@ def from_numpy(
             native = pa.Table.from_arrays(arrays, schema=Schema(schema).to_arrow())
         else:
             native = pa.Table.from_arrays(arrays, cls._numpy_column_names(data, schema))
-        return cls(
-            native,
-            backend_version=context._backend_version,
-            version=context._version,
-            validate_column_names=True,
-        )
+        return cls.from_native(native, context=context)
 
     def __narwhals_namespace__(self: Self) -> ArrowNamespace:
         from narwhals._arrow.namespace import ArrowNamespace
@@ -224,15 +222,8 @@ def rows(self: Self, *, named: bool) -> list[tuple[Any, ...]] | list[dict[str, A
         return self.native.to_pylist()
 
     def iter_columns(self) -> Iterator[ArrowSeries]:
-        from narwhals._arrow.series import ArrowSeries
-
         for name, series in zip(self.columns, self.native.itercolumns()):
-            yield ArrowSeries(
-                series,
-                name=name,
-                backend_version=self._backend_version,
-                version=self._version,
-            )
+            yield ArrowSeries.from_native(series, context=self, name=name)
 
     _iter_columns = iter_columns
 
@@ -251,18 +242,10 @@ def iter_rows(
                 yield from df[i : i + buffer_size].to_pylist()
 
     def get_column(self: Self, name: str) -> ArrowSeries:
-        from narwhals._arrow.series import ArrowSeries
-
         if not isinstance(name, str):
             msg = f"Expected str, got: {type(name)}"
             raise TypeError(msg)
-
-        return ArrowSeries(
-            self.native[name],
-            name=name,
-            backend_version=self._backend_version,
-            version=self._version,
-        )
+        return ArrowSeries.from_native(self.native[name], context=self, name=name)
 
     def __array__(self: Self, dtype: Any, *, copy: bool | None) -> _2DArray:
         return self.native.__array__(dtype, copy=copy)
@@ -304,14 +287,7 @@ def __getitem__(
             item = tuple(list(i) if is_sequence_but_not_str(i) else i for i in item)  # pyright: ignore[reportAssignmentType]
 
         if isinstance(item, str):
-            from narwhals._arrow.series import ArrowSeries
-
-            return ArrowSeries(
-                self.native[item],
-                name=item,
-                backend_version=self._backend_version,
-                version=self._version,
-            )
+            return ArrowSeries.from_native(self.native[item], context=self, name=item)
         elif (
             isinstance(item, tuple)
             and len(item) == 2
@@ -345,7 +321,6 @@ def __getitem__(
                     )
                 msg = f"Expected slice of integers or strings, got: {type(item[1])}"  # pragma: no cover
                 raise TypeError(msg)  # pragma: no cover
-            from narwhals._arrow.series import ArrowSeries
 
             # PyArrow columns are always strings
             col_name = (
@@ -357,18 +332,12 @@ def __getitem__(
                 msg = "Can not slice with tuple with the first element as a str"
                 raise TypeError(msg)
             if (isinstance(item[0], slice)) and (item[0] == slice(None)):
-                return ArrowSeries(
-                    self.native[col_name],
-                    name=col_name,
-                    backend_version=self._backend_version,
-                    version=self._version,
+                return ArrowSeries.from_native(
+                    self.native[col_name], context=self, name=col_name
                 )
             selected_rows = select_rows(self.native, item[0])
-            return ArrowSeries(
-                selected_rows[col_name],
-                name=col_name,
-                backend_version=self._backend_version,
-                version=self._version,
+            return ArrowSeries.from_native(
+                selected_rows[col_name], context=self, name=col_name
             )
 
         elif isinstance(item, slice):
@@ -589,18 +558,10 @@ def to_dict(
         self: Self, *, as_series: bool
     ) -> dict[str, ArrowSeries] | dict[str, list[Any]]:
         df = self.native
-
         names_and_values = zip(df.column_names, df.columns)
         if as_series:
-            from narwhals._arrow.series import ArrowSeries
-
             return {
-                name: ArrowSeries(
-                    col,
-                    name=name,
-                    backend_version=self._backend_version,
-                    version=self._version,
-                )
+                name: ArrowSeries.from_native(col, context=self, name=name)
                 for name, col in names_and_values
             }
         else:
@@ -778,26 +739,20 @@ def write_csv(self: Self, file: str | Path | BytesIO | None) -> str | None:
         return None
 
     def is_unique(self: Self) -> ArrowSeries:
-        from narwhals._arrow.series import ArrowSeries
-
         col_token = generate_temporary_column_name(n_bytes=8, columns=self.columns)
         row_index = pa.array(range(len(self)))
         keep_idx = (
             self.native.append_column(col_token, row_index)
             .group_by(self.columns)
             .aggregate([(col_token, "min"), (col_token, "max")])
         )
-        return ArrowSeries(
-            pa.chunked_array(
-                pc.and_(
-                    pc.is_in(row_index, keep_idx[f"{col_token}_min"]),
-                    pc.is_in(row_index, keep_idx[f"{col_token}_max"]),
-                )
-            ),
-            name="",
-            backend_version=self._backend_version,
-            version=self._version,
+        native = pa.chunked_array(
+            pc.and_(
+                pc.is_in(row_index, keep_idx[f"{col_token}_min"]),
+                pc.is_in(row_index, keep_idx[f"{col_token}_max"]),
+            )
         )
+        return ArrowSeries.from_native(native, context=self)
 
     def unique(
         self: ArrowDataFrame,

narwhals/_arrow/namespace.py

@@ -37,7 +37,11 @@
     from narwhals.utils import Version
 
 
-class ArrowNamespace(EagerNamespace[ArrowDataFrame, ArrowSeries, ArrowExpr]):
+class ArrowNamespace(
+    EagerNamespace[
+        ArrowDataFrame, ArrowSeries, ArrowExpr, "pa.Table", "ArrowChunkedArray"
+    ]
+):
     @property
     def _dataframe(self) -> type[ArrowDataFrame]:
         return ArrowDataFrame

narwhals/_arrow/series.py

@@ -43,6 +43,7 @@
     import pandas as pd
     import polars as pl
     from typing_extensions import Self
+    from typing_extensions import TypeIs
 
     from narwhals._arrow.dataframe import ArrowDataFrame
     from narwhals._arrow.namespace import ArrowNamespace
@@ -135,12 +136,7 @@ def _with_native(
         *,
         preserve_broadcast: bool = False,
     ) -> Self:
-        result = self.__class__(
-            chunked_array(series),
-            name=self._name,
-            backend_version=self._backend_version,
-            version=self._version,
-        )
+        result = self.from_native(chunked_array(series), name=self.name, context=self)
         if preserve_broadcast:
             result._broadcast = self._broadcast
         return result
@@ -156,18 +152,30 @@ def from_iterable(
     ) -> Self:
         version = context._version
         dtype_pa = narwhals_to_native_dtype(dtype, version) if dtype else None
-        return cls(
-            chunked_array([data], dtype_pa),
-            name=name,
-            backend_version=context._backend_version,
-            version=version,
+        return cls.from_native(
+            chunked_array([data], dtype_pa), name=name, context=context
         )
 
     def _from_scalar(self, value: Any) -> Self:
         if self._backend_version < (13,) and hasattr(value, "as_py"):
             value = value.as_py()
         return super()._from_scalar(value)
 
+    @staticmethod
+    def _is_native(obj: ArrowChunkedArray | Any) -> TypeIs[ArrowChunkedArray]:
+        return isinstance(obj, pa.ChunkedArray)
+
+    @classmethod
+    def from_native(
+        cls, data: ArrowChunkedArray, /, *, context: _FullContext, name: str = ""
+    ) -> Self:
+        return cls(
+            data,
+            backend_version=context._backend_version,
+            version=context._version,
+            name=name,
+        )
+
     @classmethod
     def from_numpy(cls, data: Into1DArray, /, *, context: _FullContext) -> Self:
         return cls.from_iterable(
@@ -546,7 +554,7 @@ def tail(self: Self, n: int) -> Self:
             return self._with_native(self.native.slice(abs(n)))
 
     def is_in(self: Self, other: Any) -> Self:
-        if isinstance(other, pa.ChunkedArray):
+        if self._is_native(other):
             value_set: ArrowChunkedArray | ArrowArray = other
         else:
             value_set = pa.array(other)

narwhals/_compliant/__init__.py

@@ -12,6 +12,7 @@
 from narwhals._compliant.group_by import LazyGroupBy
 from narwhals._compliant.namespace import CompliantNamespace
 from narwhals._compliant.namespace import EagerNamespace
+from narwhals._compliant.namespace import LazyNamespace
 from narwhals._compliant.selectors import CompliantSelector
 from narwhals._compliant.selectors import CompliantSelectorNamespace
 from narwhals._compliant.selectors import EagerSelectorNamespace
@@ -64,6 +65,7 @@
     "IntoCompliantExpr",
     "LazyExpr",
     "LazyGroupBy",
+    "LazyNamespace",
     "LazySelectorNamespace",
     "LazyWhen",
     "NativeFrameT_co",

narwhals/_compliant/dataframe.py

@@ -16,10 +16,11 @@
 from narwhals._compliant.typing import CompliantSeriesT
 from narwhals._compliant.typing import EagerExprT_contra
 from narwhals._compliant.typing import EagerSeriesT
-from narwhals._compliant.typing import NativeFrameT_co
+from narwhals._compliant.typing import NativeFrameT
 from narwhals._expression_parsing import evaluate_output_names_and_aliases
 from narwhals._translate import ArrowConvertible
 from narwhals._translate import DictConvertible
+from narwhals._translate import FromNative
 from narwhals._translate import NumpyConvertible
 from narwhals.utils import Version
 from narwhals.utils import _StoresNative
@@ -57,11 +58,12 @@ class CompliantDataFrame(
     NumpyConvertible["_2DArray", "_2DArray"],
     DictConvertible["_ToDict[CompliantSeriesT]", Mapping[str, Any]],
     ArrowConvertible["pa.Table", "IntoArrowTable"],
-    _StoresNative[NativeFrameT_co],
+    _StoresNative[NativeFrameT],
+    FromNative[NativeFrameT],
     Sized,
-    Protocol[CompliantSeriesT, CompliantExprT_contra, NativeFrameT_co],
+    Protocol[CompliantSeriesT, CompliantExprT_contra, NativeFrameT],
 ):
-    _native_frame: Any
+    _native_frame: NativeFrameT
     _implementation: Implementation
     _backend_version: tuple[int, ...]
     _version: Version
@@ -80,6 +82,8 @@ def from_dict(
         schema: Mapping[str, DType] | Schema | None,
     ) -> Self: ...
     @classmethod
+    def from_native(cls, data: NativeFrameT, /, *, context: _FullContext) -> Self: ...
+    @classmethod
     def from_numpy(
         cls,
         data: _2DArray,
@@ -105,8 +109,8 @@ def aggregate(self, *exprs: CompliantExprT_contra) -> Self:
     def _with_version(self, version: Version) -> Self: ...
 
     @property
-    def native(self) -> NativeFrameT_co:
-        return self._native_frame  # type: ignore[no-any-return]
+    def native(self) -> NativeFrameT:
+        return self._native_frame
 
     @property
     def columns(self) -> Sequence[str]: ...
@@ -210,16 +214,21 @@ def write_parquet(self, file: str | Path | BytesIO) -> None: ...
 
 
 class CompliantLazyFrame(
-    _StoresNative[NativeFrameT_co], Protocol[CompliantExprT_contra, NativeFrameT_co]
+    _StoresNative[NativeFrameT],
+    FromNative[NativeFrameT],
+    Protocol[CompliantExprT_contra, NativeFrameT],
 ):
-    _native_frame: Any
+    _native_frame: NativeFrameT
     _implementation: Implementation
     _backend_version: tuple[int, ...]
     _version: Version
 
     def __narwhals_lazyframe__(self) -> Self: ...
     def __narwhals_namespace__(self) -> Any: ...
 
+    @classmethod
+    def from_native(cls, data: NativeFrameT, /, *, context: _FullContext) -> Self: ...
+
     def simple_select(self, *column_names: str) -> Self:
         """`select` where all args are column names."""
         ...
@@ -234,8 +243,8 @@ def aggregate(self, *exprs: CompliantExprT_contra) -> Self:
     def _with_version(self, version: Version) -> Self: ...
 
     @property
-    def native(self) -> NativeFrameT_co:
-        return self._native_frame  # type: ignore[no-any-return]
+    def native(self) -> NativeFrameT:
+        return self._native_frame
 
     @property
     def columns(self) -> Sequence[str]: ...
@@ -307,9 +316,9 @@ def _evaluate_expr(self, expr: CompliantExprT_contra, /) -> Any:
 
 
 class EagerDataFrame(
-    CompliantDataFrame[EagerSeriesT, EagerExprT_contra, NativeFrameT_co],
-    CompliantLazyFrame[EagerExprT_contra, NativeFrameT_co],
-    Protocol[EagerSeriesT, EagerExprT_contra, NativeFrameT_co],
+    CompliantDataFrame[EagerSeriesT, EagerExprT_contra, NativeFrameT],
+    CompliantLazyFrame[EagerExprT_contra, NativeFrameT],
+    Protocol[EagerSeriesT, EagerExprT_contra, NativeFrameT],
 ):
     def _evaluate_expr(self, expr: EagerExprT_contra, /) -> EagerSeriesT:
         """Evaluate `expr` and ensure it has a **single** output."""