chore: Add Compliant*.to_narwhals (#2461)

Author: dangotbanned

narwhals/_arrow/dataframe.py

@@ -51,11 +51,11 @@
     from narwhals._arrow.typing import ChunkedArrayAny
     from narwhals._arrow.typing import Mask  # type: ignore[attr-defined]
     from narwhals._arrow.typing import Order  # type: ignore[attr-defined]
+    from narwhals._compliant.typing import CompliantDataFrameAny
+    from narwhals._compliant.typing import CompliantLazyFrameAny
     from narwhals._translate import IntoArrowTable
     from narwhals.dtypes import DType
     from narwhals.schema import Schema
-    from narwhals.typing import CompliantDataFrame
-    from narwhals.typing import CompliantLazyFrame
     from narwhals.typing import JoinStrategy
     from narwhals.typing import SizedMultiIndexSelector
     from narwhals.typing import SizedMultiNameSelector
@@ -539,9 +539,7 @@ def tail(self, n: int) -> Self:
         else:
             return self._with_native(df.slice(abs(n)), validate_column_names=False)
 
-    def lazy(
-        self, *, backend: Implementation | None = None
-    ) -> CompliantLazyFrame[Any, Any]:
+    def lazy(self, *, backend: Implementation | None = None) -> CompliantLazyFrameAny:
         if backend is None:
             return self
         elif backend is Implementation.DUCKDB:
@@ -579,10 +577,8 @@ def lazy(
         raise AssertionError  # pragma: no cover
 
     def collect(
-        self,
-        backend: Implementation | None,
-        **kwargs: Any,
-    ) -> CompliantDataFrame[Any, Any, Any]:
+        self, backend: Implementation | None, **kwargs: Any
+    ) -> CompliantDataFrameAny:
         if backend is Implementation.PYARROW or backend is None:
             from narwhals._arrow.dataframe import ArrowDataFrame
 

narwhals/_compliant/dataframe.py

@@ -12,7 +12,9 @@
 from typing import TypeVar
 from typing import overload
 
+from narwhals._compliant.typing import CompliantDataFrameAny
 from narwhals._compliant.typing import CompliantExprT_contra
+from narwhals._compliant.typing import CompliantLazyFrameAny
 from narwhals._compliant.typing import CompliantSeriesT
 from narwhals._compliant.typing import EagerExprT
 from narwhals._compliant.typing import EagerSeriesT
@@ -22,6 +24,8 @@
 from narwhals._translate import DictConvertible
 from narwhals._translate import FromNative
 from narwhals._translate import NumpyConvertible
+from narwhals._translate import ToNarwhals
+from narwhals._translate import ToNarwhalsT_co
 from narwhals.utils import Version
 from narwhals.utils import _StoresNative
 from narwhals.utils import deprecated
@@ -47,6 +51,7 @@
     from narwhals._compliant.group_by import DataFrameGroupBy
     from narwhals._compliant.namespace import EagerNamespace
     from narwhals._translate import IntoArrowTable
+    from narwhals.dataframe import DataFrame
     from narwhals.dtypes import DType
     from narwhals.schema import Schema
     from narwhals.typing import AsofJoinStrategy
@@ -81,8 +86,9 @@ class CompliantDataFrame(
     ArrowConvertible["pa.Table", "IntoArrowTable"],
     _StoresNative[NativeFrameT],
     FromNative[NativeFrameT],
+    ToNarwhals[ToNarwhalsT_co],
     Sized,
-    Protocol[CompliantSeriesT, CompliantExprT_contra, NativeFrameT],
+    Protocol[CompliantSeriesT, CompliantExprT_contra, NativeFrameT, ToNarwhalsT_co],
 ):
     _native_frame: NativeFrameT
     _implementation: Implementation
@@ -113,6 +119,7 @@ def from_numpy(
         context: _FullContext,
         schema: Mapping[str, DType] | Schema | Sequence[str] | None,
     ) -> Self: ...
+
     def __array__(self, dtype: Any, *, copy: bool | None) -> _2DArray: ...
     def __getitem__(
         self,
@@ -148,7 +155,7 @@ def shape(self) -> tuple[int, int]: ...
     def clone(self) -> Self: ...
     def collect(
         self, backend: Implementation | None, **kwargs: Any
-    ) -> CompliantDataFrame[Any, Any, Any]: ...
+    ) -> CompliantDataFrameAny: ...
     def collect_schema(self) -> Mapping[str, DType]: ...
     def drop(self, columns: Sequence[str], *, strict: bool) -> Self: ...
     def drop_nulls(self, subset: Sequence[str] | None) -> Self: ...
@@ -190,7 +197,7 @@ def join_asof(
         strategy: AsofJoinStrategy,
         suffix: str,
     ) -> Self: ...
-    def lazy(self, *, backend: Implementation | None) -> CompliantLazyFrame[Any, Any]: ...
+    def lazy(self, *, backend: Implementation | None) -> CompliantLazyFrameAny: ...
     def pivot(
         self,
         on: Sequence[str],
@@ -260,7 +267,8 @@ def _evaluate_aliases(self, *exprs: CompliantExprT_contra) -> list[str]:
 class CompliantLazyFrame(
     _StoresNative[NativeFrameT],
     FromNative[NativeFrameT],
-    Protocol[CompliantExprT_contra, NativeFrameT],
+    ToNarwhals[ToNarwhalsT_co],
+    Protocol[CompliantExprT_contra, NativeFrameT, ToNarwhalsT_co],
 ):
     _native_frame: NativeFrameT
     _implementation: Implementation
@@ -297,7 +305,7 @@ def schema(self) -> Mapping[str, DType]: ...
     def _iter_columns(self) -> Iterator[Any]: ...
     def collect(
         self, backend: Implementation | None, **kwargs: Any
-    ) -> CompliantDataFrame[Any, Any, Any]: ...
+    ) -> CompliantDataFrameAny: ...
     def collect_schema(self) -> Mapping[str, DType]: ...
     def drop(self, columns: Sequence[str], *, strict: bool) -> Self: ...
     def drop_nulls(self, subset: Sequence[str] | None) -> Self: ...
@@ -364,14 +372,17 @@ def _evaluate_aliases(self, *exprs: CompliantExprT_contra) -> list[str]:
 
 
 class EagerDataFrame(
-    CompliantDataFrame[EagerSeriesT, EagerExprT, NativeFrameT],
-    CompliantLazyFrame[EagerExprT, NativeFrameT],
+    CompliantDataFrame[EagerSeriesT, EagerExprT, NativeFrameT, "DataFrame[NativeFrameT]"],
+    CompliantLazyFrame[EagerExprT, NativeFrameT, "DataFrame[NativeFrameT]"],
     Protocol[EagerSeriesT, EagerExprT, NativeFrameT, NativeSeriesT],
 ):
     def __narwhals_namespace__(
         self,
     ) -> EagerNamespace[Self, EagerSeriesT, EagerExprT, NativeFrameT, NativeSeriesT]: ...
 
+    def to_narwhals(self) -> DataFrame[NativeFrameT]:
+        return self._version.dataframe(self, level="full")
+
     def _evaluate_expr(self, expr: EagerExprT, /) -> EagerSeriesT:
         """Evaluate `expr` and ensure it has a **single** output."""
         result: Sequence[EagerSeriesT] = expr(self)

narwhals/_compliant/namespace.py

@@ -148,6 +148,14 @@ def when(
     def from_native(self, data: NativeFrameT, /) -> EagerDataFrameT: ...
     @overload
     def from_native(self, data: NativeSeriesT, /) -> EagerSeriesT: ...
+    # TODO @dangotbanned: Align `PandasLike` typing with `_namespace`, then drop this `@overload`
+    # - Using the guards there introduces `_NativeModin`, `_NativeCuDF`
+    # - These types haven't been integrated into the backend
+    # - Most of the `pandas` stuff is still untyped
+    @overload
+    def from_native(
+        self, data: NativeFrameT | NativeSeriesT | Any, /
+    ) -> EagerDataFrameT | EagerSeriesT: ...
     def from_native(
         self, data: NativeFrameT | NativeSeriesT | Any, /
     ) -> EagerDataFrameT | EagerSeriesT:

narwhals/_compliant/series.py

@@ -21,6 +21,7 @@
 from narwhals._translate import FromIterable
 from narwhals._translate import FromNative
 from narwhals._translate import NumpyConvertible
+from narwhals._translate import ToNarwhals
 from narwhals.utils import _StoresCompliant
 from narwhals.utils import _StoresNative
 from narwhals.utils import is_compliant_series
@@ -41,6 +42,7 @@
     from narwhals._compliant.namespace import CompliantNamespace
     from narwhals._compliant.namespace import EagerNamespace
     from narwhals.dtypes import DType
+    from narwhals.series import Series
     from narwhals.typing import ClosedInterval
     from narwhals.typing import FillNullStrategy
     from narwhals.typing import Into1DArray
@@ -64,6 +66,7 @@ class CompliantSeries(
     NumpyConvertible["_1DArray", "Into1DArray"],
     FromIterable,
     FromNative[NativeSeriesT],
+    ToNarwhals["Series[NativeSeriesT]"],
     Protocol[NativeSeriesT],
 ):
     _implementation: Implementation
@@ -105,6 +108,8 @@ def from_iterable(
         name: str = "",
         dtype: DType | type[DType] | None = None,
     ) -> Self: ...
+    def to_narwhals(self) -> Series[NativeSeriesT]:
+        return self._version.series(self, level="full")
 
     # Operators
     def __add__(self, other: Any) -> Self: ...
@@ -180,7 +185,7 @@ def hist(
         *,
         bin_count: int | None,
         include_breakpoint: bool,
-    ) -> CompliantDataFrame[Self, Any, Any]: ...
+    ) -> CompliantDataFrame[Self, Any, Any, Any]: ...
     def head(self, n: int) -> Self: ...
     def is_between(
         self, lower_bound: Any, upper_bound: Any, closed: ClosedInterval
@@ -262,8 +267,8 @@ def tail(self, n: int) -> Self: ...
     def to_arrow(self) -> pa.Array[Any]: ...
     def to_dummies(
         self, *, separator: str, drop_first: bool
-    ) -> CompliantDataFrame[Self, Any, Any]: ...
-    def to_frame(self) -> CompliantDataFrame[Self, Any, Any]: ...
+    ) -> CompliantDataFrame[Self, Any, Any, Any]: ...
+    def to_frame(self) -> CompliantDataFrame[Self, Any, Any, Any]: ...
     def to_list(self) -> list[Any]: ...
     def to_pandas(self) -> pd.Series[Any]: ...
     def to_polars(self) -> pl.Series: ...
@@ -275,7 +280,7 @@ def value_counts(
         parallel: bool,
         name: str | None,
         normalize: bool,
-    ) -> CompliantDataFrame[Self, Any, Any]: ...
+    ) -> CompliantDataFrame[Self, Any, Any, Any]: ...
     def var(self, *, ddof: int) -> float: ...
     def zip_with(self, mask: Any, other: Any) -> Self: ...
 

narwhals/_compliant/typing.py

@@ -40,8 +40,8 @@
 CompliantExprAny: TypeAlias = "CompliantExpr[Any, Any]"
 CompliantSeriesAny: TypeAlias = "CompliantSeries[Any]"
 CompliantSeriesOrNativeExprAny: TypeAlias = "CompliantSeriesAny | NativeExpr"
-CompliantDataFrameAny: TypeAlias = "CompliantDataFrame[Any, Any, Any]"
-CompliantLazyFrameAny: TypeAlias = "CompliantLazyFrame[Any, Any]"
+CompliantDataFrameAny: TypeAlias = "CompliantDataFrame[Any, Any, Any, Any]"
+CompliantLazyFrameAny: TypeAlias = "CompliantLazyFrame[Any, Any, Any]"
 CompliantFrameAny: TypeAlias = "CompliantDataFrameAny | CompliantLazyFrameAny"
 CompliantNamespaceAny: TypeAlias = "CompliantNamespace[Any, Any]"
 

narwhals/_dask/dataframe.py

@@ -13,7 +13,6 @@
 from narwhals._dask.utils import evaluate_exprs
 from narwhals._pandas_like.utils import native_to_narwhals_dtype
 from narwhals._pandas_like.utils import select_columns_by_name
-from narwhals.typing import CompliantDataFrame
 from narwhals.typing import CompliantLazyFrame
 from narwhals.utils import Implementation
 from narwhals.utils import _remap_full_join_keys
@@ -32,9 +31,11 @@
     from typing_extensions import Self
     from typing_extensions import TypeIs
 
+    from narwhals._compliant.typing import CompliantDataFrameAny
     from narwhals._dask.expr import DaskExpr
     from narwhals._dask.group_by import DaskLazyGroupBy
     from narwhals._dask.namespace import DaskNamespace
+    from narwhals.dataframe import LazyFrame
     from narwhals.dtypes import DType
     from narwhals.typing import AsofJoinStrategy
     from narwhals.typing import JoinStrategy
@@ -43,7 +44,9 @@
     from narwhals.utils import _FullContext
 
 
-class DaskLazyFrame(CompliantLazyFrame["DaskExpr", "dd.DataFrame"]):
+class DaskLazyFrame(
+    CompliantLazyFrame["DaskExpr", "dd.DataFrame", "LazyFrame[dd.DataFrame]"]
+):
     def __init__(
         self,
         native_dataframe: dd.DataFrame,
@@ -69,6 +72,9 @@ def from_native(cls, data: dd.DataFrame, /, *, context: _FullContext) -> Self:
             data, backend_version=context._backend_version, version=context._version
         )
 
+    def to_narwhals(self) -> LazyFrame[dd.DataFrame]:
+        return self._version.lazyframe(self, level="lazy")
+
     def __native_namespace__(self) -> ModuleType:
         if self._implementation is Implementation.DASK:
             return self._implementation.to_native_namespace()
@@ -103,10 +109,8 @@ def with_columns(self, *exprs: DaskExpr) -> Self:
         return self._with_native(self.native.assign(**dict(new_series)))
 
     def collect(
-        self,
-        backend: Implementation | None,
-        **kwargs: Any,
-    ) -> CompliantDataFrame[Any, Any, Any]:
+        self, backend: Implementation | None, **kwargs: Any
+    ) -> CompliantDataFrameAny:
         result = self.native.compute(**kwargs)
 
         if backend is None or backend is Implementation.PANDAS:

narwhals/_duckdb/dataframe.py

@@ -21,7 +21,6 @@
 from narwhals.dependencies import get_duckdb
 from narwhals.exceptions import ColumnNotFoundError
 from narwhals.exceptions import InvalidOperationError
-from narwhals.typing import CompliantDataFrame
 from narwhals.typing import CompliantLazyFrame
 from narwhals.utils import Implementation
 from narwhals.utils import Version
@@ -39,11 +38,14 @@
     from typing_extensions import Self
     from typing_extensions import TypeIs
 
+    from narwhals._compliant.typing import CompliantDataFrameAny
     from narwhals._duckdb.expr import DuckDBExpr
     from narwhals._duckdb.group_by import DuckDBGroupBy
     from narwhals._duckdb.namespace import DuckDBNamespace
     from narwhals._duckdb.series import DuckDBInterchangeSeries
+    from narwhals.dataframe import LazyFrame
     from narwhals.dtypes import DType
+    from narwhals.stable.v1 import DataFrame as DataFrameV1
     from narwhals.typing import AsofJoinStrategy
     from narwhals.typing import JoinStrategy
     from narwhals.typing import LazyUniqueKeepStrategy
@@ -53,7 +55,13 @@
     from duckdb import SQLExpression  # type: ignore[attr-defined, unused-ignore]
 
 
-class DuckDBLazyFrame(CompliantLazyFrame["DuckDBExpr", "duckdb.DuckDBPyRelation"]):
+class DuckDBLazyFrame(
+    CompliantLazyFrame[
+        "DuckDBExpr",
+        "duckdb.DuckDBPyRelation",
+        "LazyFrame[duckdb.DuckDBPyRelation] | DataFrameV1[duckdb.DuckDBPyRelation]",
+    ]
+):
     _implementation = Implementation.DUCKDB
 
     def __init__(
@@ -82,6 +90,16 @@ def from_native(
             data, backend_version=context._backend_version, version=context._version
         )
 
+    def to_narwhals(
+        self, *args: Any, **kwds: Any
+    ) -> LazyFrame[duckdb.DuckDBPyRelation] | DataFrameV1[duckdb.DuckDBPyRelation]:
+        if self._version is Version.MAIN:
+            return self._version.lazyframe(self, level="lazy")
+
+        from narwhals.stable.v1 import DataFrame as DataFrameV1
+
+        return DataFrameV1(self, level="interchange")  # type: ignore[no-any-return]
+
     def __narwhals_dataframe__(self) -> Self:  # pragma: no cover
         # Keep around for backcompat.
         if self._version is not Version.V1:
@@ -112,10 +130,8 @@ def _iter_columns(self) -> Iterator[duckdb.Expression]:
             yield col(name)
 
     def collect(
-        self,
-        backend: ModuleType | Implementation | str | None,
-        **kwargs: Any,
-    ) -> CompliantDataFrame[Any, Any, Any]:
+        self, backend: ModuleType | Implementation | str | None, **kwargs: Any
+    ) -> CompliantDataFrameAny:
         if backend is None or backend is Implementation.PYARROW:
             import pyarrow as pa  # ignore-banned-import
 

narwhals/_pandas_like/dataframe.py

@@ -50,15 +50,15 @@
     from typing_extensions import TypeAlias
     from typing_extensions import TypeIs
 
+    from narwhals._compliant.typing import CompliantDataFrameAny
+    from narwhals._compliant.typing import CompliantLazyFrameAny
     from narwhals._pandas_like.expr import PandasLikeExpr
     from narwhals._pandas_like.group_by import PandasLikeGroupBy
     from narwhals._pandas_like.namespace import PandasLikeNamespace
     from narwhals._translate import IntoArrowTable
     from narwhals.dtypes import DType
     from narwhals.schema import Schema
     from narwhals.typing import AsofJoinStrategy
-    from narwhals.typing import CompliantDataFrame
-    from narwhals.typing import CompliantLazyFrame
     from narwhals.typing import DTypeBackend
     from narwhals.typing import JoinStrategy
     from narwhals.typing import PivotAgg
@@ -518,10 +518,8 @@ def sort(
 
     # --- convert ---
     def collect(
-        self,
-        backend: Implementation | None,
-        **kwargs: Any,
-    ) -> CompliantDataFrame[Any, Any, Any]:
+        self, backend: Implementation | None, **kwargs: Any
+    ) -> CompliantDataFrameAny:
         if backend is None:
             return PandasLikeDataFrame(
                 self.native,
@@ -773,9 +771,7 @@ def unique(
         )
 
     # --- lazy-only ---
-    def lazy(
-        self, *, backend: Implementation | None = None
-    ) -> CompliantLazyFrame[Any, Any]:
+    def lazy(self, *, backend: Implementation | None = None) -> CompliantLazyFrameAny:
         from narwhals.utils import parse_version
 
         pandas_df = self.to_pandas()