Merge remote-tracking branch 'upstream/main' into from-numpy-2d-ns

dangotbanned · dangotbanned · commit 31a525907e9e · 2025-03-24T21:18:53.000Z
diff --git a/.github/workflows/extremes.yml b/.github/workflows/extremes.yml
@@ -62,7 +62,7 @@ jobs:
           cache-suffix: ${{ matrix.python-version }}
           cache-dependency-glob: "pyproject.toml"
       - name: install-pretty-old-versions
-        run: uv pip install pipdeptree tox virtualenv setuptools pandas==1.1.5 polars==0.20.3 numpy==1.17.5 pyarrow==11.0.0 "pyarrow-stubs<17" pyspark==3.5.0 scipy==1.5.0 scikit-learn==1.1.0 duckdb==1.0 tzdata backports.zoneinfo --system
+        run: uv pip install pipdeptree tox virtualenv setuptools pandas==1.1.5 polars==0.20.3 numpy==1.17.5 pyarrow==11.0.0 "pyarrow-stubs<17" scipy==1.5.0 scikit-learn==1.1.0 duckdb==1.0 tzdata backports.zoneinfo --system
       - name: install-reqs
         run: uv pip install -e . --group tests --system
       - name: show-deps
@@ -76,7 +76,6 @@ jobs:
           echo "$DEPS" | grep 'polars==0.20.3'
           echo "$DEPS" | grep 'numpy==1.17.5'
           echo "$DEPS" | grep 'pyarrow==11.0.0'
-          echo "$DEPS" | grep 'pyspark==3.5.0'
           echo "$DEPS" | grep 'scipy==1.5.0'
           echo "$DEPS" | grep 'scikit-learn==1.1.0'
           echo "$DEPS" | grep 'duckdb==1.0'
@@ -101,7 +100,7 @@ jobs:
           cache-suffix: ${{ matrix.python-version }}
           cache-dependency-glob: "pyproject.toml"
       - name: install-not-so-old-versions
-        run: uv pip install tox virtualenv setuptools pandas==2.0.3 polars==0.20.8 numpy==1.24.4 pyarrow==15.0.0 "pyarrow-stubs<17" pyspark==3.5.0 scipy==1.8.0 scikit-learn==1.3.0 duckdb==1.0 dask[dataframe]==2024.10 tzdata --system
+        run: uv pip install tox virtualenv setuptools pandas==2.0.3 polars==0.20.8 numpy==1.24.4 pyarrow==15.0.0 "pyarrow-stubs<17" scipy==1.8.0 scikit-learn==1.3.0 duckdb==1.0 dask[dataframe]==2024.10 tzdata --system
       - name: install-reqs
         run: uv pip install -e . --group tests --system
       - name: show-deps
@@ -113,7 +112,6 @@ jobs:
           echo "$DEPS" | grep 'polars==0.20.8'
           echo "$DEPS" | grep 'numpy==1.24.4'
           echo "$DEPS" | grep 'pyarrow==15.0.0'
-          echo "$DEPS" | grep 'pyspark==3.5.0'
           echo "$DEPS" | grep 'scipy==1.8.0'
           echo "$DEPS" | grep 'scikit-learn==1.3.0'
           echo "$DEPS" | grep 'dask==2024.10'
diff --git a/.github/workflows/pytest.yml b/.github/workflows/pytest.yml
@@ -87,7 +87,7 @@ jobs:
       - name: install duckdb nightly
         run: uv pip install -U --pre duckdb --system
       - name: install pyspark
-        run: uv pip install -e ".[pyspark]" --system
+        run: echo "setuptools<78" | uv pip install -b - -e ".[pyspark]" --system
         # PySpark is not yet available on Python3.12+
         if: matrix.python-version != '3.13'
       - name: install ibis
diff --git a/.github/workflows/typing.yml b/.github/workflows/typing.yml
@@ -32,7 +32,7 @@ jobs:
         # TODO: add more dependencies/backends incrementally
         run: |
           source .venv/bin/activate
-          uv pip install -e ".[pyspark]" --group core --group typing
+          echo "setuptools<78" | uv pip install -b - -e ".[pyspark]" --group core --group typing
       - name: show-deps
         run: |
           source .venv/bin/activate
diff --git a/narwhals/_arrow/namespace.py b/narwhals/_arrow/namespace.py
@@ -72,7 +72,7 @@ def len(self: Self) -> ArrowExpr:
             version=self._version,
         )
 
-    def lit(self: Self, value: Any, dtype: DType | None) -> ArrowExpr:
+    def lit(self: Self, value: Any, dtype: DType | type[DType] | None) -> ArrowExpr:
         def _lit_arrow_series(_: ArrowDataFrame) -> ArrowSeries:
             arrow_series = ArrowSeries.from_iterable(
                 data=[value], name="literal", context=self
diff --git a/narwhals/_compliant/expr.py b/narwhals/_compliant/expr.py
@@ -106,8 +106,6 @@ def from_column_indices(
         cls: type[Self], *column_indices: int, context: _FullContext
     ) -> Self: ...
 
-    def _with_metadata(self, metadata: ExprMetadata) -> Self: ...
-
     def is_null(self) -> Self: ...
     def abs(self) -> Self: ...
     def all(self) -> Self: ...
@@ -353,21 +351,6 @@ def __narwhals_namespace__(
     ) -> EagerNamespace[EagerDataFrameT, EagerSeriesT, Self]: ...
     def __narwhals_expr__(self) -> None: ...
 
-    def _with_metadata(self, metadata: ExprMetadata) -> Self:
-        expr = self.__class__(
-            self._call,
-            function_name=self._function_name,
-            evaluate_output_names=self._evaluate_output_names,
-            alias_output_names=self._alias_output_names,
-            backend_version=self._backend_version,
-            version=self._version,
-            depth=self._depth,
-            implementation=self._implementation,
-            call_kwargs=self._call_kwargs,
-        )
-        expr._metadata = metadata
-        return expr
-
     @classmethod
     def _from_callable(
         cls,
diff --git a/narwhals/_compliant/namespace.py b/narwhals/_compliant/namespace.py
@@ -62,7 +62,7 @@ def nth(self, *column_indices: int) -> CompliantExprT:
         return self._expr.from_column_indices(*column_indices, context=self)
 
     def len(self) -> CompliantExprT: ...
-    def lit(self, value: Any, dtype: DType | None) -> CompliantExprT: ...
+    def lit(self, value: Any, dtype: DType | type[DType] | None) -> CompliantExprT: ...
     def all_horizontal(self, *exprs: CompliantExprT) -> CompliantExprT: ...
     def any_horizontal(self, *exprs: CompliantExprT) -> CompliantExprT: ...
     def sum_horizontal(self, *exprs: CompliantExprT) -> CompliantExprT: ...
diff --git a/narwhals/_dask/expr.py b/narwhals/_dask/expr.py
@@ -83,20 +83,6 @@ def __narwhals_namespace__(self) -> DaskNamespace:  # pragma: no cover
 
         return DaskNamespace(backend_version=self._backend_version, version=self._version)
 
-    def _with_metadata(self, metadata: ExprMetadata) -> Self:
-        expr = self.__class__(
-            self._call,
-            function_name=self._function_name,
-            evaluate_output_names=self._evaluate_output_names,
-            alias_output_names=self._alias_output_names,
-            backend_version=self._backend_version,
-            version=self._version,
-            depth=self._depth,
-            call_kwargs=self._call_kwargs,
-        )
-        expr._metadata = metadata
-        return expr
-
     def broadcast(self, kind: Literal[ExprKind.AGGREGATION, ExprKind.LITERAL]) -> Self:
         def func(df: DaskLazyFrame) -> list[dx.Series]:
             return [result[0] for result in self(df)]
diff --git a/narwhals/_dask/namespace.py b/narwhals/_dask/namespace.py
@@ -55,7 +55,7 @@ def __init__(
         self._backend_version = backend_version
         self._version = version
 
-    def lit(self: Self, value: Any, dtype: DType | None) -> DaskExpr:
+    def lit(self: Self, value: Any, dtype: DType | type[DType] | None) -> DaskExpr:
         def func(df: DaskLazyFrame) -> list[dx.Series]:
             if dtype is not None:
                 native_dtype = narwhals_to_native_dtype(dtype, self._version)
diff --git a/narwhals/_duckdb/expr.py b/narwhals/_duckdb/expr.py
@@ -79,19 +79,6 @@ def __narwhals_namespace__(self) -> DuckDBNamespace:  # pragma: no cover
             backend_version=self._backend_version, version=self._version
         )
 
-    def _with_metadata(self, metadata: ExprMetadata) -> Self:
-        expr = self.__class__(
-            self._call,
-            evaluate_output_names=self._evaluate_output_names,
-            alias_output_names=self._alias_output_names,
-            backend_version=self._backend_version,
-            version=self._version,
-        )
-        if func := self._window_function:
-            expr = expr._with_window_function(func)
-        expr._metadata = metadata
-        return expr
-
     def broadcast(self, kind: Literal[ExprKind.AGGREGATION, ExprKind.LITERAL]) -> Self:
         if kind is ExprKind.LITERAL:
             return self
diff --git a/narwhals/_duckdb/namespace.py b/narwhals/_duckdb/namespace.py
@@ -213,7 +213,7 @@ def func(df: DuckDBLazyFrame) -> list[duckdb.Expression]:
     def when(self: Self, predicate: DuckDBExpr) -> DuckDBWhen:
         return DuckDBWhen.from_expr(predicate, context=self)
 
-    def lit(self: Self, value: Any, dtype: DType | None) -> DuckDBExpr:
+    def lit(self: Self, value: Any, dtype: DType | type[DType] | None) -> DuckDBExpr:
         def func(_df: DuckDBLazyFrame) -> list[duckdb.Expression]:
             if dtype is not None:
                 return [
diff --git a/narwhals/_pandas_like/namespace.py b/narwhals/_pandas_like/namespace.py
@@ -61,7 +61,7 @@ def __init__(
         self._backend_version = backend_version
         self._version = version
 
-    def lit(self: Self, value: Any, dtype: DType | None) -> PandasLikeExpr:
+    def lit(self: Self, value: Any, dtype: DType | type[DType] | None) -> PandasLikeExpr:
         def _lit_pandas_series(df: PandasLikeDataFrame) -> PandasLikeSeries:
             pandas_series = self._series.from_iterable(
                 data=[value],
diff --git a/narwhals/_polars/expr.py b/narwhals/_polars/expr.py
@@ -40,15 +40,6 @@ def _from_native_expr(self: Self, expr: pl.Expr) -> Self:
             expr, version=self._version, backend_version=self._backend_version
         )
 
-    def _with_metadata(self, metadata: ExprMetadata) -> Self:
-        expr = self.__class__(
-            self._native_expr,
-            backend_version=self._backend_version,
-            version=self._version,
-        )
-        expr._metadata = metadata
-        return expr
-
     @classmethod
     def _from_series(cls, series: Any) -> Self:
         return cls(
diff --git a/narwhals/_polars/namespace.py b/narwhals/_polars/namespace.py
@@ -165,7 +165,7 @@ def concat(
             result, backend_version=self._backend_version, version=self._version
         )
 
-    def lit(self: Self, value: Any, dtype: DType | None) -> PolarsExpr:
+    def lit(self: Self, value: Any, dtype: DType | type[DType] | None) -> PolarsExpr:
         if dtype is not None:
             return self._expr(
                 pl.lit(
diff --git a/narwhals/_spark_like/expr.py b/narwhals/_spark_like/expr.py
@@ -119,20 +119,6 @@ def __narwhals_namespace__(self: Self) -> SparkLikeNamespace:  # pragma: no cove
             implementation=self._implementation,
         )
 
-    def _with_metadata(self, metadata: ExprMetadata) -> Self:
-        expr = self.__class__(
-            self._call,
-            evaluate_output_names=self._evaluate_output_names,
-            alias_output_names=self._alias_output_names,
-            backend_version=self._backend_version,
-            version=self._version,
-            implementation=self._implementation,
-        )
-        if func := self._window_function:
-            expr = expr._with_window_function(func)
-        expr._metadata = metadata
-        return expr
-
     def _with_window_function(
         self: Self,
         window_function: WindowFunction,
diff --git a/narwhals/_spark_like/namespace.py b/narwhals/_spark_like/namespace.py
@@ -46,7 +46,9 @@ def selectors(self: Self) -> SparkLikeSelectorNamespace:
     def _expr(self) -> type[SparkLikeExpr]:
         return SparkLikeExpr
 
-    def lit(self: Self, value: object, dtype: DType | None) -> SparkLikeExpr:
+    def lit(
+        self: Self, value: object, dtype: DType | type[DType] | None
+    ) -> SparkLikeExpr:
         def _lit(df: SparkLikeLazyFrame) -> list[Column]:
             column = df._F.lit(value)
             if dtype:
diff --git a/narwhals/expr.py b/narwhals/expr.py
@@ -49,12 +49,15 @@
 
 class Expr:
     def __init__(
-        self: Self,
-        to_compliant_expr: Callable[[Any], Any],
-        metadata: ExprMetadata,
+        self: Self, to_compliant_expr: _ToCompliant, metadata: ExprMetadata
     ) -> None:
         # callable from CompliantNamespace to CompliantExpr
-        self._to_compliant_expr: _ToCompliant = to_compliant_expr
+        def func(plx: CompliantNamespace[Any, Any]) -> CompliantExpr[Any, Any]:
+            result = to_compliant_expr(plx)
+            result._metadata = self._metadata
+            return result
+
+        self._to_compliant_expr: _ToCompliant = func
         self._metadata = metadata
 
     def _from_callable(self, to_compliant_expr: Callable[[Any], Any]) -> Self:
@@ -1598,9 +1601,9 @@ def over(
         )
 
         return self.__class__(
-            lambda plx: self._to_compliant_expr(plx)
-            ._with_metadata(current_meta)
-            .over(flat_partition_by, flat_order_by),
+            lambda plx: self._to_compliant_expr(plx).over(
+                flat_partition_by, flat_order_by
+            ),
             next_meta,
         )
 
diff --git a/narwhals/group_by.py b/narwhals/group_by.py
@@ -84,9 +84,9 @@ def agg(self: Self, *aggs: Expr | Iterable[Expr], **named_aggs: Expr) -> DataFra
             raise InvalidOperationError(msg)
         plx = self._df.__narwhals_namespace__()
         compliant_aggs = (
-            *(x._to_compliant_expr(plx)._with_metadata(x._metadata) for x in flat_aggs),
+            *(x._to_compliant_expr(plx) for x in flat_aggs),
             *(
-                value._to_compliant_expr(plx).alias(key)._with_metadata(value._metadata)
+                value.alias(key)._to_compliant_expr(plx)
                 for key, value in named_aggs.items()
             ),
         )
@@ -174,9 +174,9 @@ def agg(self: Self, *aggs: Expr | Iterable[Expr], **named_aggs: Expr) -> LazyFra
             raise InvalidOperationError(msg)
         plx = self._df.__narwhals_namespace__()
         compliant_aggs = (
-            *(x._to_compliant_expr(plx)._with_metadata(x._metadata) for x in flat_aggs),
+            *(x._to_compliant_expr(plx) for x in flat_aggs),
             *(
-                value._to_compliant_expr(plx).alias(key)._with_metadata(value._metadata)
+                value.alias(key)._to_compliant_expr(plx)
                 for key, value in named_aggs.items()
             ),
         )

Original file line number	Diff line number	Diff line change
`@@ -72,7 +72,7 @@ def len(self: Self) -> ArrowExpr:`
`72`	`72`	`version=self._version,`
`73`	`73`	`)`
`74`	`74`
`75`		`- def lit(self: Self, value: Any, dtype: DType \| None) -> ArrowExpr:`
	`75`	`+ def lit(self: Self, value: Any, dtype: DType \| type[DType] \| None) -> ArrowExpr:`
`76`	`76`	`def _lit_arrow_series(_: ArrowDataFrame) -> ArrowSeries:`
`77`	`77`	`arrow_series = ArrowSeries.from_iterable(`
`78`	`78`	`data=[value], name="literal", context=self`