Merge remote-tracking branch 'upstream/main' into oh-nodes

Author: dangotbanned

.github/workflows/downstream_tests.yml

@@ -367,7 +367,7 @@ jobs:
       - name: install-deps
         run: |
           cd hierarchicalforecast
-          uv pip install --system ".[dev,polars]"
+          uv pip install . --group dev --group polars --system
       - name: install-narwhals-dev
         run: |
           uv pip uninstall narwhals --system
@@ -627,7 +627,8 @@ jobs:
       - name: install-deps
         run: |
           cd fairlearn
-          uv pip install -e . -r requirements.txt matplotlib polars pyarrow pytest typing-extensions --system
+          # TODO(FBruzzesi): Align with fairlearn team to get a minimal requirement to test narwhals features
+          uv pip install -e . -r requirements.txt lightgbm matplotlib polars pyarrow pytest typing-extensions --system
       - name: install-narwhals-dev
         run: |
           cd fairlearn
@@ -637,6 +638,4 @@ jobs:
       - name: run-pytest
         run: |
           cd fairlearn
-          # TODO(FBruzzesi): I hope this will be simplified once there is a decision on
-          # https://github.com/fairlearn/fairlearn/issues/1555
-          pytest test/unit/preprocessing test/unit/metrics
+          pytest test/unit -m "narwhals"

docs/api-reference/expr_list.md

@@ -11,6 +11,7 @@
         - mean
         - median
         - min
+        - sort
         - sum
         - unique
       show_source: false

docs/api-reference/series_list.md

@@ -11,6 +11,7 @@
         - mean
         - median
         - min
+        - sort
         - sum
         - unique
       show_source: false

docs/javascripts/table.js

@@ -1,4 +1,10 @@
 document$.subscribe(function() {
+  // Only run on api-completeness pages
+  var currentPath = window.location.pathname
+  if (!currentPath.includes('/api-completeness/')) {
+    return
+  }
+  
   var tables = document.querySelectorAll("article table:not([class])")
   tables.forEach(function(table) {
     new Tablesort(table)

narwhals/_arrow/series_list.py

@@ -5,7 +5,7 @@
 import pyarrow as pa
 import pyarrow.compute as pc
 
-from narwhals._arrow.utils import ArrowSeriesNamespace, list_agg
+from narwhals._arrow.utils import ArrowSeriesNamespace, list_agg, list_sort
 from narwhals._compliant.any_namespace import ListNamespace
 from narwhals._utils import not_implemented
 
@@ -35,5 +35,10 @@ def median(self) -> ArrowSeries:
     def sum(self) -> ArrowSeries:
         return self.with_native(list_agg(self.native, "sum"))
 
+    def sort(self, *, descending: bool, nulls_last: bool) -> ArrowSeries:
+        return self.with_native(
+            list_sort(self.native, descending=descending, nulls_last=nulls_last)
+        )
+
     unique = not_implemented()
     contains = not_implemented()

narwhals/_arrow/utils.py

@@ -532,3 +532,35 @@ def list_agg(
             )
         ]
     )
+
+
+def list_sort(
+    array: ChunkedArrayAny, *, descending: bool, nulls_last: bool
+) -> ChunkedArrayAny:
+    sort_direction: Literal["ascending", "descending"] = (
+        "descending" if descending else "ascending"
+    )
+    nulls_position: Literal["at_start", "at_end"] = "at_end" if nulls_last else "at_start"
+    idx, v = "idx", "values"
+    is_not_sorted = pc.greater(pc.list_value_length(array), lit(0))
+    indexed = pa.Table.from_arrays(
+        [arange(start=0, end=len(array), step=1), array], names=[idx, v]
+    )
+    not_sorted_part = indexed.filter(is_not_sorted)
+    pass_through = indexed.filter(pc.fill_null(pc.invert(is_not_sorted), lit(True)))  # pyright: ignore[reportArgumentType]
+    exploded = pa.Table.from_arrays(
+        [pc.list_flatten(array), pc.list_parent_indices(array)], names=[v, idx]
+    )
+    sorted_indices = pc.sort_indices(
+        exploded,
+        sort_keys=[(idx, "ascending"), (v, sort_direction)],
+        null_placement=nulls_position,
+    )
+    offsets = not_sorted_part.column(v).combine_chunks().offsets  # type: ignore[attr-defined]
+    sorted_imploded = pa.ListArray.from_arrays(
+        offsets, pa.array(exploded.take(sorted_indices).column(v))
+    )
+    imploded_by_idx = pa.Table.from_arrays(
+        [not_sorted_part.column(idx), sorted_imploded], names=[idx, v]
+    )
+    return pa.concat_tables([imploded_by_idx, pass_through]).sort_by(idx).column(v)

narwhals/_compliant/any_namespace.py

@@ -75,6 +75,7 @@ def max(self) -> CompliantT_co: ...
     def mean(self) -> CompliantT_co: ...
     def median(self) -> CompliantT_co: ...
     def sum(self) -> CompliantT_co: ...
+    def sort(self, *, descending: bool, nulls_last: bool) -> CompliantT_co: ...
 
 
 class NameNamespace(_StoresCompliant[CompliantT_co], Protocol[CompliantT_co]):

narwhals/_compliant/expr.py

@@ -1014,6 +1014,11 @@ def median(self) -> EagerExprT:
     def sum(self) -> EagerExprT:
         return self.compliant._reuse_series_namespace("list", "sum")
 
+    def sort(self, *, descending: bool, nulls_last: bool) -> EagerExprT:
+        return self.compliant._reuse_series_namespace(
+            "list", "sort", descending=descending, nulls_last=nulls_last
+        )
+
 
 class CompliantExprNameNamespace(  # type: ignore[misc]
     _ExprNamespace[CompliantExprT_co],

narwhals/_duckdb/expr_list.py

@@ -64,3 +64,10 @@ def func(expr: Expression) -> Expression:
             )
 
         return self.compliant._with_callable(func)
+
+    def sort(self, *, descending: bool, nulls_last: bool) -> DuckDBExpr:
+        sort_direction = "DESC" if descending else "ASC"
+        nulls_position = "NULLS LAST" if nulls_last else "NULLS FIRST"
+        return self.compliant._with_elementwise(
+            lambda expr: F("list_sort", expr, lit(sort_direction), lit(nulls_position))
+        )

narwhals/_ibis/expr_list.py

@@ -52,4 +52,18 @@ def func(expr: ir.ArrayColumn) -> ir.Value:
 
         return self.compliant._with_callable(func)
 
+    def sort(self, *, descending: bool, nulls_last: bool) -> IbisExpr:
+        if descending:
+            msg = "Descending sort is not currently supported for Ibis."
+            raise NotImplementedError(msg)
+
+        def func(expr: ir.ArrayColumn) -> ir.ArrayValue:
+            if nulls_last:
+                return expr.sort()
+            expr_no_nulls = expr.filter(lambda x: x.notnull())
+            expr_nulls = expr.filter(lambda x: x.isnull())
+            return expr_nulls.concat(expr_no_nulls.sort())
+
+        return self.compliant._with_callable(func)
+
     median = not_implemented()