fix: pyspark unpivot with index None (#2090)

Author: EdAbati

narwhals/_spark_like/dataframe.py

@@ -420,11 +420,16 @@ def unpivot(
         variable_name: str,
         value_name: str,
     ) -> Self:
-        return self._from_native_frame(
-            self._native_frame.unpivot(
-                ids=index,
-                values=on,
-                variableColumnName=variable_name,
-                valueColumnName=value_name,
-            )
+        ids = tuple(self.columns) if index is None else tuple(index)
+        values = (
+            tuple(set(self.columns).difference(set(ids))) if on is None else tuple(on)
+        )
+        unpivoted_native_frame = self._native_frame.unpivot(
+            ids=ids,
+            values=values,
+            variableColumnName=variable_name,
+            valueColumnName=value_name,
         )
+        if index is None:
+            unpivoted_native_frame = unpivoted_native_frame.drop(*ids)
+        return self._from_native_frame(unpivoted_native_frame)

tests/frame/unpivot_test.py

@@ -20,30 +20,58 @@
     "c": [2, 4, 6],
 }
 
-expected_b_only = {
+expected_on_b_idx_a = {
     "a": ["x", "y", "z"],
     "variable": ["b", "b", "b"],
     "value": [1, 3, 5],
 }
 
-expected_b_c = {
+expected_on_b_c_idx_a = {
     "a": ["x", "y", "z", "x", "y", "z"],
     "variable": ["b", "b", "b", "c", "c", "c"],
     "value": [1, 3, 5, 2, 4, 6],
 }
 
+expected_on_none_idx_a = {
+    "a": ["x", "y", "z", "x", "y", "z"],
+    "variable": ["b", "b", "b", "c", "c", "c"],
+    "value": [1, 3, 5, 2, 4, 6],
+}
+
+expected_on_b_c_idx_none = {
+    "variable": ["b", "b", "b", "c", "c", "c"],
+    "value": [1, 3, 5, 2, 4, 6],
+}
+
+expected_on_none_idx_none = {
+    "variable": ["a", "a", "a", "b", "b", "b", "c", "c", "c"],
+    "value": ["x", "y", "z", "1", "3", "5", "2", "4", "6"],
+}
+
 
 @pytest.mark.parametrize(
-    ("on", "expected"),
-    [("b", expected_b_only), (["b", "c"], expected_b_c), (None, expected_b_c)],
+    ("on", "index", "expected"),
+    [
+        ("b", ["a"], expected_on_b_idx_a),
+        (["b", "c"], ["a"], expected_on_b_c_idx_a),
+        (None, ["a"], expected_on_none_idx_a),
+        (["b", "c"], None, expected_on_b_c_idx_none),
+        (None, None, expected_on_none_idx_none),
+    ],
 )
-def test_unpivot_on(
+def test_unpivot(
     constructor: Constructor,
     on: str | list[str] | None,
+    index: list[str] | None,
     expected: dict[str, list[float]],
+    request: pytest.FixtureRequest,
 ) -> None:
+    if on is None and index is None and "polars" not in str(constructor):
+        # TODO(2082): add support in other backends
+        request.applymarker(pytest.mark.xfail)
     df = nw.from_native(constructor(data))
-    result = df.unpivot(on=on, index=["a"]).sort("variable", "a")
+    sort_columns = ["variable"] if index is None else ["variable", "a"]
+    result = df.unpivot(on=on, index=index).sort(by=sort_columns)
     assert_equal_data(result, expected)