ci: unxfail truncate tests for pyspark by checking local results (#2794)

Author: MarcoGorelli

tests/expr_and_series/dt/truncate_test.py

@@ -53,11 +53,6 @@ def test_truncate(
     every: str,
     expected: list[datetime],
 ) -> None:
-    if any(x in str(constructor) for x in ("sqlframe", "pyspark")):
-        # TODO(marco): investigate pyspark, it also localizes to UTC here.
-        request.applymarker(
-            pytest.mark.xfail(reason="https://github.com/eakmanrq/sqlframe/issues/383")
-        )
     if every.endswith("ns") and any(
         x in str(constructor) for x in ("polars", "duckdb", "pyspark", "ibis")
     ):
@@ -109,11 +104,10 @@ def test_truncate_multiples(
     every: str,
     expected: list[datetime],
 ) -> None:
-    if any(x in str(constructor) for x in ("sqlframe", "cudf", "pyspark", "duckdb")):
+    if any(x in str(constructor) for x in ("cudf", "pyspark", "duckdb")):
         # Reasons:
-        # - sqlframe: https://github.com/eakmanrq/sqlframe/issues/383
         # - cudf: https://github.com/rapidsai/cudf/issues/18654
-        # - pyspark: Only multiple 1 is currently supported
+        # - pyspark/sqlframe: Only multiple 1 is currently supported
         request.applymarker(pytest.mark.xfail())
     if every.endswith("ns") and any(
         x in str(constructor) for x in ("polars", "duckdb", "ibis")

tests/utils.py

@@ -77,6 +77,10 @@ def assert_equal_data(result: Any, expected: Mapping[str, Any]) -> None:
         hasattr(result, "_compliant_frame")
         and result._compliant_frame._implementation is Implementation.IBIS
     )
+    is_spark_like = (
+        hasattr(result, "_compliant_frame")
+        and result._compliant_frame._implementation.is_spark_like()
+    )
     if is_duckdb:
         result = from_native(result.to_native().arrow())
     if is_ibis:
@@ -122,6 +126,17 @@ def assert_equal_data(result: Any, expected: Mapping[str, Any]) -> None:
                 are_equivalent_values = pd.isna(rhs)
             elif type(lhs) is date and type(rhs) is datetime:
                 are_equivalent_values = datetime(lhs.year, lhs.month, lhs.day) == rhs
+            elif (
+                is_spark_like
+                and isinstance(lhs, datetime)
+                and isinstance(rhs, datetime)
+                and rhs.tzinfo is None
+                and lhs.tzinfo
+            ):
+                # PySpark converts timezone-naive to timezone-aware by default in many cases.
+                # For now, we just assert that the local result matches the expected one.
+                # https://github.com/narwhals-dev/narwhals/issues/2793
+                are_equivalent_values = lhs.replace(tzinfo=None) == rhs
             else:
                 are_equivalent_values = lhs == rhs