Merge branch 'pandas-dev:main' into main

Author: LirongMa

doc/source/whatsnew/v3.0.0.rst

@@ -933,6 +933,7 @@ Bug fixes
 Categorical
 ^^^^^^^^^^^
 - Bug in :func:`Series.apply` where ``nan`` was ignored for :class:`CategoricalDtype` (:issue:`59938`)
+- Bug in :func:`testing.assert_index_equal` raising ``TypeError`` instead of ``AssertionError`` for incomparable ``CategoricalIndex`` when ``check_categorical=True`` and ``exact=False`` (:issue:`61935`)
 - Bug in :meth:`Categorical.astype` where ``copy=False`` would still trigger a copy of the codes (:issue:`62000`)
 - Bug in :meth:`DataFrame.pivot` and :meth:`DataFrame.set_index` raising an ``ArrowNotImplementedError`` for columns with pyarrow dictionary dtype (:issue:`53051`)
 - Bug in :meth:`Series.convert_dtypes` with ``dtype_backend="pyarrow"`` where empty :class:`CategoricalDtype` :class:`Series` raised an error or got converted to ``null[pyarrow]`` (:issue:`59934`)

pandas/_libs/index.pyx

@@ -838,7 +838,7 @@ cdef class BaseMultiIndexCodesEngine:
             raise KeyError(key)
         try:
             indices = [1 if checknull(v) else lev.get_loc(v) + multiindex_nulls_shift
-                       for lev, v in zip(self.levels, key)]
+                       for lev, v in zip(self.levels, key, strict=True)]
         except KeyError:
             raise KeyError(key)
 

pandas/_libs/missing.pyx

@@ -72,7 +72,7 @@ cpdef bint check_na_tuples_nonequal(object left, object right):
     if len(left) != len(right):
         return False
 
-    for left_element, right_element in zip(left, right):
+    for left_element, right_element in zip(left, right, strict=True):
         if left_element is C_NA and right_element is not C_NA:
             return True
         elif right_element is C_NA and left_element is not C_NA:

pandas/_libs/tslibs/fields.pyx

@@ -109,7 +109,7 @@ def month_position_check(fields, weekdays) -> str | None:
         int32_t[:] months = fields["M"]
         int32_t[:] days = fields["D"]
 
-    for y, m, d, wd in zip(years, months, days, weekdays):
+    for y, m, d, wd in zip(years, months, days, weekdays, strict=True):
         if calendar_start:
             calendar_start &= d == 1
         if business_start:

pandas/_libs/tslibs/offsets.pyx

@@ -2217,7 +2217,7 @@ cdef class BusinessHour(BusinessMixin):
         # Use python string formatting to be faster than strftime
         hours = ",".join(
             f"{st.hour:02d}:{st.minute:02d}-{en.hour:02d}:{en.minute:02d}"
-            for st, en in zip(self.start, self.end)
+            for st, en in zip(self.start, self.end, strict=True)
         )
         attrs = [f"{self._prefix}={hours}"]
         out += ": " + ", ".join(attrs)
@@ -2414,7 +2414,7 @@ cdef class BusinessHour(BusinessMixin):
         # get total business hours by sec in one business day
         businesshours = sum(
             self._get_business_hours_by_sec(st, en)
-            for st, en in zip(self.start, self.end)
+            for st, en in zip(self.start, self.end, strict=True)
         )
 
         bd, r = divmod(abs(n * 60), businesshours // 60)
@@ -5357,7 +5357,7 @@ cpdef to_offset(freq, bint is_period=False):
                 # the last element must be blank
                 raise ValueError("last element must be blank")
 
-            tups = zip(split[0::4], split[1::4], split[2::4])
+            tups = zip(split[0::4], split[1::4], split[2::4], strict=False)
             for n, (sep, stride, name) in enumerate(tups):
                 name = _warn_about_deprecated_aliases(name, is_period)
                 _validate_to_offset_alias(name, is_period)

pandas/_libs/tslibs/timezones.pyx

@@ -252,7 +252,7 @@ cdef object _get_utc_trans_times_from_dateutil_tz(tzinfo tz):
     """
     new_trans = list(tz._trans_list)
     last_std_offset = 0
-    for i, (trans, tti) in enumerate(zip(tz._trans_list, tz._trans_idx)):
+    for i, (trans, tti) in enumerate(zip(tz._trans_list, tz._trans_idx, strict=True)):
         if not tti.isdst:
             last_std_offset = tti.offset
         new_trans[i] = trans - last_std_offset

pandas/_testing/asserters.py

@@ -325,7 +325,17 @@ def _check_types(left, right, obj: str = "Index") -> None:
     # skip exact index checking when `check_categorical` is False
     elif check_exact and check_categorical:
         if not left.equals(right):
-            mismatch = left._values != right._values
+            # _values compare can raise TypeError (non-comparable
+            # categoricals (GH#61935)
+            try:
+                mismatch = left._values != right._values
+            except TypeError:
+                raise_assert_detail(
+                    obj,
+                    "types are not comparable (non-matching categorical categories)",
+                    left,
+                    right,
+                )
 
             if not isinstance(mismatch, np.ndarray):
                 mismatch = cast("ExtensionArray", mismatch).fillna(True)

pandas/core/frame.py

@@ -107,6 +107,7 @@
     is_list_like,
     is_scalar,
     is_sequence,
+    is_string_dtype,
     needs_i8_conversion,
     pandas_dtype,
 )
@@ -4454,8 +4455,12 @@ def _set_item_frame_value(self, key, value: DataFrame) -> None:
                 cols_droplevel = maybe_droplevels(cols, key)
                 if (
                     not isinstance(cols_droplevel, MultiIndex)
+                    and is_string_dtype(cols_droplevel.dtype)
                     and not cols_droplevel.any()
                 ):
+                    # if cols_droplevel contains only empty strings,
+                    # value.reindex(cols_droplevel, axis=1) would be full of NaNs
+                    # see GH#62518 and GH#61841
                     return
                 if len(cols_droplevel) and not cols_droplevel.equals(value.columns):
                     value = value.reindex(cols_droplevel, axis=1)

pandas/core/window/rolling.py

@@ -351,7 +351,7 @@ def __iter__(self) -> Iterator:
         )
         self._check_window_bounds(start, end, len(obj))
 
-        for s, e in zip(start, end):
+        for s, e in zip(start, end, strict=True):
             result = obj.iloc[slice(s, e)]
             yield result
 
@@ -802,7 +802,7 @@ def _apply_pairwise(
             groupby_codes = []
             groupby_levels = []
             # e.g. [[1, 2], [4, 5]] as [[1, 4], [2, 5]]
-            for gb_level_pair in map(list, zip(*gb_pairs)):
+            for gb_level_pair in map(list, zip(*gb_pairs, strict=True)):
                 labels = np.repeat(np.array(gb_level_pair), old_result_len)
                 codes, levels = factorize(labels)
                 groupby_codes.append(codes)

pandas/tests/frame/test_arithmetic.py

@@ -11,6 +11,8 @@
 import numpy as np
 import pytest
 
+from pandas.compat._optional import import_optional_dependency
+
 import pandas as pd
 from pandas import (
     DataFrame,
@@ -24,6 +26,7 @@
     _check_mixed_float,
     _check_mixed_int,
 )
+from pandas.util.version import Version
 
 
 @pytest.fixture
@@ -1114,6 +1117,8 @@ def test_binop_other(self, op, value, dtype, switch_numexpr_min_elements):
             (operator.mod, "complex128"),
         }
 
+        ne = import_optional_dependency("numexpr", errors="ignore")
+        ne_warns_on_op = ne is not None and Version(ne.__version__) < Version("2.13.1")
         if (op, dtype) in invalid:
             warn = None
             if (dtype == "<M8[ns]" and op == operator.add) or (
@@ -1142,7 +1147,11 @@ def test_binop_other(self, op, value, dtype, switch_numexpr_min_elements):
 
         elif (op, dtype) in skip:
             if op in [operator.add, operator.mul]:
-                if expr.USE_NUMEXPR and switch_numexpr_min_elements == 0:
+                if (
+                    expr.USE_NUMEXPR
+                    and switch_numexpr_min_elements == 0
+                    and ne_warns_on_op
+                ):
                     warn = UserWarning
                 else:
                     warn = None