Merge remote-tracking branch 'upstream/main' into add-typing-aliases

Author: Dr-Irv

ci/deps/actions-311-downstream_compat.yaml

@@ -50,7 +50,8 @@ dependencies:
   - pytz>=2023.4
   - pyxlsb>=1.0.10
   - s3fs>=2023.12.2
-  - scipy>=1.12.0
+  # TEMP upper pin for scipy (https://github.com/statsmodels/statsmodels/issues/9584)
+  - scipy>=1.12.0,<1.16
   - sqlalchemy>=2.0.0
   - tabulate>=0.9.0
   - xarray>=2024.1.1

doc/source/whatsnew/v3.0.0.rst

@@ -753,6 +753,7 @@ Indexing
 - Bug in :meth:`DataFrame.loc` with inconsistent behavior of loc-set with 2 given indexes to Series (:issue:`59933`)
 - Bug in :meth:`Index.get_indexer` and similar methods when ``NaN`` is located at or after position 128 (:issue:`58924`)
 - Bug in :meth:`MultiIndex.insert` when a new value inserted to a datetime-like level gets cast to ``NaT`` and fails indexing (:issue:`60388`)
+- Bug in :meth:`Series.__setitem__` when assigning boolean series with boolean indexer will raise ``LossySetitemError`` (:issue:`57338`)
 - Bug in printing :attr:`Index.names` and :attr:`MultiIndex.levels` would not escape single quotes (:issue:`60190`)
 - Bug in reindexing of :class:`DataFrame` with :class:`PeriodDtype` columns in case of consolidated block (:issue:`60980`, :issue:`60273`)
 

pandas/core/dtypes/cast.py

@@ -1926,6 +1926,10 @@ def np_can_hold_element(dtype: np.dtype, element: Any) -> Any:
                         # i.e. there are pd.NA elements
                         raise LossySetitemError
                 return element
+            # GH 57338 check boolean array set as object type
+            if tipo.kind == "O" and isinstance(element, np.ndarray):
+                if lib.is_bool_array(element):
+                    return element.astype("bool")
             raise LossySetitemError
         if lib.is_bool(element):
             return element

pandas/core/indexes/base.py

@@ -7026,6 +7026,23 @@ def infer_objects(self, copy: bool = True) -> Index:
         ----------
         copy : bool, default True
             Whether to make a copy in cases where no inference occurs.
+
+        Returns
+        -------
+        Index
+            An Index with a new dtype if the dtype was inferred
+            or a shallow copy if the dtype could not be inferred.
+
+        See Also
+        --------
+        Index.inferred_type: Return a string of the type inferred from the values.
+
+        Examples
+        --------
+        >>> pd.Index(["a", 1]).infer_objects()
+        Index(['a', '1'], dtype='object')
+        >>> pd.Index([1, 2], dtype="object").infer_objects()
+        Index([1, 2], dtype='int64')
         """
         if self._is_multi:
             raise NotImplementedError(

pandas/tests/base/test_misc.py

@@ -3,12 +3,9 @@
 import numpy as np
 import pytest
 
-from pandas._config import using_string_dtype
-
 from pandas.compat import PYPY
 
 from pandas.core.dtypes.common import (
-    is_dtype_equal,
     is_object_dtype,
 )
 
@@ -81,10 +78,7 @@ def test_ndarray_compat_properties(index_or_series_obj):
     assert Series([1]).item() == 1
 
 
-@pytest.mark.skipif(
-    PYPY or using_string_dtype(),
-    reason="not relevant for PyPy doesn't work properly for arrow strings",
-)
+@pytest.mark.skipif(PYPY, reason="not relevant for PyPy")
 def test_memory_usage(index_or_series_memory_obj):
     obj = index_or_series_memory_obj
     # Clear index caches so that len(obj) == 0 report 0 memory usage
@@ -98,18 +92,21 @@ def test_memory_usage(index_or_series_memory_obj):
     res = obj.memory_usage()
     res_deep = obj.memory_usage(deep=True)
 
-    is_object = is_object_dtype(obj) or (is_ser and is_object_dtype(obj.index))
-    is_categorical = isinstance(obj.dtype, pd.CategoricalDtype) or (
-        is_ser and isinstance(obj.index.dtype, pd.CategoricalDtype)
-    )
-    is_object_string = is_dtype_equal(obj, "string[python]") or (
-        is_ser and is_dtype_equal(obj.index.dtype, "string[python]")
-    )
+    def _is_object_dtype(obj):
+        if isinstance(obj, pd.MultiIndex):
+            return any(_is_object_dtype(level) for level in obj.levels)
+        elif isinstance(obj.dtype, pd.CategoricalDtype):
+            return _is_object_dtype(obj.dtype.categories)
+        elif isinstance(obj.dtype, pd.StringDtype):
+            return obj.dtype.storage == "python"
+        return is_object_dtype(obj)
+
+    has_objects = _is_object_dtype(obj) or (is_ser and _is_object_dtype(obj.index))
 
     if len(obj) == 0:
         expected = 0
         assert res_deep == res == expected
-    elif is_object or is_categorical or is_object_string:
+    elif has_objects:
         # only deep will pick them up
         assert res_deep > res
     else:

pandas/tests/dtypes/cast/test_can_hold_element.py

@@ -77,3 +77,22 @@ def test_can_hold_element_int8_int():
     assert not can_hold_element(arr, np.uint32(element))
     assert not can_hold_element(arr, np.int64(element))
     assert not can_hold_element(arr, np.uint64(element))
+
+
+def test_can_hold_element_bool():
+    arr = np.array([], dtype=bool)
+
+    element = True
+    assert can_hold_element(arr, element)
+    assert can_hold_element(arr, np.array([element]))
+    assert can_hold_element(arr, np.array([element], dtype=object))
+
+    element = 1
+    assert not can_hold_element(arr, element)
+    assert not can_hold_element(arr, np.array([element]))
+    assert not can_hold_element(arr, np.array([element], dtype=object))
+
+    element = np.nan
+    assert not can_hold_element(arr, element)
+    assert not can_hold_element(arr, np.array([element]))
+    assert not can_hold_element(arr, np.array([element], dtype=object))

pandas/tests/extension/test_arrow.py

@@ -3295,7 +3295,7 @@ def test_decimal_parse_raises():
     # GH 56984
     ser = pd.Series(["1.2345"], dtype=ArrowDtype(pa.string()))
     with pytest.raises(
-        pa.lib.ArrowInvalid, match="Rescaling Decimal128 value would cause data loss"
+        pa.lib.ArrowInvalid, match="Rescaling Decimal(128)? value would cause data loss"
     ):
         ser.astype(ArrowDtype(pa.decimal128(1, 0)))
 

pandas/tests/series/indexing/test_setitem.py

@@ -1838,3 +1838,13 @@ def test_setitem_empty_mask_dont_upcast_dt64():
     ser.mask(mask, "foo", inplace=True)
     assert ser.dtype == dti.dtype  # no-op -> dont upcast
     tm.assert_series_equal(ser, orig)
+
+
+def test_setitem_bool_dtype_with_boolean_indexer():
+    # GH 57338
+    s1 = Series([True, True, True], dtype=bool)
+    s2 = Series([False, False, False], dtype=bool)
+    condition = [False, True, False]
+    s1[condition] = s2[condition]
+    expected = Series([True, False, True], dtype=bool)
+    tm.assert_series_equal(s1, expected)