Merge remote-tracking branch 'upstream/main' into read-csv-from-directory

Author: fangchenli

ci/deps/actions-311-downstream_compat.yaml

@@ -50,7 +50,8 @@ dependencies:
   - pytz>=2023.4
   - pyxlsb>=1.0.10
   - s3fs>=2023.12.2
-  - scipy>=1.12.0
+  # TEMP upper pin for scipy (https://github.com/statsmodels/statsmodels/issues/9584)
+  - scipy>=1.12.0,<1.16
   - sqlalchemy>=2.0.0
   - tabulate>=0.9.0
   - xarray>=2024.1.1

doc/source/whatsnew/v3.0.0.rst

@@ -752,6 +752,7 @@ Indexing
 - Bug in :meth:`DataFrame.loc` with inconsistent behavior of loc-set with 2 given indexes to Series (:issue:`59933`)
 - Bug in :meth:`Index.get_indexer` and similar methods when ``NaN`` is located at or after position 128 (:issue:`58924`)
 - Bug in :meth:`MultiIndex.insert` when a new value inserted to a datetime-like level gets cast to ``NaT`` and fails indexing (:issue:`60388`)
+- Bug in :meth:`Series.__setitem__` when assigning boolean series with boolean indexer will raise ``LossySetitemError`` (:issue:`57338`)
 - Bug in printing :attr:`Index.names` and :attr:`MultiIndex.levels` would not escape single quotes (:issue:`60190`)
 - Bug in reindexing of :class:`DataFrame` with :class:`PeriodDtype` columns in case of consolidated block (:issue:`60980`, :issue:`60273`)
 

pandas/core/apply.py

@@ -10,6 +10,7 @@
     TYPE_CHECKING,
     Any,
     Literal,
+    TypeAlias,
     cast,
 )
 
@@ -71,7 +72,7 @@
     from pandas.core.resample import Resampler
     from pandas.core.window.rolling import BaseWindow
 
-ResType = dict[int, Any]
+ResType: TypeAlias = dict[int, Any]
 
 
 class BaseExecutionEngine(abc.ABC):

pandas/core/arrays/datetimelike.py

@@ -10,6 +10,7 @@
     TYPE_CHECKING,
     Any,
     Literal,
+    TypeAlias,
     Union,
     cast,
     final,
@@ -161,7 +162,7 @@
         TimedeltaArray,
     )
 
-DTScalarOrNaT = Union[DatetimeLikeScalar, NaTType]
+DTScalarOrNaT: TypeAlias = DatetimeLikeScalar | NaTType
 
 
 def _make_unpacked_invalid_op(op_name: str):
@@ -386,7 +387,7 @@ def __getitem__(self, key: PositionalIndexer2D) -> Self | DTScalarOrNaT:
         # Use cast as we know we will get back a DatetimeLikeArray or DTScalar,
         # but skip evaluating the Union at runtime for performance
         # (see https://github.com/pandas-dev/pandas/pull/44624)
-        result = cast("Union[Self, DTScalarOrNaT]", super().__getitem__(key))
+        result = cast(Union[Self, DTScalarOrNaT], super().__getitem__(key))
         if lib.is_scalar(result):
             return result
         else:

pandas/core/arrays/interval.py

@@ -9,7 +9,7 @@
 from typing import (
     TYPE_CHECKING,
     Literal,
-    Union,
+    TypeAlias,
     overload,
 )
 
@@ -109,8 +109,8 @@
     )
 
 
-IntervalSide = Union[TimeArrayLike, np.ndarray]
-IntervalOrNA = Union[Interval, float]
+IntervalSide: TypeAlias = TimeArrayLike | np.ndarray
+IntervalOrNA: TypeAlias = Interval | float
 
 _interval_shared_docs: dict[str, str] = {}
 

pandas/core/arrays/string_arrow.py

@@ -4,7 +4,6 @@
 import re
 from typing import (
     TYPE_CHECKING,
-    Union,
 )
 import warnings
 
@@ -63,9 +62,6 @@
     from pandas import Series
 
 
-ArrowStringScalarOrNAT = Union[str, libmissing.NAType]
-
-
 def _chk_pyarrow_available() -> None:
     if pa_version_under10p1:
         msg = "pyarrow>=10.0.1 is required for PyArrow backed ArrowExtensionArray."

pandas/core/dtypes/cast.py

@@ -1926,6 +1926,10 @@ def np_can_hold_element(dtype: np.dtype, element: Any) -> Any:
                         # i.e. there are pd.NA elements
                         raise LossySetitemError
                 return element
+            # GH 57338 check boolean array set as object type
+            if tipo.kind == "O" and isinstance(element, np.ndarray):
+                if lib.is_bool_array(element):
+                    return element.astype("bool")
             raise LossySetitemError
         if lib.is_bool(element):
             return element

pandas/core/groupby/generic.py

@@ -17,8 +17,8 @@
     Any,
     Literal,
     NamedTuple,
+    TypeAlias,
     TypeVar,
-    Union,
     cast,
 )
 import warnings
@@ -102,7 +102,7 @@
     from pandas.core.generic import NDFrame
 
 # TODO(typing) the return value on this callable should be any *scalar*.
-AggScalar = Union[str, Callable[..., Any]]
+AggScalar: TypeAlias = str | Callable[..., Any]
 # TODO: validate types on ScalarResult and move to _typing
 # Blocked from using by https://github.com/python/mypy/issues/1484
 # See note at _mangle_lambda_list

pandas/core/groupby/groupby.py

@@ -26,6 +26,7 @@ class providing the base-class of operations.
 from typing import (
     TYPE_CHECKING,
     Literal,
+    TypeAlias,
     TypeVar,
     Union,
     cast,
@@ -449,13 +450,13 @@ def f(self):
         return attr
 
 
-_KeysArgType = Union[
-    Hashable,
-    list[Hashable],
-    Callable[[Hashable], Hashable],
-    list[Callable[[Hashable], Hashable]],
-    Mapping[Hashable, Hashable],
-]
+_KeysArgType: TypeAlias = (
+    Hashable
+    | list[Hashable]
+    | Callable[[Hashable], Hashable]
+    | list[Callable[[Hashable], Hashable]]
+    | Mapping[Hashable, Hashable]
+)
 
 
 class BaseGroupBy(PandasObject, SelectionMixin[NDFrameT], GroupByIndexingMixin):
@@ -957,9 +958,8 @@ def __iter__(self) -> Iterator[tuple[Hashable, NDFrameT]]:
         level = self.level
         result = self._grouper.get_iterator(self._selected_obj)
         # mypy: Argument 1 to "len" has incompatible type "Hashable"; expected "Sized"
-        if (
-            (is_list_like(level) and len(level) == 1)  # type: ignore[arg-type]
-            or (isinstance(keys, list) and len(keys) == 1)
+        if (is_list_like(level) and len(level) == 1) or (  # type: ignore[arg-type]
+            isinstance(keys, list) and len(keys) == 1
         ):
             # GH#42795 - when keys is a list, return tuples even when length is 1
             result = (((key,), group) for key, group in result)

pandas/core/indexes/base.py

@@ -7026,6 +7026,23 @@ def infer_objects(self, copy: bool = True) -> Index:
         ----------
         copy : bool, default True
             Whether to make a copy in cases where no inference occurs.
+
+        Returns
+        -------
+        Index
+            An Index with a new dtype if the dtype was inferred
+            or a shallow copy if the dtype could not be inferred.
+
+        See Also
+        --------
+        Index.inferred_type: Return a string of the type inferred from the values.
+
+        Examples
+        --------
+        >>> pd.Index(["a", 1]).infer_objects()
+        Index(['a', '1'], dtype='object')
+        >>> pd.Index([1, 2], dtype="object").infer_objects()
+        Index([1, 2], dtype='int64')
         """
         if self._is_multi:
             raise NotImplementedError(