Merge remote-tracking branch 'upstream/main' into ref/index_equiv

Author: mroeschke

doc/source/whatsnew/v3.0.0.rst

@@ -272,11 +272,11 @@ Performance improvements
 - Performance improvement in :meth:`Index.join` by propagating cached attributes in cases where the result matches one of the inputs (:issue:`57023`)
 - Performance improvement in :meth:`Index.take` when ``indices`` is a full range indexer from zero to length of index (:issue:`56806`)
 - Performance improvement in :meth:`MultiIndex.equals` for equal length indexes (:issue:`56990`)
-- Performance improvement in :meth:`RangeIndex.__getitem__` with a boolean mask returning a :class:`RangeIndex` instead of a :class:`Index` when possible. (:issue:`57588`)
+- Performance improvement in :meth:`RangeIndex.__getitem__` with a boolean mask or integers returning a :class:`RangeIndex` instead of a :class:`Index` when possible. (:issue:`57588`)
 - Performance improvement in :meth:`RangeIndex.append` when appending the same index (:issue:`57252`)
-- Performance improvement in :meth:`RangeIndex.join` returning a :class:`RangeIndex` instead of a :class:`Index` when possible. (:issue:`57651`)
-- Performance improvement in :meth:`RangeIndex.reindex` returning a :class:`RangeIndex` instead of a :class:`Index` when possible. (:issue:`57647`)
-- Performance improvement in :meth:`RangeIndex.take` returning a :class:`RangeIndex` instead of a :class:`Index` when possible. (:issue:`57445`)
+- Performance improvement in :meth:`RangeIndex.join` returning a :class:`RangeIndex` instead of a :class:`Index` when possible. (:issue:`57651`, :issue:`57752`)
+- Performance improvement in :meth:`RangeIndex.reindex` returning a :class:`RangeIndex` instead of a :class:`Index` when possible. (:issue:`57647`, :issue:`57752`)
+- Performance improvement in :meth:`RangeIndex.take` returning a :class:`RangeIndex` instead of a :class:`Index` when possible. (:issue:`57445`, :issue:`57752`)
 - Performance improvement in ``DataFrameGroupBy.__len__`` and ``SeriesGroupBy.__len__`` (:issue:`57595`)
 - Performance improvement in indexing operations for string dtypes (:issue:`56997`)
 
@@ -289,6 +289,7 @@ Bug fixes
 - Fixed bug in :meth:`DataFrame.join` inconsistently setting result index name (:issue:`55815`)
 - Fixed bug in :meth:`DataFrame.to_string` that raised ``StopIteration`` with nested DataFrames. (:issue:`16098`)
 - Fixed bug in :meth:`DataFrame.update` bool dtype being converted to object (:issue:`55509`)
+- Fixed bug in :meth:`DataFrameGroupBy.apply` that was returning a completely empty DataFrame when all return values of ``func`` were ``None`` instead of returning an empty DataFrame with the original columns and dtypes. (:issue:`57775`)
 - Fixed bug in :meth:`Series.diff` allowing non-integer values for the ``periods`` argument. (:issue:`56607`)
 - Fixed bug in :meth:`Series.rank` that doesn't preserve missing values for nullable integers when ``na_option='keep'``. (:issue:`56976`)
 
@@ -393,6 +394,7 @@ Other
 ^^^^^
 - Bug in :class:`DataFrame` when passing a ``dict`` with a NA scalar and ``columns`` that would always return ``np.nan`` (:issue:`57205`)
 - Bug in :func:`tseries.api.guess_datetime_format` would fail to infer time format when "%Y" == "%H%M" (:issue:`57452`)
+- Bug in :func:`unique` on :class:`Index` not always returning :class:`Index` (:issue:`57043`)
 - Bug in :meth:`DataFrame.sort_index` when passing ``axis="columns"`` and ``ignore_index=True`` and ``ascending=False`` not returning a :class:`RangeIndex` columns (:issue:`57293`)
 - Bug in :meth:`DataFrame.where` where using a non-bool type array in the function would return a ``ValueError`` instead of a ``TypeError`` (:issue:`56330`)
 - Bug in Dataframe Interchange Protocol implementation was returning incorrect results for data buffers' associated dtype, for string and datetime columns (:issue:`54781`)

environment.yml

@@ -62,6 +62,7 @@ dependencies:
   # downstream packages
   - dask-core
   - seaborn-base
+  - dask-expr
 
   # local testing dependencies
   - moto

pandas/core/algorithms.py

@@ -439,6 +439,10 @@ def unique_with_mask(values, mask: npt.NDArray[np.bool_] | None = None):
         # Dispatch to extension dtype's unique.
         return values.unique()
 
+    if isinstance(values, ABCIndex):
+        # Dispatch to Index's unique.
+        return values.unique()
+
     original = values
     hashtable, values = _get_hashtable_algo(values)
 

pandas/core/groupby/generic.py

@@ -1642,8 +1642,11 @@ def _wrap_applied_output(
         first_not_none = next(com.not_none(*values), None)
 
         if first_not_none is None:
-            # GH9684 - All values are None, return an empty frame.
-            return self.obj._constructor()
+            # GH9684 - All values are None, return an empty frame
+            # GH57775 - Ensure that columns and dtypes from original frame are kept.
+            result = self.obj._constructor(columns=data.columns)
+            result = result.astype(data.dtypes)
+            return result
         elif isinstance(first_not_none, DataFrame):
             return self._concat_objects(
                 values,

pandas/core/groupby/groupby.py

@@ -1636,6 +1636,14 @@ def apply(self, func, *args, include_groups: bool = True, **kwargs) -> NDFrameT:
         a    5
         b    2
         dtype: int64
+
+        Example 4: The function passed to ``apply`` returns ``None`` for one of the
+        group. This group is filtered from the result:
+
+        >>> g1.apply(lambda x: None if x.iloc[0, 0] == 3 else x, include_groups=False)
+           B  C
+        0  1  4
+        1  2  6
         """
         if isinstance(func, str):
             if hasattr(self, func):

pandas/core/indexes/base.py

@@ -4235,7 +4235,6 @@ def join(
 
         return self._join_via_get_indexer(other, how, sort)
 
-    @final
     def _join_empty(
         self, other: Index, how: JoinHow, sort: bool
     ) -> tuple[Index, npt.NDArray[np.intp] | None, npt.NDArray[np.intp] | None]:

pandas/core/indexes/range.py

@@ -472,18 +472,31 @@ def _shallow_copy(self, values, name: Hashable = no_default):
 
         if values.dtype.kind == "f":
             return Index(values, name=name, dtype=np.float64)
-        if values.dtype.kind == "i" and values.ndim == 1 and len(values) > 1:
+        if values.dtype.kind == "i" and values.ndim == 1:
             # GH 46675 & 43885: If values is equally spaced, return a
             # more memory-compact RangeIndex instead of Index with 64-bit dtype
+            if len(values) == 0:
+                return type(self)._simple_new(_empty_range, name=name)
+            elif len(values) == 1:
+                start = values[0]
+                new_range = range(start, start + self.step, self.step)
+                return type(self)._simple_new(new_range, name=name)
             diff = values[1] - values[0]
             if not missing.isna(diff) and diff != 0:
-                maybe_range_indexer, remainder = np.divmod(values - values[0], diff)
-                if (
-                    lib.is_range_indexer(maybe_range_indexer, len(maybe_range_indexer))
-                    and not remainder.any()
-                ):
+                if len(values) == 2:
+                    # Can skip is_range_indexer check
                     new_range = range(values[0], values[-1] + diff, diff)
                     return type(self)._simple_new(new_range, name=name)
+                else:
+                    maybe_range_indexer, remainder = np.divmod(values - values[0], diff)
+                    if (
+                        lib.is_range_indexer(
+                            maybe_range_indexer, len(maybe_range_indexer)
+                        )
+                        and not remainder.any()
+                    ):
+                        new_range = range(values[0], values[-1] + diff, diff)
+                        return type(self)._simple_new(new_range, name=name)
         return self._constructor._simple_new(values, name=name)
 
     def _view(self) -> Self:
@@ -894,12 +907,19 @@ def symmetric_difference(
             result = result.rename(result_name)
         return result
 
+    def _join_empty(
+        self, other: Index, how: JoinHow, sort: bool
+    ) -> tuple[Index, npt.NDArray[np.intp] | None, npt.NDArray[np.intp] | None]:
+        if other.dtype.kind == "i":
+            other = self._shallow_copy(other._values, name=other.name)
+        return super()._join_empty(other, how=how, sort=sort)
+
     def _join_monotonic(
         self, other: Index, how: JoinHow = "left"
     ) -> tuple[Index, npt.NDArray[np.intp] | None, npt.NDArray[np.intp] | None]:
         # This currently only gets called for the monotonic increasing case
         if not isinstance(other, type(self)):
-            maybe_ri = self._shallow_copy(other._values)
+            maybe_ri = self._shallow_copy(other._values, name=other.name)
             if not isinstance(maybe_ri, type(self)):
                 return super()._join_monotonic(other, how=how)
             other = maybe_ri
@@ -1075,6 +1095,8 @@ def __getitem__(self, key):
         """
         Conserve RangeIndex type for scalar and slice keys.
         """
+        if key is Ellipsis:
+            key = slice(None)
         if isinstance(key, slice):
             return self._getitem_slice(key)
         elif is_integer(key):
@@ -1094,17 +1116,20 @@ def __getitem__(self, key):
             )
         elif com.is_bool_indexer(key):
             if isinstance(getattr(key, "dtype", None), ExtensionDtype):
-                np_key = key.to_numpy(dtype=bool, na_value=False)
+                key = key.to_numpy(dtype=bool, na_value=False)
             else:
-                np_key = np.asarray(key, dtype=bool)
-            check_array_indexer(self._range, np_key)  # type: ignore[arg-type]
+                key = np.asarray(key, dtype=bool)
+            check_array_indexer(self._range, key)  # type: ignore[arg-type]
             # Short circuit potential _shallow_copy check
-            if np_key.all():
+            if key.all():
                 return self._simple_new(self._range, name=self.name)
-            elif not np_key.any():
+            elif not key.any():
                 return self._simple_new(_empty_range, name=self.name)
-            return self.take(np.flatnonzero(np_key))
-        return super().__getitem__(key)
+            key = np.flatnonzero(key)
+        try:
+            return self.take(key)
+        except (TypeError, ValueError):
+            return super().__getitem__(key)
 
     def _getitem_slice(self, slobj: slice) -> Self:
         """

pandas/core/internals/__init__.py

@@ -35,8 +35,6 @@ def __getattr__(name: str):
         return create_block_manager_from_blocks
 
     if name in [
-        "NumericBlock",
-        "ObjectBlock",
         "Block",
         "ExtensionBlock",
         "DatetimeTZBlock",
@@ -49,25 +47,17 @@ def __getattr__(name: str):
             # on hard-coding stacklevel
             stacklevel=2,
         )
-        if name == "NumericBlock":
-            from pandas.core.internals.blocks import NumericBlock
-
-            return NumericBlock
-        elif name == "DatetimeTZBlock":
+        if name == "DatetimeTZBlock":
             from pandas.core.internals.blocks import DatetimeTZBlock
 
             return DatetimeTZBlock
         elif name == "ExtensionBlock":
             from pandas.core.internals.blocks import ExtensionBlock
 
             return ExtensionBlock
-        elif name == "Block":
+        else:
             from pandas.core.internals.blocks import Block
 
             return Block
-        else:
-            from pandas.core.internals.blocks import ObjectBlock
-
-            return ObjectBlock
 
     raise AttributeError(f"module 'pandas.core.internals' has no attribute '{name}'")

pandas/core/internals/blocks.py

@@ -2148,18 +2148,6 @@ def is_numeric(self) -> bool:  # type: ignore[override]
         return kind in "fciub"
 
 
-class NumericBlock(NumpyBlock):
-    # this Block type is kept for backwards-compatibility
-    # TODO(3.0): delete and remove deprecation in __init__.py.
-    __slots__ = ()
-
-
-class ObjectBlock(NumpyBlock):
-    # this Block type is kept for backwards-compatibility
-    # TODO(3.0): delete and remove deprecation in __init__.py.
-    __slots__ = ()
-
-
 class NDArrayBackedExtensionBlock(EABackedBlock):
     """
     Block backed by an NDArrayBackedExtensionArray

pandas/core/methods/selectn.py

@@ -213,7 +213,7 @@ def compute(self, method: str) -> DataFrame:
                     f"cannot use method {method!r} with this dtype"
                 )
 
-        def get_indexer(current_indexer, other_indexer):
+        def get_indexer(current_indexer: Index, other_indexer: Index) -> Index:
             """
             Helper function to concat `current_indexer` and `other_indexer`
             depending on `method`