Merge branch 'main' into doc/clarify-categorical-map

Author: Talyahav17

.github/workflows/unit-tests.yml

@@ -358,8 +358,6 @@ jobs:
 
       - name: Run Tests
         uses: ./.github/actions/run-tests
-        # TEMP allow this to fail until we fixed all test failures (related to chained assignment warnings)
-        continue-on-error: true
 
   # NOTE: this job must be kept in sync with the Pyodide build job in wheels.yml
   emscripten:

doc/source/whatsnew/v3.0.0.rst

@@ -603,6 +603,7 @@ Other API changes
   an empty ``RangeIndex`` or empty ``Index`` with object dtype when determining
   the dtype of the resulting Index (:issue:`60797`)
 - :class:`IncompatibleFrequency` now subclasses ``TypeError`` instead of ``ValueError``. As a result, joins with mismatched frequencies now cast to object like other non-comparable joins, and arithmetic with indexes with mismatched frequencies align (:issue:`55782`)
+- :meth:`CategoricalIndex.append` no longer attempts to cast different-dtype indexes to the caller's dtype (:issue:`41626`)
 - :meth:`ExtensionDtype.construct_array_type` is now a regular method instead of a ``classmethod`` (:issue:`58663`)
 - Comparison operations between :class:`Index` and :class:`Series` now consistently return :class:`Series` regardless of which object is on the left or right (:issue:`36759`)
 - Numpy functions like ``np.isinf`` that return a bool dtype when called on a :class:`Index` object now return a bool-dtype :class:`Index` instead of ``np.ndarray`` (:issue:`52676`)
@@ -974,8 +975,8 @@ Indexing
 - Bug in reindexing of :class:`DataFrame` with :class:`PeriodDtype` columns in case of consolidated block (:issue:`60980`, :issue:`60273`)
 - Bug in :meth:`DataFrame.loc.__getitem__` and :meth:`DataFrame.iloc.__getitem__` with a :class:`CategoricalDtype` column with integer categories raising when trying to index a row containing a ``NaN`` entry (:issue:`58954`)
 - Bug in :meth:`Index.__getitem__` incorrectly raising with a 0-dim ``np.ndarray`` key (:issue:`55601`)
+- Bug in adding new rows with :meth:`DataFrame.loc.__setitem__` or :class:`Series.loc.__setitem__` which failed to retain dtype on the object's index in some cases (:issue:`41626`)
 - Bug in indexing on a :class:`DatetimeIndex` with a ``timestamp[pyarrow]`` dtype or on a :class:`TimedeltaIndex` with a ``duration[pyarrow]`` dtype (:issue:`62277`)
--
 
 Missing
 ^^^^^^^
@@ -1083,6 +1084,7 @@ Reshaping
 - Bug in :meth:`DataFrame.join` when a :class:`DataFrame` with a :class:`MultiIndex` would raise an ``AssertionError`` when :attr:`MultiIndex.names` contained ``None``. (:issue:`58721`)
 - Bug in :meth:`DataFrame.merge` where merging on a column containing only ``NaN`` values resulted in an out-of-bounds array access (:issue:`59421`)
 - Bug in :meth:`DataFrame.unstack` producing incorrect results when ``sort=False`` (:issue:`54987`, :issue:`55516`)
+- Bug in :meth:`DataFrame.unstack` raising an error with indexes containing ``NaN`` with ``sort=False`` (:issue:`61221`)
 - Bug in :meth:`DataFrame.merge` when merging two :class:`DataFrame` on ``intc`` or ``uintc`` types on Windows (:issue:`60091`, :issue:`58713`)
 - Bug in :meth:`DataFrame.pivot_table` incorrectly subaggregating results when called without an ``index`` argument (:issue:`58722`)
 - Bug in :meth:`DataFrame.pivot_table` incorrectly ignoring the ``values`` argument when also supplied to the ``index`` or ``columns`` parameters (:issue:`57876`, :issue:`61292`)
@@ -1093,7 +1095,7 @@ Reshaping
 - Bug in :func:`melt` where calling with duplicate column names in ``id_vars`` raised a misleading ``AttributeError`` (:issue:`61475`)
 - Bug in :meth:`DataFrame.merge` where user-provided suffixes could result in duplicate column names if the resulting names matched existing columns. Now raises a :class:`MergeError` in such cases. (:issue:`61402`)
 - Bug in :meth:`DataFrame.merge` with :class:`CategoricalDtype` columns incorrectly raising ``RecursionError`` (:issue:`56376`)
--
+- Bug in :meth:`DataFrame.merge` with a ``float32`` index incorrectly casting the index to ``float64`` (:issue:`41626`)
 
 Sparse
 ^^^^^^

environment.yml

@@ -91,6 +91,9 @@ dependencies:
   - sphinx
   - sphinx-design
   - sphinx-copybutton
+
+  # static typing
+  - scipy-stubs
   - types-python-dateutil
   - types-PyMySQL
   - types-pytz

pandas/_testing/asserters.py

@@ -584,19 +584,13 @@ def raise_assert_detail(
 
     if isinstance(left, np.ndarray):
         left = pprint_thing(left)
-    elif isinstance(left, (CategoricalDtype, NumpyEADtype)):
+    elif isinstance(left, (CategoricalDtype, StringDtype, NumpyEADtype)):
         left = repr(left)
-    elif isinstance(left, StringDtype):
-        # TODO(infer_string) this special case could be avoided if we have
-        # a more informative repr https://github.com/pandas-dev/pandas/issues/59342
-        left = f"StringDtype(storage={left.storage}, na_value={left.na_value})"
 
     if isinstance(right, np.ndarray):
         right = pprint_thing(right)
-    elif isinstance(right, (CategoricalDtype, NumpyEADtype)):
+    elif isinstance(right, (CategoricalDtype, StringDtype, NumpyEADtype)):
         right = repr(right)
-    elif isinstance(right, StringDtype):
-        right = f"StringDtype(storage={right.storage}, na_value={right.na_value})"
 
     msg += f"""
 [left]:  {left}

pandas/_testing/contexts.py

@@ -12,7 +12,10 @@
 )
 import uuid
 
-from pandas.compat import PYPY
+from pandas.compat import (
+    PYPY,
+    WARNING_CHECK_DISABLED,
+)
 from pandas.errors import ChainedAssignmentError
 
 from pandas.io.common import get_handle
@@ -163,7 +166,7 @@ def with_csv_dialect(name: str, **kwargs) -> Generator[None]:
 def raises_chained_assignment_error(extra_warnings=(), extra_match=()):
     from pandas._testing import assert_produces_warning
 
-    if PYPY:
+    if PYPY or WARNING_CHECK_DISABLED:
         if not extra_warnings:
             from contextlib import nullcontext
 

pandas/compat/__init__.py

@@ -21,6 +21,7 @@
     PY312,
     PY314,
     PYPY,
+    WARNING_CHECK_DISABLED,
     WASM,
 )
 from pandas.compat.numpy import is_numpy_dev
@@ -158,6 +159,7 @@ def is_ci_environment() -> bool:
     "PY314",
     "PYARROW_MIN_VERSION",
     "PYPY",
+    "WARNING_CHECK_DISABLED",
     "WASM",
     "is_numpy_dev",
     "pa_version_under14p0",

pandas/compat/_constants.py

@@ -19,6 +19,8 @@
 WASM = (sys.platform == "emscripten") or (platform.machine() in ["wasm32", "wasm64"])
 ISMUSL = "musl" in (sysconfig.get_config_var("HOST_GNU_TYPE") or "")
 REF_COUNT = 2
+WARNING_CHECK_DISABLED = PY314
+
 
 __all__ = [
     "IS64",

pandas/core/arrays/boolean.py

@@ -14,6 +14,7 @@
     lib,
     missing as libmissing,
 )
+from pandas.util._decorators import set_module
 
 from pandas.core.dtypes.common import is_list_like
 from pandas.core.dtypes.dtypes import register_extension_dtype
@@ -39,6 +40,7 @@
 
 
 @register_extension_dtype
+@set_module("pandas")
 class BooleanDtype(BaseMaskedDtype):
     """
     Extension dtype for boolean data.

pandas/core/arrays/floating.py

@@ -8,6 +8,8 @@
 
 import numpy as np
 
+from pandas.util._decorators import set_module
+
 from pandas.core.dtypes.base import register_extension_dtype
 from pandas.core.dtypes.common import is_float_dtype
 
@@ -168,13 +170,15 @@ class FloatingArray(NumericArray):
 
 
 @register_extension_dtype
+@set_module("pandas")
 class Float32Dtype(FloatingDtype):
     type = np.float32
     name: ClassVar[str] = "Float32"
     __doc__ = _dtype_docstring.format(dtype="float32")
 
 
 @register_extension_dtype
+@set_module("pandas")
 class Float64Dtype(FloatingDtype):
     type = np.float64
     name: ClassVar[str] = "Float64"

pandas/core/arrays/integer.py

@@ -8,6 +8,8 @@
 
 import numpy as np
 
+from pandas.util._decorators import set_module
+
 from pandas.core.dtypes.base import register_extension_dtype
 from pandas.core.dtypes.common import is_integer_dtype
 
@@ -218,55 +220,63 @@ class IntegerArray(NumericArray):
 
 
 @register_extension_dtype
+@set_module("pandas")
 class Int8Dtype(IntegerDtype):
     type = np.int8
     name: ClassVar[str] = "Int8"
     __doc__ = _dtype_docstring.format(dtype="int8")
 
 
 @register_extension_dtype
+@set_module("pandas")
 class Int16Dtype(IntegerDtype):
     type = np.int16
     name: ClassVar[str] = "Int16"
     __doc__ = _dtype_docstring.format(dtype="int16")
 
 
 @register_extension_dtype
+@set_module("pandas")
 class Int32Dtype(IntegerDtype):
     type = np.int32
     name: ClassVar[str] = "Int32"
     __doc__ = _dtype_docstring.format(dtype="int32")
 
 
 @register_extension_dtype
+@set_module("pandas")
 class Int64Dtype(IntegerDtype):
     type = np.int64
     name: ClassVar[str] = "Int64"
     __doc__ = _dtype_docstring.format(dtype="int64")
 
 
 @register_extension_dtype
+@set_module("pandas")
 class UInt8Dtype(IntegerDtype):
     type = np.uint8
     name: ClassVar[str] = "UInt8"
     __doc__ = _dtype_docstring.format(dtype="uint8")
 
 
 @register_extension_dtype
+@set_module("pandas")
 class UInt16Dtype(IntegerDtype):
     type = np.uint16
     name: ClassVar[str] = "UInt16"
     __doc__ = _dtype_docstring.format(dtype="uint16")
 
 
 @register_extension_dtype
+@set_module("pandas")
 class UInt32Dtype(IntegerDtype):
     type = np.uint32
     name: ClassVar[str] = "UInt32"
     __doc__ = _dtype_docstring.format(dtype="uint32")
 
 
 @register_extension_dtype
+@set_module("pandas")
 class UInt64Dtype(IntegerDtype):
     type = np.uint64
     name: ClassVar[str] = "UInt64"