Merge remote-tracking branch 'upstream/main' into doc-whatsnew-2.3.1

Author: jorisvandenbossche

doc/source/whatsnew/v2.3.1.rst

@@ -61,6 +61,7 @@ correctly, rather than defaulting to ``object`` dtype. For example:
 Bug fixes
 ^^^^^^^^^
 - Bug in :meth:`.DataFrameGroupBy.min`, :meth:`.DataFrameGroupBy.max`, :meth:`.Resampler.min`, :meth:`.Resampler.max` where all NA values of string dtype would return float instead of string dtype (:issue:`60810`)
+- Bug in :meth:`DataFrame.join` incorrectly downcasting object-dtype indexes (:issue:`61771`)
 - Bug in :meth:`DataFrame.sum` with ``axis=1``, :meth:`.DataFrameGroupBy.sum` or :meth:`.SeriesGroupBy.sum` with ``skipna=True``, and :meth:`.Resampler.sum` with all NA values of :class:`StringDtype` resulted in ``0`` instead of the empty string ``""`` (:issue:`60229`)
 - Fixed bug in :meth:`DataFrame.explode` and :meth:`Series.explode` where methods would fail with ``dtype="str"`` (:issue:`61623`)
 - Fixed bug in unpickling objects pickled in pandas versions pre-2.3.0 that used :class:`StringDtype` (:issue:`61763`)

pandas/compat/__init__.py

@@ -35,6 +35,7 @@
     pa_version_under18p0,
     pa_version_under19p0,
     pa_version_under20p0,
+    pa_version_under21p0,
 )
 
 if TYPE_CHECKING:
@@ -168,4 +169,5 @@ def is_ci_environment() -> bool:
     "pa_version_under18p0",
     "pa_version_under19p0",
     "pa_version_under20p0",
+    "pa_version_under21p0",
 ]

pandas/compat/pyarrow.py

@@ -18,6 +18,7 @@
     pa_version_under18p0 = _palv < Version("18.0.0")
     pa_version_under19p0 = _palv < Version("19.0.0")
     pa_version_under20p0 = _palv < Version("20.0.0")
+    pa_version_under21p0 = _palv < Version("21.0.0")
     HAS_PYARROW = _palv >= Version("12.0.1")
 except ImportError:
     pa_version_under12p1 = True
@@ -30,4 +31,5 @@
     pa_version_under18p0 = True
     pa_version_under19p0 = True
     pa_version_under20p0 = True
+    pa_version_under21p0 = True
     HAS_PYARROW = False

pandas/core/reshape/merge.py

@@ -1328,13 +1328,13 @@ def _maybe_add_join_keys(
                 # if we have an all missing left_indexer
                 # make sure to just use the right values or vice-versa
                 if left_indexer is not None and (left_indexer == -1).all():
-                    key_col = Index(rvals)
+                    key_col = Index(rvals, dtype=rvals.dtype, copy=False)
                     result_dtype = rvals.dtype
                 elif right_indexer is not None and (right_indexer == -1).all():
-                    key_col = Index(lvals)
+                    key_col = Index(lvals, dtype=lvals.dtype, copy=False)
                     result_dtype = lvals.dtype
                 else:
-                    key_col = Index(lvals)
+                    key_col = Index(lvals, dtype=lvals.dtype, copy=False)
                     if left_indexer is not None:
                         mask_left = left_indexer == -1
                         key_col = key_col.where(~mask_left, rvals)

pandas/tests/copy_view/test_functions.py

@@ -1,10 +1,6 @@
 import numpy as np
 import pytest
 
-from pandas._config import using_string_dtype
-
-from pandas.compat import HAS_PYARROW
-
 from pandas import (
     DataFrame,
     Index,
@@ -247,13 +243,9 @@ def test_merge_copy_keyword():
     assert np.shares_memory(get_array(df2, "b"), get_array(result, "b"))
 
 
-@pytest.mark.xfail(
-    using_string_dtype() and HAS_PYARROW,
-    reason="TODO(infer_string); result.index infers str dtype while both "
-    "df1 and df2 index are object.",
-)
-def test_join_on_key():
-    df_index = Index(["a", "b", "c"], name="key", dtype=object)
+@pytest.mark.parametrize("dtype", [object, "str"])
+def test_join_on_key(dtype):
+    df_index = Index(["a", "b", "c"], name="key", dtype=dtype)
 
     df1 = DataFrame({"a": [1, 2, 3]}, index=df_index.copy(deep=True))
     df2 = DataFrame({"b": [4, 5, 6]}, index=df_index.copy(deep=True))
@@ -265,7 +257,7 @@ def test_join_on_key():
 
     assert np.shares_memory(get_array(result, "a"), get_array(df1, "a"))
     assert np.shares_memory(get_array(result, "b"), get_array(df2, "b"))
-    assert np.shares_memory(get_array(result.index), get_array(df1.index))
+    assert tm.shares_memory(get_array(result.index), get_array(df1.index))
     assert not np.shares_memory(get_array(result.index), get_array(df2.index))
 
     result.iloc[0, 0] = 0

pandas/tests/extension/test_arrow.py

@@ -43,6 +43,7 @@
     pa_version_under14p0,
     pa_version_under19p0,
     pa_version_under20p0,
+    pa_version_under21p0,
 )
 
 from pandas.core.dtypes.dtypes import (
@@ -542,7 +543,10 @@ def _get_expected_reduction_dtype(self, arr, op_name: str, skipna: bool):
             else:
                 cmp_dtype = arr.dtype
         elif arr.dtype.name == "decimal128(7, 3)[pyarrow]":
-            if op_name not in ["median", "var", "std", "sem", "skew"]:
+            if op_name == "sum" and not pa_version_under21p0:
+                # https://github.com/apache/arrow/pull/44184
+                cmp_dtype = ArrowDtype(pa.decimal128(38, 3))
+            elif op_name not in ["median", "var", "std", "sem", "skew"]:
                 cmp_dtype = arr.dtype
             else:
                 cmp_dtype = "float64[pyarrow]"