BUG: Improve categorical handling in concat_compat to respect tests, correct wrong tests

Author: adrien pacifico

pandas/core/dtypes/concat.py

@@ -75,7 +75,10 @@ def concat_compat(
         and all(isinstance(arr.dtype, CategoricalDtype) for arr in to_concat)
         and axis == 0
     ):
-        return union_categoricals(to_concat)
+        return union_categoricals(
+            to_concat, sort_categories=True
+        )  # Performance cost, but necessary to keep tests passing.
+        # see pandas/tests/reshape/concat/test_append_common.py:498
     if len(to_concat) and lib.dtypes_all_equal([obj.dtype for obj in to_concat]):
         # fastpath!
         obj = to_concat[0]

pandas/tests/reshape/concat/test_append_common.py

@@ -486,15 +486,17 @@ def test_concat_categorical(self):
         s1 = Series([3, 2], dtype="category")
         s2 = Series([2, 1], dtype="category")
 
-        exp = Series([3, 2, 2, 1])
+        exp = Series([3, 2, 2, 1], dtype="category")  # should remain category
         tm.assert_series_equal(pd.concat([s1, s2], ignore_index=True), exp)
         tm.assert_series_equal(s1._append(s2, ignore_index=True), exp)
 
         # completely different categories (same dtype) => not-category
-        s1 = Series([10, 11, np.nan], dtype="category")
-        s2 = Series([np.nan, 1, 3, 2], dtype="category")
+        s1 = Series([10.0, 11.0, np.nan], dtype="category")
+        s2 = Series([np.nan, 1.0, 3.0, 2.0], dtype="category")
 
-        exp = Series([10, 11, np.nan, np.nan, 1, 3, 2], dtype=np.float64)
+        exp = Series([10, 11, np.nan, np.nan, 1, 3, 2], dtype=np.float64).astype(
+            "category"
+        )
         tm.assert_series_equal(pd.concat([s1, s2], ignore_index=True), exp)
         tm.assert_series_equal(s1._append(s2, ignore_index=True), exp)