Fix behavior for DataFrame.sum and Series.sum

Author: snitish

doc/source/whatsnew/v3.0.0.rst

@@ -750,13 +750,13 @@ Groupby/resample/rolling
 - Bug in :meth:`.DataFrameGroupBy.quantile` when ``interpolation="nearest"`` is inconsistent with :meth:`DataFrame.quantile` (:issue:`47942`)
 - Bug in :meth:`.Resampler.interpolate` on a :class:`DataFrame` with non-uniform sampling and/or indices not aligning with the resulting resampled index would result in wrong interpolation (:issue:`21351`)
 - Bug in :meth:`DataFrame.ewm` and :meth:`Series.ewm` when passed ``times`` and aggregation functions other than mean (:issue:`51695`)
+- Bug in :meth:`DataFrame.sum`, :meth:`Series.sum`, :meth:`DataFrameGroupBy.sum` and :math:`SeriesGroupBy.sum` where in case of all-nan values for object dtype the result is incorrectly set to 0 instead of ``nan``. (:issue:`60229`)
 - Bug in :meth:`DataFrameGroupBy.agg` that raises ``AttributeError`` when there is dictionary input and duplicated columns, instead of returning a DataFrame with the aggregation of all duplicate columns. (:issue:`55041`)
 - Bug in :meth:`DataFrameGroupBy.apply` and :meth:`SeriesGroupBy.apply` for empty data frame with ``group_keys=False`` still creating output index using group keys. (:issue:`60471`)
 - Bug in :meth:`DataFrameGroupBy.apply` that was returning a completely empty DataFrame when all return values of ``func`` were ``None`` instead of returning an empty DataFrame with the original columns and dtypes. (:issue:`57775`)
 - Bug in :meth:`DataFrameGroupBy.apply` with ``as_index=False`` that was returning :class:`MultiIndex` instead of returning :class:`Index`. (:issue:`58291`)
 - Bug in :meth:`DataFrameGroupBy.cumsum` and :meth:`DataFrameGroupBy.cumprod` where ``numeric_only`` parameter was passed indirectly through kwargs instead of passing directly. (:issue:`58811`)
 - Bug in :meth:`DataFrameGroupBy.cumsum` where it did not return the correct dtype when the label contained ``None``. (:issue:`58811`)
-- Bug in :meth:`DataFrameGroupBy.sum` and :math:`SeriesGroupBy.sum` where in case of all-nan values for object dtype the result is incorrectly set to 0 instead of ``None``. (:issue:`58811`)
 - Bug in :meth:`DataFrameGroupby.transform` and :meth:`SeriesGroupby.transform` with a reducer and ``observed=False`` that coerces dtype to float when there are unobserved categories. (:issue:`55326`)
 - Bug in :meth:`Rolling.apply` for ``method="table"`` where column order was not being respected due to the columns getting sorted by default. (:issue:`59666`)
 - Bug in :meth:`Rolling.apply` where the applied function could be called on fewer than ``min_period`` periods if ``method="table"``. (:issue:`58868`)

pandas/core/nanops.py

@@ -638,6 +638,13 @@ def nansum(
     the_sum = values.sum(axis, dtype=dtype_sum)
     the_sum = _maybe_null_out(the_sum, axis, mask, values.shape, min_count=min_count)
 
+    if dtype.kind == "O" and skipna and min_count == 0:
+        # GH#60229 For object dtype, sum of all-NA array should be nan
+        if isinstance(the_sum, np.ndarray):
+            the_sum[mask.sum(axis=axis) == mask.shape[axis]] = np.nan
+        elif mask.all():
+            the_sum = np.nan
+
     return the_sum
 
 

pandas/tests/frame/test_arithmetic.py

@@ -1186,7 +1186,7 @@ def test_frame_single_columns_object_sum_axis_1():
     }
     df = DataFrame(data)
     result = df.sum(axis=1)
-    expected = Series(["A", 1.2, 0])
+    expected = Series(["A", 1.2, np.nan])
     tm.assert_series_equal(result, expected)
 
 

pandas/tests/frame/test_reductions.py

@@ -420,10 +420,25 @@ def test_stat_operators_attempt_obj_array(self, method, df, axis):
         assert df.values.dtype == np.object_
         result = getattr(df, method)(axis=axis)
         expected = getattr(df.astype("f8"), method)(axis=axis).astype(object)
+        if method == "sum":
+            # GH#60229 in case of all-NA object array, sum should be nan
+            expected[df.isna().all(axis=axis)] = np.nan
         if axis in [1, "columns"] and method in ["min", "max"]:
             expected[expected.isna()] = None
         tm.assert_series_equal(result, expected)
 
+    def test_object_sum_allna(self):
+        # GH#60229
+        df = DataFrame({"a": [np.nan] * 5, "b": [pd.NA] * 5}, dtype=object)
+
+        result = df.sum(axis=0, skipna=True)
+        expected = Series([np.nan, np.nan], index=["a", "b"], dtype=object)
+        tm.assert_series_equal(result, expected)
+
+        result = df.sum(axis=0, skipna=False)
+        expected = Series([np.nan, pd.NA], index=["a", "b"], dtype=object)
+        tm.assert_series_equal(result, expected)
+
     @pytest.mark.parametrize("op", ["mean", "std", "var", "skew", "kurt", "sem"])
     def test_mixed_ops(self, op):
         # GH#16116

pandas/tests/series/test_reductions.py

@@ -111,6 +111,20 @@ def test_prod_numpy16_bug():
     assert not isinstance(result, Series)
 
 
+@pytest.mark.parametrize("nan_val", [np.nan, pd.NA])
+def test_object_sum_allna(nan_val):
+    # GH#60229
+    ser = Series([nan_val] * 5, dtype=object)
+
+    result = ser.sum(axis=0, skipna=True)
+    expected = np.nan
+    tm.assert_equal(result, expected)
+
+    result = ser.sum(axis=0, skipna=False)
+    expected = nan_val
+    tm.assert_equal(result, expected)
+
+
 @pytest.mark.parametrize("func", [np.any, np.all])
 @pytest.mark.parametrize("kwargs", [{"keepdims": True}, {"out": object()}])
 def test_validate_any_all_out_keepdims_raises(kwargs, func):