Skip to content

Commit 9c5e29a

Browse files
committed
BUG: Preserve dtype in DataFrame.fillna when filling from another DataFrame
1 parent cfe54bd commit 9c5e29a

File tree

3 files changed

+38
-2
lines changed

3 files changed

+38
-2
lines changed

doc/source/whatsnew/v3.0.0.rst

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -748,7 +748,7 @@ Indexing
748748
Missing
749749
^^^^^^^
750750
- Bug in :meth:`DataFrame.fillna` and :meth:`Series.fillna` that would ignore the ``limit`` argument on :class:`.ExtensionArray` dtypes (:issue:`58001`)
751-
-
751+
- Bug in :meth:`DataFrame.fillna` where filling from another ``DataFrame`` of the same dtype could incorrectly cast the result to ``object`` dtype. (:issue:`61568`)
752752

753753
MultiIndex
754754
^^^^^^^^^^

pandas/core/generic.py

Lines changed: 18 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -7145,7 +7145,24 @@ def fillna(
71457145
else:
71467146
new_data = self._mgr.fillna(value=value, limit=limit, inplace=inplace)
71477147
elif isinstance(value, ABCDataFrame) and self.ndim == 2:
7148-
new_data = self.where(self.notna(), value)._mgr
7148+
filled_columns = {}
7149+
for col in self.columns:
7150+
lhs = self[col]
7151+
if col in value.columns:
7152+
rhs = value[col]
7153+
filled = lhs.where(notna(lhs), rhs)
7154+
7155+
# restore original dtype if fallback to object occurred
7156+
if lhs.dtype == rhs.dtype and filled.dtype == object:
7157+
try:
7158+
filled = filled.astype(lhs.dtype)
7159+
except Exception:
7160+
pass
7161+
else:
7162+
filled = lhs
7163+
filled_columns[col] = filled
7164+
7165+
new_data = type(self)(filled_columns, index=self.index)._mgr
71497166
else:
71507167
raise ValueError(f"invalid fill value with a {type(value)}")
71517168

pandas/tests/frame/methods/test_fillna.py

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -795,3 +795,22 @@ def test_fillna_out_of_bounds_datetime():
795795
msg = "Cannot cast 0001-01-01 00:00:00 to unit='ns' without overflow"
796796
with pytest.raises(OutOfBoundsDatetime, match=msg):
797797
df.fillna(Timestamp("0001-01-01"))
798+
799+
800+
def test_fillna_dataframe_preserves_dtypes_mixed_columns():
801+
# GH#61568
802+
empty = DataFrame([[None] * 4] * 4, columns=list("ABCD"), dtype=np.float64)
803+
full = DataFrame(
804+
[
805+
[1.0, 2.0, "3.0", 4.0],
806+
[5.0, 6.0, "7.0", 8.0],
807+
[9.0, 10.0, "11.0", 12.0],
808+
[13.0, 14.0, "15.0", 16.0],
809+
],
810+
columns=list("ABCD"),
811+
)
812+
result = empty.fillna(full)
813+
expected_dtypes = Series(
814+
{"A": "float64", "B": "float64", "C": "object", "D": "float64"}
815+
)
816+
tm.assert_series_equal(result.dtypes, expected_dtypes)

0 commit comments

Comments
 (0)