diff --git a/doc/source/whatsnew/v3.0.0.rst b/doc/source/whatsnew/v3.0.0.rst index a619b08a8676f..a43e3b878862b 100644 --- a/doc/source/whatsnew/v3.0.0.rst +++ b/doc/source/whatsnew/v3.0.0.rst @@ -660,6 +660,7 @@ Other Deprecations - Deprecated the ``arg`` parameter of ``Series.map``; pass the added ``func`` argument instead. (:issue:`61260`) - Deprecated using ``epoch`` date format in :meth:`DataFrame.to_json` and :meth:`Series.to_json`, use ``iso`` instead. (:issue:`57063`) - Deprecated allowing ``fill_value`` that cannot be held in the original dtype (excepting NA values for integer and bool dtypes) in :meth:`Series.unstack` and :meth:`DataFrame.unstack` (:issue:`12189`, :issue:`53868`) +- Deprecated allowing ``fill_value`` that cannot be held in the original dtype (excepting NA values for integer and bool dtypes) in :meth:`Series.shift` and :meth:`DataFrame.shift` (:issue:`53802`) .. --------------------------------------------------------------------------- .. _whatsnew_300.prior_deprecations: diff --git a/pandas/core/internals/blocks.py b/pandas/core/internals/blocks.py index 5a11eb88160bb..766c45fe1de6b 100644 --- a/pandas/core/internals/blocks.py +++ b/pandas/core/internals/blocks.py @@ -1449,6 +1449,18 @@ def shift(self, periods: int, fill_value: Any = None) -> list[Block]: fill_value, ) except LossySetitemError: + if self.dtype.kind not in "iub" or not is_valid_na_for_dtype( + fill_value, self.dtype + ): + # GH#53802 + warnings.warn( + "shifting with a fill value that cannot be held by " + "original dtype is deprecated and will raise in a future " + "version. Explicitly cast to the desired dtype before " + "shifting instead.", + Pandas4Warning, + stacklevel=find_stack_level(), + ) nb = self.coerce_to_target_dtype(fill_value, raise_on_upcast=False) return nb.shift(periods, fill_value=fill_value) diff --git a/pandas/tests/frame/methods/test_shift.py b/pandas/tests/frame/methods/test_shift.py index b52240c208493..3f67caaa6ce26 100644 --- a/pandas/tests/frame/methods/test_shift.py +++ b/pandas/tests/frame/methods/test_shift.py @@ -1,6 +1,8 @@ import numpy as np import pytest +from pandas.errors import Pandas4Warning + import pandas as pd from pandas import ( CategoricalIndex, @@ -766,3 +768,29 @@ def test_series_shift_interval_preserves_closed(self): result = ser.shift(1) expected = Series([np.nan, pd.Interval(1, 2, closed="right")]) tm.assert_series_equal(result, expected) + + def test_shift_invalid_fill_value_deprecation(self): + # GH#53802 + df = DataFrame( + { + "a": [1, 2, 3], + "b": [True, False, True], + } + ) + + msg = "shifting with a fill value that cannot" + with tm.assert_produces_warning(Pandas4Warning, match=msg): + df.shift(1, fill_value="foo") + + with tm.assert_produces_warning(Pandas4Warning, match=msg): + df["a"].shift(1, fill_value="foo") + with tm.assert_produces_warning(Pandas4Warning, match=msg): + df["b"].shift(1, fill_value="foo") + + # An incompatible null value + with tm.assert_produces_warning(Pandas4Warning, match=msg): + df.shift(1, fill_value=NaT) + with tm.assert_produces_warning(Pandas4Warning, match=msg): + df["a"].shift(1, fill_value=NaT) + with tm.assert_produces_warning(Pandas4Warning, match=msg): + df["b"].shift(1, fill_value=NaT)