wip

Author: MarcoGorelli

pandas/_testing/__init__.py

@@ -57,6 +57,7 @@
     assert_indexing_slices_equivalent,
     assert_interval_array_equal,
     assert_is_sorted,
+    assert_maybe_raises,
     assert_metadata_equivalent,
     assert_numpy_array_equal,
     assert_period_array_equal,
@@ -557,6 +558,7 @@ def shares_memory(left, right) -> bool:
     "assert_indexing_slices_equivalent",
     "assert_interval_array_equal",
     "assert_is_sorted",
+    "assert_maybe_raises",
     "assert_metadata_equivalent",
     "assert_numpy_array_equal",
     "assert_period_array_equal",

pandas/_testing/asserters.py

@@ -1,5 +1,6 @@
 from __future__ import annotations
 
+import contextlib
 import operator
 from typing import (
     TYPE_CHECKING,
@@ -9,6 +10,7 @@
 )
 
 import numpy as np
+import pytest
 
 from pandas._libs import lib
 from pandas._libs.missing import is_matching_na
@@ -1443,3 +1445,9 @@ def assert_metadata_equivalent(
             assert val is None
         else:
             assert val == getattr(right, attr, None)
+
+
+def assert_maybe_raises(condition, exception, match):
+    if condition:
+        return pytest.raises(exception, match=match)
+    return contextlib.nullcontext()

pandas/tests/copy_view/test_indexing.py

@@ -716,11 +716,24 @@ def test_column_as_series_set_with_upcast(backend):
     # Case: selecting a single column now also uses Copy-on-Write -> when
     # setting a value causes an upcast, we don't need to update the parent
     # DataFrame through the cache mechanism
-    _dtype_backend, DataFrame, _Series = backend
+    dtype_backend, DataFrame, Series = backend
     df = DataFrame({"a": [1, 2, 3], "b": [4, 5, 6], "c": [0.1, 0.2, 0.3]})
+    df_orig = df.copy()
+
     s = df["a"]
-    with pytest.raises(TypeError, match="Invalid value"):
-        s[0] = "foo"
+    if dtype_backend == "nullable":
+        with pytest.raises(TypeError, match="Invalid value"):
+            s[0] = "foo"
+        expected = Series([1, 2, 3], name="a")
+    else:
+        with pytest.raises(TypeError, match="Invalid value"):
+            s[0] = "foo"
+            return
+
+    tm.assert_series_equal(s, expected)
+    tm.assert_frame_equal(df, df_orig)
+    # ensure cached series on getitem is not the changed series
+    tm.assert_series_equal(df["a"], df_orig["a"])
 
 
 @pytest.mark.parametrize(
@@ -769,6 +782,7 @@ def test_dataframe_add_column_from_series(backend):
     tm.assert_frame_equal(df, expected)
 
 
+@pytest.mark.parametrize("val", [100, "a"])
 @pytest.mark.parametrize(
     "indexer_func, indexer",
     [
@@ -783,13 +797,20 @@ def test_dataframe_add_column_from_series(backend):
 @pytest.mark.parametrize(
     "col", [[0.1, 0.2, 0.3], [7, 8, 9]], ids=["mixed-block", "single-block"]
 )
-def test_set_value_copy_only_necessary_column(indexer_func, indexer, col):
+def test_set_value_copy_only_necessary_column(indexer_func, indexer, val, col):
     # When setting inplace, only copy column that is modified instead of the whole
     # block (by splitting the block)
     df = DataFrame({"a": [1, 2, 3], "b": [4, 5, 6], "c": col})
     df_orig = df.copy()
     view = df[:]
-    indexer_func(df)[indexer] = 100
+
+    if val == "a":
+        with tm.assert_produces_warning(
+            FutureWarning, match="Setting an item of incompatible dtype is deprecated"
+        ):
+            indexer_func(df)[indexer] = val
+
+    indexer_func(df)[indexer] = val
 
     assert np.shares_memory(get_array(df, "b"), get_array(view, "b"))
     assert not np.shares_memory(get_array(df, "a"), get_array(view, "a"))

pandas/tests/copy_view/test_methods.py

@@ -1105,6 +1105,32 @@ def test_putmask_aligns_rhs_no_reference(dtype):
     assert np.shares_memory(arr_a, get_array(df, "a"))
 
 
+@pytest.mark.parametrize(
+    "val, exp, warn", [(5.5, True, FutureWarning), (5, False, None)]
+)
+def test_putmask_dont_copy_some_blocks(val, exp, warn):
+    df = DataFrame({"a": [1, 2], "b": 1, "c": 1.5})
+    view = df[:]
+    df_orig = df.copy()
+    indexer = DataFrame(
+        [[True, False, False], [True, False, False]], columns=list("abc")
+    )
+    if warn:
+        with pytest.raises(TypeError, match="Invalid value"):
+            df[indexer] = val
+        return
+    else:
+        df[indexer] = val
+
+    assert not np.shares_memory(get_array(view, "a"), get_array(df, "a"))
+    # TODO(CoW): Could split blocks to avoid copying the whole block
+    assert np.shares_memory(get_array(view, "b"), get_array(df, "b")) is exp
+    assert np.shares_memory(get_array(view, "c"), get_array(df, "c"))
+    assert df._mgr._has_no_reference(1) is not exp
+    assert not df._mgr._has_no_reference(2)
+    tm.assert_frame_equal(view, df_orig)
+
+
 @pytest.mark.parametrize("dtype", ["int64", "Int64"])
 @pytest.mark.parametrize(
     "func",

pandas/tests/frame/indexing/test_coercion.py

@@ -46,6 +46,40 @@ def test_loc_setitem_multiindex_columns(self, consolidate):
         # TODO: i think this isn't about MultiIndex and could be done with iloc?
 
 
+def test_37477():
+    # fixed by GH#45121
+    orig = DataFrame({"A": [1, 2, 3], "B": [3, 4, 5]})
+    expected = DataFrame({"A": [1, 2, 3], "B": [3, 1.2, 5]})
+
+    df = orig.copy()
+    with tm.assert_produces_warning(
+        FutureWarning, match="Setting an item of incompatible dtype"
+    ):
+        df.at[1, "B"] = 1.2
+    tm.assert_frame_equal(df, expected)
+
+    df = orig.copy()
+    with tm.assert_produces_warning(
+        FutureWarning, match="Setting an item of incompatible dtype"
+    ):
+        df.loc[1, "B"] = 1.2
+    tm.assert_frame_equal(df, expected)
+
+    df = orig.copy()
+    with tm.assert_produces_warning(
+        FutureWarning, match="Setting an item of incompatible dtype"
+    ):
+        df.iat[1, 1] = 1.2
+    tm.assert_frame_equal(df, expected)
+
+    df = orig.copy()
+    with tm.assert_produces_warning(
+        FutureWarning, match="Setting an item of incompatible dtype"
+    ):
+        df.iloc[1, 1] = 1.2
+    tm.assert_frame_equal(df, expected)
+
+
 def test_6942(indexer_al):
     # check that the .at __setitem__ after setting "Live" actually sets the data
     start = Timestamp("2014-04-01")
@@ -73,6 +107,20 @@ def test_26395(indexer_al):
     expected = DataFrame({"D": [0, 0, 2]}, index=["A", "B", "C"], dtype=np.int64)
     tm.assert_frame_equal(df, expected)
 
+    with tm.assert_produces_warning(
+        FutureWarning, match="Setting an item of incompatible dtype"
+    ):
+        indexer_al(df)["C", "D"] = 44.5
+    expected = DataFrame({"D": [0, 0, 44.5]}, index=["A", "B", "C"], dtype=np.float64)
+    tm.assert_frame_equal(df, expected)
+
+    with tm.assert_produces_warning(
+        FutureWarning, match="Setting an item of incompatible dtype"
+    ):
+        indexer_al(df)["C", "D"] = "hello"
+    expected = DataFrame({"D": [0, 0, "hello"]}, index=["A", "B", "C"], dtype=object)
+    tm.assert_frame_equal(df, expected)
+
 
 @pytest.mark.xfail(reason="unwanted upcast")
 def test_15231():

pandas/tests/frame/indexing/test_indexing.py

@@ -9,6 +9,7 @@
 import numpy as np
 import pytest
 
+from pandas._libs import iNaT
 from pandas.errors import InvalidIndexError
 
 from pandas.core.dtypes.common import is_integer
@@ -24,6 +25,7 @@
     Timestamp,
     date_range,
     isna,
+    notna,
     to_datetime,
 )
 import pandas._testing as tm
@@ -830,6 +832,15 @@ def test_setitem_single_column_mixed_datetime(self):
         )
         tm.assert_series_equal(result, expected)
 
+        # GH#16674 iNaT is treated as an integer when given by the user
+        with tm.assert_produces_warning(
+            FutureWarning, match="Setting an item of incompatible dtype"
+        ):
+            df.loc["b", "timestamp"] = iNaT
+        assert not isna(df.loc["b", "timestamp"])
+        assert df["timestamp"].dtype == np.object_
+        assert df.loc["b", "timestamp"] == iNaT
+
         # allow this syntax (as of GH#3216)
         df.loc["c", "timestamp"] = np.nan
         assert isna(df.loc["c", "timestamp"])
@@ -838,6 +849,38 @@ def test_setitem_single_column_mixed_datetime(self):
         df.loc["d", :] = np.nan
         assert not isna(df.loc["c", :]).all()
 
+    def test_setitem_mixed_datetime(self):
+        # GH 9336
+        expected = DataFrame(
+            {
+                "a": [0, 0, 0, 0, 13, 14],
+                "b": [
+                    datetime(2012, 1, 1),
+                    1,
+                    "x",
+                    "y",
+                    datetime(2013, 1, 1),
+                    datetime(2014, 1, 1),
+                ],
+            }
+        )
+        df = DataFrame(0, columns=list("ab"), index=range(6))
+        df["b"] = pd.NaT
+        df.loc[0, "b"] = datetime(2012, 1, 1)
+        with tm.assert_produces_warning(
+            FutureWarning, match="Setting an item of incompatible dtype"
+        ):
+            df.loc[1, "b"] = 1
+        df.loc[[2, 3], "b"] = "x", "y"
+        A = np.array(
+            [
+                [13, np.datetime64("2013-01-01T00:00:00")],
+                [14, np.datetime64("2014-01-01T00:00:00")],
+            ]
+        )
+        df.loc[[4, 5], ["a", "b"]] = A
+        tm.assert_frame_equal(df, expected)
+
     def test_setitem_frame_float(self, float_frame):
         piece = float_frame.loc[float_frame.index[:2], ["A", "B"]]
         float_frame.loc[float_frame.index[-2] :, ["A", "B"]] = piece.values
@@ -889,6 +932,17 @@ def test_setitem_frame_mixed_ndarray(self, float_string_frame):
         f.loc[key] = piece.values
         tm.assert_almost_equal(f.loc[f.index[-2:], ["A", "B"]].values, piece.values)
 
+    def test_setitem_frame_upcast(self):
+        # needs upcasting
+        df = DataFrame([[1, 2, "foo"], [3, 4, "bar"]], columns=["A", "B", "C"])
+        df2 = df.copy()
+        with tm.assert_produces_warning(FutureWarning, match="incompatible dtype"):
+            df2.loc[:, ["A", "B"]] = df.loc[:, ["A", "B"]] + 0.5
+        expected = df.reindex(columns=["A", "B"])
+        expected += 0.5
+        expected["C"] = df["C"]
+        tm.assert_frame_equal(df2, expected)
+
     def test_setitem_frame_align(self, float_frame):
         piece = float_frame.loc[float_frame.index[:2], ["A", "B"]]
         piece.index = float_frame.index[-2:]
@@ -1299,6 +1353,26 @@ def test_loc_expand_empty_frame_keep_midx_names(self):
         )
         tm.assert_frame_equal(df, expected)
 
+    @pytest.mark.parametrize(
+        "val, idxr",
+        [
+            ("x", "a"),
+            ("x", ["a"]),
+            (1, "a"),
+            (1, ["a"]),
+        ],
+    )
+    def test_loc_setitem_rhs_frame(self, idxr, val):
+        # GH#47578
+        df = DataFrame({"a": [1, 2]})
+
+        with tm.assert_produces_warning(
+            FutureWarning, match="Setting an item of incompatible dtype"
+        ):
+            df.loc[:, idxr] = DataFrame({"a": [val, 11]}, index=[1, 2])
+        expected = DataFrame({"a": [np.nan, val]})
+        tm.assert_frame_equal(df, expected)
+
     def test_iloc_setitem_enlarge_no_warning(self):
         # GH#47381
         df = DataFrame(columns=["a", "b"])
@@ -1502,6 +1576,22 @@ def test_setitem(self):
         tm.assert_series_equal(df["D"], Series(idx, name="D"))
         del df["D"]
 
+        # With NaN: because uint64 has no NaN element,
+        # the column should be cast to object.
+        df2 = df.copy()
+        with tm.assert_produces_warning(FutureWarning, match="incompatible dtype"):
+            df2.iloc[1, 1] = pd.NaT
+            df2.iloc[1, 2] = pd.NaT
+        result = df2["B"]
+        tm.assert_series_equal(notna(result), Series([True, False, True], name="B"))
+        tm.assert_series_equal(
+            df2.dtypes,
+            Series(
+                [np.dtype("uint64"), np.dtype("O"), np.dtype("O")],
+                index=["A", "B", "C"],
+            ),
+        )
+
 
 def test_object_casting_indexing_wraps_datetimelike():
     # GH#31649, check the indexing methods all the way down the stack
@@ -1836,31 +1926,22 @@ def test_add_new_column_infer_string():
 class TestSetitemValidation:
     # This is adapted from pandas/tests/arrays/masked/test_indexing.py
     # but checks for warnings instead of errors.
-    def _check_setitem_invalid(self, df, invalid, indexer):
+    def _check_setitem_invalid(self, df, invalid, indexer, warn):
+        msg = "Setting an item of incompatible dtype is deprecated"
+        msg = re.escape(msg)
+
         orig_df = df.copy()
 
         # iloc
-        with pytest.raises(TypeError, match="Invalid value"):
+        with tm.assert_produces_warning(warn, match=msg):
             df.iloc[indexer, 0] = invalid
             df = orig_df.copy()
 
         # loc
-        with pytest.raises(TypeError, match="Invalid value"):
+        with tm.assert_produces_warning(warn, match=msg):
             df.loc[indexer, "a"] = invalid
             df = orig_df.copy()
 
-    def _check_setitem_valid(self, df, invalid, indexer):
-        # Just execute, verify no error is raised
-        orig_df = df.copy()
-
-        # iloc
-        df.iloc[indexer, 0] = invalid
-        df = orig_df.copy()
-
-        # loc
-        df.loc[indexer, "a"] = invalid
-        df = orig_df.copy()
-
     _invalid_scalars = [
         1 + 2j,
         "True",
@@ -1878,19 +1959,20 @@ def _check_setitem_valid(self, df, invalid, indexer):
     @pytest.mark.parametrize("indexer", _indexers)
     def test_setitem_validation_scalar_bool(self, invalid, indexer):
         df = DataFrame({"a": [True, False, False]}, dtype="bool")
-        self._check_setitem_invalid(df, invalid, indexer)
+        self._check_setitem_invalid(df, invalid, indexer, FutureWarning)
 
     @pytest.mark.parametrize("invalid", _invalid_scalars + [True, 1.5, np.float64(1.5)])
     @pytest.mark.parametrize("indexer", _indexers)
     def test_setitem_validation_scalar_int(self, invalid, any_int_numpy_dtype, indexer):
         df = DataFrame({"a": [1, 2, 3]}, dtype=any_int_numpy_dtype)
         if isna(invalid) and invalid is not pd.NaT and not np.isnat(invalid):
-            self._check_setitem_valid(df, invalid, indexer)
+            warn = None
         else:
-            self._check_setitem_invalid(df, invalid, indexer)
+            warn = FutureWarning
+        self._check_setitem_invalid(df, invalid, indexer, warn)
 
     @pytest.mark.parametrize("invalid", _invalid_scalars + [True])
     @pytest.mark.parametrize("indexer", _indexers)
     def test_setitem_validation_scalar_float(self, invalid, float_numpy_dtype, indexer):
         df = DataFrame({"a": [1, 2, None]}, dtype=float_numpy_dtype)
-        self._check_setitem_invalid(df, invalid, indexer)
+        self._check_setitem_invalid(df, invalid, indexer, FutureWarning)