Merge branch 'main' into plot

Author: MartinBraquet

doc/source/whatsnew/v3.0.0.rst

@@ -68,6 +68,7 @@ Other enhancements
 - :func:`read_parquet` accepts ``to_pandas_kwargs`` which are forwarded to :meth:`pyarrow.Table.to_pandas` which enables passing additional keywords to customize the conversion to pandas, such as ``maps_as_pydicts`` to read the Parquet map data type as python dictionaries (:issue:`56842`)
 - :meth:`.DataFrameGroupBy.transform`, :meth:`.SeriesGroupBy.transform`, :meth:`.DataFrameGroupBy.agg`, :meth:`.SeriesGroupBy.agg`, :meth:`.SeriesGroupBy.apply`, :meth:`.DataFrameGroupBy.apply` now support ``kurt`` (:issue:`40139`)
 - :meth:`DataFrame.apply` supports using third-party execution engines like the Bodo.ai JIT compiler (:issue:`60668`)
+- :meth:`DataFrame.iloc` and :meth:`Series.iloc` now support boolean masks in ``__getitem__`` for more consistent indexing behavior (:issue:`60994`)
 - :meth:`DataFrameGroupBy.transform`, :meth:`SeriesGroupBy.transform`, :meth:`DataFrameGroupBy.agg`, :meth:`SeriesGroupBy.agg`, :meth:`RollingGroupby.apply`, :meth:`ExpandingGroupby.apply`, :meth:`Rolling.apply`, :meth:`Expanding.apply`, :meth:`DataFrame.apply` with ``engine="numba"`` now supports positional arguments passed as kwargs (:issue:`58995`)
 - :meth:`Rolling.agg`, :meth:`Expanding.agg` and :meth:`ExponentialMovingWindow.agg` now accept :class:`NamedAgg` aggregations through ``**kwargs`` (:issue:`28333`)
 - :meth:`Series.map` can now accept kwargs to pass on to func (:issue:`59814`)
@@ -761,8 +762,9 @@ Plotting
 - Bug in :meth:`.DataFrameGroupBy.boxplot` failed when there were multiple groupings (:issue:`14701`)
 - Bug in :meth:`DataFrame.plot.bar` with ``stacked=True`` where labels on stacked bars with zero-height segments were incorrectly positioned at the base instead of the label position of the previous segment (:issue:`59429`)
 - Bug in :meth:`DataFrame.plot.line` raising ``ValueError`` when set both color and a ``dict`` style (:issue:`59461`)
-- Bug in :meth:`DataFrame.plot` that causes a shift to the right when the frequency multiplier is greater than one. (:issue:`57587`)
-- Bug in :meth:`Series.plot` preventing a line and bar from being aligned on the same plot (:issue:`61161`)
+- Bug in :meth:`DataFrame.plot` that causes a shift to the right when the frequency multiplier is greater than one. (:issue:`57587`
+- Bug in :meth:`Series.plot` preventing a line and bar from being aligned on the same plot (:issue:`611
+- Bug in :meth:`Series.plot` preventing a line and scatter plot from being aligned (:issue:`6100
 - Bug in :meth:`Series.plot` with ``kind="pie"`` with :class:`ArrowDtype` (:issue:`59192`)
 
 Groupby/resample/rolling

pandas/core/indexing.py

@@ -1582,11 +1582,7 @@ def _validate_key(self, key, axis: AxisInt) -> None:
         if com.is_bool_indexer(key):
             if hasattr(key, "index") and isinstance(key.index, Index):
                 if key.index.inferred_type == "integer":
-                    raise NotImplementedError(
-                        "iLocation based boolean "
-                        "indexing on an integer type "
-                        "is not available"
-                    )
+                    return
                 raise ValueError(
                     "iLocation based boolean indexing cannot use an indexable as a mask"
                 )

pandas/plotting/_matplotlib/core.py

@@ -67,10 +67,9 @@
 from pandas.plotting._matplotlib.misc import unpack_single_str_list
 from pandas.plotting._matplotlib.style import get_standard_colors
 from pandas.plotting._matplotlib.timeseries import (
-    decorate_axes,
     format_dateaxis,
     maybe_convert_index,
-    maybe_resample,
+    prepare_ts_data,
     use_dynamic_x,
 )
 from pandas.plotting._matplotlib.tools import (
@@ -304,7 +303,7 @@ def _is_ts_plot(self) -> bool:
 
     @final
     def _use_dynamic_x(self) -> bool:
-        return use_dynamic_x(self._get_ax(0), self.data)
+        return use_dynamic_x(self._get_ax(0), self.data.index)
 
     @final
     @staticmethod
@@ -1342,10 +1341,20 @@ def __init__(
             c = self.data.columns[c]
         self.c = c
 
+    @register_pandas_matplotlib_converters
     def _make_plot(self, fig: Figure) -> None:
         x, y, c, data = self.x, self.y, self.c, self.data
         ax = self.axes[0]
 
+        from pandas import Series
+
+        x_data = data[x]
+        s = Series(index=x_data)
+        if use_dynamic_x(ax, s.index):
+            s = maybe_convert_index(ax, s)
+            freq, s = prepare_ts_data(s, ax, self.kwds)
+            x_data = s.index
+
         c_is_column = is_hashable(c) and c in self.data.columns
 
         color_by_categorical = c_is_column and isinstance(
@@ -1362,7 +1371,7 @@ def _make_plot(self, fig: Figure) -> None:
         else:
             label = None
 
-        # if a list of non color strings is passed in as c, color points
+        # if a list of non-color strings is passed in as c, color points
         # by uniqueness of the strings, such same strings get same color
         create_colors = not self._are_valid_colors(c_values)
         if create_colors:
@@ -1378,7 +1387,7 @@ def _make_plot(self, fig: Figure) -> None:
             )
 
         scatter = ax.scatter(
-            data[x].values,
+            x_data.values,
             data[y].values,
             c=c_values,
             label=label,
@@ -1630,15 +1639,8 @@ def _ts_plot(self, ax: Axes, x, data: Series, style=None, **kwds):
         # accept x to be consistent with normal plot func,
         # x is not passed to tsplot as it uses data.index as x coordinate
         # column_num must be in kwds for stacking purpose
-        freq, data = maybe_resample(data, ax, kwds)
+        freq, data = prepare_ts_data(data, ax, kwds)
 
-        # Set ax with freq info
-        decorate_axes(ax, freq)
-        # digging deeper
-        if hasattr(ax, "left_ax"):
-            decorate_axes(ax.left_ax, freq)
-        if hasattr(ax, "right_ax"):
-            decorate_axes(ax.right_ax, freq)
         # TODO #54485
         ax._plot_data.append((data, self._kind, kwds))  # type: ignore[attr-defined]
 

pandas/plotting/_matplotlib/timeseries.py

@@ -48,7 +48,6 @@
     from pandas._typing import NDFrameT
 
     from pandas import (
-        DataFrame,
         DatetimeIndex,
         Index,
         PeriodIndex,
@@ -231,8 +230,8 @@ def _get_freq(ax: Axes, series: Series):
     return freq, ax_freq
 
 
-def use_dynamic_x(ax: Axes, data: DataFrame | Series) -> bool:
-    freq = _get_index_freq(data.index)
+def use_dynamic_x(ax: Axes, index: Index) -> bool:
+    freq = _get_index_freq(index)
     ax_freq = _get_ax_freq(ax)
 
     if freq is None:  # convert irregular if axes has freq info
@@ -250,16 +249,15 @@ def use_dynamic_x(ax: Axes, data: DataFrame | Series) -> bool:
         return False
 
     # FIXME: hack this for 0.10.1, creating more technical debt...sigh
-    if isinstance(data.index, ABCDatetimeIndex):
+    if isinstance(index, ABCDatetimeIndex):
         # error: "BaseOffset" has no attribute "_period_dtype_code"
         freq_str = OFFSET_TO_PERIOD_FREQSTR.get(freq_str, freq_str)
         base = to_offset(freq_str, is_period=True)._period_dtype_code  # type: ignore[attr-defined]
-        x = data.index
         if base <= FreqGroup.FR_DAY.value:
-            return x[:1].is_normalized
-        period = Period(x[0], freq_str)
+            return index[:1].is_normalized
+        period = Period(index[0], freq_str)
         assert isinstance(period, Period)
-        return period.to_timestamp().tz_localize(x.tz) == x[0]
+        return period.to_timestamp().tz_localize(index.tz) == index[0]
     return True
 
 
@@ -366,3 +364,19 @@ def format_dateaxis(
         raise TypeError("index type not supported")
 
     plt.draw_if_interactive()
+
+
+def prepare_ts_data(
+    series: Series, ax: Axes, kwargs: dict[str, Any]
+) -> tuple[BaseOffset | str, Series]:
+    freq, data = maybe_resample(series, ax, kwargs)
+
+    # Set ax with freq info
+    decorate_axes(ax, freq)
+    # digging deeper
+    if hasattr(ax, "left_ax"):
+        decorate_axes(ax.left_ax, freq)
+    if hasattr(ax, "right_ax"):
+        decorate_axes(ax.right_ax, freq)
+
+    return freq, data

pandas/tests/indexing/test_iloc.py

@@ -726,15 +726,16 @@ def test_iloc_setitem_with_scalar_index(self, indexer, value):
 
     @pytest.mark.filterwarnings("ignore::UserWarning")
     def test_iloc_mask(self):
-        # GH 3631, iloc with a mask (of a series) should raise
+        # GH 60994, iloc with a mask (of a series) should return accordingly
         df = DataFrame(list(range(5)), index=list("ABCDE"), columns=["a"])
         mask = df.a % 2 == 0
         msg = "iLocation based boolean indexing cannot use an indexable as a mask"
         with pytest.raises(ValueError, match=msg):
             df.iloc[mask]
+
         mask.index = range(len(mask))
-        msg = "iLocation based boolean indexing on an integer type is not available"
-        with pytest.raises(NotImplementedError, match=msg):
+        msg = "Unalignable boolean Series provided as indexer"
+        with pytest.raises(IndexingError, match=msg):
             df.iloc[mask]
 
         # ndarray ok
@@ -753,18 +754,13 @@ def test_iloc_mask(self):
             (None, ".iloc"): "0b1100",
             ("index", ""): "0b11",
             ("index", ".loc"): "0b11",
-            ("index", ".iloc"): (
-                "iLocation based boolean indexing cannot use an indexable as a mask"
-            ),
-            ("locs", ""): "Unalignable boolean Series provided as indexer "
-            "(index of the boolean Series and of the indexed "
-            "object do not match).",
-            ("locs", ".loc"): "Unalignable boolean Series provided as indexer "
-            "(index of the boolean Series and of the "
-            "indexed object do not match).",
-            ("locs", ".iloc"): (
-                "iLocation based boolean indexing on an integer type is not available"
-            ),
+            (
+                "index",
+                ".iloc",
+            ): "iLocation based boolean indexing cannot use an indexable as a mask",
+            ("locs", ""): "Unalignable boolean Series provided as indexer",
+            ("locs", ".loc"): "Unalignable boolean Series provided as indexer",
+            ("locs", ".iloc"): "Unalignable boolean Series provided as indexer",
         }
 
         # UserWarnings from reindex of a boolean mask
@@ -780,18 +776,52 @@ def test_iloc_mask(self):
                     else:
                         accessor = df
                     answer = str(bin(accessor[mask]["nums"].sum()))
-                except (ValueError, IndexingError, NotImplementedError) as err:
+                except (ValueError, IndexingError) as err:
                     answer = str(err)
 
                 key = (
                     idx,
                     method,
                 )
-                r = expected.get(key)
-                if r != answer:
-                    raise AssertionError(
-                        f"[{key}] does not match [{answer}], received [{r}]"
+                expected_result = expected.get(key)
+
+                # Fix the assertion to check for substring match
+                if (
+                    idx is None or (idx == "index" and method != ".iloc")
+                ) and "0b" in expected_result:
+                    # For successful numeric results, exact match is needed
+                    assert expected_result == answer, (
+                        f"[{key}] does not match [{answer}]"
                     )
+                else:
+                    # For error messages, substring match is sufficient
+                    assert expected_result in answer, f"[{key}] not found in [{answer}]"
+
+    def test_iloc_with_numpy_bool_array(self):
+        df = DataFrame(list(range(5)), index=list("ABCDE"), columns=["a"])
+        result = df.iloc[np.array([True, False, True, False, True], dtype=bool)]
+        expected = DataFrame({"a": [0, 2, 4]}, index=["A", "C", "E"])
+        tm.assert_frame_equal(result, expected)
+
+    def test_iloc_series_mask_with_index_mismatch_raises(self):
+        df = DataFrame(list(range(5)), index=list("ABCDE"), columns=["a"])
+        mask = df.a % 2 == 0
+        msg = "Unalignable boolean Series provided as indexer"
+        with pytest.raises(IndexingError, match=msg):
+            df.iloc[Series([True] * len(mask), dtype=bool)]
+
+    def test_iloc_series_mask_all_true(self):
+        df = DataFrame(list(range(5)), columns=["a"])
+        mask = Series([True] * len(df), dtype=bool)
+        result = df.iloc[mask]
+        tm.assert_frame_equal(result, df)
+
+    def test_iloc_series_mask_alternate_true(self):
+        df = DataFrame(list(range(5)), columns=["a"])
+        mask = Series([True, False, True, False, True], dtype=bool)
+        result = df.iloc[mask]
+        expected = DataFrame({"a": [0, 2, 4]}, index=[0, 2, 4])
+        tm.assert_frame_equal(result, expected)
 
     def test_iloc_non_unique_indexing(self):
         # GH 4017, non-unique indexing (on the axis)

pandas/tests/plotting/frame/test_frame.py

@@ -840,14 +840,26 @@ def test_plot_scatter_shape(self):
         axes = df.plot(x="x", y="y", kind="scatter", subplots=True)
         _check_axes_shape(axes, axes_num=1, layout=(1, 1))
 
-    def test_raise_error_on_datetime_time_data(self):
-        # GH 8113, datetime.time type is not supported by matplotlib in scatter
+    def test_scatter_on_datetime_time_data(self):
+        # datetime.time type is now supported in scatter, since a converter
+        # is implemented in ScatterPlot
         df = DataFrame(np.random.default_rng(2).standard_normal(10), columns=["a"])
         df["dtime"] = date_range(start="2014-01-01", freq="h", periods=10).time
-        msg = "must be a string or a (real )?number, not 'datetime.time'"
+        df.plot(kind="scatter", x="dtime", y="a")
 
-        with pytest.raises(TypeError, match=msg):
-            df.plot(kind="scatter", x="dtime", y="a")
+    def test_scatter_line_xticks(self):
+        # GH#61005
+        df = DataFrame(
+            [(datetime(year=2025, month=1, day=1, hour=n), n) for n in range(3)],
+            columns=["datetime", "y"],
+        )
+        fig, ax = plt.subplots(2, sharex=True)
+        df.plot.scatter(x="datetime", y="y", ax=ax[0])
+        scatter_xticks = ax[0].get_xticks()
+        df.plot(x="datetime", y="y", ax=ax[1])
+        line_xticks = ax[1].get_xticks()
+        assert scatter_xticks[0] == line_xticks[0]
+        assert scatter_xticks[-1] == line_xticks[-1]
 
     @pytest.mark.parametrize("x, y", [("dates", "vals"), (0, 1)])
     def test_scatterplot_datetime_data(self, x, y):

web/pandas/community/ecosystem.md

@@ -469,7 +469,7 @@ read_record.data
 df.dtypes
 ```
 
-ArcticDB also supports appending, updating, and querying data from storage to a pandas DataFrame. Please find more information [here](https://docs.arcticdb.io/latest/api/query_builder/).
+ArcticDB also supports appending, updating, and querying data from storage to a pandas DataFrame. Please find more information [here](https://docs.arcticdb.io/latest/api/processing/#arcticdb.QueryBuilder).
 
 ### [Hugging Face](https://huggingface.co/datasets)