Merge remote-tracking branch 'upstream/main' into enh-pd-cut

Author: chaarvii

.github/workflows/unit-tests.yml

@@ -391,6 +391,7 @@ jobs:
         env:
           PYTHON_GIL: 0
 
+  # NOTE: this job must be kept in sync with the Pyodide build job in wheels.yml
   emscripten:
     # Note: the Python version, Emscripten toolchain version are determined
     # by the Pyodide version. The appropriate versions can be found in the

.github/workflows/wheels.yml

@@ -100,6 +100,13 @@ jobs:
         - [windows-2022, win_amd64]
         # TODO: support PyPy?
         python: [["cp310", "3.10"], ["cp311", "3.11"], ["cp312", "3.12"]]
+
+        # Build Pyodide wheels and upload them to Anaconda.org
+        # NOTE: this job is similar to the one in unit-tests.yml except for the fact
+        # that it uses cibuildwheel instead of a standard Pyodide xbuildenv setup.
+        include:
+          - buildplat: [ubuntu-22.04, pyodide_wasm32]
+            python: ["cp312", "3.12"]
     env:
       IS_PUSH: ${{ github.event_name == 'push' && startsWith(github.ref, 'refs/tags/v') }}
       IS_SCHEDULE_DISPATCH: ${{ github.event_name == 'schedule' || github.event_name == 'workflow_dispatch' }}
@@ -140,12 +147,13 @@ jobs:
         run: echo "sdist_name=$(cd ./dist && ls -d */)" >> "$GITHUB_ENV"
 
       - name: Build wheels
-        uses: pypa/[email protected].1
+        uses: pypa/[email protected].2
         with:
          package-dir: ./dist/${{ startsWith(matrix.buildplat[1], 'macosx') && env.sdist_name || needs.build_sdist.outputs.sdist_file }}
         env:
           CIBW_PRERELEASE_PYTHONS: True
           CIBW_BUILD: ${{ matrix.python[0] }}-${{ matrix.buildplat[1] }}
+          CIBW_PLATFORM: ${{ matrix.buildplat[1] == 'pyodide_wasm32' && 'pyodide' || 'auto' }}
 
       - name: Set up Python
         uses: mamba-org/setup-micromamba@v1

asv_bench/benchmarks/indexing.py

@@ -546,24 +546,17 @@ def time_chained_indexing(self, mode):
 
 
 class Block:
-    params = [
-        (True, "True"),
-        (np.array(True), "np.array(True)"),
-    ]
-
-    def setup(self, true_value, mode):
+    def setup(self):
         self.df = DataFrame(
             False,
             columns=np.arange(500).astype(str),
             index=date_range("2010-01-01", "2011-01-01"),
         )
 
-        self.true_value = true_value
-
-    def time_test(self, true_value, mode):
+    def time_test(self):
         start = datetime(2010, 5, 1)
         end = datetime(2010, 9, 1)
-        self.df.loc[start:end, :] = true_value
+        self.df.loc[start:end, :] = True
 
 
 from .pandas_vb_common import setup  # noqa: F401 isort:skip

ci/code_checks.sh

@@ -70,10 +70,6 @@ if [[ -z "$CHECK" || "$CHECK" == "docstrings" ]]; then
         --format=actions \
         -i ES01 `# For now it is ok if docstrings are missing the extended summary` \
         -i "pandas.Series.dt PR01" `# Accessors are implemented as classes, but we do not document the Parameters section` \
-        -i "pandas.DataFrame.max RT03" \
-        -i "pandas.DataFrame.mean RT03" \
-        -i "pandas.DataFrame.median RT03" \
-        -i "pandas.DataFrame.min RT03" \
         -i "pandas.DataFrame.plot PR02" \
         -i "pandas.Grouper PR02" \
         -i "pandas.MultiIndex.append PR07,SA01" \
@@ -95,7 +91,6 @@ if [[ -z "$CHECK" || "$CHECK" == "docstrings" ]]; then
         -i "pandas.Period.freq GL08" \
         -i "pandas.Period.freqstr SA01" \
         -i "pandas.Period.month SA01" \
-        -i "pandas.Period.now SA01" \
         -i "pandas.Period.ordinal GL08" \
         -i "pandas.Period.strftime PR01,SA01" \
         -i "pandas.Period.to_timestamp SA01" \
@@ -167,7 +162,7 @@ if [[ -z "$CHECK" || "$CHECK" == "docstrings" ]]; then
         -i "pandas.Series.product RT03" \
         -i "pandas.Series.reorder_levels RT03,SA01" \
         -i "pandas.Series.sem PR01,RT03,SA01" \
-        -i "pandas.Series.skew RT03,SA01" \
+        -i "pandas.Series.skew SA01" \
         -i "pandas.Series.sparse PR01,SA01" \
         -i "pandas.Series.sparse.density SA01" \
         -i "pandas.Series.sparse.fill_value SA01" \

doc/source/getting_started/intro_tutorials/04_plotting.rst

@@ -32,8 +32,10 @@ How do I create plots in pandas?
     air_quality.head()
 
 .. note::
-    The usage of the ``index_col`` and ``parse_dates`` parameters of the ``read_csv`` function to define the first (0th) column as
-    index of the resulting ``DataFrame`` and convert the dates in the column to :class:`Timestamp` objects, respectively.
+    The ``index_col=0`` and ``parse_dates=True`` parameters passed to the ``read_csv`` function define
+    the first (0th) column as index of the resulting ``DataFrame`` and convert the dates in the column
+    to :class:`Timestamp`  objects, respectively.
+
 
 .. raw:: html
 

doc/source/getting_started/overview.rst

@@ -6,11 +6,11 @@
 Package overview
 ****************
 
-pandas is a `Python <https://www.python.org>`__ package providing fast,
+pandas is a `Python <https://www.python.org>`__ package that provides fast,
 flexible, and expressive data structures designed to make working with
 "relational" or "labeled" data both easy and intuitive. It aims to be the
-fundamental high-level building block for doing practical, **real-world** data
-analysis in Python. Additionally, it has the broader goal of becoming **the
+fundamental high-level building block for Python's practical, **real-world** data
+analysis. Additionally, it seeks to become **the
 most powerful and flexible open source data analysis/manipulation tool
 available in any language**. It is already well on its way toward this goal.
 

doc/source/user_guide/categorical.rst

@@ -793,7 +793,7 @@ Assigning a ``Categorical`` to parts of a column of other types will use the val
     :okwarning:
 
     df = pd.DataFrame({"a": [1, 1, 1, 1, 1], "b": ["a", "a", "a", "a", "a"]})
-    df.loc[1:2, "a"] = pd.Categorical(["b", "b"], categories=["a", "b"])
+    df.loc[1:2, "a"] = pd.Categorical([2, 2], categories=[2, 3])
     df.loc[2:3, "b"] = pd.Categorical(["b", "b"], categories=["a", "b"])
     df
     df.dtypes

doc/source/whatsnew/v3.0.0.rst

@@ -43,6 +43,7 @@ Other enhancements
 - :func:`cut` now supports a string for ``bins`` kwarg by dispatching to ``numpy.histogram_bin_edges``. (:issue:`59165`)
 - :meth:`DataFrame.corrwith` now accepts ``min_periods`` as optional arguments, as in :meth:`DataFrame.corr` and :meth:`Series.corr` (:issue:`9490`)
 - :meth:`DataFrame.cummin`, :meth:`DataFrame.cummax`, :meth:`DataFrame.cumprod` and :meth:`DataFrame.cumsum` methods now have a ``numeric_only`` parameter (:issue:`53072`)
+- :meth:`DataFrame.ewm` now allows ``adjust=False`` when ``times`` is provided (:issue:`54328`)
 - :meth:`DataFrame.fillna` and :meth:`Series.fillna` can now accept ``value=None``; for non-object dtype the corresponding NA value will be used (:issue:`57723`)
 - :meth:`DataFrame.pivot_table` and :func:`pivot_table` now allow the passing of keyword arguments to ``aggfunc`` through ``**kwargs`` (:issue:`57884`)
 - :meth:`Series.cummin` and :meth:`Series.cummax` now supports :class:`CategoricalDtype` (:issue:`52335`)
@@ -361,6 +362,7 @@ Other Removals
 - Changed the default value of ``na_action`` in :meth:`Categorical.map` to ``None`` (:issue:`51645`)
 - Changed the default value of ``observed`` in :meth:`DataFrame.groupby` and :meth:`Series.groupby` to ``True`` (:issue:`51811`)
 - Enforce deprecation in :func:`testing.assert_series_equal` and :func:`testing.assert_frame_equal` with object dtype and mismatched null-like values, which are now considered not-equal (:issue:`18463`)
+- Enforce banning of upcasting in in-place setitem-like operations (:issue:`59007`) (see `PDEP6 <https://pandas.pydata.org/pdeps/0006-ban-upcasting.html>`_)
 - Enforced deprecation ``all`` and ``any`` reductions with ``datetime64``, :class:`DatetimeTZDtype`, and :class:`PeriodDtype` dtypes (:issue:`58029`)
 - Enforced deprecation disallowing ``float`` "periods" in :func:`date_range`, :func:`period_range`, :func:`timedelta_range`, :func:`interval_range`,  (:issue:`56036`)
 - Enforced deprecation disallowing parsing datetimes with mixed time zones unless user passes ``utc=True`` to :func:`to_datetime` (:issue:`57275`)
@@ -382,6 +384,8 @@ Other Removals
 - Enforced deprecation of string ``A`` denoting frequency in :class:`YearEnd` and strings ``A-DEC``, ``A-JAN``, etc. denoting annual frequencies with various fiscal year ends (:issue:`57699`)
 - Enforced deprecation of string ``BAS`` denoting frequency in :class:`BYearBegin` and strings ``BAS-DEC``, ``BAS-JAN``, etc. denoting annual frequencies with various fiscal year starts (:issue:`57793`)
 - Enforced deprecation of string ``BA`` denoting frequency in :class:`BYearEnd` and strings ``BA-DEC``, ``BA-JAN``, etc. denoting annual frequencies with various fiscal year ends (:issue:`57793`)
+- Enforced deprecation of strings ``H``, ``BH``, and ``CBH`` denoting frequencies in :class:`Hour`, :class:`BusinessHour`, :class:`CustomBusinessHour` (:issue:`59143`)
+- Enforced deprecation of strings ``H``, ``BH``, and ``CBH`` denoting units in :class:`Timedelta` (:issue:`59143`)
 - Enforced deprecation of strings ``T``, ``L``, ``U``, and ``N`` denoting frequencies in :class:`Minute`, :class:`Milli`, :class:`Micro`, :class:`Nano` (:issue:`57627`)
 - Enforced deprecation of strings ``T``, ``L``, ``U``, and ``N`` denoting units in :class:`Timedelta` (:issue:`57627`)
 - Enforced deprecation of the behavior of :func:`concat` when ``len(keys) != len(objs)`` would truncate to the shorter of the two. Now this raises a ``ValueError`` (:issue:`43485`)
@@ -503,6 +507,7 @@ Datetimelike
 - Bug in :meth:`DatetimeIndex.is_year_start` and :meth:`DatetimeIndex.is_quarter_start` does not raise on Custom business days frequencies bigger then "1C" (:issue:`58664`)
 - Bug in :meth:`DatetimeIndex.is_year_start` and :meth:`DatetimeIndex.is_quarter_start` returning ``False`` on double-digit frequencies (:issue:`58523`)
 - Bug in :meth:`DatetimeIndex.union` when ``unit`` was non-nanosecond (:issue:`59036`)
+- Bug in :meth:`Series.dt.microsecond` producing incorrect results for pyarrow backed :class:`Series`. (:issue:`59154`)
 - Bug in setting scalar values with mismatched resolution into arrays with non-nanosecond ``datetime64``, ``timedelta64`` or :class:`DatetimeTZDtype` incorrectly truncating those scalars (:issue:`56410`)
 
 Timedelta
@@ -552,11 +557,13 @@ MultiIndex
 - :func:`DataFrame.loc` with ``axis=0``  and :class:`MultiIndex` when setting a value adds extra columns (:issue:`58116`)
 - :meth:`DataFrame.melt` would not accept multiple names in ``var_name`` when the columns were a :class:`MultiIndex` (:issue:`58033`)
 - :meth:`MultiIndex.insert` would not insert NA value correctly at unified location of index -1 (:issue:`59003`)
+- :func:`MultiIndex.get_level_values` accessing a :class:`DatetimeIndex` does not carry the frequency attribute along (:issue:`58327`, :issue:`57949`)
 -
 
 I/O
 ^^^
 - Bug in :class:`DataFrame` and :class:`Series` ``repr`` of :py:class:`collections.abc.Mapping`` elements. (:issue:`57915`)
+- Bug in :meth:`.DataFrame.to_json` when ``"index"`` was a value in the :attr:`DataFrame.column` and :attr:`Index.name` was ``None``. Now, this will fail with a ``ValueError`` (:issue:`58925`)
 - Bug in :meth:`DataFrame.to_dict` raises unnecessary ``UserWarning`` when columns are not unique and ``orient='tight'``. (:issue:`58281`)
 - Bug in :meth:`DataFrame.to_excel` when writing empty :class:`DataFrame` with :class:`MultiIndex` on both axes (:issue:`57696`)
 - Bug in :meth:`DataFrame.to_stata` when writing :class:`DataFrame` and ``byteorder=`big```. (:issue:`58969`)

pandas/_libs/tslibs/dtypes.pxd

@@ -14,12 +14,12 @@ cdef bint is_supported_unit(NPY_DATETIMEUNIT reso)
 cdef dict c_OFFSET_TO_PERIOD_FREQSTR
 cdef dict c_PERIOD_TO_OFFSET_FREQSTR
 cdef dict c_OFFSET_RENAMED_FREQSTR
-cdef dict c_DEPR_ABBREVS
 cdef dict c_DEPR_UNITS
 cdef dict c_PERIOD_AND_OFFSET_DEPR_FREQSTR
 cdef dict attrname_to_abbrevs
 cdef dict npy_unit_to_attrname
 cdef dict attrname_to_npy_unit
+cdef str INVALID_FREQ_ERR_MSG
 
 cdef enum c_FreqGroup:
     # Mirrors FreqGroup in the .pyx file

pandas/_libs/tslibs/dtypes.pyx

@@ -1,9 +1,6 @@
 # period frequency constants corresponding to scikits timeseries
 # originals
 from enum import Enum
-import warnings
-
-from pandas.util._exceptions import find_stack_level
 
 from pandas._libs.tslibs.ccalendar cimport c_MONTH_NUMBERS
 from pandas._libs.tslibs.np_datetime cimport (
@@ -338,14 +335,6 @@ PERIOD_TO_OFFSET_FREQSTR = {
 cdef dict c_OFFSET_TO_PERIOD_FREQSTR = OFFSET_TO_PERIOD_FREQSTR
 cdef dict c_PERIOD_TO_OFFSET_FREQSTR = PERIOD_TO_OFFSET_FREQSTR
 
-# Map deprecated resolution abbreviations to correct resolution abbreviations
-cdef dict c_DEPR_ABBREVS = {
-    "H": "h",
-    "BH": "bh",
-    "CBH": "cbh",
-    "S": "s",
-}
-
 cdef dict c_DEPR_UNITS = {
     "w": "W",
     "d": "D",
@@ -372,6 +361,8 @@ cdef dict c_PERIOD_AND_OFFSET_DEPR_FREQSTR = {
     "MIN": "min",
 }
 
+cdef str INVALID_FREQ_ERR_MSG = "Invalid frequency: {0}"
+
 
 class FreqGroup(Enum):
     # Mirrors c_FreqGroup in the .pxd file
@@ -461,39 +452,18 @@ class Resolution(Enum):
         >>> Resolution.get_reso_from_freqstr('h') == Resolution.RESO_HR
         True
         """
-        cdef:
-            str abbrev
         try:
-            if freq in c_DEPR_ABBREVS:
-                abbrev = c_DEPR_ABBREVS[freq]
-                warnings.warn(
-                    f"\'{freq}\' is deprecated and will be removed in a future "
-                    f"version. Please use \'{abbrev}\' "
-                    f"instead of \'{freq}\'.",
-                    FutureWarning,
-                    stacklevel=find_stack_level(),
-                )
-                freq = abbrev
             attr_name = _abbrev_to_attrnames[freq]
-        except KeyError:
+        except KeyError as exc:
+            msg = INVALID_FREQ_ERR_MSG.format(freq)
             # For quarterly and yearly resolutions, we need to chop off
             #  a month string.
             split_freq = freq.split("-")
             if len(split_freq) != 2:
-                raise
+                raise ValueError(msg) from exc
             if split_freq[1] not in _month_names:
                 # i.e. we want e.g. "Q-DEC", not "Q-INVALID"
-                raise
-            if split_freq[0] in c_DEPR_ABBREVS:
-                abbrev = c_DEPR_ABBREVS[split_freq[0]]
-                warnings.warn(
-                    f"\'{split_freq[0]}\' is deprecated and will be removed in a "
-                    f"future version. Please use \'{abbrev}\' "
-                    f"instead of \'{split_freq[0]}\'.",
-                    FutureWarning,
-                    stacklevel=find_stack_level(),
-                )
-                split_freq[0] = abbrev
+                raise ValueError(msg) from exc
             attr_name = _abbrev_to_attrnames[split_freq[0]]
 
         return cls.from_attrname(attr_name)