Merge remote-tracking branch 'upstream/main' into read-csv-from-directory

fangchenli · fangchenli · commit 19a747127fcd · 2025-07-16T13:59:14.000-07:00
diff --git a/.github/workflows/docbuild-and-upload.yml b/.github/workflows/docbuild-and-upload.yml
@@ -57,6 +57,8 @@ jobs:
       run: python web/pandas_web.py web/pandas --target-path=web/build
 
     - name: Build documentation
+      # TEMP don't let errors fail the build until all string dtype changes are fixed
+      continue-on-error: true
       run: doc/make.py --warnings-are-errors
 
     - name: Build the interactive terminal
diff --git a/.github/workflows/unit-tests.yml b/.github/workflows/unit-tests.yml
@@ -30,7 +30,7 @@ jobs:
         env_file: [actions-310.yaml, actions-311.yaml, actions-312.yaml, actions-313.yaml]
         # Prevent the include jobs from overriding other jobs
         pattern: [""]
-        pandas_future_infer_string: ["0"]
+        pandas_future_infer_string: ["1"]
         include:
           - name: "Downstream Compat"
             env_file: actions-311-downstream_compat.yaml
@@ -45,6 +45,10 @@ jobs:
             env_file: actions-313-freethreading.yaml
             pattern: "not slow and not network and not single_cpu"
             platform: ubuntu-24.04
+          - name: "Without PyArrow"
+            env_file: actions-312.yaml
+            pattern: "not slow and not network and not single_cpu"
+            platform: ubuntu-24.04
           - name: "Locale: it_IT"
             env_file: actions-311.yaml
             pattern: "not slow and not network and not single_cpu"
@@ -67,13 +71,9 @@ jobs:
             # It will be temporarily activated during tests with locale.setlocale
             extra_loc: "zh_CN"
             platform: ubuntu-24.04
-          - name: "Future infer strings"
+          - name: "Past no infer strings"
             env_file: actions-312.yaml
-            pandas_future_infer_string: "1"
-            platform: ubuntu-24.04
-          - name: "Future infer strings (without pyarrow)"
-            env_file: actions-311.yaml
-            pandas_future_infer_string: "1"
+            pandas_future_infer_string: "0"
             platform: ubuntu-24.04
           - name: "Numpy Dev"
             env_file: actions-311-numpydev.yaml
@@ -83,7 +83,6 @@ jobs:
           - name: "Pyarrow Nightly"
             env_file: actions-311-pyarrownightly.yaml
             pattern: "not slow and not network and not single_cpu"
-            pandas_future_infer_string: "1"
             platform: ubuntu-24.04
       fail-fast: false
     name: ${{ matrix.name || format('{0} {1}', matrix.platform, matrix.env_file) }}
@@ -98,7 +97,7 @@ jobs:
       PYTEST_TARGET: ${{ matrix.pytest_target || 'pandas' }}
       # Clipboard tests
       QT_QPA_PLATFORM: offscreen
-      REMOVE_PYARROW: ${{ matrix.name == 'Future infer strings (without pyarrow)' && '1' || '0' }}
+      REMOVE_PYARROW: ${{ matrix.name == 'Without PyArrow' && '1' || '0' }}
     concurrency:
       # https://github.community/t/concurrecy-not-work-for-push/183068/7
       group: ${{ github.event_name == 'push' && github.run_number || github.ref }}-${{ matrix.env_file }}-${{ matrix.pattern }}-${{ matrix.extra_apt || '' }}-${{ matrix.pandas_future_infer_string }}-${{ matrix.platform }}
diff --git a/README.md b/README.md
@@ -175,7 +175,7 @@ All contributions, bug reports, bug fixes, documentation improvements, enhanceme
 
 A detailed overview on how to contribute can be found in the **[contributing guide](https://pandas.pydata.org/docs/dev/development/contributing.html)**.
 
-If you are simply looking to start working with the pandas codebase, navigate to the [GitHub "issues" tab](https://github.com/pandas-dev/pandas/issues) and start looking through interesting issues. There are a number of issues listed under [Docs](https://github.com/pandas-dev/pandas/issues?labels=Docs&sort=updated&state=open) and [good first issue](https://github.com/pandas-dev/pandas/issues?labels=good+first+issue&sort=updated&state=open) where you could start out.
+If you are simply looking to start working with the pandas codebase, navigate to the [GitHub "issues" tab](https://github.com/pandas-dev/pandas/issues) and start looking through interesting issues. There are a number of issues listed under [Docs](https://github.com/pandas-dev/pandas/issues?q=is%3Aissue%20state%3Aopen%20label%3ADocs%20sort%3Aupdated-desc) and [good first issue](https://github.com/pandas-dev/pandas/issues?q=is%3Aissue%20state%3Aopen%20label%3A%22good%20first%20issue%22%20sort%3Aupdated-desc) where you could start out.
 
 You can also triage issues which may include reproducing bug reports, or asking for vital information such as version numbers or reproduction instructions. If you would like to start triaging issues, one easy way to get started is to [subscribe to pandas on CodeTriage](https://www.codetriage.com/pandas-dev/pandas).
 
diff --git a/ci/code_checks.sh b/ci/code_checks.sh
@@ -58,7 +58,9 @@ if [[ -z "$CHECK" || "$CHECK" == "doctests" ]]; then
 
     MSG='Python and Cython Doctests' ; echo "$MSG"
     python -c 'import pandas as pd; pd.test(run_doctests=True)'
-    RET=$(($RET + $?)) ; echo "$MSG" "DONE"
+    # TEMP don't let doctests fail the build until all string dtype changes are fixed
+    # RET=$(($RET + $?)) ; echo "$MSG" "DONE"
+    echo "$MSG" "DONE"
 
 fi
 
diff --git a/doc/source/whatsnew/v0.4.x.rst b/doc/source/whatsnew/v0.4.x.rst
@@ -11,7 +11,7 @@ New features
 - Added Python 3 support using 2to3 (:issue:`200`)
 - :ref:`Added <dsintro.name_attribute>` ``name`` attribute to ``Series``, now
   prints as part of ``Series.__repr__``
-- :meth:`Series.isnull`` and :meth:`Series.notnull` (:issue:`209`, :issue:`203`)
+- :meth:`Series.isnull` and :meth:`Series.notnull` (:issue:`209`, :issue:`203`)
 - :ref:`Added <basics.align>` ``Series.align`` method for aligning two series
   with choice of join method (ENH56_)
 - :ref:`Added <advanced.get_level_values>` method ``get_level_values`` to
diff --git a/doc/source/whatsnew/v2.0.0.rst b/doc/source/whatsnew/v2.0.0.rst
@@ -984,7 +984,7 @@ Removal of prior version deprecations/changes
 - Removed :meth:`Series.str.__iter__` (:issue:`28277`)
 - Removed ``pandas.SparseArray`` in favor of :class:`arrays.SparseArray` (:issue:`30642`)
 - Removed ``pandas.SparseSeries`` and ``pandas.SparseDataFrame``, including pickle support. (:issue:`30642`)
-- Enforced disallowing passing an integer ``fill_value`` to :meth:`DataFrame.shift` and :meth:`Series.shift`` with datetime64, timedelta64, or period dtypes (:issue:`32591`)
+- Enforced disallowing passing an integer ``fill_value`` to :meth:`DataFrame.shift` and :meth:`Series.shift` with datetime64, timedelta64, or period dtypes (:issue:`32591`)
 - Enforced disallowing a string column label into ``times`` in :meth:`DataFrame.ewm` (:issue:`43265`)
 - Enforced disallowing passing ``True`` and ``False`` into ``inclusive`` in :meth:`Series.between` in favor of ``"both"`` and ``"neither"`` respectively (:issue:`40628`)
 - Enforced disallowing using ``usecols`` with out of bounds indices for ``read_csv`` with ``engine="c"`` (:issue:`25623`)
@@ -1045,7 +1045,7 @@ Removal of prior version deprecations/changes
 - Enforced deprecation of silently dropping columns that raised a ``TypeError`` in :class:`Series.transform` and :class:`DataFrame.transform` when used with a list or dictionary (:issue:`43740`)
 - Changed behavior of :meth:`DataFrame.apply` with list-like so that any partial failure will raise an error (:issue:`43740`)
 - Changed behaviour of :meth:`DataFrame.to_latex` to now use the Styler implementation via :meth:`.Styler.to_latex` (:issue:`47970`)
-- Changed behavior of :meth:`Series.__setitem__` with an integer key and a :class:`Float64Index` when the key is not present in the index; previously we treated the key as positional (behaving like ``series.iloc[key] = val``), now we treat it is a label (behaving like ``series.loc[key] = val``), consistent with :meth:`Series.__getitem__`` behavior (:issue:`33469`)
+- Changed behavior of :meth:`Series.__setitem__` with an integer key and a :class:`Float64Index` when the key is not present in the index; previously we treated the key as positional (behaving like ``series.iloc[key] = val``), now we treat it is a label (behaving like ``series.loc[key] = val``), consistent with :meth:`Series.__getitem__` behavior (:issue:`33469`)
 - Removed ``na_sentinel`` argument from :func:`factorize`, :meth:`.Index.factorize`, and :meth:`.ExtensionArray.factorize` (:issue:`47157`)
 - Changed behavior of :meth:`Series.diff` and :meth:`DataFrame.diff` with :class:`ExtensionDtype` dtypes whose arrays do not implement ``diff``, these now raise ``TypeError`` rather than casting to numpy (:issue:`31025`)
 - Enforced deprecation of calling numpy "ufunc"s on :class:`DataFrame` with ``method="outer"``; this now raises ``NotImplementedError`` (:issue:`36955`)
diff --git a/doc/source/whatsnew/v2.0.3.rst b/doc/source/whatsnew/v2.0.3.rst
@@ -13,7 +13,7 @@ including other versions of pandas.
 
 Fixed regressions
 ~~~~~~~~~~~~~~~~~
-- Bug in :meth:`Timestamp.weekday`` was returning incorrect results before ``'0000-02-29'`` (:issue:`53738`)
+- Bug in :meth:`Timestamp.weekday` was returning incorrect results before ``'0000-02-29'`` (:issue:`53738`)
 - Fixed performance regression in merging on datetime-like columns (:issue:`53231`)
 - Fixed regression when :meth:`DataFrame.to_string` creates extra space for string dtypes (:issue:`52690`)
 
diff --git a/doc/source/whatsnew/v2.1.0.rst b/doc/source/whatsnew/v2.1.0.rst
@@ -721,7 +721,7 @@ Conversion
 Strings
 ^^^^^^^
 - Bug in :meth:`Series.str` that did not raise a  ``TypeError`` when iterated (:issue:`54173`)
-- Bug in ``repr`` for :class:`DataFrame`` with string-dtype columns (:issue:`54797`)
+- Bug in ``repr`` for :class:`DataFrame` with string-dtype columns (:issue:`54797`)
 
 Interval
 ^^^^^^^^
diff --git a/doc/source/whatsnew/v3.0.0.rst b/doc/source/whatsnew/v3.0.0.rst
@@ -414,6 +414,7 @@ Other API changes
 - Index set operations (like union or intersection) will now ignore the dtype of
   an empty ``RangeIndex`` or empty ``Index`` with object dtype when determining
   the dtype of the resulting Index (:issue:`60797`)
+- Numpy functions like ``np.isinf`` that return a bool dtype when called on a :class:`Index` object now return a bool-dtype :class:`Index` instead of ``np.ndarray`` (:issue:`52676`)
 
 .. ---------------------------------------------------------------------------
 .. _whatsnew_300.deprecations:
@@ -784,7 +785,7 @@ MultiIndex
 
 I/O
 ^^^
-- Bug in :class:`DataFrame` and :class:`Series` ``repr`` of :py:class:`collections.abc.Mapping`` elements. (:issue:`57915`)
+- Bug in :class:`DataFrame` and :class:`Series` ``repr`` of :py:class:`collections.abc.Mapping` elements. (:issue:`57915`)
 - Bug in :meth:`.DataFrame.to_json` when ``"index"`` was a value in the :attr:`DataFrame.column` and :attr:`Index.name` was ``None``. Now, this will fail with a ``ValueError`` (:issue:`58925`)
 - Bug in :meth:`.io.common.is_fsspec_url` not recognizing chained fsspec URLs (:issue:`48978`)
 - Bug in :meth:`DataFrame._repr_html_` which ignored the ``"display.float_format"`` option (:issue:`59876`)
@@ -869,6 +870,7 @@ Reshaping
 - Bug in :meth:`DataFrame.merge` when merging two :class:`DataFrame` on ``intc`` or ``uintc`` types on Windows (:issue:`60091`, :issue:`58713`)
 - Bug in :meth:`DataFrame.pivot_table` incorrectly subaggregating results when called without an ``index`` argument (:issue:`58722`)
 - Bug in :meth:`DataFrame.pivot_table` incorrectly ignoring the ``values`` argument when also supplied to the ``index`` or ``columns`` parameters (:issue:`57876`, :issue:`61292`)
+- Bug in :meth:`DataFrame.pivot_table` where ``margins=True`` did not correctly include groups with ``NaN`` values in the index or columns when ``dropna=False`` was explicitly passed. (:issue:`61509`)
 - Bug in :meth:`DataFrame.stack` with the new implementation where ``ValueError`` is raised when ``level=[]`` (:issue:`60740`)
 - Bug in :meth:`DataFrame.unstack` producing incorrect results when manipulating empty :class:`DataFrame` with an :class:`ExtentionDtype` (:issue:`59123`)
 - Bug in :meth:`concat` where concatenating DataFrame and Series with ``ignore_index = True`` drops the series name (:issue:`60723`, :issue:`56257`)
diff --git a/pandas/core/config_init.py b/pandas/core/config_init.py
@@ -880,7 +880,7 @@ def register_converter_cb(key: str) -> None:
 with cf.config_prefix("future"):
     cf.register_option(
         "infer_string",
-        True if os.environ.get("PANDAS_FUTURE_INFER_STRING", "0") == "1" else False,
+        False if os.environ.get("PANDAS_FUTURE_INFER_STRING", "1") == "0" else True,
         "Whether to infer sequence of str objects as pyarrow string "
         "dtype, which will be the default in pandas 3.0 "
         "(at which point this option will be deprecated).",
diff --git a/pandas/core/indexes/base.py b/pandas/core/indexes/base.py
@@ -965,12 +965,8 @@ def __array_wrap__(self, result, context=None, return_scalar=False):
         Gets called after a ufunc and other functions e.g. np.split.
         """
         result = lib.item_from_zerodim(result)
-        if (not isinstance(result, Index) and is_bool_dtype(result.dtype)) or np.ndim(
-            result
-        ) > 1:
-            # exclude Index to avoid warning from is_bool_dtype deprecation;
-            #  in the Index case it doesn't matter which path we go down.
-            # reached in plotting tests with e.g. np.nonzero(index)
+        if np.ndim(result) > 1:
+            # Reached in plotting tests with e.g. np.nonzero(index)
             return result
 
         return Index(result, name=self.name)
diff --git a/pandas/core/reshape/encoding.py b/pandas/core/reshape/encoding.py
@@ -390,7 +390,9 @@ def from_dummies(
         The default category is the implied category when a value has none of the
         listed categories specified with a one, i.e. if all dummies in a row are
         zero. Can be a single value for all variables or a dict directly mapping
-        the default categories to a prefix of a variable.
+        the default categories to a prefix of a variable. The default category
+        will be coerced to the dtype of ``data.columns`` if such coercion is
+        lossless, and will raise otherwise.
 
     Returns
     -------
diff --git a/pandas/core/reshape/pivot.py b/pandas/core/reshape/pivot.py
@@ -396,6 +396,7 @@ def __internal_pivot_table(
             observed=dropna,
             margins_name=margins_name,
             fill_value=fill_value,
+            dropna=dropna,
         )
 
     # discard the top level
@@ -422,6 +423,7 @@ def _add_margins(
     observed: bool,
     margins_name: Hashable = "All",
     fill_value=None,
+    dropna: bool = True,
 ):
     if not isinstance(margins_name, str):
         raise ValueError("margins_name argument must be a string")
@@ -461,6 +463,7 @@ def _add_margins(
             kwargs,
             observed,
             margins_name,
+            dropna,
         )
         if not isinstance(marginal_result_set, tuple):
             return marginal_result_set
@@ -469,7 +472,7 @@ def _add_margins(
         # no values, and table is a DataFrame
         assert isinstance(table, ABCDataFrame)
         marginal_result_set = _generate_marginal_results_without_values(
-            table, data, rows, cols, aggfunc, kwargs, observed, margins_name
+            table, data, rows, cols, aggfunc, kwargs, observed, margins_name, dropna
         )
         if not isinstance(marginal_result_set, tuple):
             return marginal_result_set
@@ -538,6 +541,7 @@ def _generate_marginal_results(
     kwargs,
     observed: bool,
     margins_name: Hashable = "All",
+    dropna: bool = True,
 ):
     margin_keys: list | Index
     if len(cols) > 0:
@@ -551,7 +555,7 @@ def _all_key(key):
         if len(rows) > 0:
             margin = (
                 data[rows + values]
-                .groupby(rows, observed=observed)
+                .groupby(rows, observed=observed, dropna=dropna)
                 .agg(aggfunc, **kwargs)
             )
             cat_axis = 1
@@ -567,7 +571,7 @@ def _all_key(key):
         else:
             margin = (
                 data[cols[:1] + values]
-                .groupby(cols[:1], observed=observed)
+                .groupby(cols[:1], observed=observed, dropna=dropna)
                 .agg(aggfunc, **kwargs)
                 .T
             )
@@ -610,7 +614,9 @@ def _all_key(key):
 
     if len(cols) > 0:
         row_margin = (
-            data[cols + values].groupby(cols, observed=observed).agg(aggfunc, **kwargs)
+            data[cols + values]
+            .groupby(cols, observed=observed, dropna=dropna)
+            .agg(aggfunc, **kwargs)
         )
         row_margin = row_margin.stack()
 
@@ -633,6 +639,7 @@ def _generate_marginal_results_without_values(
     kwargs,
     observed: bool,
     margins_name: Hashable = "All",
+    dropna: bool = True,
 ):
     margin_keys: list | Index
     if len(cols) > 0:
@@ -645,7 +652,7 @@ def _all_key():
             return (margins_name,) + ("",) * (len(cols) - 1)
 
         if len(rows) > 0:
-            margin = data.groupby(rows, observed=observed)[rows].apply(
+            margin = data.groupby(rows, observed=observed, dropna=dropna)[rows].apply(
                 aggfunc, **kwargs
             )
             all_key = _all_key()
@@ -654,7 +661,9 @@ def _all_key():
             margin_keys.append(all_key)
 
         else:
-            margin = data.groupby(level=0, observed=observed).apply(aggfunc, **kwargs)
+            margin = data.groupby(level=0, observed=observed, dropna=dropna).apply(
+                aggfunc, **kwargs
+            )
             all_key = _all_key()
             table[all_key] = margin
             result = table
@@ -665,7 +674,7 @@ def _all_key():
         margin_keys = table.columns
 
     if len(cols):
-        row_margin = data.groupby(cols, observed=observed)[cols].apply(
+        row_margin = data.groupby(cols, observed=observed, dropna=dropna)[cols].apply(
             aggfunc, **kwargs
         )
     else:
diff --git a/pandas/core/series.py b/pandas/core/series.py
@@ -263,7 +263,6 @@ class Series(base.IndexOpsMixin, NDFrame):  # type: ignore[misc]
         Data type for the output Series. If not specified, this will be
         inferred from `data`.
         See the :ref:`user guide <basics.dtypes>` for more usages.
-        If ``data`` is Series then is ignored.
     name : Hashable, default None
         The name to give to the Series.
     copy : bool, default False
diff --git a/pandas/core/tools/numeric.py b/pandas/core/tools/numeric.py
@@ -114,6 +114,14 @@ def to_numeric(
         Numeric if parsing succeeded.
         Return type depends on input.  Series if Series, otherwise ndarray.
 
+    Raises
+    ------
+    ValueError
+        If the input contains non-numeric values and `errors='raise'`.
+    TypeError
+        If the input is not list-like, 1D, or scalar convertible to numeric,
+        such as nested lists or unsupported input types (e.g., dict).
+
     See Also
     --------
     DataFrame.astype : Cast argument to a specified dtype.
diff --git a/pandas/errors/__init__.py b/pandas/errors/__init__.py
@@ -379,7 +379,7 @@ def __init__(self, class_instance, methodtype: str = "method") -> None:
         types = {"method", "classmethod", "staticmethod", "property"}
         if methodtype not in types:
             raise ValueError(
-                f"methodtype must be one of {methodtype}, got {types} instead."
+                f"methodtype must be one of {types}, got {methodtype} instead."
             )
         self.methodtype = methodtype
         self.class_instance = class_instance
diff --git a/pandas/tests/indexes/test_numpy_compat.py b/pandas/tests/indexes/test_numpy_compat.py
@@ -2,6 +2,7 @@
 import pytest
 
 from pandas import (
+    BooleanDtype,
     CategoricalIndex,
     DatetimeIndex,
     Index,
@@ -14,7 +15,6 @@
     is_complex_dtype,
     is_numeric_dtype,
 )
-from pandas.core.arrays import BooleanArray
 from pandas.core.indexes.datetimelike import DatetimeIndexOpsMixin
 
 
@@ -111,11 +111,10 @@ def test_numpy_ufuncs_other(index, func):
         if func in (np.isfinite, np.isinf, np.isnan):
             # numpy 1.18 changed isinf and isnan to not raise on dt64/td64
             result = func(index)
-            assert isinstance(result, np.ndarray)
 
             out = np.empty(index.shape, dtype=bool)
             func(index, out=out)
-            tm.assert_numpy_array_equal(out, result)
+            tm.assert_index_equal(Index(out), result)
         else:
             with tm.external_error_raised(TypeError):
                 func(index)
@@ -129,19 +128,20 @@ def test_numpy_ufuncs_other(index, func):
     ):
         # Results in bool array
         result = func(index)
+        assert isinstance(result, Index)
         if not isinstance(index.dtype, np.dtype):
             # e.g. Int64 we expect to get BooleanArray back
-            assert isinstance(result, BooleanArray)
+            assert isinstance(result.dtype, BooleanDtype)
         else:
-            assert isinstance(result, np.ndarray)
+            assert isinstance(result.dtype, np.dtype)
 
         out = np.empty(index.shape, dtype=bool)
         func(index, out=out)
 
         if not isinstance(index.dtype, np.dtype):
-            tm.assert_numpy_array_equal(out, result._data)
+            tm.assert_index_equal(result, Index(out, dtype="boolean"))
         else:
-            tm.assert_numpy_array_equal(out, result)
+            tm.assert_index_equal(result, Index(out))
 
     elif len(index) == 0:
         pass
diff --git a/pandas/tests/reshape/test_crosstab.py b/pandas/tests/reshape/test_crosstab.py
diff --git a/pandas/tests/reshape/test_from_dummies.py b/pandas/tests/reshape/test_from_dummies.py
diff --git a/pandas/tests/reshape/test_pivot.py b/pandas/tests/reshape/test_pivot.py
