Merge branch 'main' into deps/pytz/optional

Author: mroeschke

.gitattributes

@@ -68,7 +68,7 @@ ci export-ignore
 doc export-ignore
 gitpod export-ignore
 MANIFEST.in export-ignore
-scripts export-ignore
+scripts/** export-ignore
 typings export-ignore
 web export-ignore
 CITATION.cff export-ignore
@@ -82,3 +82,7 @@ setup.py export-ignore
 # csv_dir_path fixture checks the existence of the directory
 # exclude the whole directory to avoid running related tests in sdist
 pandas/tests/io/parser/data export-ignore
+
+# Include cibw script in sdist since it's needed for building wheels
+scripts/cibw_before_build.sh -export-ignore
+scripts/cibw_before_test.sh -export-ignore

.github/workflows/wheels.yml

@@ -99,14 +99,25 @@ jobs:
         - [macos-14, macosx_arm64]
         - [windows-2022, win_amd64]
         # TODO: support PyPy?
-        python: [["cp310", "3.10"], ["cp311", "3.11"], ["cp312", "3.12"]]
-
+        python: [["cp310", "3.10"], ["cp311", "3.11"], ["cp312", "3.12"], ["cp313", "3.13"], ["cp313t", "3.13"]]
+        include:
+        # TODO: Remove this plus installing build deps in cibw_before_build.sh
+        # and test deps in cibw_before_test.sh after pandas can be built with a released NumPy/Cython
+        - python: ["cp313", "3.13"]
+          cibw_build_frontend: 'pip; args: --no-build-isolation'
+        - python: ["cp313t", "3.13"]
+          cibw_build_frontend: 'pip; args: --no-build-isolation'
         # Build Pyodide wheels and upload them to Anaconda.org
         # NOTE: this job is similar to the one in unit-tests.yml except for the fact
         # that it uses cibuildwheel instead of a standard Pyodide xbuildenv setup.
-        include:
-          - buildplat: [ubuntu-22.04, pyodide_wasm32]
-            python: ["cp312", "3.12"]
+        - buildplat: [ubuntu-22.04, pyodide_wasm32]
+          python: ["cp312", "3.12"]
+          cibw_build_frontend: 'build'
+        # TODO: Build free-threaded wheels for Windows
+        exclude:
+        - buildplat: [windows-2022, win_amd64]
+          python: ["cp313t", "3.13"]
+
     env:
       IS_PUSH: ${{ github.event_name == 'push' && startsWith(github.ref, 'refs/tags/v') }}
       IS_SCHEDULE_DISPATCH: ${{ github.event_name == 'schedule' || github.event_name == 'workflow_dispatch' }}
@@ -153,6 +164,7 @@ jobs:
         env:
           CIBW_PRERELEASE_PYTHONS: True
           CIBW_BUILD: ${{ matrix.python[0] }}-${{ matrix.buildplat[1] }}
+          CIBW_BUILD_FRONTEND: ${{ matrix.cibw_build_frontend || 'pip' }}
           CIBW_PLATFORM: ${{ matrix.buildplat[1] == 'pyodide_wasm32' && 'pyodide' || 'auto' }}
 
       - name: Set up Python
@@ -176,15 +188,17 @@ jobs:
       - name: Test Windows Wheels
         if: ${{ matrix.buildplat[1] == 'win_amd64' }}
         shell: pwsh
+        # TODO: Remove NumPy nightly install when there's a 3.13 wheel on PyPI
         run: |
           $TST_CMD = @"
           python -m pip install hypothesis>=6.84.0 pytest>=7.3.2 pytest-xdist>=3.4.0;
+          ${{ matrix.python[1] == '3.13' && 'python -m pip install -i https://pypi.anaconda.org/scientific-python-nightly-wheels/simple numpy;' }}
           python -m pip install `$(Get-Item pandas\wheelhouse\*.whl);
           python -c `'import pandas as pd; pd.test(extra_args=[`\"--no-strict-data-files`\", `\"-m not clipboard and not single_cpu and not slow and not network and not db`\"])`';
           "@
           # add rc to the end of the image name if the Python version is unreleased
-          docker pull python:${{ matrix.python[1] == '3.12' && '3.12-rc' || format('{0}-windowsservercore', matrix.python[1]) }}
-          docker run --env PANDAS_CI='1' -v ${PWD}:C:\pandas python:${{ matrix.python[1] == '3.12' && '3.12-rc' || format('{0}-windowsservercore', matrix.python[1]) }} powershell -Command $TST_CMD
+          docker pull python:${{ matrix.python[1] == '3.13' && '3.13-rc' || format('{0}-windowsservercore', matrix.python[1]) }}
+          docker run --env PANDAS_CI='1' -v ${PWD}:C:\pandas python:${{ matrix.python[1] == '3.13' && '3.13-rc' || format('{0}-windowsservercore', matrix.python[1]) }} powershell -Command $TST_CMD
 
       - uses: actions/upload-artifact@v4
         with:

MANIFEST.in

@@ -62,3 +62,7 @@ prune pandas/tests/io/parser/data
 # Selectively re-add *.cxx files that were excluded above
 graft pandas/_libs/src
 graft pandas/_libs/include
+
+# Include cibw script in sdist since it's needed for building wheels
+include scripts/cibw_before_build.sh
+include scripts/cibw_before_test.sh

ci/code_checks.sh

@@ -304,7 +304,6 @@ if [[ -z "$CHECK" || "$CHECK" == "docstrings" ]]; then
         -i "pandas.api.types.is_re PR07,SA01" \
         -i "pandas.api.types.is_re_compilable PR07,SA01" \
         -i "pandas.api.types.is_sparse SA01" \
-        -i "pandas.api.types.is_string_dtype SA01" \
         -i "pandas.api.types.is_timedelta64_ns_dtype SA01" \
         -i "pandas.api.types.pandas_dtype PR07,RT03,SA01" \
         -i "pandas.api.types.union_categoricals RT03,SA01" \

doc/source/getting_started/comparison/comparison_with_sql.rst

@@ -505,7 +505,7 @@ DELETE
     DELETE FROM tips
     WHERE tip > 9;
 
-In pandas we select the rows that should remain instead of deleting them:
+In pandas we select the rows that should remain instead of deleting the rows that should be removed:
 
 .. ipython:: python
 

doc/source/user_guide/pyarrow.rst

@@ -159,9 +159,11 @@ PyArrow also provides IO reading functionality that has been integrated into sev
 functions provide an ``engine`` keyword that can dispatch to PyArrow to accelerate reading from an IO source.
 
 * :func:`read_csv`
+* :func:`read_feather`
 * :func:`read_json`
 * :func:`read_orc`
-* :func:`read_feather`
+* :func:`read_parquet`
+* :func:`read_table` (experimental)
 
 .. ipython:: python
 

doc/source/whatsnew/v3.0.0.rst

@@ -33,6 +33,7 @@ Other enhancements
 - :func:`DataFrame.to_excel` now raises an ``UserWarning`` when the character count in a cell exceeds Excel's limitation of 32767 characters (:issue:`56954`)
 - :func:`read_stata` now returns ``datetime64`` resolutions better matching those natively stored in the stata format (:issue:`55642`)
 - :meth:`DataFrame.agg` called with ``axis=1`` and a ``func`` which relabels the result index now raises a ``NotImplementedError`` (:issue:`58807`).
+- :meth:`Index.get_loc` now accepts also subclasses of ``tuple`` as keys (:issue:`57922`)
 - :meth:`Styler.set_tooltips` provides alternative method to storing tooltips by using title attribute of td elements. (:issue:`56981`)
 - Allow dictionaries to be passed to :meth:`pandas.Series.str.replace` via ``pat`` parameter (:issue:`51748`)
 - Support passing a :class:`Series` input to :func:`json_normalize` that retains the :class:`Series` :class:`Index` (:issue:`51452`)
@@ -49,6 +50,7 @@ Other enhancements
 - :meth:`Series.cummin` and :meth:`Series.cummax` now supports :class:`CategoricalDtype` (:issue:`52335`)
 - :meth:`Series.plot` now correctly handle the ``ylabel`` parameter for pie charts, allowing for explicit control over the y-axis label (:issue:`58239`)
 - Restore support for reading Stata 104-format and enable reading 103-format dta files (:issue:`58554`)
+- Support reading Stata 102-format (Stata 1) dta files (:issue:`58978`)
 - Support reading Stata 110-format (Stata 7) dta files (:issue:`47176`)
 
 .. ---------------------------------------------------------------------------
@@ -264,8 +266,10 @@ Other API changes
 ^^^^^^^^^^^^^^^^^
 - 3rd party ``py.path`` objects are no longer explicitly supported in IO methods. Use :py:class:`pathlib.Path` objects instead (:issue:`57091`)
 - :func:`read_table`'s ``parse_dates`` argument defaults to ``None`` to improve consistency with :func:`read_csv` (:issue:`57476`)
+- All classes inheriting from builtin ``tuple`` (including types created with :func:`collections.namedtuple`) are now hashed and compared as builtin ``tuple`` during indexing operations (:issue:`57922`)
 - Made ``dtype`` a required argument in :meth:`ExtensionArray._from_sequence_of_strings` (:issue:`56519`)
 - Passing a :class:`Series` input to :func:`json_normalize` will now retain the :class:`Series` :class:`Index`, previously output had a new :class:`RangeIndex` (:issue:`51452`)
+- Removed :meth:`Index.sort` which always raised a ``TypeError``. This attribute is not defined and will raise an ``AttributeError`` (:issue:`59283`)
 - Updated :meth:`DataFrame.to_excel` so that the output spreadsheet has no styling. Custom styling can still be done using :meth:`Styler.to_excel` (:issue:`54154`)
 - pickle and HDF (``.h5``) files created with Python 2 are no longer explicitly supported (:issue:`57387`)
 - pickled objects from pandas version less than ``1.0.0`` are no longer supported (:issue:`57155`)
@@ -313,6 +317,7 @@ Other Deprecations
 - Deprecated allowing non-keyword arguments in :meth:`DataFrame.all`, :meth:`DataFrame.min`, :meth:`DataFrame.max`, :meth:`DataFrame.sum`, :meth:`DataFrame.prod`, :meth:`DataFrame.mean`, :meth:`DataFrame.median`, :meth:`DataFrame.sem`, :meth:`DataFrame.var`, :meth:`DataFrame.std`, :meth:`DataFrame.skew`, :meth:`DataFrame.kurt`, :meth:`Series.all`,  :meth:`Series.min`, :meth:`Series.max`, :meth:`Series.sum`, :meth:`Series.prod`, :meth:`Series.mean`, :meth:`Series.median`, :meth:`Series.sem`, :meth:`Series.var`, :meth:`Series.std`, :meth:`Series.skew`, and :meth:`Series.kurt`. (:issue:`57087`)
 - Deprecated allowing non-keyword arguments in :meth:`Series.to_markdown` except ``buf``. (:issue:`57280`)
 - Deprecated allowing non-keyword arguments in :meth:`Series.to_string` except ``buf``. (:issue:`57280`)
+- Deprecated behavior of :meth:`.DataFrameGroupBy.groups` and :meth:`.SeriesGroupBy.groups`, in a future version ``groups`` by one element list will return tuple instead of scalar. (:issue:`58858`)
 - Deprecated behavior of :meth:`Series.dt.to_pytimedelta`, in a future version this will return a :class:`Series` containing python ``datetime.timedelta`` objects instead of an ``ndarray`` of timedelta; this matches the behavior of other :meth:`Series.dt` properties. (:issue:`57463`)
 - Deprecated lowercase strings ``d``, ``b`` and ``c`` denoting frequencies in :class:`Day`, :class:`BusinessDay` and :class:`CustomBusinessDay` in favour of ``D``, ``B`` and ``C`` (:issue:`58998`)
 - Deprecated lowercase strings ``w``, ``w-mon``, ``w-tue``, etc. denoting frequencies in :class:`Week` in favour of ``W``, ``W-MON``, ``W-TUE``, etc. (:issue:`58998`)

pandas/_libs/include/pandas/vendored/klib/khash_python.h

@@ -207,7 +207,8 @@ static inline int pyobject_cmp(PyObject *a, PyObject *b) {
     if (PyComplex_CheckExact(a)) {
       return complexobject_cmp((PyComplexObject *)a, (PyComplexObject *)b);
     }
-    if (PyTuple_CheckExact(a)) {
+    if (PyTuple_Check(a)) {
+      // compare tuple subclasses as builtin tuples
       return tupleobject_cmp((PyTupleObject *)a, (PyTupleObject *)b);
     }
     // frozenset isn't yet supported
@@ -311,7 +312,8 @@ static inline khuint32_t kh_python_hash_func(PyObject *key) {
     // because complex(k,0) == k holds for any int-object k
     // and kh_complex128_hash_func doesn't respect it
     hash = complexobject_hash((PyComplexObject *)key);
-  } else if (PyTuple_CheckExact(key)) {
+  } else if (PyTuple_Check(key)) {
+    // hash tuple subclasses as builtin tuples
     hash = tupleobject_hash((PyTupleObject *)key);
   } else {
     hash = PyObject_Hash(key);

pandas/core/apply.py

@@ -486,20 +486,14 @@ def compute_dict_like(
                 cols = df[key]
 
                 if cols.ndim == 1:
-                    series_list = [obj._gotitem(key, ndim=1, subset=cols)]
+                    series = obj._gotitem(key, ndim=1, subset=cols)
+                    results.append(getattr(series, op_name)(how, **kwargs))
+                    keys.append(key)
                 else:
-                    series_list = []
-                    for index in range(cols.shape[1]):
-                        col = cols.iloc[:, index]
-
+                    for _, col in cols.items():
                         series = obj._gotitem(key, ndim=1, subset=col)
-                        series_list.append(series)
-
-                for series in series_list:
-                    result = getattr(series, op_name)(how, **kwargs)
-                    results.append(result)
-                    keys.append(key)
-
+                        results.append(getattr(series, op_name)(how, **kwargs))
+                        keys.append(key)
         else:
             results = [
                 getattr(obj._gotitem(key, ndim=1), op_name)(how, **kwargs)

pandas/core/dtypes/common.py

@@ -558,6 +558,11 @@ def is_string_dtype(arr_or_dtype) -> bool:
     boolean
         Whether or not the array or dtype is of the string dtype.
 
+    See Also
+    --------
+    api.types.is_string_dtype : Check whether the provided array or dtype
+                                is of the string dtype.
+
     Examples
     --------
     >>> from pandas.api.types import is_string_dtype