Merge remote-tracking branch 'upstream/main' into string-dtype-disallow-str-pyarrow-alias

Author: jorisvandenbossche

.github/workflows/wheels.yml

@@ -94,7 +94,7 @@ jobs:
         buildplat:
         - [ubuntu-22.04, manylinux_x86_64]
         - [ubuntu-22.04, musllinux_x86_64]
-        - [macos-12, macosx_x86_64]
+        - [macos-13, macosx_x86_64]
         # Note: M1 images on Github Actions start from macOS 14
         - [macos-14, macosx_arm64]
         - [windows-2022, win_amd64]

.pre-commit-config.yaml

@@ -19,7 +19,7 @@ ci:
     skip: [pyright, mypy]
 repos:
 -   repo: https://github.com/astral-sh/ruff-pre-commit
-    rev: v0.8.1
+    rev: v0.8.6
     hooks:
     -   id: ruff
         args: [--exit-non-zero-on-fix]
@@ -34,7 +34,7 @@ repos:
     -   id: ruff-format
         exclude: ^scripts|^pandas/tests/frame/test_query_eval.py
 -   repo: https://github.com/jendrikseipp/vulture
-    rev: 'v2.13'
+    rev: 'v2.14'
     hooks:
       - id: vulture
         entry: python scripts/run_vulture.py
@@ -74,7 +74,7 @@ repos:
     hooks:
     -   id: isort
 -   repo: https://github.com/asottile/pyupgrade
-    rev: v3.19.0
+    rev: v3.19.1
     hooks:
     -   id: pyupgrade
         args: [--py310-plus]
@@ -95,12 +95,17 @@ repos:
     - id: sphinx-lint
       args: ["--enable", "all", "--disable", "line-too-long"]
 -   repo: https://github.com/pre-commit/mirrors-clang-format
-    rev: v19.1.4
+    rev: v19.1.6
     hooks:
     - id: clang-format
       files: ^pandas/_libs/src|^pandas/_libs/include
       args: [-i]
       types_or: [c, c++]
+-   repo: https://github.com/trim21/pre-commit-mirror-meson
+    rev: v1.6.1
+    hooks:
+    - id: meson-fmt
+      args: ['--inplace']
 -   repo: local
     hooks:
     -   id: pyright

asv_bench/benchmarks/io/csv.py

@@ -594,7 +594,7 @@ def setup(self):
         self.StringIO_input = StringIO(data)
 
     def time_read_csv_index_col(self):
-        read_csv(self.StringIO_input, index_col="a")
+        read_csv(self.data(self.StringIO_input), index_col="a")
 
 
 class ReadCSVDatePyarrowEngine(StringIORewind):
@@ -605,7 +605,7 @@ def setup(self):
 
     def time_read_csv_index_col(self):
         read_csv(
-            self.StringIO_input,
+            self.data(self.StringIO_input),
             parse_dates=["a"],
             engine="pyarrow",
             dtype_backend="pyarrow",

doc/source/user_guide/io.rst

@@ -2340,6 +2340,7 @@ Read a URL with no options:
 .. code-block:: ipython
 
    In [320]: url = "https://www.fdic.gov/resources/resolutions/bank-failures/failed-bank-list"
+
    In [321]: pd.read_html(url)
    Out[321]:
    [                         Bank NameBank           CityCity StateSt  ...              Acquiring InstitutionAI Closing DateClosing FundFund
@@ -2366,6 +2367,7 @@ Read a URL while passing headers alongside the HTTP request:
 .. code-block:: ipython
 
    In [322]: url = 'https://www.sump.org/notes/request/' # HTTP request reflector
+
    In [323]: pd.read_html(url)
    Out[323]:
    [                   0                    1
@@ -2378,14 +2380,16 @@ Read a URL while passing headers alongside the HTTP request:
     1              Host:         www.sump.org
     2        User-Agent:    Python-urllib/3.8
     3        Connection:                close]
+
    In [324]: headers = {
-   In [325]:    'User-Agent':'Mozilla Firefox v14.0',
-   In [326]:    'Accept':'application/json',
-   In [327]:    'Connection':'keep-alive',
-   In [328]:    'Auth':'Bearer 2*/f3+fe68df*4'
-   In [329]: }
-   In [340]: pd.read_html(url, storage_options=headers)
-   Out[340]:
+      .....:    'User-Agent':'Mozilla Firefox v14.0',
+      .....:    'Accept':'application/json',
+      .....:    'Connection':'keep-alive',
+      .....:    'Auth':'Bearer 2*/f3+fe68df*4'
+      .....: }
+
+   In [325]: pd.read_html(url, storage_options=headers)
+   Out[325]:
    [                   0                    1
     0     Remote Socket:  51.15.105.256:51760
     1  Protocol Version:             HTTP/1.1

doc/source/whatsnew/v3.0.0.rst

@@ -55,6 +55,7 @@ Other enhancements
 - :meth:`Series.plot` now correctly handle the ``ylabel`` parameter for pie charts, allowing for explicit control over the y-axis label (:issue:`58239`)
 - :meth:`DataFrame.plot.scatter` argument ``c`` now accepts a column of strings, where rows with the same string are colored identically (:issue:`16827` and :issue:`16485`)
 - :func:`read_parquet` accepts ``to_pandas_kwargs`` which are forwarded to :meth:`pyarrow.Table.to_pandas` which enables passing additional keywords to customize the conversion to pandas, such as ``maps_as_pydicts`` to read the Parquet map data type as python dictionaries (:issue:`56842`)
+- :meth:`.DataFrameGroupBy.transform`, :meth:`.SeriesGroupBy.transform`, :meth:`.DataFrameGroupBy.agg`, :meth:`.SeriesGroupBy.agg`, :meth:`.SeriesGroupBy.apply`, :meth:`.DataFrameGroupBy.apply` now support ``kurt`` (:issue:`40139`)
 - :meth:`DataFrameGroupBy.transform`, :meth:`SeriesGroupBy.transform`, :meth:`DataFrameGroupBy.agg`, :meth:`SeriesGroupBy.agg`, :meth:`RollingGroupby.apply`, :meth:`ExpandingGroupby.apply`, :meth:`Rolling.apply`, :meth:`Expanding.apply`, :meth:`DataFrame.apply` with ``engine="numba"`` now supports positional arguments passed as kwargs (:issue:`58995`)
 - :meth:`Rolling.agg`, :meth:`Expanding.agg` and :meth:`ExponentialMovingWindow.agg` now accept :class:`NamedAgg` aggregations through ``**kwargs`` (:issue:`28333`)
 - :meth:`Series.map` can now accept kwargs to pass on to func (:issue:`59814`)
@@ -798,6 +799,7 @@ Other
 - Bug in :meth:`Series.dt` methods in :class:`ArrowDtype` that were returning incorrect values. (:issue:`57355`)
 - Bug in :meth:`Series.rank` that doesn't preserve missing values for nullable integers when ``na_option='keep'``. (:issue:`56976`)
 - Bug in :meth:`Series.replace` and :meth:`DataFrame.replace` inconsistently replacing matching instances when ``regex=True`` and missing values are present. (:issue:`56599`)
+- Bug in :meth:`Series.replace` and :meth:`DataFrame.replace` throwing ``ValueError`` when ``regex=True`` and all NA values. (:issue:`60688`)
 - Bug in :meth:`Series.to_string` when series contains complex floats with exponents (:issue:`60405`)
 - Bug in :meth:`read_csv` where chained fsspec TAR file and ``compression="infer"`` fails with ``tarfile.ReadError`` (:issue:`60028`)
 - Bug in Dataframe Interchange Protocol implementation was returning incorrect results for data buffers' associated dtype, for string and datetime columns (:issue:`54781`)

meson.build

@@ -1,15 +1,13 @@
 # This file is adapted from https://github.com/scipy/scipy/blob/main/meson.build
 project(
     'pandas',
-    'c', 'cpp', 'cython',
+    'c',
+    'cpp',
+    'cython',
     version: run_command(['generate_version.py', '--print'], check: true).stdout().strip(),
     license: 'BSD-3',
     meson_version: '>=1.2.1',
-    default_options: [
-        'buildtype=release',
-        'c_std=c11',
-        'warning_level=2',
-    ]
+    default_options: ['buildtype=release', 'c_std=c11', 'warning_level=2'],
 )
 
 fs = import('fs')
@@ -18,41 +16,40 @@ tempita = files('generate_pxi.py')
 versioneer = files('generate_version.py')
 
 
-add_project_arguments('-DNPY_NO_DEPRECATED_API=0', language : 'c')
-add_project_arguments('-DNPY_NO_DEPRECATED_API=0', language : 'cpp')
+add_project_arguments('-DNPY_NO_DEPRECATED_API=0', language: 'c')
+add_project_arguments('-DNPY_NO_DEPRECATED_API=0', language: 'cpp')
 
 # Allow supporting older numpys than the version compiled against
 # Set the define to the min supported version of numpy for pandas
 # e.g. right now this is targeting numpy 1.21+
-add_project_arguments('-DNPY_TARGET_VERSION=NPY_1_21_API_VERSION', language : 'c')
-add_project_arguments('-DNPY_TARGET_VERSION=NPY_1_21_API_VERSION', language : 'cpp')
+add_project_arguments('-DNPY_TARGET_VERSION=NPY_1_21_API_VERSION', language: 'c')
+add_project_arguments(
+    '-DNPY_TARGET_VERSION=NPY_1_21_API_VERSION',
+    language: 'cpp',
+)
 
 
 if fs.exists('_version_meson.py')
     py.install_sources('_version_meson.py', subdir: 'pandas')
 else
-    custom_target('write_version_file',
+    custom_target(
+        'write_version_file',
         output: '_version_meson.py',
-        command: [
-            py, versioneer, '-o', '@OUTPUT@'
-        ],
+        command: [py, versioneer, '-o', '@OUTPUT@'],
         build_by_default: true,
         build_always_stale: true,
         install: true,
-        install_dir: py.get_install_dir() / 'pandas'
+        install_dir: py.get_install_dir() / 'pandas',
     )
     meson.add_dist_script(py, versioneer, '-o', '_version_meson.py')
 endif
 
 cy = meson.get_compiler('cython')
 if cy.version().version_compare('>=3.1.0')
-  add_project_arguments('-Xfreethreading_compatible=true', language : 'cython')
+    add_project_arguments('-Xfreethreading_compatible=true', language: 'cython')
 endif
 
 # Needed by pandas.test() when it looks for the pytest ini options
-py.install_sources(
-    'pyproject.toml',
-    subdir: 'pandas'
-)
+py.install_sources('pyproject.toml', subdir: 'pandas')
 
 subdir('pandas')

pandas/_libs/byteswap.pyx

@@ -15,7 +15,7 @@ from libc.string cimport memcpy
 
 def read_float_with_byteswap(bytes data, Py_ssize_t offset, bint byteswap):
     cdef uint32_t value
-    assert offset + sizeof(value) < len(data)
+    assert offset + <Py_ssize_t>sizeof(value) < len(data)
     cdef const void *ptr = <unsigned char*>(data) + offset
     memcpy(&value, ptr, sizeof(value))
     if byteswap:
@@ -28,7 +28,7 @@ def read_float_with_byteswap(bytes data, Py_ssize_t offset, bint byteswap):
 
 def read_double_with_byteswap(bytes data, Py_ssize_t offset, bint byteswap):
     cdef uint64_t value
-    assert offset + sizeof(value) < len(data)
+    assert offset + <Py_ssize_t>sizeof(value) < len(data)
     cdef const void *ptr = <unsigned char*>(data) + offset
     memcpy(&value, ptr, sizeof(value))
     if byteswap:
@@ -41,7 +41,7 @@ def read_double_with_byteswap(bytes data, Py_ssize_t offset, bint byteswap):
 
 def read_uint16_with_byteswap(bytes data, Py_ssize_t offset, bint byteswap):
     cdef uint16_t res
-    assert offset + sizeof(res) < len(data)
+    assert offset + <Py_ssize_t>sizeof(res) < len(data)
     memcpy(&res, <const unsigned char*>(data) + offset, sizeof(res))
     if byteswap:
         res = _byteswap2(res)
@@ -50,7 +50,7 @@ def read_uint16_with_byteswap(bytes data, Py_ssize_t offset, bint byteswap):
 
 def read_uint32_with_byteswap(bytes data, Py_ssize_t offset, bint byteswap):
     cdef uint32_t res
-    assert offset + sizeof(res) < len(data)
+    assert offset + <Py_ssize_t>sizeof(res) < len(data)
     memcpy(&res, <const unsigned char*>(data) + offset, sizeof(res))
     if byteswap:
         res = _byteswap4(res)
@@ -59,7 +59,7 @@ def read_uint32_with_byteswap(bytes data, Py_ssize_t offset, bint byteswap):
 
 def read_uint64_with_byteswap(bytes data, Py_ssize_t offset, bint byteswap):
     cdef uint64_t res
-    assert offset + sizeof(res) < len(data)
+    assert offset + <Py_ssize_t>sizeof(res) < len(data)
     memcpy(&res, <const unsigned char*>(data) + offset, sizeof(res))
     if byteswap:
         res = _byteswap8(res)

pandas/_libs/groupby.pyi

@@ -97,6 +97,15 @@ def group_skew(
     result_mask: np.ndarray | None = ...,
     skipna: bool = ...,
 ) -> None: ...
+def group_kurt(
+    out: np.ndarray,  # float64_t[:, ::1]
+    counts: np.ndarray,  # int64_t[::1]
+    values: np.ndarray,  # ndarray[float64_T, ndim=2]
+    labels: np.ndarray,  # const intp_t[::1]
+    mask: np.ndarray | None = ...,
+    result_mask: np.ndarray | None = ...,
+    skipna: bool = ...,
+) -> None: ...
 def group_mean(
     out: np.ndarray,  # floating[:, ::1]
     counts: np.ndarray,  # int64_t[::1]

pandas/_libs/groupby.pyx

@@ -910,7 +910,7 @@ def group_var(
 @cython.wraparound(False)
 @cython.boundscheck(False)
 @cython.cdivision(True)
-@cython.cpow
+@cython.cpow(True)
 def group_skew(
     float64_t[:, ::1] out,
     int64_t[::1] counts,
@@ -961,7 +961,7 @@ def group_skew(
                     isna_entry = _treat_as_na(val, False)
 
                 if not isna_entry:
-                    # Based on RunningStats::Push from
+                    # Running stats update based on RunningStats::Push from
                     #  https://www.johndcook.com/blog/skewness_kurtosis/
                     n1 = nobs[lab, j]
                     n = n1 + 1
@@ -995,6 +995,100 @@ def group_skew(
                     )
 
 
+@cython.wraparound(False)
+@cython.boundscheck(False)
+@cython.cdivision(True)
+@cython.cpow(True)
+def group_kurt(
+    float64_t[:, ::1] out,
+    int64_t[::1] counts,
+    ndarray[float64_t, ndim=2] values,
+    const intp_t[::1] labels,
+    const uint8_t[:, ::1] mask=None,
+    uint8_t[:, ::1] result_mask=None,
+    bint skipna=True,
+) -> None:
+    cdef:
+        Py_ssize_t i, j, N, K, lab, ngroups = len(counts)
+        int64_t[:, ::1] nobs
+        Py_ssize_t len_values = len(values), len_labels = len(labels)
+        bint isna_entry, uses_mask = mask is not None
+        float64_t[:, ::1] M1, M2, M3, M4
+        float64_t delta, delta_n, delta_n2, term1, val
+        int64_t n1, n
+        float64_t ct, num, den, adj
+
+    if len_values != len_labels:
+        raise ValueError("len(index) != len(labels)")
+
+    nobs = np.zeros((<object>out).shape, dtype=np.int64)
+
+    # M1, M2, M3 and M4 correspond to 1st, 2nd, 3rd and 4th Moments
+    M1 = np.zeros((<object>out).shape, dtype=np.float64)
+    M2 = np.zeros((<object>out).shape, dtype=np.float64)
+    M3 = np.zeros((<object>out).shape, dtype=np.float64)
+    M4 = np.zeros((<object>out).shape, dtype=np.float64)
+
+    N, K = (<object>values).shape
+
+    out[:, :] = 0.0
+
+    with nogil:
+        for i in range(N):
+            lab = labels[i]
+            if lab < 0:
+                continue
+
+            counts[lab] += 1
+
+            for j in range(K):
+                val = values[i, j]
+
+                if uses_mask:
+                    isna_entry = mask[i, j]
+                else:
+                    isna_entry = _treat_as_na(val, False)
+
+                if not isna_entry:
+                    # Running stats update based on RunningStats::Push from
+                    #  https://www.johndcook.com/blog/skewness_kurtosis/
+                    n1 = nobs[lab, j]
+                    n = n1 + 1
+
+                    nobs[lab, j] = n
+                    delta = val - M1[lab, j]
+                    delta_n = delta / n
+                    delta_n2 = delta_n * delta_n
+                    term1 = delta * delta_n * n1
+
+                    M1[lab, j] += delta_n
+                    M4[lab, j] += (term1 * delta_n2 * (n*n - 3*n + 3)
+                                   + 6 * delta_n2 * M2[lab, j]
+                                   - 4 * delta_n * M3[lab, j])
+                    M3[lab, j] += term1 * delta_n * (n - 2) - 3 * delta_n * M2[lab, j]
+                    M2[lab, j] += term1
+                elif not skipna:
+                    M1[lab, j] = NaN
+                    M2[lab, j] = NaN
+                    M3[lab, j] = NaN
+                    M4[lab, j] = NaN
+
+        for i in range(ngroups):
+            for j in range(K):
+                ct = <float64_t>nobs[i, j]
+                if ct < 4:
+                    if result_mask is not None:
+                        result_mask[i, j] = 1
+                    out[i, j] = NaN
+                elif M2[i, j] == 0:
+                    out[i, j] = 0
+                else:
+                    num = ct * (ct + 1) * (ct - 1) * M4[i, j]
+                    den = (ct - 2) * (ct - 3) * M2[i, j] ** 2
+                    adj = 3.0 * (ct - 1) ** 2 / ((ct - 2) * (ct - 3))
+                    out[i, j] = num / den - adj
+
+
 @cython.wraparound(False)
 @cython.boundscheck(False)
 def group_mean(

pandas/_libs/include/pandas/vendored/klib/khash_python.h

@@ -34,11 +34,9 @@ static void *traced_calloc(size_t num, size_t size) {
 }
 
 static void *traced_realloc(void *old_ptr, size_t size) {
+  PyTraceMalloc_Untrack(KHASH_TRACE_DOMAIN, (uintptr_t)old_ptr);
   void *ptr = realloc(old_ptr, size);
   if (ptr != NULL) {
-    if (old_ptr != ptr) {
-      PyTraceMalloc_Untrack(KHASH_TRACE_DOMAIN, (uintptr_t)old_ptr);
-    }
     PyTraceMalloc_Track(KHASH_TRACE_DOMAIN, (uintptr_t)ptr, size);
   }
   return ptr;