Merge remote-tracking branch 'upstream/main' into cln/comps

Author: mroeschke

.circleci/config.yml

@@ -54,7 +54,7 @@ jobs:
             /opt/python/cp311-cp311/bin/python -m venv ~/virtualenvs/pandas-dev
             . ~/virtualenvs/pandas-dev/bin/activate
             python -m pip install --no-cache-dir -U pip wheel setuptools meson-python==0.13.1 meson[ninja]==1.2.1
-            python -m pip install --no-cache-dir versioneer[toml] cython numpy python-dateutil pytz pytest>=7.3.2 pytest-xdist>=2.2.0 hypothesis>=6.46.1
+            python -m pip install --no-cache-dir versioneer[toml] cython numpy python-dateutil pytz pytest>=7.3.2 pytest-xdist>=3.4.0 hypothesis>=6.84.0
             python -m pip install --no-cache-dir --no-build-isolation -e . --config-settings=setup-args="--werror"
             python -m pip list --no-cache-dir
             export PANDAS_CI=1
@@ -141,11 +141,9 @@ workflows:
               only: /^v.*/
           matrix:
             parameters:
-              cibw-build: ["cp39-manylinux_aarch64",
-                           "cp310-manylinux_aarch64",
+              cibw-build: ["cp310-manylinux_aarch64",
                            "cp311-manylinux_aarch64",
                            "cp312-manylinux_aarch64",
-                           "cp39-musllinux_aarch64",
                            "cp310-musllinux_aarch64",
                            "cp311-musllinux_aarch64",
                            "cp312-musllinux_aarch64",]

.github/actions/build_pandas/action.yml

@@ -4,12 +4,6 @@ inputs:
   editable:
     description: Whether to build pandas in editable mode (default true)
     default: true
-  meson_args:
-    description: Extra flags to pass to meson
-    required: false
-  cflags_adds:
-    description: Items to append to the CFLAGS variable
-    required: false
 runs:
   using: composite
   steps:
@@ -30,12 +24,11 @@ runs:
 
     - name: Build Pandas
       run: |
-        export CFLAGS="$CFLAGS ${{ inputs.cflags_adds }}"
         if [[ ${{ inputs.editable }} == "true" ]]; then
-          pip install -e . --no-build-isolation -v --no-deps ${{ inputs.meson_args }} \
+          pip install -e . --no-build-isolation -v --no-deps \
             --config-settings=setup-args="--werror"
         else
-          pip install . --no-build-isolation -v --no-deps ${{ inputs.meson_args }} \
+          pip install . --no-build-isolation -v --no-deps \
             --config-settings=setup-args="--werror"
         fi
       shell: bash -el {0}

.github/actions/run-tests/action.yml

@@ -7,14 +7,14 @@ runs:
       shell: bash -el {0}
 
     - name: Publish test results
-      uses: actions/upload-artifact@v3
+      uses: actions/upload-artifact@v4
       with:
         name: Test results
         path: test-data.xml
       if: failure()
 
     - name: Upload coverage to Codecov
-      uses: codecov/codecov-action@v3
+      uses: codecov/codecov-action@v4
       with:
         flags: unittests
         name: codecov-pandas

.github/workflows/package-checks.yml

@@ -53,7 +53,7 @@ jobs:
     runs-on: ubuntu-22.04
     strategy:
       matrix:
-        python-version: ['3.9', '3.10', '3.11']
+        python-version: ['3.10', '3.11']
       fail-fast: false
     name: Test Conda Forge Recipe - Python ${{ matrix.python-version }}
     concurrency:

.github/workflows/unit-tests.yml

@@ -26,7 +26,7 @@ jobs:
     timeout-minutes: 90
     strategy:
       matrix:
-        env_file: [actions-39.yaml, actions-310.yaml, actions-311.yaml, actions-312.yaml]
+        env_file: [actions-310.yaml, actions-311.yaml, actions-312.yaml]
         # Prevent the include jobs from overriding other jobs
         pattern: [""]
         include:
@@ -35,7 +35,7 @@ jobs:
             pattern: "not slow and not network and not single_cpu"
             pytest_target: "pandas/tests/test_downstream.py"
           - name: "Minimum Versions"
-            env_file: actions-39-minimum_versions.yaml
+            env_file: actions-310-minimum_versions.yaml
             pattern: "not slow and not network and not single_cpu"
           - name: "Locale: it_IT"
             env_file: actions-311.yaml
@@ -74,9 +74,9 @@ jobs:
       PATTERN: ${{ matrix.pattern }}
       LANG: ${{ matrix.lang || 'C.UTF-8' }}
       LC_ALL: ${{ matrix.lc_all || '' }}
-      PANDAS_CI: ${{ matrix.pandas_ci || '1' }}
+      PANDAS_CI: '1'
       TEST_ARGS: ${{ matrix.test_args || '' }}
-      PYTEST_WORKERS: ${{ matrix.pytest_workers || 'auto' }}
+      PYTEST_WORKERS: 'auto'
       PYTEST_TARGET: ${{ matrix.pytest_target || 'pandas' }}
       NPY_PROMOTION_STATE: ${{ matrix.env_file == 'actions-311-numpydev.yaml' && 'weak' || 'legacy' }}
       # Clipboard tests
@@ -88,7 +88,7 @@ jobs:
 
     services:
       mysql:
-        image: mysql:8.0.33
+        image: mysql:8
         env:
           MYSQL_ALLOW_EMPTY_PASSWORD: yes
           MYSQL_DATABASE: pandas
@@ -101,7 +101,7 @@ jobs:
           - 3306:3306
 
       postgres:
-        image: postgres:13
+        image: postgres:16
         env:
           PGUSER: postgres
           POSTGRES_USER: postgres
@@ -116,7 +116,7 @@ jobs:
           - 5432:5432
 
       moto:
-        image: motoserver/moto:4.1.13
+        image: motoserver/moto:5.0.0
         env:
           AWS_ACCESS_KEY_ID: foobar_key
           AWS_SECRET_ACCESS_KEY: foobar_secret
@@ -146,9 +146,8 @@ jobs:
     - name: Build Pandas
       id: build
       uses: ./.github/actions/build_pandas
-      with:
-        meson_args: ${{ matrix.meson_args }}
-        cflags_adds: ${{ matrix.cflags_adds }}
+      # TODO: Re-enable once Pypy has Pypy 3.10 on conda-forge
+      if: ${{ matrix.name != 'Pypy' }}
 
     - name: Test (not single_cpu)
       uses: ./.github/actions/run-tests
@@ -170,7 +169,7 @@ jobs:
       matrix:
         # Note: Don't use macOS latest since macos 14 appears to be arm64 only
         os: [macos-13, macos-14, windows-latest]
-        env_file: [actions-39.yaml, actions-310.yaml, actions-311.yaml, actions-312.yaml]
+        env_file: [actions-310.yaml, actions-311.yaml, actions-312.yaml]
       fail-fast: false
     runs-on: ${{ matrix.os }}
     name: ${{ format('{0} {1}', matrix.os, matrix.env_file) }}
@@ -228,7 +227,7 @@ jobs:
           . ~/virtualenvs/pandas-dev/bin/activate
           python -m pip install --no-cache-dir -U pip wheel setuptools meson[ninja]==1.2.1 meson-python==0.13.1
           python -m pip install numpy --config-settings=setup-args="-Dallow-noblas=true"
-          python -m pip install --no-cache-dir versioneer[toml] cython python-dateutil pytz pytest>=7.3.2 pytest-xdist>=2.2.0 hypothesis>=6.46.1
+          python -m pip install --no-cache-dir versioneer[toml] cython python-dateutil pytz pytest>=7.3.2 pytest-xdist>=3.4.0 hypothesis>=6.84.0
           python -m pip install --no-cache-dir --no-build-isolation -e . --config-settings=setup-args="--werror"
           python -m pip list --no-cache-dir
           export PANDAS_CI=1
@@ -266,7 +265,7 @@ jobs:
           /opt/python/cp311-cp311/bin/python -m venv ~/virtualenvs/pandas-dev
           . ~/virtualenvs/pandas-dev/bin/activate
           python -m pip install --no-cache-dir -U pip wheel setuptools meson-python==0.13.1 meson[ninja]==1.2.1
-          python -m pip install --no-cache-dir versioneer[toml] cython numpy python-dateutil pytz pytest>=7.3.2 pytest-xdist>=2.2.0 hypothesis>=6.46.1
+          python -m pip install --no-cache-dir versioneer[toml] cython numpy python-dateutil pytz pytest>=7.3.2 pytest-xdist>=3.4.0 hypothesis>=6.84.0
           python -m pip install --no-cache-dir --no-build-isolation -e . --config-settings=setup-args="--werror"
           python -m pip list --no-cache-dir
 
@@ -300,7 +299,7 @@ jobs:
     #    To freeze this file, uncomment out the ``if: false`` condition, and migrate the jobs
     #    to the corresponding posix/windows-macos/sdist etc. workflows.
     # Feel free to modify this comment as necessary.
-    if: false # Uncomment this to freeze the workflow, comment it to unfreeze
+    # if: false # Uncomment this to freeze the workflow, comment it to unfreeze
     defaults:
       run:
         shell: bash -eou pipefail {0}
@@ -315,7 +314,7 @@ jobs:
 
     concurrency:
       # https://github.community/t/concurrecy-not-work-for-push/183068/7
-      group: ${{ github.event_name == 'push' && github.run_number || github.ref }}-${{ matrix.os }}-${{ matrix.pytest_target }}-dev
+      group: ${{ github.event_name == 'push' && github.run_number || github.ref }}-${{ matrix.os }}-python-dev
       cancel-in-progress: true
 
     env:
@@ -332,21 +331,65 @@ jobs:
       - name: Set up Python Dev Version
         uses: actions/setup-python@v5
         with:
-          python-version: '3.12-dev'
+          python-version: '3.13-dev'
 
       - name: Build Environment
         run: |
           python --version
           python -m pip install --upgrade pip setuptools wheel meson[ninja]==1.2.1 meson-python==0.13.1
           python -m pip install --pre --extra-index-url https://pypi.anaconda.org/scientific-python-nightly-wheels/simple numpy
           python -m pip install versioneer[toml]
-          python -m pip install python-dateutil pytz tzdata cython hypothesis>=6.46.1 pytest>=7.3.2 pytest-xdist>=2.2.0 pytest-cov
+          python -m pip install python-dateutil pytz tzdata cython hypothesis>=6.84.0 pytest>=7.3.2 pytest-xdist>=3.4.0 pytest-cov
+          python -m pip install -ve . --no-build-isolation --no-index --no-deps --config-settings=setup-args="--werror"
+          python -m pip list
+
+      - name: Run Tests
+        uses: ./.github/actions/run-tests
+
+  python-freethreading:
+    defaults:
+      run:
+        shell: bash -eou pipefail {0}
+    runs-on: ubuntu-22.04
+
+    timeout-minutes: 90
+
+    concurrency:
+      # https://github.community/t/concurrecy-not-work-for-push/183068/7
+      group: ${{ github.event_name == 'push' && github.run_number || github.ref }}-${{ matrix.os }}-python-freethreading-dev
+      cancel-in-progress: true
+
+    env:
+      PYTEST_WORKERS: "auto"
+      PANDAS_CI: 1
+      PATTERN: "not slow and not network and not clipboard and not single_cpu"
+      PYTEST_TARGET: pandas
+
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+
+      - name: Set up Python Free-threading Version
+        uses: deadsnakes/[email protected]
+        with:
+          python-version: 3.13-dev
+          nogil: true
+
+      - name: Build Environment
+        run: |
+          python --version
+          python -m pip install --upgrade pip setuptools wheel meson[ninja]==1.2.1 meson-python==0.13.1
+          python -m pip install --pre --extra-index-url https://pypi.anaconda.org/scientific-python-nightly-wheels/simple numpy cython
+          python -m pip install versioneer[toml]
+          python -m pip install python-dateutil pytz tzdata hypothesis>=6.84.0 pytest>=7.3.2 pytest-xdist>=3.4.0 pytest-cov
           python -m pip install -ve . --no-build-isolation --no-index --no-deps --config-settings=setup-args="--werror"
           python -m pip list
 
       - name: Run Tests
         uses: ./.github/actions/run-tests
 
+  # NOTE: this job must be kept in sync with the Pyodide build job in wheels.yml
   emscripten:
     # Note: the Python version, Emscripten toolchain version are determined
     # by the Pyodide version. The appropriate versions can be found in the

.github/workflows/wheels.yml

@@ -99,7 +99,14 @@ jobs:
         - [macos-14, macosx_arm64]
         - [windows-2022, win_amd64]
         # TODO: support PyPy?
-        python: [["cp39", "3.9"], ["cp310", "3.10"], ["cp311", "3.11"], ["cp312", "3.12"]]
+        python: [["cp310", "3.10"], ["cp311", "3.11"], ["cp312", "3.12"]]
+
+        # Build Pyodide wheels and upload them to Anaconda.org
+        # NOTE: this job is similar to the one in unit-tests.yml except for the fact
+        # that it uses cibuildwheel instead of a standard Pyodide xbuildenv setup.
+        include:
+          - buildplat: [ubuntu-22.04, pyodide_wasm32]
+            python: ["cp312", "3.12"]
     env:
       IS_PUSH: ${{ github.event_name == 'push' && startsWith(github.ref, 'refs/tags/v') }}
       IS_SCHEDULE_DISPATCH: ${{ github.event_name == 'schedule' || github.event_name == 'workflow_dispatch' }}
@@ -140,12 +147,13 @@ jobs:
         run: echo "sdist_name=$(cd ./dist && ls -d */)" >> "$GITHUB_ENV"
 
       - name: Build wheels
-        uses: pypa/cibuildwheel@v2.18.1
+        uses: pypa/cibuildwheel@v2.19.2
         with:
          package-dir: ./dist/${{ startsWith(matrix.buildplat[1], 'macosx') && env.sdist_name || needs.build_sdist.outputs.sdist_file }}
         env:
           CIBW_PRERELEASE_PYTHONS: True
           CIBW_BUILD: ${{ matrix.python[0] }}-${{ matrix.buildplat[1] }}
+          CIBW_PLATFORM: ${{ matrix.buildplat[1] == 'pyodide_wasm32' && 'pyodide' || 'auto' }}
 
       - name: Set up Python
         uses: mamba-org/setup-micromamba@v1
@@ -170,7 +178,7 @@ jobs:
         shell: pwsh
         run: |
           $TST_CMD = @"
-          python -m pip install hypothesis>=6.46.1 pytest>=7.3.2 pytest-xdist>=2.2.0;
+          python -m pip install hypothesis>=6.84.0 pytest>=7.3.2 pytest-xdist>=3.4.0;
           python -m pip install `$(Get-Item pandas\wheelhouse\*.whl);
           python -c `'import pandas as pd; pd.test(extra_args=[`\"--no-strict-data-files`\", `\"-m not clipboard and not single_cpu and not slow and not network and not db`\"])`';
           "@

.pre-commit-config.yaml

@@ -19,7 +19,7 @@ ci:
     skip: [pyright, mypy]
 repos:
 -   repo: https://github.com/astral-sh/ruff-pre-commit
-    rev: v0.4.7
+    rev: v0.5.0
     hooks:
     -   id: ruff
         args: [--exit-non-zero-on-fix]
@@ -67,15 +67,16 @@ repos:
     -   id: fix-encoding-pragma
         args: [--remove]
     -   id: trailing-whitespace
+        args: [--markdown-linebreak-ext=md]
 -   repo: https://github.com/PyCQA/isort
     rev: 5.13.2
     hooks:
     -   id: isort
 -   repo: https://github.com/asottile/pyupgrade
-    rev: v3.15.2
+    rev: v3.16.0
     hooks:
     -   id: pyupgrade
-        args: [--py39-plus]
+        args: [--py310-plus]
 -   repo: https://github.com/pre-commit/pygrep-hooks
     rev: v1.10.0
     hooks:
@@ -92,7 +93,7 @@ repos:
     - id: sphinx-lint
       args: ["--enable", "all", "--disable", "line-too-long"]
 -   repo: https://github.com/pre-commit/mirrors-clang-format
-    rev: v18.1.5
+    rev: v18.1.8
     hooks:
     - id: clang-format
       files: ^pandas/_libs/src|^pandas/_libs/include

asv_bench/benchmarks/indexing.py

@@ -546,24 +546,17 @@ def time_chained_indexing(self, mode):
 
 
 class Block:
-    params = [
-        (True, "True"),
-        (np.array(True), "np.array(True)"),
-    ]
-
-    def setup(self, true_value, mode):
+    def setup(self):
         self.df = DataFrame(
             False,
             columns=np.arange(500).astype(str),
             index=date_range("2010-01-01", "2011-01-01"),
         )
 
-        self.true_value = true_value
-
-    def time_test(self, true_value, mode):
+    def time_test(self):
         start = datetime(2010, 5, 1)
         end = datetime(2010, 9, 1)
-        self.df.loc[start:end, :] = true_value
+        self.df.loc[start:end, :] = True
 
 
 from .pandas_vb_common import setup  # noqa: F401 isort:skip

asv_bench/benchmarks/tslibs/timedelta.py

@@ -20,7 +20,7 @@ def time_from_int(self):
         Timedelta(123456789)
 
     def time_from_unit(self):
-        Timedelta(1, unit="d")
+        Timedelta(1, unit="D")
 
     def time_from_components(self):
         Timedelta(