Merge branch 'main' into pdep-17

Author: Aloqeely

.circleci/config.yml

@@ -15,15 +15,16 @@ jobs:
       - checkout
       - run:
           name: Install Environment and Run Tests
-          shell: /bin/bash -exuo pipefail
+          shell: /bin/bash -exo pipefail
+          # https://pytest-qt.readthedocs.io/en/latest/troubleshooting.html#github-actions-azure-pipelines-travis-ci-and-gitlab-ci-cd
           command: |
-            MAMBA_URL="https://github.com/conda-forge/miniforge/releases/download/24.3.0-0/Mambaforge-24.3.0-0-Linux-aarch64.sh"
-            wget -q $MAMBA_URL -O minimamba.sh
-            chmod +x minimamba.sh
-            MAMBA_DIR="$HOME/miniconda3"
-            rm -rf $MAMBA_DIR
-            ./minimamba.sh -b -p $MAMBA_DIR
-            export PATH=$MAMBA_DIR/bin:$PATH
+            MINI_URL="https://github.com/conda-forge/miniforge/releases/download/24.3.0-0/Miniforge3-24.3.0-0-Linux-aarch64.sh"
+            wget -q $MINI_URL -O Miniforge3.sh
+            chmod +x Miniforge3.sh
+            MINI_DIR="$HOME/miniconda3"
+            rm -rf $MINI_DIR
+            ./Miniforge3.sh -b -p $MINI_DIR
+            export PATH=$MINI_DIR/bin:$PATH
             conda info -a
             conda env create -q -n pandas-dev -f $ENV_FILE
             conda list -n pandas-dev
@@ -33,6 +34,7 @@ jobs:
             fi
             python -m pip install --no-build-isolation -ve . --config-settings=setup-args="--werror"
             PATH=$HOME/miniconda3/envs/pandas-dev/bin:$HOME/miniconda3/condabin:$PATH
+            sudo apt-get update && sudo apt-get install -y libegl1 libopengl0
             ci/run_tests.sh
   test-linux-musl:
     docker:
@@ -54,7 +56,7 @@ jobs:
             /opt/python/cp311-cp311/bin/python -m venv ~/virtualenvs/pandas-dev
             . ~/virtualenvs/pandas-dev/bin/activate
             python -m pip install --no-cache-dir -U pip wheel setuptools meson-python==0.13.1 meson[ninja]==1.2.1
-            python -m pip install --no-cache-dir versioneer[toml] cython numpy python-dateutil pytz pytest>=7.3.2 pytest-xdist>=2.2.0 hypothesis>=6.46.1
+            python -m pip install --no-cache-dir versioneer[toml] cython numpy python-dateutil pytz pytest>=7.3.2 pytest-xdist>=3.4.0 hypothesis>=6.84.0
             python -m pip install --no-cache-dir --no-build-isolation -e . --config-settings=setup-args="--werror"
             python -m pip list --no-cache-dir
             export PANDAS_CI=1
@@ -89,29 +91,24 @@ jobs:
           name: Build aarch64 wheels
           no_output_timeout: 30m # Sometimes the tests won't generate any output, make sure the job doesn't get killed by that
           command: |
-            pip3 install cibuildwheel==2.18.1
-            cibuildwheel --prerelease-pythons --output-dir wheelhouse
+            pip3 install cibuildwheel==2.20.0
+            cibuildwheel --output-dir wheelhouse
 
           environment:
             CIBW_BUILD: << parameters.cibw-build >>
 
       - run:
           name: Install Anaconda Client & Upload Wheels
+          shell: /bin/bash -exo pipefail
           command: |
-            echo "Install Mambaforge"
-            MAMBA_URL="https://github.com/conda-forge/miniforge/releases/download/24.3.0-0/Mambaforge-24.3.0-0-Linux-aarch64.sh"
-            echo "Downloading $MAMBA_URL"
-            wget -q $MAMBA_URL -O minimamba.sh
-            chmod +x minimamba.sh
-
-            MAMBA_DIR="$HOME/miniconda3"
-            rm -rf $MAMBA_DIR
-            ./minimamba.sh -b -p $MAMBA_DIR
-
-            export PATH=$MAMBA_DIR/bin:$PATH
-
-            mamba install -y -c conda-forge anaconda-client
-
+            MINI_URL="https://github.com/conda-forge/miniforge/releases/download/24.3.0-0/Miniforge3-24.3.0-0-Linux-aarch64.sh"
+            wget -q $MINI_URL -O Miniforge3.sh
+            chmod +x Miniforge3.sh
+            MINI_DIR="$HOME/miniconda3"
+            rm -rf $MINI_DIR
+            ./Miniforge3.sh -b -p $MINI_DIR
+            export PATH=$MINI_DIR/bin:$PATH
+            conda install -y -c conda-forge anaconda-client
             source ci/upload_wheels.sh
             set_upload_vars
             upload_wheels

.gitattributes

@@ -68,7 +68,7 @@ ci export-ignore
 doc export-ignore
 gitpod export-ignore
 MANIFEST.in export-ignore
-scripts export-ignore
+scripts/** export-ignore
 typings export-ignore
 web export-ignore
 CITATION.cff export-ignore
@@ -82,3 +82,7 @@ setup.py export-ignore
 # csv_dir_path fixture checks the existence of the directory
 # exclude the whole directory to avoid running related tests in sdist
 pandas/tests/io/parser/data export-ignore
+
+# Include cibw script in sdist since it's needed for building wheels
+scripts/cibw_before_build.sh -export-ignore
+scripts/cibw_before_test.sh -export-ignore

.github/workflows/code-checks.yml

@@ -51,6 +51,11 @@ jobs:
     # TODO: The doctests have to be run first right now, since the Cython doctests only work
     # with pandas installed in non-editable mode
     # This can be removed once pytest-cython doesn't require C extensions to be installed inplace
+
+    - name: Extra installs
+      # https://pytest-qt.readthedocs.io/en/latest/troubleshooting.html#github-actions-azure-pipelines-travis-ci-and-gitlab-ci-cd
+      run: sudo apt-get update && sudo apt-get install -y libegl1 libopengl0
+
     - name: Run doctests
       run: cd ci && ./code_checks.sh doctests
       if: ${{ steps.build.outcome == 'success' && always() }}

.github/workflows/docbuild-and-upload.yml

@@ -46,6 +46,10 @@ jobs:
     - name: Build Pandas
       uses: ./.github/actions/build_pandas
 
+    - name: Extra installs
+      # https://pytest-qt.readthedocs.io/en/latest/troubleshooting.html#github-actions-azure-pipelines-travis-ci-and-gitlab-ci-cd
+      run: sudo apt-get update && sudo apt-get install -y libegl1 libopengl0
+
     - name: Test website
       run: python -m pytest web/
 

.github/workflows/unit-tests.yml

@@ -57,6 +57,9 @@ jobs:
             # Also install zh_CN (its encoding is gb2312) but do not activate it.
             # It will be temporarily activated during tests with locale.setlocale
             extra_loc: "zh_CN"
+          - name: "Future infer strings"
+            env_file: actions-311.yaml
+            pandas_future_infer_string: "1"
           - name: "Pypy"
             env_file: actions-pypy-39.yaml
             pattern: "not slow and not network and not single_cpu"
@@ -75,6 +78,7 @@ jobs:
       LANG: ${{ matrix.lang || 'C.UTF-8' }}
       LC_ALL: ${{ matrix.lc_all || '' }}
       PANDAS_CI: '1'
+      PANDAS_FUTURE_INFER_STRING: ${{ matrix.pandas_future_infer_string || '0' }}
       TEST_ARGS: ${{ matrix.test_args || '' }}
       PYTEST_WORKERS: 'auto'
       PYTEST_TARGET: ${{ matrix.pytest_target || 'pandas' }}
@@ -130,8 +134,8 @@ jobs:
         fetch-depth: 0
 
     - name: Extra installs
-      run: sudo apt-get update && sudo apt-get install -y ${{ matrix.extra_apt }}
-      if: ${{ matrix.extra_apt }}
+      # https://pytest-qt.readthedocs.io/en/latest/troubleshooting.html#github-actions-azure-pipelines-travis-ci-and-gitlab-ci-cd
+      run: sudo apt-get update && sudo apt-get install -y libegl1 libopengl0 ${{ matrix.extra_apt || ''}}
 
     - name: Generate extra locales
       # These extra locales will be available for locale.setlocale() calls in tests
@@ -227,7 +231,7 @@ jobs:
           . ~/virtualenvs/pandas-dev/bin/activate
           python -m pip install --no-cache-dir -U pip wheel setuptools meson[ninja]==1.2.1 meson-python==0.13.1
           python -m pip install numpy --config-settings=setup-args="-Dallow-noblas=true"
-          python -m pip install --no-cache-dir versioneer[toml] cython python-dateutil pytz pytest>=7.3.2 pytest-xdist>=2.2.0 hypothesis>=6.46.1
+          python -m pip install --no-cache-dir versioneer[toml] cython python-dateutil pytz pytest>=7.3.2 pytest-xdist>=3.4.0 hypothesis>=6.84.0
           python -m pip install --no-cache-dir --no-build-isolation -e . --config-settings=setup-args="--werror"
           python -m pip list --no-cache-dir
           export PANDAS_CI=1
@@ -265,7 +269,7 @@ jobs:
           /opt/python/cp311-cp311/bin/python -m venv ~/virtualenvs/pandas-dev
           . ~/virtualenvs/pandas-dev/bin/activate
           python -m pip install --no-cache-dir -U pip wheel setuptools meson-python==0.13.1 meson[ninja]==1.2.1
-          python -m pip install --no-cache-dir versioneer[toml] cython numpy python-dateutil pytz pytest>=7.3.2 pytest-xdist>=2.2.0 hypothesis>=6.46.1
+          python -m pip install --no-cache-dir versioneer[toml] cython numpy python-dateutil pytz pytest>=7.3.2 pytest-xdist>=3.4.0 hypothesis>=6.84.0
           python -m pip install --no-cache-dir --no-build-isolation -e . --config-settings=setup-args="--werror"
           python -m pip list --no-cache-dir
 
@@ -339,7 +343,7 @@ jobs:
           python -m pip install --upgrade pip setuptools wheel meson[ninja]==1.2.1 meson-python==0.13.1
           python -m pip install --pre --extra-index-url https://pypi.anaconda.org/scientific-python-nightly-wheels/simple numpy
           python -m pip install versioneer[toml]
-          python -m pip install python-dateutil pytz tzdata cython hypothesis>=6.46.1 pytest>=7.3.2 pytest-xdist>=2.2.0 pytest-cov
+          python -m pip install python-dateutil pytz tzdata cython hypothesis>=6.84.0 pytest>=7.3.2 pytest-xdist>=3.4.0 pytest-cov
           python -m pip install -ve . --no-build-isolation --no-index --no-deps --config-settings=setup-args="--werror"
           python -m pip list
 
@@ -382,15 +386,14 @@ jobs:
           python -m pip install --upgrade pip setuptools wheel meson[ninja]==1.2.1 meson-python==0.13.1
           python -m pip install --pre --extra-index-url https://pypi.anaconda.org/scientific-python-nightly-wheels/simple numpy cython
           python -m pip install versioneer[toml]
-          python -m pip install python-dateutil pytz tzdata hypothesis>=6.46.1 pytest>=7.3.2 pytest-xdist>=2.2.0 pytest-cov
+          python -m pip install python-dateutil pytz tzdata hypothesis>=6.84.0 pytest>=7.3.2 pytest-xdist>=3.4.0 pytest-cov
           python -m pip install -ve . --no-build-isolation --no-index --no-deps --config-settings=setup-args="--werror"
           python -m pip list
 
       - name: Run Tests
         uses: ./.github/actions/run-tests
-        env:
-          PYTHON_GIL: 0
 
+  # NOTE: this job must be kept in sync with the Pyodide build job in wheels.yml
   emscripten:
     # Note: the Python version, Emscripten toolchain version are determined
     # by the Pyodide version. The appropriate versions can be found in the

.github/workflows/wheels.yml

@@ -99,7 +99,25 @@ jobs:
         - [macos-14, macosx_arm64]
         - [windows-2022, win_amd64]
         # TODO: support PyPy?
-        python: [["cp310", "3.10"], ["cp311", "3.11"], ["cp312", "3.12"]]
+        python: [["cp310", "3.10"], ["cp311", "3.11"], ["cp312", "3.12"], ["cp313", "3.13"], ["cp313t", "3.13"]]
+        include:
+        # TODO: Remove this plus installing build deps in cibw_before_build.sh
+        # and test deps in cibw_before_test.sh after pandas can be built with a released NumPy/Cython
+        - python: ["cp313", "3.13"]
+          cibw_build_frontend: 'pip; args: --no-build-isolation'
+        - python: ["cp313t", "3.13"]
+          cibw_build_frontend: 'pip; args: --no-build-isolation'
+        # Build Pyodide wheels and upload them to Anaconda.org
+        # NOTE: this job is similar to the one in unit-tests.yml except for the fact
+        # that it uses cibuildwheel instead of a standard Pyodide xbuildenv setup.
+        - buildplat: [ubuntu-22.04, pyodide_wasm32]
+          python: ["cp312", "3.12"]
+          cibw_build_frontend: 'build'
+        # TODO: Build free-threaded wheels for Windows
+        exclude:
+        - buildplat: [windows-2022, win_amd64]
+          python: ["cp313t", "3.13"]
+
     env:
       IS_PUSH: ${{ github.event_name == 'push' && startsWith(github.ref, 'refs/tags/v') }}
       IS_SCHEDULE_DISPATCH: ${{ github.event_name == 'schedule' || github.event_name == 'workflow_dispatch' }}
@@ -140,12 +158,13 @@ jobs:
         run: echo "sdist_name=$(cd ./dist && ls -d */)" >> "$GITHUB_ENV"
 
       - name: Build wheels
-        uses: pypa/cibuildwheel@v2.19.1
+        uses: pypa/cibuildwheel@v2.20.0
         with:
          package-dir: ./dist/${{ startsWith(matrix.buildplat[1], 'macosx') && env.sdist_name || needs.build_sdist.outputs.sdist_file }}
         env:
-          CIBW_PRERELEASE_PYTHONS: True
           CIBW_BUILD: ${{ matrix.python[0] }}-${{ matrix.buildplat[1] }}
+          CIBW_BUILD_FRONTEND: ${{ matrix.cibw_build_frontend || 'pip' }}
+          CIBW_PLATFORM: ${{ matrix.buildplat[1] == 'pyodide_wasm32' && 'pyodide' || 'auto' }}
 
       - name: Set up Python
         uses: mamba-org/setup-micromamba@v1
@@ -168,15 +187,17 @@ jobs:
       - name: Test Windows Wheels
         if: ${{ matrix.buildplat[1] == 'win_amd64' }}
         shell: pwsh
+        # TODO: Remove NumPy nightly install when there's a 3.13 wheel on PyPI
         run: |
           $TST_CMD = @"
-          python -m pip install hypothesis>=6.46.1 pytest>=7.3.2 pytest-xdist>=2.2.0;
+          python -m pip install hypothesis>=6.84.0 pytest>=7.3.2 pytest-xdist>=3.4.0;
+          ${{ matrix.python[1] == '3.13' && 'python -m pip install -i https://pypi.anaconda.org/scientific-python-nightly-wheels/simple numpy;' }}
           python -m pip install `$(Get-Item pandas\wheelhouse\*.whl);
           python -c `'import pandas as pd; pd.test(extra_args=[`\"--no-strict-data-files`\", `\"-m not clipboard and not single_cpu and not slow and not network and not db`\"])`';
           "@
           # add rc to the end of the image name if the Python version is unreleased
-          docker pull python:${{ matrix.python[1] == '3.12' && '3.12-rc' || format('{0}-windowsservercore', matrix.python[1]) }}
-          docker run --env PANDAS_CI='1' -v ${PWD}:C:\pandas python:${{ matrix.python[1] == '3.12' && '3.12-rc' || format('{0}-windowsservercore', matrix.python[1]) }} powershell -Command $TST_CMD
+          docker pull python:${{ matrix.python[1] == '3.13' && '3.13-rc' || format('{0}-windowsservercore', matrix.python[1]) }}
+          docker run --env PANDAS_CI='1' -v ${PWD}:C:\pandas python:${{ matrix.python[1] == '3.13' && '3.13-rc' || format('{0}-windowsservercore', matrix.python[1]) }} powershell -Command $TST_CMD
 
       - uses: actions/upload-artifact@v4
         with:

.pre-commit-config.yaml

@@ -19,7 +19,7 @@ ci:
     skip: [pyright, mypy]
 repos:
 -   repo: https://github.com/astral-sh/ruff-pre-commit
-    rev: v0.4.7
+    rev: v0.5.0
     hooks:
     -   id: ruff
         args: [--exit-non-zero-on-fix]
@@ -73,7 +73,7 @@ repos:
     hooks:
     -   id: isort
 -   repo: https://github.com/asottile/pyupgrade
-    rev: v3.15.2
+    rev: v3.16.0
     hooks:
     -   id: pyupgrade
         args: [--py310-plus]
@@ -93,7 +93,7 @@ repos:
     - id: sphinx-lint
       args: ["--enable", "all", "--disable", "line-too-long"]
 -   repo: https://github.com/pre-commit/mirrors-clang-format
-    rev: v18.1.5
+    rev: v18.1.8
     hooks:
     - id: clang-format
       files: ^pandas/_libs/src|^pandas/_libs/include

MANIFEST.in

@@ -62,3 +62,7 @@ prune pandas/tests/io/parser/data
 # Selectively re-add *.cxx files that were excluded above
 graft pandas/_libs/src
 graft pandas/_libs/include
+
+# Include cibw script in sdist since it's needed for building wheels
+include scripts/cibw_before_build.sh
+include scripts/cibw_before_test.sh

README.md

@@ -96,7 +96,7 @@ The source code is currently hosted on GitHub at:
 https://github.com/pandas-dev/pandas
 
 Binary installers for the latest released version are available at the [Python
-Package Index (PyPI)](https://pypi.org/project/pandas) and on [Conda](https://docs.conda.io/en/latest/).
+Package Index (PyPI)](https://pypi.org/project/pandas) and on [Conda](https://anaconda.org/conda-forge/pandas).
 
 ```sh
 # conda

asv_bench/benchmarks/indexing.py

@@ -546,24 +546,17 @@ def time_chained_indexing(self, mode):
 
 
 class Block:
-    params = [
-        (True, "True"),
-        (np.array(True), "np.array(True)"),
-    ]
-
-    def setup(self, true_value, mode):
+    def setup(self):
         self.df = DataFrame(
             False,
             columns=np.arange(500).astype(str),
             index=date_range("2010-01-01", "2011-01-01"),
         )
 
-        self.true_value = true_value
-
-    def time_test(self, true_value, mode):
+    def time_test(self):
         start = datetime(2010, 5, 1)
         end = datetime(2010, 9, 1)
-        self.df.loc[start:end, :] = true_value
+        self.df.loc[start:end, :] = True
 
 
 from .pandas_vb_common import setup  # noqa: F401 isort:skip