Merge remote-tracking branch 'upstream/main' into 55837

Author: jorisvandenbossche

.circleci/config.yml

@@ -1,29 +1,72 @@
 version: 2.1
 
 jobs:
-  test-arm:
+  test-linux-arm:
     machine:
-      image: ubuntu-2004:2022.04.1
+      image: default
     resource_class: arm.large
     environment:
-      ENV_FILE: ci/deps/circle-310-arm64.yaml
+      ENV_FILE: ci/deps/circle-311-arm64.yaml
       PYTEST_WORKERS: auto
       PATTERN: "not single_cpu and not slow and not network and not clipboard and not arm_slow and not db"
       PYTEST_TARGET: "pandas"
       PANDAS_CI: "1"
     steps:
       - checkout
-      - run: .circleci/setup_env.sh
-      - run: >
-          PATH=$HOME/miniconda3/envs/pandas-dev/bin:$HOME/miniconda3/condabin:$PATH
-          LD_PRELOAD=$HOME/miniconda3/envs/pandas-dev/lib/libgomp.so.1:$LD_PRELOAD
-          ci/run_tests.sh
+      - run:
+          name: Install Environment and Run Tests
+          shell: /bin/bash -exo pipefail
+          # https://pytest-qt.readthedocs.io/en/latest/troubleshooting.html#github-actions-azure-pipelines-travis-ci-and-gitlab-ci-cd
+          command: |
+            MINI_URL="https://github.com/conda-forge/miniforge/releases/download/24.3.0-0/Miniforge3-24.3.0-0-Linux-aarch64.sh"
+            wget -q $MINI_URL -O Miniforge3.sh
+            chmod +x Miniforge3.sh
+            MINI_DIR="$HOME/miniconda3"
+            rm -rf $MINI_DIR
+            ./Miniforge3.sh -b -p $MINI_DIR
+            export PATH=$MINI_DIR/bin:$PATH
+            conda info -a
+            conda env create -q -n pandas-dev -f $ENV_FILE
+            conda list -n pandas-dev
+            source activate pandas-dev
+            if pip show pandas 1>/dev/null; then
+                pip uninstall -y pandas
+            fi
+            python -m pip install --no-build-isolation -ve . -Csetup-args="--werror"
+            PATH=$HOME/miniconda3/envs/pandas-dev/bin:$HOME/miniconda3/condabin:$PATH
+            sudo apt-get update && sudo apt-get install -y libegl1 libopengl0
+            ci/run_tests.sh
+  test-linux-musl:
+    docker:
+      - image: quay.io/pypa/musllinux_1_1_aarch64
+    resource_class: arm.large
+    steps:
+      # Install pkgs first to have git in the image
+      # (needed for checkout)
+      - run:
+          name: Install System Packages
+          command: |
+            apk update
+            apk add git
+            apk add musl-locales
+      - checkout
+      - run:
+          name: Install Environment and Run Tests
+          command: |
+            /opt/python/cp311-cp311/bin/python -m venv ~/virtualenvs/pandas-dev
+            . ~/virtualenvs/pandas-dev/bin/activate
+            python -m pip install --no-cache-dir -U pip wheel setuptools meson-python==0.13.1 meson[ninja]==1.2.1
+            python -m pip install --no-cache-dir versioneer[toml] cython numpy python-dateutil pytest>=7.3.2 pytest-xdist>=3.4.0 hypothesis>=6.84.0
+            python -m pip install --no-cache-dir --no-build-isolation -e . -Csetup-args="--werror"
+            python -m pip list --no-cache-dir
+            export PANDAS_CI=1
+            python -m pytest -m 'not slow and not network and not clipboard and not single_cpu' pandas --junitxml=test-data.xml
   build-aarch64:
     parameters:
       cibw-build:
         type: string
     machine:
-      image: ubuntu-2004:2022.04.1
+      image: default
     resource_class: arm.large
     environment:
       TRIGGER_SOURCE: << pipeline.trigger_source >>
@@ -48,33 +91,30 @@ jobs:
           name: Build aarch64 wheels
           no_output_timeout: 30m # Sometimes the tests won't generate any output, make sure the job doesn't get killed by that
           command: |
-            pip3 install cibuildwheel==2.15.0
-            # When this is a nightly wheel build, allow picking up NumPy 2.0 dev wheels:
-            if [[ "$IS_SCHEDULE_DISPATCH" == "true" || "$IS_PUSH" != 'true' ]]; then
-                export CIBW_ENVIRONMENT="PIP_EXTRA_INDEX_URL=https://pypi.anaconda.org/scientific-python-nightly-wheels/simple"
+            pip3 install cibuildwheel==2.20.0
+            if [[ $CIBW_BUILD == cp313t* ]]; then
+              # TODO: temporarily run 3.13 free threaded builds without build isolation
+              # since we need pre-release cython
+              CIBW_BUILD_FRONTEND="pip; args: --no-build-isolation" cibuildwheel --output-dir wheelhouse
+            else
+              cibuildwheel --output-dir wheelhouse
             fi
-            cibuildwheel --prerelease-pythons --output-dir wheelhouse
 
           environment:
             CIBW_BUILD: << parameters.cibw-build >>
 
       - run:
           name: Install Anaconda Client & Upload Wheels
+          shell: /bin/bash -exo pipefail
           command: |
-            echo "Install Mambaforge"
-            MAMBA_URL="https://github.com/conda-forge/miniforge/releases/download/23.1.0-0/Mambaforge-23.1.0-0-Linux-aarch64.sh"
-            echo "Downloading $MAMBA_URL"
-            wget -q $MAMBA_URL -O minimamba.sh
-            chmod +x minimamba.sh
-
-            MAMBA_DIR="$HOME/miniconda3"
-            rm -rf $MAMBA_DIR
-            ./minimamba.sh -b -p $MAMBA_DIR
-
-            export PATH=$MAMBA_DIR/bin:$PATH
-
-            mamba install -y -c conda-forge anaconda-client
-
+            MINI_URL="https://github.com/conda-forge/miniforge/releases/download/24.3.0-0/Miniforge3-24.3.0-0-Linux-aarch64.sh"
+            wget -q $MINI_URL -O Miniforge3.sh
+            chmod +x Miniforge3.sh
+            MINI_DIR="$HOME/miniconda3"
+            rm -rf $MINI_DIR
+            ./Miniforge3.sh -b -p $MINI_DIR
+            export PATH=$MINI_DIR/bin:$PATH
+            conda install -y -c conda-forge anaconda-client
             source ci/upload_wheels.sh
             set_upload_vars
             upload_wheels
@@ -88,7 +128,14 @@ workflows:
       not:
         equal: [ scheduled_pipeline, << pipeline.trigger_source >> ]
     jobs:
-      - test-arm
+      - test-linux-arm
+  test-musl:
+    # Don't run trigger this one when scheduled pipeline runs
+    when:
+      not:
+        equal: [ scheduled_pipeline, << pipeline.trigger_source >> ]
+    jobs:
+      - test-linux-musl
   build-wheels:
     jobs:
       - build-aarch64:
@@ -97,4 +144,13 @@ workflows:
               only: /^v.*/
           matrix:
             parameters:
-              cibw-build: ["cp39-manylinux_aarch64", "cp310-manylinux_aarch64", "cp311-manylinux_aarch64", "cp312-manylinux_aarch64"]
+              cibw-build: ["cp310-manylinux_aarch64",
+                           "cp311-manylinux_aarch64",
+                           "cp312-manylinux_aarch64",
+                           "cp313-manylinux_aarch64",
+                           "cp313t-manylinux_aarch64",
+                           "cp310-musllinux_aarch64",
+                           "cp311-musllinux_aarch64",
+                           "cp312-musllinux_aarch64",
+                           "cp313-musllinux_aarch64",
+                           "cp313t-musllinux_aarch64"]

.circleci/setup_env.sh

@@ -8,7 +8,6 @@
 	// Use 'settings' to set *default* container specific settings.json values on container create.
 	// You can edit these settings after create using File > Preferences > Settings > Remote.
 	"settings": {
-		"terminal.integrated.shell.linux": "/bin/bash",
 		"python.pythonPath": "/usr/local/bin/python",
 		"python.formatting.provider": "black",
 		"python.linting.enabled": true,

.devcontainer.json

@@ -68,7 +68,7 @@ ci export-ignore
 doc export-ignore
 gitpod export-ignore
 MANIFEST.in export-ignore
-scripts export-ignore
+scripts/** export-ignore
 typings export-ignore
 web export-ignore
 CITATION.cff export-ignore
@@ -82,3 +82,7 @@ setup.py export-ignore
 # csv_dir_path fixture checks the existence of the directory
 # exclude the whole directory to avoid running related tests in sdist
 pandas/tests/io/parser/data export-ignore
+
+# Include cibw script in sdist since it's needed for building wheels
+scripts/cibw_before_build.sh -export-ignore
+scripts/cibw_before_test.sh -export-ignore

.gitattributes

@@ -0,0 +1,74 @@
+name: PDEP Vote
+description: Call for a vote on a PDEP
+title: "VOTE: "
+labels: [Vote]
+
+body:
+  - type: markdown
+    attributes:
+      value: >
+        As per [PDEP-1](https://pandas.pydata.org/pdeps/0001-purpose-and-guidelines.html), the following issue template should be used when a
+        maintainer has opened a PDEP discussion and is ready to call for a vote.
+  - type: checkboxes
+    attributes:
+      label: Locked issue
+      options:
+        - label: >
+            I locked this voting issue so that only voting members are able to cast their votes or
+            comment on this issue.
+          required: true
+  - type: input
+    id: PDEP-name
+    attributes:
+      label: PDEP number and title
+      placeholder: >
+        PDEP-1: Purpose and guidelines
+    validations:
+      required: true
+  - type: input
+    id: PDEP-link
+    attributes:
+      label: Pull request with discussion
+      description: e.g. https://github.com/pandas-dev/pandas/pull/47444
+    validations:
+      required: true
+  - type: input
+    id: PDEP-rendered-link
+    attributes:
+      label: Rendered PDEP for easy reading
+      description: e.g. https://github.com/pandas-dev/pandas/pull/47444/files?short_path=7c449e6#diff-7c449e698132205b235c501f7e47ebba38da4d2b7f9492c98f16745dba787041
+    validations:
+      required: true
+  - type: input
+    id: PDEP-number-of-discussion-participants
+    attributes:
+      label: Discussion participants
+      description: >
+        You may find it useful to list or total the number of participating members in the
+        PDEP discussion PR. This would be the maximum possible disapprove votes.
+      placeholder: >
+        14 voting members participated in the PR discussion thus far.
+  - type: input
+    id: PDEP-vote-end
+    attributes:
+      label: Voting will close in 15 days.
+      description: The voting period end date. ('Voting will close in 15 days.' will be automatically written)
+  - type: markdown
+    attributes:
+      value: ---
+  - type: textarea
+    id: Vote
+    attributes:
+      label: Vote
+      value: |
+        Cast your vote in a comment below.
+        * +1: approve.
+        * 0: abstain.
+            * Reason: A one sentence reason is required.
+        * -1: disapprove
+            * Reason: A one sentence reason is required.
+        A disapprove vote requires prior participation in the linked discussion PR.
+
+        @pandas-dev/pandas-core
+    validations:
+      required: true

.github/ISSUE_TEMPLATE/pdep_vote.yaml

@@ -22,11 +22,20 @@ runs:
         fi
       shell: bash -el {0}
 
+    - name: Uninstall nomkl
+      run: |
+        if conda list nomkl | grep nomkl 1>/dev/null; then
+          conda remove nomkl -y
+        fi
+      shell: bash -el {0}
+
     - name: Build Pandas
       run: |
         if [[ ${{ inputs.editable }} == "true" ]]; then
-          pip install -e . --no-build-isolation -v --no-deps
+          pip install -e . --no-build-isolation -v --no-deps \
+            -Csetup-args="--werror"
         else
-          pip install . --no-build-isolation -v --no-deps
+          pip install . --no-build-isolation -v --no-deps \
+            -Csetup-args="--werror"
         fi
       shell: bash -el {0}

.github/actions/build_pandas/action.yml

@@ -7,19 +7,14 @@ runs:
       shell: bash -el {0}
 
     - name: Publish test results
-      uses: actions/upload-artifact@v3
+      uses: actions/upload-artifact@v4
       with:
         name: Test results
         path: test-data.xml
       if: failure()
 
-    - name: Report Coverage
-      run: coverage report -m
-      shell: bash -el {0}
-      if: failure()
-
     - name: Upload coverage to Codecov
-      uses: codecov/codecov-action@v3
+      uses: codecov/codecov-action@v4
       with:
         flags: unittests
         name: codecov-pandas

.github/actions/run-tests/action.yml

@@ -9,8 +9,16 @@ runs:
     - name: Install ${{ inputs.environment-file }}
       uses: mamba-org/setup-micromamba@v1
       with:
+        # Pinning to avoid 2.0 failures
+        micromamba-version: '1.5.10-0'
         environment-file: ${{ inputs.environment-file }}
         environment-name: test
-        condarc-file: ci/condarc.yml
+        condarc-file: ci/.condarc
         cache-environment: true
         cache-downloads: true
+
+    - name: Uninstall pyarrow
+      if: ${{ env.REMOVE_PYARROW == '1' }}
+      run: |
+        micromamba remove -y pyarrow
+      shell: bash -el {0}

.github/actions/setup-conda/action.yml

@@ -9,6 +9,7 @@ on:
       - "doc/make.py"
 jobs:
   linkcheck:
+    if: false
     runs-on: ubuntu-latest
     defaults:
       run: