Merge remote-tracking branch 'upstream/main' into set-__module__-scalars

Author: jorisvandenbossche

.circleci/config.yml

@@ -1,46 +1,64 @@
 version: 2.1
 
 jobs:
-  test-arm:
+  test-linux-arm:
     machine:
       image: default
     resource_class: arm.large
     environment:
-      ENV_FILE: ci/deps/circle-310-arm64.yaml
+      ENV_FILE: ci/deps/circle-311-arm64.yaml
       PYTEST_WORKERS: auto
       PATTERN: "not single_cpu and not slow and not network and not clipboard and not arm_slow and not db"
       PYTEST_TARGET: "pandas"
       PANDAS_CI: "1"
     steps:
       - checkout
-      - run: .circleci/setup_env.sh
-      - run: >
-          PATH=$HOME/miniconda3/envs/pandas-dev/bin:$HOME/miniconda3/condabin:$PATH
-          LD_PRELOAD=$HOME/miniconda3/envs/pandas-dev/lib/libgomp.so.1:$LD_PRELOAD
-          ci/run_tests.sh
-  linux-musl:
+      - run:
+          name: Install Environment and Run Tests
+          shell: /bin/bash -exuo pipefail
+          command: |
+            MAMBA_URL="https://github.com/conda-forge/miniforge/releases/download/24.3.0-0/Mambaforge-24.3.0-0-Linux-aarch64.sh"
+            wget -q $MAMBA_URL -O minimamba.sh
+            chmod +x minimamba.sh
+            MAMBA_DIR="$HOME/miniconda3"
+            rm -rf $MAMBA_DIR
+            ./minimamba.sh -b -p $MAMBA_DIR
+            export PATH=$MAMBA_DIR/bin:$PATH
+            conda info -a
+            conda env create -q -n pandas-dev -f $ENV_FILE
+            conda list -n pandas-dev
+            source activate pandas-dev
+            if pip show pandas 1>/dev/null; then
+                pip uninstall -y pandas
+            fi
+            python -m pip install --no-build-isolation -ve . --config-settings=setup-args="--werror"
+            PATH=$HOME/miniconda3/envs/pandas-dev/bin:$HOME/miniconda3/condabin:$PATH
+            ci/run_tests.sh
+  test-linux-musl:
     docker:
       - image: quay.io/pypa/musllinux_1_1_aarch64
     resource_class: arm.large
     steps:
       # Install pkgs first to have git in the image
       # (needed for checkout)
-      - run: |
-          apk update
-          apk add git
-          apk add musl-locales
+      - run:
+          name: Install System Packages
+          command: |
+            apk update
+            apk add git
+            apk add musl-locales
       - checkout
-      - run: |
-          /opt/python/cp311-cp311/bin/python -m venv ~/virtualenvs/pandas-dev
-          . ~/virtualenvs/pandas-dev/bin/activate
-          python -m pip install --no-cache-dir -U pip wheel setuptools meson-python==0.13.1 meson[ninja]==1.2.1
-          python -m pip install --no-cache-dir versioneer[toml] cython numpy python-dateutil pytz pytest>=7.3.2 pytest-xdist>=2.2.0 hypothesis>=6.46.1
-          python -m pip install --no-cache-dir --no-build-isolation -e . --config-settings=setup-args="--werror"
-          python -m pip list --no-cache-dir
-      - run: |
-          . ~/virtualenvs/pandas-dev/bin/activate
-          export PANDAS_CI=1
-          python -m pytest -m 'not slow and not network and not clipboard and not single_cpu' pandas --junitxml=test-data.xml
+      - run:
+          name: Install Environment and Run Tests
+          command: |
+            /opt/python/cp311-cp311/bin/python -m venv ~/virtualenvs/pandas-dev
+            . ~/virtualenvs/pandas-dev/bin/activate
+            python -m pip install --no-cache-dir -U pip wheel setuptools meson-python==0.13.1 meson[ninja]==1.2.1
+            python -m pip install --no-cache-dir versioneer[toml] cython numpy python-dateutil pytz pytest>=7.3.2 pytest-xdist>=2.2.0 hypothesis>=6.46.1
+            python -m pip install --no-cache-dir --no-build-isolation -e . --config-settings=setup-args="--werror"
+            python -m pip list --no-cache-dir
+            export PANDAS_CI=1
+            python -m pytest -m 'not slow and not network and not clipboard and not single_cpu' pandas --junitxml=test-data.xml
   build-aarch64:
     parameters:
       cibw-build:
@@ -71,11 +89,7 @@ jobs:
           name: Build aarch64 wheels
           no_output_timeout: 30m # Sometimes the tests won't generate any output, make sure the job doesn't get killed by that
           command: |
-            pip3 install cibuildwheel==2.15.0
-            # When this is a nightly wheel build, allow picking up NumPy 2.0 dev wheels:
-            if [[ "$IS_SCHEDULE_DISPATCH" == "true" || "$IS_PUSH" != 'true' ]]; then
-                export CIBW_ENVIRONMENT="PIP_EXTRA_INDEX_URL=https://pypi.anaconda.org/scientific-python-nightly-wheels/simple"
-            fi
+            pip3 install cibuildwheel==2.18.1
             cibuildwheel --prerelease-pythons --output-dir wheelhouse
 
           environment:
@@ -85,7 +99,7 @@ jobs:
           name: Install Anaconda Client & Upload Wheels
           command: |
             echo "Install Mambaforge"
-            MAMBA_URL="https://github.com/conda-forge/miniforge/releases/download/23.1.0-0/Mambaforge-23.1.0-0-Linux-aarch64.sh"
+            MAMBA_URL="https://github.com/conda-forge/miniforge/releases/download/24.3.0-0/Mambaforge-24.3.0-0-Linux-aarch64.sh"
             echo "Downloading $MAMBA_URL"
             wget -q $MAMBA_URL -O minimamba.sh
             chmod +x minimamba.sh
@@ -111,14 +125,14 @@ workflows:
       not:
         equal: [ scheduled_pipeline, << pipeline.trigger_source >> ]
     jobs:
-      - test-arm
+      - test-linux-arm
   test-musl:
     # Don't run trigger this one when scheduled pipeline runs
     when:
       not:
         equal: [ scheduled_pipeline, << pipeline.trigger_source >> ]
     jobs:
-      - linux-musl
+      - test-linux-musl
   build-wheels:
     jobs:
       - build-aarch64:

.circleci/setup_env.sh

@@ -0,0 +1,74 @@
+name: PDEP Vote
+description: Call for a vote on a PDEP
+title: "VOTE: "
+labels: [Vote]
+
+body:
+  - type: markdown
+    attributes:
+      value: >
+        As per [PDEP-1](https://pandas.pydata.org/pdeps/0001-purpose-and-guidelines.html), the following issue template should be used when a
+        maintainer has opened a PDEP discussion and is ready to call for a vote.
+  - type: checkboxes
+    attributes:
+      label: Locked issue
+      options:
+        - label: >
+            I locked this voting issue so that only voting members are able to cast their votes or
+            comment on this issue.
+          required: true
+  - type: input
+    id: PDEP-name
+    attributes:
+      label: PDEP number and title
+      placeholder: >
+        PDEP-1: Purpose and guidelines
+    validations:
+      required: true
+  - type: input
+    id: PDEP-link
+    attributes:
+      label: Pull request with discussion
+      description: e.g. https://github.com/pandas-dev/pandas/pull/47444
+    validations:
+      required: true
+  - type: input
+    id: PDEP-rendered-link
+    attributes:
+      label: Rendered PDEP for easy reading
+      description: e.g. https://github.com/pandas-dev/pandas/pull/47444/files?short_path=7c449e6#diff-7c449e698132205b235c501f7e47ebba38da4d2b7f9492c98f16745dba787041
+    validations:
+      required: true
+  - type: input
+    id: PDEP-number-of-discussion-participants
+    attributes:
+      label: Discussion participants
+      description: >
+        You may find it useful to list or total the number of participating members in the
+        PDEP discussion PR. This would be the maximum possible disapprove votes.
+      placeholder: >
+        14 voting members participated in the PR discussion thus far.
+  - type: input
+    id: PDEP-vote-end
+    attributes:
+      label: Voting will close in 15 days.
+      description: The voting period end date. ('Voting will close in 15 days.' will be automatically written)
+  - type: markdown
+    attributes:
+      value: ---
+  - type: textarea
+    id: Vote
+    attributes:
+      label: Vote
+      value: |
+        Cast your vote in a comment below.
+        * +1: approve.
+        * 0: abstain.
+            * Reason: A one sentence reason is required.
+        * -1: disapprove
+            * Reason: A one sentence reason is required.
+        A disapprove vote requires prior participation in the linked discussion PR.
+
+        @pandas-dev/pandas-core
+    validations:
+      required: true

.github/ISSUE_TEMPLATE/pdep_vote.yaml

@@ -1,8 +1,8 @@
-name: Purge caches once a week
+name: Purge caches daily
 on:
   schedule:
-    # 4:10 UTC on Sunday
-    - cron: "10 4 * * 0"
+    # 4:10 UTC daily
+    - cron: "10 4 * * *"
 
 jobs:
   cleanup:

.github/workflows/cache-cleanup-weekly.yml renamed to .github/workflows/cache-cleanup-daily.yml

@@ -85,7 +85,7 @@ jobs:
         echo "PYTHONPATH=$PYTHONPATH" >> $GITHUB_ENV
       if: ${{ steps.build.outcome == 'success' && always() }}
 
-    - name: Typing + pylint
+    - name: Typing
       uses: pre-commit/[email protected]
       with:
         extra_args: --verbose --hook-stage manual --all-files

.github/workflows/code-checks.yml

@@ -314,7 +314,7 @@ jobs:
     timeout-minutes: 90
 
     concurrency:
-      #https://github.community/t/concurrecy-not-work-for-push/183068/7
+      # https://github.community/t/concurrecy-not-work-for-push/183068/7
       group: ${{ github.event_name == 'push' && github.run_number || github.ref }}-${{ matrix.os }}-${{ matrix.pytest_target }}-dev
       cancel-in-progress: true
 
@@ -346,3 +346,62 @@ jobs:
 
       - name: Run Tests
         uses: ./.github/actions/run-tests
+
+  emscripten:
+    # Note: the Python version, Emscripten toolchain version are determined
+    # by the Pyodide version. The appropriate versions can be found in the
+    # Pyodide repodata.json "info" field, or in the Makefile.envs file:
+    # https://github.com/pyodide/pyodide/blob/stable/Makefile.envs#L2
+    # The Node.js version can be determined via Pyodide:
+    # https://pyodide.org/en/stable/usage/index.html#node-js
+    name: Pyodide build
+    runs-on: ubuntu-22.04
+    concurrency:
+      # https://github.community/t/concurrecy-not-work-for-push/183068/7
+      group: ${{ github.event_name == 'push' && github.run_number || github.ref }}-wasm
+      cancel-in-progress: true
+    steps:
+      - name: Checkout pandas Repo
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+
+      - name: Set up Python for Pyodide
+        id: setup-python
+        uses: actions/setup-python@v5
+        with:
+          python-version: '3.11.3'
+
+      - name: Set up Emscripten toolchain
+        uses: mymindstorm/setup-emsdk@v14
+        with:
+          version: '3.1.46'
+          actions-cache-folder: emsdk-cache
+
+      - name: Install pyodide-build
+        run: pip install "pyodide-build==0.25.1"
+
+      - name: Build pandas for Pyodide
+        run: |
+          pyodide build
+
+      - name: Set up Node.js
+        uses: actions/setup-node@v4
+        with:
+          node-version: '18'
+
+      - name: Set up Pyodide virtual environment
+        run: |
+          pyodide venv .venv-pyodide
+          source .venv-pyodide/bin/activate
+          pip install dist/*.whl
+
+      - name: Test pandas for Pyodide
+        env:
+          PANDAS_CI: 1
+        run: |
+          source .venv-pyodide/bin/activate
+          pip install pytest hypothesis
+          # do not import pandas from the checked out repo
+          cd ..
+          python -c 'import pandas as pd; pd.test(extra_args=["-m not clipboard and not single_cpu and not slow and not network and not db"])'

.github/workflows/unit-tests.yml

@@ -139,27 +139,14 @@ jobs:
         shell: bash -el {0}
         run: echo "sdist_name=$(cd ./dist && ls -d */)" >> "$GITHUB_ENV"
 
-      - name: Build normal wheels
-        if: ${{ (env.IS_SCHEDULE_DISPATCH != 'true' || env.IS_PUSH == 'true') }}
-        uses: pypa/[email protected]
+      - name: Build wheels
+        uses: pypa/[email protected]
         with:
          package-dir: ./dist/${{ startsWith(matrix.buildplat[1], 'macosx') && env.sdist_name || needs.build_sdist.outputs.sdist_file }}
         env:
           CIBW_PRERELEASE_PYTHONS: True
           CIBW_BUILD: ${{ matrix.python[0] }}-${{ matrix.buildplat[1] }}
 
-      - name: Build nightly wheels (with NumPy pre-release)
-        if: ${{ (env.IS_SCHEDULE_DISPATCH == 'true' && env.IS_PUSH != 'true') }}
-        uses: pypa/[email protected]
-        with:
-         package-dir: ./dist/${{ startsWith(matrix.buildplat[1], 'macosx') && env.sdist_name || needs.build_sdist.outputs.sdist_file }}
-        env:
-          # The nightly wheels should be build witht he NumPy 2.0 pre-releases
-          # which requires the additional URL.
-          CIBW_ENVIRONMENT: PIP_EXTRA_INDEX_URL=https://pypi.anaconda.org/scientific-python-nightly-wheels/simple
-          CIBW_PRERELEASE_PYTHONS: True
-          CIBW_BUILD: ${{ matrix.python[0] }}-${{ matrix.buildplat[1] }}
-
       - name: Set up Python
         uses: mamba-org/setup-micromamba@v1
         with: