Merge branch 'main' into daydst2

Author: jbrockmendel

.circleci/config.yml

@@ -3,7 +3,7 @@ version: 2.1
 jobs:
   test-arm:
     machine:
-      image: ubuntu-2004:2022.04.1
+      image: default
     resource_class: arm.large
     environment:
       ENV_FILE: ci/deps/circle-310-arm64.yaml
@@ -18,12 +18,35 @@ jobs:
           PATH=$HOME/miniconda3/envs/pandas-dev/bin:$HOME/miniconda3/condabin:$PATH
           LD_PRELOAD=$HOME/miniconda3/envs/pandas-dev/lib/libgomp.so.1:$LD_PRELOAD
           ci/run_tests.sh
+  linux-musl:
+    docker:
+      - image: quay.io/pypa/musllinux_1_1_aarch64
+    resource_class: arm.large
+    steps:
+      # Install pkgs first to have git in the image
+      # (needed for checkout)
+      - run: |
+          apk update
+          apk add git
+          apk add musl-locales
+      - checkout
+      - run: |
+          /opt/python/cp311-cp311/bin/python -m venv ~/virtualenvs/pandas-dev
+          . ~/virtualenvs/pandas-dev/bin/activate
+          python -m pip install --no-cache-dir -U pip wheel setuptools meson-python==0.13.1 meson[ninja]==1.2.1
+          python -m pip install --no-cache-dir versioneer[toml] cython numpy python-dateutil pytz pytest>=7.3.2 pytest-xdist>=2.2.0 hypothesis>=6.46.1
+          python -m pip install --no-cache-dir --no-build-isolation -e . --config-settings=setup-args="--werror"
+          python -m pip list --no-cache-dir
+      - run: |
+          . ~/virtualenvs/pandas-dev/bin/activate
+          export PANDAS_CI=1
+          python -m pytest -m 'not slow and not network and not clipboard and not single_cpu' pandas --junitxml=test-data.xml
   build-aarch64:
     parameters:
       cibw-build:
         type: string
     machine:
-      image: ubuntu-2004:2022.04.1
+      image: default
     resource_class: arm.large
     environment:
       TRIGGER_SOURCE: << pipeline.trigger_source >>
@@ -89,6 +112,13 @@ workflows:
         equal: [ scheduled_pipeline, << pipeline.trigger_source >> ]
     jobs:
       - test-arm
+  test-musl:
+    # Don't run trigger this one when scheduled pipeline runs
+    when:
+      not:
+        equal: [ scheduled_pipeline, << pipeline.trigger_source >> ]
+    jobs:
+      - linux-musl
   build-wheels:
     jobs:
       - build-aarch64:
@@ -97,4 +127,11 @@ workflows:
               only: /^v.*/
           matrix:
             parameters:
-              cibw-build: ["cp39-manylinux_aarch64", "cp310-manylinux_aarch64", "cp311-manylinux_aarch64", "cp312-manylinux_aarch64"]
+              cibw-build: ["cp39-manylinux_aarch64",
+                           "cp310-manylinux_aarch64",
+                           "cp311-manylinux_aarch64",
+                           "cp312-manylinux_aarch64",
+                           "cp39-musllinux_aarch64",
+                           "cp310-musllinux_aarch64",
+                           "cp311-musllinux_aarch64",
+                           "cp312-musllinux_aarch64",]

.github/actions/build_pandas/action.yml

@@ -4,6 +4,12 @@ inputs:
   editable:
     description: Whether to build pandas in editable mode (default true)
     default: true
+  meson_args:
+    description: Extra flags to pass to meson
+    required: false
+  cflags_adds:
+    description: Items to append to the CFLAGS variable
+    required: false
 runs:
   using: composite
   steps:
@@ -24,11 +30,12 @@ runs:
 
     - name: Build Pandas
       run: |
+        export CFLAGS="$CFLAGS ${{ inputs.cflags_adds }}"
         if [[ ${{ inputs.editable }} == "true" ]]; then
-          pip install -e . --no-build-isolation -v --no-deps \
+          pip install -e . --no-build-isolation -v --no-deps ${{ inputs.meson_args }} \
             --config-settings=setup-args="--werror"
         else
-          pip install . --no-build-isolation -v --no-deps \
+          pip install . --no-build-isolation -v --no-deps ${{ inputs.meson_args }} \
             --config-settings=setup-args="--werror"
         fi
       shell: bash -el {0}

.github/actions/run-tests/action.yml

@@ -13,11 +13,6 @@ runs:
         path: test-data.xml
       if: failure()
 
-    - name: Report Coverage
-      run: coverage report -m
-      shell: bash -el {0}
-      if: failure()
-
     - name: Upload coverage to Codecov
       uses: codecov/codecov-action@v3
       with:

.github/workflows/broken-linkcheck.yml

@@ -9,6 +9,7 @@ on:
       - "doc/make.py"
 jobs:
   linkcheck:
+    if: false
     runs-on: ubuntu-latest
     defaults:
       run:

.github/workflows/code-checks.yml

@@ -4,11 +4,11 @@ on:
   push:
     branches:
       - main
-      - 2.1.x
+      - 2.2.x
   pull_request:
     branches:
       - main
-      - 2.1.x
+      - 2.2.x
 
 env:
   ENV_FILE: environment.yml
@@ -86,7 +86,7 @@ jobs:
       if: ${{ steps.build.outcome == 'success' && always() }}
 
     - name: Typing + pylint
-      uses: pre-commit/[email protected].0
+      uses: pre-commit/[email protected].1
       with:
         extra_args: --verbose --hook-stage manual --all-files
       if: ${{ steps.build.outcome == 'success' && always() }}

.github/workflows/comment-commands.yml

@@ -24,12 +24,10 @@ jobs:
     concurrency:
       group: ${{ github.actor }}-preview-docs
     steps:
-      - run: |
-          if curl --output /dev/null --silent --head --fail "https://pandas.pydata.org/preview/${{ github.event.issue.number }}/"; then
-            curl -H "Authorization: token ${{ secrets.GITHUB_TOKEN }}" -d '{"body": "Website preview of this PR available at: https://pandas.pydata.org/preview/${{ github.event.issue.number }}/"}' https://api.github.com/repos/${{ github.repository }}/issues/${{ github.event.issue.number }}/comments
-          else
-            curl -H "Authorization: token ${{ secrets.GITHUB_TOKEN }}" -d '{"body": "No preview found for PR #${{ github.event.issue.number }}. Did the docs build complete?"}' https://api.github.com/repos/${{ github.repository }}/issues/${{ github.event.issue.number }}/comments
-          fi
+      - uses: pandas-dev/[email protected]
+        with:
+          previewer-server: "https://pandas.pydata.org/preview"
+          artifact-job: "Doc Build and Upload"
   asv_run:
     runs-on: ubuntu-22.04
     # TODO: Support more benchmarking options later, against different branches, against self, etc

.github/workflows/deprecation-tracking-bot.yml

@@ -19,7 +19,7 @@ jobs:
       issues: write
     runs-on: ubuntu-22.04
     env:
-      DEPRECATION_TRACKER_ISSUE: 50578
+      DEPRECATION_TRACKER_ISSUE: 56596
     steps:
     - uses: actions/github-script@v7
       id: update-deprecation-issue

.github/workflows/docbuild-and-upload.yml

@@ -4,13 +4,13 @@ on:
   push:
     branches:
       - main
-      - 2.1.x
+      - 2.2.x
     tags:
       - '*'
   pull_request:
     branches:
       - main
-      - 2.1.x
+      - 2.2.x
 
 env:
   ENV_FILE: environment.yml
@@ -90,10 +90,3 @@ jobs:
         name: website
         path: web/build
         retention-days: 14
-
-    - name: Trigger web/doc preview
-      run: curl -X POST https://pandas.pydata.org/preview/submit/$RUN_ID/$PR_ID/
-      env:
-        RUN_ID: ${{ github.run_id }}
-        PR_ID: ${{ github.event.pull_request.number }}
-      if: github.event_name == 'pull_request'

.github/workflows/package-checks.yml

@@ -4,11 +4,11 @@ on:
   push:
     branches:
       - main
-      - 2.1.x
+      - 2.2.x
   pull_request:
     branches:
       - main
-      - 2.1.x
+      - 2.2.x
     types: [ labeled, opened, synchronize, reopened ]
 
 permissions:
@@ -24,7 +24,7 @@ jobs:
     runs-on: ubuntu-22.04
     strategy:
       matrix:
-        extra: ["test", "performance", "computation", "fss", "aws", "gcp", "excel", "parquet", "feather", "hdf5", "spss", "postgresql", "mysql", "sql-other", "html", "xml", "plot", "output-formatting", "clipboard", "compression", "consortium-standard", "all"]
+        extra: ["test", "pyarrow", "performance", "computation", "fss", "aws", "gcp", "excel", "parquet", "feather", "hdf5", "spss", "postgresql", "mysql", "sql-other", "html", "xml", "plot", "output-formatting", "clipboard", "compression", "all"]
       fail-fast: false
     name: Install Extras - ${{ matrix.extra }}
     concurrency:

.github/workflows/unit-tests.yml

@@ -4,11 +4,11 @@ on:
   push:
     branches:
       - main
-      - 2.1.x
+      - 2.2.x
   pull_request:
     branches:
       - main
-      - 2.1.x
+      - 2.2.x
     paths-ignore:
       - "doc/**"
       - "web/**"
@@ -57,34 +57,6 @@ jobs:
             # Also install zh_CN (its encoding is gb2312) but do not activate it.
             # It will be temporarily activated during tests with locale.setlocale
             extra_loc: "zh_CN"
-          - name: "Copy-on-Write 3.9"
-            env_file: actions-39.yaml
-            pattern: "not slow and not network and not single_cpu"
-            pandas_copy_on_write: "1"
-          - name: "Copy-on-Write 3.10"
-            env_file: actions-310.yaml
-            pattern: "not slow and not network and not single_cpu"
-            pandas_copy_on_write: "1"
-          - name: "Copy-on-Write 3.11"
-            env_file: actions-311.yaml
-            pattern: "not slow and not network and not single_cpu"
-            pandas_copy_on_write: "1"
-          - name: "Copy-on-Write 3.12"
-            env_file: actions-312.yaml
-            pattern: "not slow and not network and not single_cpu"
-            pandas_copy_on_write: "1"
-          - name: "Copy-on-Write 3.11 (warnings)"
-            env_file: actions-311.yaml
-            pattern: "not slow and not network and not single_cpu"
-            pandas_copy_on_write: "warn"
-          - name: "Copy-on-Write 3.10 (warnings)"
-            env_file: actions-310.yaml
-            pattern: "not slow and not network and not single_cpu"
-            pandas_copy_on_write: "warn"
-          - name: "Copy-on-Write 3.9 (warnings)"
-            env_file: actions-39.yaml
-            pattern: "not slow and not network and not single_cpu"
-            pandas_copy_on_write: "warn"
           - name: "Pypy"
             env_file: actions-pypy-39.yaml
             pattern: "not slow and not network and not single_cpu"
@@ -102,16 +74,16 @@ jobs:
       PATTERN: ${{ matrix.pattern }}
       LANG: ${{ matrix.lang || 'C.UTF-8' }}
       LC_ALL: ${{ matrix.lc_all || '' }}
-      PANDAS_COPY_ON_WRITE: ${{ matrix.pandas_copy_on_write || '0' }}
       PANDAS_CI: ${{ matrix.pandas_ci || '1' }}
       TEST_ARGS: ${{ matrix.test_args || '' }}
-      PYTEST_WORKERS: 'auto'
+      PYTEST_WORKERS: ${{ matrix.pytest_workers || 'auto' }}
       PYTEST_TARGET: ${{ matrix.pytest_target || 'pandas' }}
+      NPY_PROMOTION_STATE: ${{ matrix.env_file == 'actions-311-numpydev.yaml' && 'weak' || 'legacy' }}
       # Clipboard tests
       QT_QPA_PLATFORM: offscreen
     concurrency:
       # https://github.community/t/concurrecy-not-work-for-push/183068/7
-      group: ${{ github.event_name == 'push' && github.run_number || github.ref }}-${{ matrix.env_file }}-${{ matrix.pattern }}-${{ matrix.extra_apt || '' }}-${{ matrix.pandas_copy_on_write || '' }}
+      group: ${{ github.event_name == 'push' && github.run_number || github.ref }}-${{ matrix.env_file }}-${{ matrix.pattern }}-${{ matrix.extra_apt || '' }}}
       cancel-in-progress: true
 
     services:
@@ -174,6 +146,9 @@ jobs:
     - name: Build Pandas
       id: build
       uses: ./.github/actions/build_pandas
+      with:
+        meson_args: ${{ matrix.meson_args }}
+        cflags_adds: ${{ matrix.cflags_adds }}
 
     - name: Test (not single_cpu)
       uses: ./.github/actions/run-tests
@@ -193,7 +168,8 @@ jobs:
     timeout-minutes: 90
     strategy:
       matrix:
-        os: [macos-latest, windows-latest]
+        # Note: Don't use macOS latest since macos 14 appears to be arm64 only
+        os: [macos-13, macos-14, windows-latest]
         env_file: [actions-39.yaml, actions-310.yaml, actions-311.yaml, actions-312.yaml]
       fail-fast: false
     runs-on: ${{ matrix.os }}
@@ -206,8 +182,7 @@ jobs:
       PANDAS_CI: 1
       PYTEST_TARGET: pandas
       PATTERN: "not slow and not db and not network and not single_cpu"
-      # GH 47443: PYTEST_WORKERS > 0 crashes Windows builds with memory related errors
-      PYTEST_WORKERS: ${{ matrix.os == 'macos-latest' && 'auto' || '0' }}
+      PYTEST_WORKERS: 'auto'
 
     steps:
       - name: Checkout
@@ -333,7 +308,8 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        os: [ubuntu-22.04, macOS-latest, windows-latest]
+        # Separate out macOS 13 and 14, since macOS 14 is arm64 only
+        os: [ubuntu-22.04, macOS-13, macOS-14, windows-latest]
 
     timeout-minutes: 90