Merge remote-tracking branch 'upstream/main' into regr_index_map_tz

# Conflicts:
#	doc/source/whatsnew/v2.2.1.rst

Author: rhshadrach

.circleci/config.yml

@@ -3,7 +3,7 @@ version: 2.1
 jobs:
   test-arm:
     machine:
-      image: ubuntu-2004:2022.04.1
+      image: default
     resource_class: arm.large
     environment:
       ENV_FILE: ci/deps/circle-310-arm64.yaml
@@ -46,7 +46,7 @@ jobs:
       cibw-build:
         type: string
     machine:
-      image: ubuntu-2004:2022.04.1
+      image: default
     resource_class: arm.large
     environment:
       TRIGGER_SOURCE: << pipeline.trigger_source >>

.github/actions/run-tests/action.yml

@@ -1,16 +1,9 @@
 name: Run tests and report results
-inputs:
-  preload:
-    description: Preload arguments for sanitizer
-    required: false
-  asan_options:
-    description: Arguments for Address Sanitizer (ASAN)
-    required: false
 runs:
   using: composite
   steps:
     - name: Test
-      run: ${{ inputs.asan_options }} ${{ inputs.preload }} ci/run_tests.sh
+      run: ci/run_tests.sh
       shell: bash -el {0}
 
     - name: Publish test results

.github/workflows/comment-commands.yml

@@ -24,7 +24,7 @@ jobs:
     concurrency:
       group: ${{ github.actor }}-preview-docs
     steps:
-      - uses: pandas-dev/[email protected].1
+      - uses: pandas-dev/[email protected].2
         with:
           previewer-server: "https://pandas.pydata.org/preview"
           artifact-job: "Doc Build and Upload"

.github/workflows/package-checks.yml

@@ -24,7 +24,7 @@ jobs:
     runs-on: ubuntu-22.04
     strategy:
       matrix:
-        extra: ["test", "performance", "computation", "fss", "aws", "gcp", "excel", "parquet", "feather", "hdf5", "spss", "postgresql", "mysql", "sql-other", "html", "xml", "plot", "output-formatting", "clipboard", "compression", "consortium-standard", "all"]
+        extra: ["test", "pyarrow", "performance", "computation", "fss", "aws", "gcp", "excel", "parquet", "feather", "hdf5", "spss", "postgresql", "mysql", "sql-other", "html", "xml", "plot", "output-formatting", "clipboard", "compression", "all"]
       fail-fast: false
     name: Install Extras - ${{ matrix.extra }}
     concurrency:

.github/workflows/unit-tests.yml

@@ -64,21 +64,10 @@ jobs:
           - name: "Numpy Dev"
             env_file: actions-311-numpydev.yaml
             pattern: "not slow and not network and not single_cpu"
-            # Currently restricted the warnings that error to Deprecation Warnings from numpy
-            # done since pyarrow isn't compatible with numpydev always
-            # TODO: work with pyarrow to revert this?
-            test_args: "-W error::DeprecationWarning:numpy -W error::FutureWarning:numpy"
+            test_args: "-W error::DeprecationWarning -W error::FutureWarning"
           - name: "Pyarrow Nightly"
             env_file: actions-311-pyarrownightly.yaml
             pattern: "not slow and not network and not single_cpu"
-          - name: "ASAN / UBSAN"
-            env_file: actions-311-sanitizers.yaml
-            pattern: "not slow and not network and not single_cpu and not skip_ubsan"
-            asan_options: "ASAN_OPTIONS=detect_leaks=0"
-            preload: LD_PRELOAD=$(gcc -print-file-name=libasan.so)
-            meson_args: --config-settings=setup-args="-Db_sanitize=address,undefined"
-            cflags_adds: -fno-sanitize-recover=all
-            pytest_workers: -1  # disable pytest-xdist as it swallows stderr from ASAN
       fail-fast: false
     name: ${{ matrix.name || format('ubuntu-latest {0}', matrix.env_file) }}
     env:
@@ -164,18 +153,12 @@ jobs:
     - name: Test (not single_cpu)
       uses: ./.github/actions/run-tests
       if: ${{ matrix.name != 'Pypy' }}
-      with:
-        preload: ${{ matrix.preload }}
-        asan_options: ${{ matrix.asan_options }}
       env:
         # Set pattern to not single_cpu if not already set
         PATTERN: ${{ env.PATTERN == '' && 'not single_cpu' || matrix.pattern }}
 
     - name: Test (single_cpu)
       uses: ./.github/actions/run-tests
-      with:
-        preload: ${{ matrix.preload }}
-        asan_options: ${{ matrix.asan_options }}
       env:
         PATTERN: 'single_cpu'
         PYTEST_WORKERS: 0

.github/workflows/wheels.yml

@@ -141,7 +141,7 @@ jobs:
 
       - name: Build normal wheels
         if: ${{ (env.IS_SCHEDULE_DISPATCH != 'true' || env.IS_PUSH == 'true') }}
-        uses: pypa/cibuildwheel@v2.16.5
+        uses: pypa/cibuildwheel@v2.17.0
         with:
          package-dir: ./dist/${{ startsWith(matrix.buildplat[1], 'macosx') && env.sdist_name || needs.build_sdist.outputs.sdist_file }}
         env:
@@ -150,7 +150,7 @@ jobs:
 
       - name: Build nightly wheels (with NumPy pre-release)
         if: ${{ (env.IS_SCHEDULE_DISPATCH == 'true' && env.IS_PUSH != 'true') }}
-        uses: pypa/cibuildwheel@v2.16.5
+        uses: pypa/cibuildwheel@v2.17.0
         with:
          package-dir: ./dist/${{ startsWith(matrix.buildplat[1], 'macosx') && env.sdist_name || needs.build_sdist.outputs.sdist_file }}
         env:

.pre-commit-config.yaml

@@ -19,7 +19,7 @@ ci:
     skip: [pylint, pyright, mypy]
 repos:
 -   repo: https://github.com/astral-sh/ruff-pre-commit
-    rev: v0.1.13
+    rev: v0.3.1
     hooks:
     -   id: ruff
         args: [--exit-non-zero-on-fix]
@@ -30,6 +30,12 @@ repos:
         files: ^pandas
         exclude: ^pandas/tests
         args: [--select, "ANN001,ANN2", --fix-only, --exit-non-zero-on-fix]
+    -   id: ruff
+        name: ruff-use-pd_array-in-core
+        alias: ruff-use-pd_array-in-core
+        files: ^pandas/core/
+        exclude: ^pandas/core/api\.py$
+        args: [--select, "ICN001", --exit-non-zero-on-fix]
     -   id: ruff-format
         exclude: ^scripts
 -   repo: https://github.com/jendrikseipp/vulture
@@ -53,13 +59,11 @@ repos:
 -   repo: https://github.com/pre-commit/pre-commit-hooks
     rev: v4.5.0
     hooks:
-    -   id: check-ast
     -   id: check-case-conflict
     -   id: check-toml
     -   id: check-xml
     -   id: check-yaml
         exclude: ^ci/meta.yaml$
-    -   id: debug-statements
     -   id: end-of-file-fixer
         exclude: \.txt$
     -   id: mixed-line-ending
@@ -74,7 +78,7 @@ repos:
     hooks:
     -   id: pylint
         stages: [manual]
-        args: [--load-plugins=pylint.extensions.redefined_loop_name]
+        args: [--load-plugins=pylint.extensions.redefined_loop_name, --fail-on=I0021]
     -   id: pylint
         alias: redefined-outer-name
         name: Redefining name from outer scope
@@ -129,7 +133,7 @@ repos:
         types: [python]
         stages: [manual]
         additional_dependencies: &pyright_dependencies
-        - [email protected].347
+        - [email protected].351
     -   id: pyright
         # note: assumes python env is setup and activated
         name: pyright reportGeneralTypeIssues
@@ -274,13 +278,6 @@ repos:
         language: python
         entry: python scripts/validate_unwanted_patterns.py --validation-type="nodefault_used_not_only_for_typing"
         types: [python]
-    -   id: use-pd_array-in-core
-        name: Import pandas.array as pd_array in core
-        language: python
-        entry: python scripts/use_pd_array_in_core.py
-        files: ^pandas/core/
-        exclude: ^pandas/core/api\.py$
-        types: [python]
     -   id: no-return-exception
         name: Use raise instead of return for exceptions
         language: pygrep

Dockerfile

@@ -9,6 +9,7 @@ RUN apt-get install -y build-essential
 RUN apt-get install -y libhdf5-dev libgles2-mesa-dev
 
 RUN python -m pip install --upgrade pip
-RUN python -m pip install \
-    -r https://raw.githubusercontent.com/pandas-dev/pandas/main/requirements-dev.txt
+COPY requirements-dev.txt /tmp
+RUN python -m pip install -r /tmp/requirements-dev.txt
+RUN git config --global --add safe.directory /home/pandas
 CMD ["/bin/bash"]

asv_bench/benchmarks/categoricals.py

@@ -88,7 +88,7 @@ def setup(self):
         )
 
         for col in ("int", "float", "timestamp"):
-            self.df[col + "_as_str"] = self.df[col].astype(str)
+            self.df[f"{col}_as_str"] = self.df[col].astype(str)
 
         for col in self.df.columns:
             self.df[col] = self.df[col].astype("category")

asv_bench/benchmarks/indexing.py

@@ -3,6 +3,7 @@
 lower-level methods directly on Index and subclasses, see index_object.py,
 indexing_engine.py, and index_cached.py
 """
+
 from datetime import datetime
 import warnings