Merge branch 'pandas-dev:main' into niruta-61662-setitem-doc

Author: niruta25

.github/workflows/unit-tests.yml

@@ -75,11 +75,6 @@ jobs:
             env_file: actions-311.yaml
             pandas_future_infer_string: "1"
             platform: ubuntu-24.04
-          - name: "Pypy"
-            env_file: actions-pypy-39.yaml
-            pattern: "not slow and not network and not single_cpu"
-            test_args: "--max-worker-restart 0"
-            platform: ubuntu-24.04
           - name: "Numpy Dev"
             env_file: actions-311-numpydev.yaml
             pattern: "not slow and not network and not single_cpu"
@@ -140,9 +135,6 @@ jobs:
 
       moto:
         image: motoserver/moto:5.0.27
-        env:
-          AWS_ACCESS_KEY_ID: foobar_key
-          AWS_SECRET_ACCESS_KEY: foobar_secret
         ports:
           - 5000:5000
 
@@ -172,12 +164,9 @@ jobs:
       with:
         # xref https://github.com/cython/cython/issues/6870
         werror: ${{ matrix.name != 'Freethreading' }}
-      # TODO: Re-enable once Pypy has Pypy 3.10 on conda-forge
-      if: ${{ matrix.name != 'Pypy' }}
 
     - name: Test (not single_cpu)
       uses: ./.github/actions/run-tests
-      if: ${{ matrix.name != 'Pypy' }}
       env:
         # Set pattern to not single_cpu if not already set
         PATTERN: ${{ env.PATTERN == '' && 'not single_cpu' || matrix.pattern }}

.github/workflows/wheels.yml

@@ -101,7 +101,6 @@ jobs:
         - [macos-14, macosx_arm64]
         - [windows-2022, win_amd64]
         - [windows-11-arm, win_arm64]
-        # TODO: support PyPy?
         python: [["cp310", "3.10"], ["cp311", "3.11"], ["cp312", "3.12"], ["cp313", "3.13"], ["cp313t", "3.13"]]
         include:
         # Build Pyodide wheels and upload them to Anaconda.org

.pre-commit-config.yaml

@@ -19,7 +19,7 @@ ci:
     skip: [pyright, mypy]
 repos:
 -   repo: https://github.com/astral-sh/ruff-pre-commit
-    rev: v0.11.12
+    rev: v0.12.2
     hooks:
     -   id: ruff
         args: [--exit-non-zero-on-fix]
@@ -47,7 +47,7 @@ repos:
         types_or: [python, rst, markdown, cython, c]
         additional_dependencies: [tomli]
 -   repo: https://github.com/MarcoGorelli/cython-lint
-    rev: v0.16.6
+    rev: v0.16.7
     hooks:
     -   id: cython-lint
     -   id: double-quote-cython-strings
@@ -95,14 +95,14 @@ repos:
     - id: sphinx-lint
       args: ["--enable", "all", "--disable", "line-too-long"]
 -   repo: https://github.com/pre-commit/mirrors-clang-format
-    rev: v20.1.5
+    rev: v20.1.7
     hooks:
     - id: clang-format
       files: ^pandas/_libs/src|^pandas/_libs/include
       args: [-i]
       types_or: [c, c++]
 -   repo: https://github.com/trim21/pre-commit-mirror-meson
-    rev: v1.8.1
+    rev: v1.8.2
     hooks:
     - id: meson-fmt
       args: ['--inplace']

AUTHORS.md

@@ -7,12 +7,12 @@ About the Copyright Holders
     led by Wes McKinney. AQR released the source under this license in 2009.
 *   Copyright (c) 2011-2012, Lambda Foundry, Inc.
 
-    Wes is now an employee of Lambda Foundry, and remains the pandas project
+    Wes became an employee of Lambda Foundry, and remained the pandas project
     lead.
 *   Copyright (c) 2011-2012, PyData Development Team
 
     The PyData Development Team is the collection of developers of the PyData
-    project. This includes all of the PyData sub-projects, including pandas. The
+    project. This includes all of the PyData sub-projects, such as pandas. The
     core team that coordinates development on GitHub can be found here:
     https://github.com/pydata.
 
@@ -23,11 +23,11 @@ Our Copyright Policy
 
 PyData uses a shared copyright model. Each contributor maintains copyright
 over their contributions to PyData. However, it is important to note that
-these contributions are typically only changes to the repositories. Thus,
+these contributions are typically limited to changes to the repositories. Thus,
 the PyData source code, in its entirety, is not the copyright of any single
 person or institution. Instead, it is the collective copyright of the
 entire PyData Development Team. If individual contributors want to maintain
-a record of what changes/contributions they have specific copyright on,
+a record of the specific changes or contributions they hold copyright to,
 they should indicate their copyright in the commit message of the change
 when they commit the change to one of the PyData repositories.
 
@@ -50,7 +50,7 @@ Other licenses can be found in the LICENSES directory.
 License
 =======
 
-pandas is distributed under a 3-clause ("Simplified" or "New") BSD
+pandas is distributed under the 3-clause ("Simplified" or "New") BSD
 license. Parts of NumPy, SciPy, numpydoc, bottleneck, which all have
-BSD-compatible licenses, are included. Their licenses follow the pandas
+BSD-compatible licenses, are included. Their licenses are compatible with the pandas
 license.

asv_bench/benchmarks/gil.py

@@ -36,7 +36,7 @@
 from .pandas_vb_common import BaseIO  # isort:skip
 
 
-def test_parallel(num_threads=2, kwargs_list=None):
+def run_parallel(num_threads=2, kwargs_list=None):
     """
     Decorator to run the same function multiple times in parallel.
 
@@ -95,7 +95,7 @@ def setup(self, threads, method):
             {"key": np.random.randint(0, ngroups, size=N), "data": np.random.randn(N)}
         )
 
-        @test_parallel(num_threads=threads)
+        @run_parallel(num_threads=threads)
         def parallel():
             getattr(df.groupby("key")["data"], method)()
 
@@ -123,7 +123,7 @@ def setup(self, threads):
         ngroups = 10**3
         data = Series(np.random.randint(0, ngroups, size=size))
 
-        @test_parallel(num_threads=threads)
+        @run_parallel(num_threads=threads)
         def get_groups():
             data.groupby(data).groups
 
@@ -142,7 +142,7 @@ def setup(self, dtype):
         df = DataFrame({"col": np.arange(N, dtype=dtype)})
         indexer = np.arange(100, len(df) - 100)
 
-        @test_parallel(num_threads=2)
+        @run_parallel(num_threads=2)
         def parallel_take1d():
             take_nd(df["col"].values, indexer)
 
@@ -163,7 +163,7 @@ def setup(self):
         k = 5 * 10**5
         kwargs_list = [{"arr": np.random.randn(N)}, {"arr": np.random.randn(N)}]
 
-        @test_parallel(num_threads=2, kwargs_list=kwargs_list)
+        @run_parallel(num_threads=2, kwargs_list=kwargs_list)
         def parallel_kth_smallest(arr):
             algos.kth_smallest(arr, k)
 
@@ -180,42 +180,42 @@ def setup(self):
         self.period = self.dti.to_period("D")
 
     def time_datetime_field_year(self):
-        @test_parallel(num_threads=2)
+        @run_parallel(num_threads=2)
         def run(dti):
             dti.year
 
         run(self.dti)
 
     def time_datetime_field_day(self):
-        @test_parallel(num_threads=2)
+        @run_parallel(num_threads=2)
         def run(dti):
             dti.day
 
         run(self.dti)
 
     def time_datetime_field_daysinmonth(self):
-        @test_parallel(num_threads=2)
+        @run_parallel(num_threads=2)
         def run(dti):
             dti.days_in_month
 
         run(self.dti)
 
     def time_datetime_field_normalize(self):
-        @test_parallel(num_threads=2)
+        @run_parallel(num_threads=2)
         def run(dti):
             dti.normalize()
 
         run(self.dti)
 
     def time_datetime_to_period(self):
-        @test_parallel(num_threads=2)
+        @run_parallel(num_threads=2)
         def run(dti):
             dti.to_period("s")
 
         run(self.dti)
 
     def time_period_to_datetime(self):
-        @test_parallel(num_threads=2)
+        @run_parallel(num_threads=2)
         def run(period):
             period.to_timestamp()
 
@@ -232,7 +232,7 @@ def setup(self, method):
         if hasattr(DataFrame, "rolling"):
             df = DataFrame(arr).rolling(win)
 
-            @test_parallel(num_threads=2)
+            @run_parallel(num_threads=2)
             def parallel_rolling():
                 getattr(df, method)()
 
@@ -249,7 +249,7 @@ def parallel_rolling():
                 "std": rolling_std,
             }
 
-            @test_parallel(num_threads=2)
+            @run_parallel(num_threads=2)
             def parallel_rolling():
                 rolling[method](arr, win)
 
@@ -286,7 +286,7 @@ def setup(self, dtype):
         self.fname = f"__test_{dtype}__.csv"
         df.to_csv(self.fname)
 
-        @test_parallel(num_threads=2)
+        @run_parallel(num_threads=2)
         def parallel_read_csv():
             read_csv(self.fname)
 
@@ -305,7 +305,7 @@ class ParallelFactorize:
     def setup(self, threads):
         strings = Index([f"i-{i}" for i in range(100000)], dtype=object)
 
-        @test_parallel(num_threads=threads)
+        @run_parallel(num_threads=threads)
         def parallel():
             factorize(strings)
 

asv_bench/benchmarks/io/csv.py

@@ -53,6 +53,25 @@ def time_frame(self, kind):
         self.df.to_csv(self.fname)
 
 
+class ToCSVFloatFormatVariants(BaseIO):
+    fname = "__test__.csv"
+
+    def setup(self):
+        self.df = DataFrame(np.random.default_rng(seed=42).random((1000, 1000)))
+
+    def time_old_style_percent_format(self):
+        self.df.to_csv(self.fname, float_format="%.6f")
+
+    def time_new_style_brace_format(self):
+        self.df.to_csv(self.fname, float_format="{:.6f}")
+
+    def time_new_style_thousands_format(self):
+        self.df.to_csv(self.fname, float_format="{:,.2f}")
+
+    def time_callable_format(self):
+        self.df.to_csv(self.fname, float_format=lambda x: f"{x:.6f}")
+
+
 class ToCSVMultiIndexUnusedLevels(BaseIO):
     fname = "__test__.csv"
 

ci/deps/actions-310-minimum_versions.yaml

@@ -22,7 +22,7 @@ dependencies:
 
   # required dependencies
   - python-dateutil=2.8.2
-  - numpy=1.23.5
+  - numpy=1.26.0
 
   # optional dependencies
   - beautifulsoup4=4.12.3
@@ -41,7 +41,7 @@ dependencies:
   - qtpy=2.3.0
   - openpyxl=3.1.2
   - psycopg2=2.9.6
-  - pyarrow=10.0.1
+  - pyarrow=12.0.1
   - pyiceberg=0.7.1
   - pymysql=1.1.0
   - pyqt=5.15.9
@@ -62,4 +62,4 @@ dependencies:
   - pip:
     - adbc-driver-postgresql==0.10.0
     - adbc-driver-sqlite==0.8.0
-    - tzdata==2022.7
+    - tzdata==2023.3

ci/deps/actions-310.yaml

@@ -39,7 +39,7 @@ dependencies:
   - qtpy>=2.3.0
   - openpyxl>=3.1.2
   - psycopg2>=2.9.6
-  - pyarrow>=10.0.1
+  - pyarrow>=12.0.1
   - pyiceberg>=0.7.1
   - pymysql>=1.1.0
   - pyqt>=5.15.9
@@ -60,4 +60,4 @@ dependencies:
   - pip:
     - adbc-driver-postgresql>=0.10.0
     - adbc-driver-sqlite>=0.8.0
-    - tzdata>=2022.7
+    - tzdata>=2023.3

ci/deps/actions-311-downstream_compat.yaml

@@ -40,7 +40,7 @@ dependencies:
   - qtpy>=2.3.0
   - openpyxl>=3.1.2
   - psycopg2>=2.9.6
-  - pyarrow>=10.0.1
+  - pyarrow>=12.0.1
   - pyiceberg>=0.7.1
   - pymysql>=1.1.0
   - pyqt>=5.15.9
@@ -50,7 +50,8 @@ dependencies:
   - pytz>=2023.4
   - pyxlsb>=1.0.10
   - s3fs>=2023.12.2
-  - scipy>=1.12.0
+  # TEMP upper pin for scipy (https://github.com/statsmodels/statsmodels/issues/9584)
+  - scipy>=1.12.0,<1.16
   - sqlalchemy>=2.0.0
   - tabulate>=0.9.0
   - xarray>=2024.1.1
@@ -72,4 +73,4 @@ dependencies:
   - pip:
     - adbc-driver-postgresql>=0.10.0
     - adbc-driver-sqlite>=0.8.0
-    - tzdata>=2022.7
+    - tzdata>=2023.3

ci/deps/actions-311-numpydev.yaml

@@ -24,4 +24,4 @@ dependencies:
     - "--extra-index-url https://pypi.anaconda.org/scientific-python-nightly-wheels/simple"
     - "--pre"
     - "numpy"
-    - "tzdata>=2022.7"
+    - "tzdata>=2023.3"