Merge remote-tracking branch 'upstream/main' into string-dtype-switch-default

Author: jorisvandenbossche

.github/workflows/unit-tests.yml

@@ -75,11 +75,6 @@ jobs:
             env_file: actions-312.yaml
             pandas_future_infer_string: "0"
             platform: ubuntu-24.04
-          - name: "Pypy"
-            env_file: actions-pypy-39.yaml
-            pattern: "not slow and not network and not single_cpu"
-            test_args: "--max-worker-restart 0"
-            platform: ubuntu-24.04
           - name: "Numpy Dev"
             env_file: actions-311-numpydev.yaml
             pattern: "not slow and not network and not single_cpu"
@@ -168,12 +163,9 @@ jobs:
       with:
         # xref https://github.com/cython/cython/issues/6870
         werror: ${{ matrix.name != 'Freethreading' }}
-      # TODO: Re-enable once Pypy has Pypy 3.10 on conda-forge
-      if: ${{ matrix.name != 'Pypy' }}
 
     - name: Test (not single_cpu)
       uses: ./.github/actions/run-tests
-      if: ${{ matrix.name != 'Pypy' }}
       env:
         # Set pattern to not single_cpu if not already set
         PATTERN: ${{ env.PATTERN == '' && 'not single_cpu' || matrix.pattern }}

.github/workflows/wheels.yml

@@ -101,7 +101,6 @@ jobs:
         - [macos-14, macosx_arm64]
         - [windows-2022, win_amd64]
         - [windows-11-arm, win_arm64]
-        # TODO: support PyPy?
         python: [["cp310", "3.10"], ["cp311", "3.11"], ["cp312", "3.12"], ["cp313", "3.13"], ["cp313t", "3.13"]]
         include:
         # Build Pyodide wheels and upload them to Anaconda.org

.pre-commit-config.yaml

@@ -19,7 +19,7 @@ ci:
     skip: [pyright, mypy]
 repos:
 -   repo: https://github.com/astral-sh/ruff-pre-commit
-    rev: v0.11.12
+    rev: v0.12.2
     hooks:
     -   id: ruff
         args: [--exit-non-zero-on-fix]
@@ -47,7 +47,7 @@ repos:
         types_or: [python, rst, markdown, cython, c]
         additional_dependencies: [tomli]
 -   repo: https://github.com/MarcoGorelli/cython-lint
-    rev: v0.16.6
+    rev: v0.16.7
     hooks:
     -   id: cython-lint
     -   id: double-quote-cython-strings
@@ -95,14 +95,14 @@ repos:
     - id: sphinx-lint
       args: ["--enable", "all", "--disable", "line-too-long"]
 -   repo: https://github.com/pre-commit/mirrors-clang-format
-    rev: v20.1.5
+    rev: v20.1.7
     hooks:
     - id: clang-format
       files: ^pandas/_libs/src|^pandas/_libs/include
       args: [-i]
       types_or: [c, c++]
 -   repo: https://github.com/trim21/pre-commit-mirror-meson
-    rev: v1.8.1
+    rev: v1.8.2
     hooks:
     - id: meson-fmt
       args: ['--inplace']

README.md

@@ -175,7 +175,7 @@ All contributions, bug reports, bug fixes, documentation improvements, enhanceme
 
 A detailed overview on how to contribute can be found in the **[contributing guide](https://pandas.pydata.org/docs/dev/development/contributing.html)**.
 
-If you are simply looking to start working with the pandas codebase, navigate to the [GitHub "issues" tab](https://github.com/pandas-dev/pandas/issues) and start looking through interesting issues. There are a number of issues listed under [Docs](https://github.com/pandas-dev/pandas/issues?labels=Docs&sort=updated&state=open) and [good first issue](https://github.com/pandas-dev/pandas/issues?labels=good+first+issue&sort=updated&state=open) where you could start out.
+If you are simply looking to start working with the pandas codebase, navigate to the [GitHub "issues" tab](https://github.com/pandas-dev/pandas/issues) and start looking through interesting issues. There are a number of issues listed under [Docs](https://github.com/pandas-dev/pandas/issues?q=is%3Aissue%20state%3Aopen%20label%3ADocs%20sort%3Aupdated-desc) and [good first issue](https://github.com/pandas-dev/pandas/issues?q=is%3Aissue%20state%3Aopen%20label%3A%22good%20first%20issue%22%20sort%3Aupdated-desc) where you could start out.
 
 You can also triage issues which may include reproducing bug reports, or asking for vital information such as version numbers or reproduction instructions. If you would like to start triaging issues, one easy way to get started is to [subscribe to pandas on CodeTriage](https://www.codetriage.com/pandas-dev/pandas).
 

asv_bench/benchmarks/gil.py

@@ -36,7 +36,7 @@
 from .pandas_vb_common import BaseIO  # isort:skip
 
 
-def test_parallel(num_threads=2, kwargs_list=None):
+def run_parallel(num_threads=2, kwargs_list=None):
     """
     Decorator to run the same function multiple times in parallel.
 
@@ -95,7 +95,7 @@ def setup(self, threads, method):
             {"key": np.random.randint(0, ngroups, size=N), "data": np.random.randn(N)}
         )
 
-        @test_parallel(num_threads=threads)
+        @run_parallel(num_threads=threads)
         def parallel():
             getattr(df.groupby("key")["data"], method)()
 
@@ -123,7 +123,7 @@ def setup(self, threads):
         ngroups = 10**3
         data = Series(np.random.randint(0, ngroups, size=size))
 
-        @test_parallel(num_threads=threads)
+        @run_parallel(num_threads=threads)
         def get_groups():
             data.groupby(data).groups
 
@@ -142,7 +142,7 @@ def setup(self, dtype):
         df = DataFrame({"col": np.arange(N, dtype=dtype)})
         indexer = np.arange(100, len(df) - 100)
 
-        @test_parallel(num_threads=2)
+        @run_parallel(num_threads=2)
         def parallel_take1d():
             take_nd(df["col"].values, indexer)
 
@@ -163,7 +163,7 @@ def setup(self):
         k = 5 * 10**5
         kwargs_list = [{"arr": np.random.randn(N)}, {"arr": np.random.randn(N)}]
 
-        @test_parallel(num_threads=2, kwargs_list=kwargs_list)
+        @run_parallel(num_threads=2, kwargs_list=kwargs_list)
         def parallel_kth_smallest(arr):
             algos.kth_smallest(arr, k)
 
@@ -180,42 +180,42 @@ def setup(self):
         self.period = self.dti.to_period("D")
 
     def time_datetime_field_year(self):
-        @test_parallel(num_threads=2)
+        @run_parallel(num_threads=2)
         def run(dti):
             dti.year
 
         run(self.dti)
 
     def time_datetime_field_day(self):
-        @test_parallel(num_threads=2)
+        @run_parallel(num_threads=2)
         def run(dti):
             dti.day
 
         run(self.dti)
 
     def time_datetime_field_daysinmonth(self):
-        @test_parallel(num_threads=2)
+        @run_parallel(num_threads=2)
         def run(dti):
             dti.days_in_month
 
         run(self.dti)
 
     def time_datetime_field_normalize(self):
-        @test_parallel(num_threads=2)
+        @run_parallel(num_threads=2)
         def run(dti):
             dti.normalize()
 
         run(self.dti)
 
     def time_datetime_to_period(self):
-        @test_parallel(num_threads=2)
+        @run_parallel(num_threads=2)
         def run(dti):
             dti.to_period("s")
 
         run(self.dti)
 
     def time_period_to_datetime(self):
-        @test_parallel(num_threads=2)
+        @run_parallel(num_threads=2)
         def run(period):
             period.to_timestamp()
 
@@ -232,7 +232,7 @@ def setup(self, method):
         if hasattr(DataFrame, "rolling"):
             df = DataFrame(arr).rolling(win)
 
-            @test_parallel(num_threads=2)
+            @run_parallel(num_threads=2)
             def parallel_rolling():
                 getattr(df, method)()
 
@@ -249,7 +249,7 @@ def parallel_rolling():
                 "std": rolling_std,
             }
 
-            @test_parallel(num_threads=2)
+            @run_parallel(num_threads=2)
             def parallel_rolling():
                 rolling[method](arr, win)
 
@@ -286,7 +286,7 @@ def setup(self, dtype):
         self.fname = f"__test_{dtype}__.csv"
         df.to_csv(self.fname)
 
-        @test_parallel(num_threads=2)
+        @run_parallel(num_threads=2)
         def parallel_read_csv():
             read_csv(self.fname)
 
@@ -305,7 +305,7 @@ class ParallelFactorize:
     def setup(self, threads):
         strings = Index([f"i-{i}" for i in range(100000)], dtype=object)
 
-        @test_parallel(num_threads=threads)
+        @run_parallel(num_threads=threads)
         def parallel():
             factorize(strings)
 

asv_bench/benchmarks/io/csv.py

@@ -53,6 +53,25 @@ def time_frame(self, kind):
         self.df.to_csv(self.fname)
 
 
+class ToCSVFloatFormatVariants(BaseIO):
+    fname = "__test__.csv"
+
+    def setup(self):
+        self.df = DataFrame(np.random.default_rng(seed=42).random((1000, 1000)))
+
+    def time_old_style_percent_format(self):
+        self.df.to_csv(self.fname, float_format="%.6f")
+
+    def time_new_style_brace_format(self):
+        self.df.to_csv(self.fname, float_format="{:.6f}")
+
+    def time_new_style_thousands_format(self):
+        self.df.to_csv(self.fname, float_format="{:,.2f}")
+
+    def time_callable_format(self):
+        self.df.to_csv(self.fname, float_format=lambda x: f"{x:.6f}")
+
+
 class ToCSVMultiIndexUnusedLevels(BaseIO):
     fname = "__test__.csv"
 

ci/deps/actions-310-minimum_versions.yaml

@@ -22,7 +22,7 @@ dependencies:
 
   # required dependencies
   - python-dateutil=2.8.2
-  - numpy=1.23.5
+  - numpy=1.26.0
 
   # optional dependencies
   - beautifulsoup4=4.12.3
@@ -62,4 +62,4 @@ dependencies:
   - pip:
     - adbc-driver-postgresql==0.10.0
     - adbc-driver-sqlite==0.8.0
-    - tzdata==2022.7
+    - tzdata==2023.3

ci/deps/actions-310.yaml

@@ -60,4 +60,4 @@ dependencies:
   - pip:
     - adbc-driver-postgresql>=0.10.0
     - adbc-driver-sqlite>=0.8.0
-    - tzdata>=2022.7
+    - tzdata>=2023.3

ci/deps/actions-311-downstream_compat.yaml

@@ -73,4 +73,4 @@ dependencies:
   - pip:
     - adbc-driver-postgresql>=0.10.0
     - adbc-driver-sqlite>=0.8.0
-    - tzdata>=2022.7
+    - tzdata>=2023.3

ci/deps/actions-311-numpydev.yaml

@@ -24,4 +24,4 @@ dependencies:
     - "--extra-index-url https://pypi.anaconda.org/scientific-python-nightly-wheels/simple"
     - "--pre"
     - "numpy"
-    - "tzdata>=2022.7"
+    - "tzdata>=2023.3"