Merge remote-tracking branch 'upstream/main' into read-csv-from-directory

Author: fangchenli

.github/workflows/unit-tests.yml

@@ -75,11 +75,6 @@ jobs:
             env_file: actions-311.yaml
             pandas_future_infer_string: "1"
             platform: ubuntu-24.04
-          - name: "Pypy"
-            env_file: actions-pypy-39.yaml
-            pattern: "not slow and not network and not single_cpu"
-            test_args: "--max-worker-restart 0"
-            platform: ubuntu-24.04
           - name: "Numpy Dev"
             env_file: actions-311-numpydev.yaml
             pattern: "not slow and not network and not single_cpu"
@@ -169,12 +164,9 @@ jobs:
       with:
         # xref https://github.com/cython/cython/issues/6870
         werror: ${{ matrix.name != 'Freethreading' }}
-      # TODO: Re-enable once Pypy has Pypy 3.10 on conda-forge
-      if: ${{ matrix.name != 'Pypy' }}
 
     - name: Test (not single_cpu)
       uses: ./.github/actions/run-tests
-      if: ${{ matrix.name != 'Pypy' }}
       env:
         # Set pattern to not single_cpu if not already set
         PATTERN: ${{ env.PATTERN == '' && 'not single_cpu' || matrix.pattern }}

.github/workflows/wheels.yml

@@ -101,7 +101,6 @@ jobs:
         - [macos-14, macosx_arm64]
         - [windows-2022, win_amd64]
         - [windows-11-arm, win_arm64]
-        # TODO: support PyPy?
         python: [["cp310", "3.10"], ["cp311", "3.11"], ["cp312", "3.12"], ["cp313", "3.13"], ["cp313t", "3.13"]]
         include:
         # Build Pyodide wheels and upload them to Anaconda.org

.pre-commit-config.yaml

@@ -19,7 +19,7 @@ ci:
     skip: [pyright, mypy]
 repos:
 -   repo: https://github.com/astral-sh/ruff-pre-commit
-    rev: v0.11.12
+    rev: v0.12.2
     hooks:
     -   id: ruff
         args: [--exit-non-zero-on-fix]
@@ -47,7 +47,7 @@ repos:
         types_or: [python, rst, markdown, cython, c]
         additional_dependencies: [tomli]
 -   repo: https://github.com/MarcoGorelli/cython-lint
-    rev: v0.16.6
+    rev: v0.16.7
     hooks:
     -   id: cython-lint
     -   id: double-quote-cython-strings
@@ -95,14 +95,14 @@ repos:
     - id: sphinx-lint
       args: ["--enable", "all", "--disable", "line-too-long"]
 -   repo: https://github.com/pre-commit/mirrors-clang-format
-    rev: v20.1.5
+    rev: v20.1.7
     hooks:
     - id: clang-format
       files: ^pandas/_libs/src|^pandas/_libs/include
       args: [-i]
       types_or: [c, c++]
 -   repo: https://github.com/trim21/pre-commit-mirror-meson
-    rev: v1.8.1
+    rev: v1.8.2
     hooks:
     - id: meson-fmt
       args: ['--inplace']

asv_bench/benchmarks/gil.py

@@ -36,7 +36,7 @@
 from .pandas_vb_common import BaseIO  # isort:skip
 
 
-def test_parallel(num_threads=2, kwargs_list=None):
+def run_parallel(num_threads=2, kwargs_list=None):
     """
     Decorator to run the same function multiple times in parallel.
 
@@ -95,7 +95,7 @@ def setup(self, threads, method):
             {"key": np.random.randint(0, ngroups, size=N), "data": np.random.randn(N)}
         )
 
-        @test_parallel(num_threads=threads)
+        @run_parallel(num_threads=threads)
         def parallel():
             getattr(df.groupby("key")["data"], method)()
 
@@ -123,7 +123,7 @@ def setup(self, threads):
         ngroups = 10**3
         data = Series(np.random.randint(0, ngroups, size=size))
 
-        @test_parallel(num_threads=threads)
+        @run_parallel(num_threads=threads)
         def get_groups():
             data.groupby(data).groups
 
@@ -142,7 +142,7 @@ def setup(self, dtype):
         df = DataFrame({"col": np.arange(N, dtype=dtype)})
         indexer = np.arange(100, len(df) - 100)
 
-        @test_parallel(num_threads=2)
+        @run_parallel(num_threads=2)
         def parallel_take1d():
             take_nd(df["col"].values, indexer)
 
@@ -163,7 +163,7 @@ def setup(self):
         k = 5 * 10**5
         kwargs_list = [{"arr": np.random.randn(N)}, {"arr": np.random.randn(N)}]
 
-        @test_parallel(num_threads=2, kwargs_list=kwargs_list)
+        @run_parallel(num_threads=2, kwargs_list=kwargs_list)
         def parallel_kth_smallest(arr):
             algos.kth_smallest(arr, k)
 
@@ -180,42 +180,42 @@ def setup(self):
         self.period = self.dti.to_period("D")
 
     def time_datetime_field_year(self):
-        @test_parallel(num_threads=2)
+        @run_parallel(num_threads=2)
         def run(dti):
             dti.year
 
         run(self.dti)
 
     def time_datetime_field_day(self):
-        @test_parallel(num_threads=2)
+        @run_parallel(num_threads=2)
         def run(dti):
             dti.day
 
         run(self.dti)
 
     def time_datetime_field_daysinmonth(self):
-        @test_parallel(num_threads=2)
+        @run_parallel(num_threads=2)
         def run(dti):
             dti.days_in_month
 
         run(self.dti)
 
     def time_datetime_field_normalize(self):
-        @test_parallel(num_threads=2)
+        @run_parallel(num_threads=2)
         def run(dti):
             dti.normalize()
 
         run(self.dti)
 
     def time_datetime_to_period(self):
-        @test_parallel(num_threads=2)
+        @run_parallel(num_threads=2)
         def run(dti):
             dti.to_period("s")
 
         run(self.dti)
 
     def time_period_to_datetime(self):
-        @test_parallel(num_threads=2)
+        @run_parallel(num_threads=2)
         def run(period):
             period.to_timestamp()
 
@@ -232,7 +232,7 @@ def setup(self, method):
         if hasattr(DataFrame, "rolling"):
             df = DataFrame(arr).rolling(win)
 
-            @test_parallel(num_threads=2)
+            @run_parallel(num_threads=2)
             def parallel_rolling():
                 getattr(df, method)()
 
@@ -249,7 +249,7 @@ def parallel_rolling():
                 "std": rolling_std,
             }
 
-            @test_parallel(num_threads=2)
+            @run_parallel(num_threads=2)
             def parallel_rolling():
                 rolling[method](arr, win)
 
@@ -286,7 +286,7 @@ def setup(self, dtype):
         self.fname = f"__test_{dtype}__.csv"
         df.to_csv(self.fname)
 
-        @test_parallel(num_threads=2)
+        @run_parallel(num_threads=2)
         def parallel_read_csv():
             read_csv(self.fname)
 
@@ -305,7 +305,7 @@ class ParallelFactorize:
     def setup(self, threads):
         strings = Index([f"i-{i}" for i in range(100000)], dtype=object)
 
-        @test_parallel(num_threads=threads)
+        @run_parallel(num_threads=threads)
         def parallel():
             factorize(strings)
 

asv_bench/benchmarks/io/csv.py

@@ -53,6 +53,25 @@ def time_frame(self, kind):
         self.df.to_csv(self.fname)
 
 
+class ToCSVFloatFormatVariants(BaseIO):
+    fname = "__test__.csv"
+
+    def setup(self):
+        self.df = DataFrame(np.random.default_rng(seed=42).random((1000, 1000)))
+
+    def time_old_style_percent_format(self):
+        self.df.to_csv(self.fname, float_format="%.6f")
+
+    def time_new_style_brace_format(self):
+        self.df.to_csv(self.fname, float_format="{:.6f}")
+
+    def time_new_style_thousands_format(self):
+        self.df.to_csv(self.fname, float_format="{:,.2f}")
+
+    def time_callable_format(self):
+        self.df.to_csv(self.fname, float_format=lambda x: f"{x:.6f}")
+
+
 class ToCSVMultiIndexUnusedLevels(BaseIO):
     fname = "__test__.csv"
 

ci/deps/actions-310-minimum_versions.yaml

@@ -22,7 +22,7 @@ dependencies:
 
   # required dependencies
   - python-dateutil=2.8.2
-  - numpy=1.23.5
+  - numpy=1.26.0
 
   # optional dependencies
   - beautifulsoup4=4.12.3
@@ -62,4 +62,4 @@ dependencies:
   - pip:
     - adbc-driver-postgresql==0.10.0
     - adbc-driver-sqlite==0.8.0
-    - tzdata==2022.7
+    - tzdata==2023.3

ci/deps/actions-310.yaml

@@ -60,4 +60,4 @@ dependencies:
   - pip:
     - adbc-driver-postgresql>=0.10.0
     - adbc-driver-sqlite>=0.8.0
-    - tzdata>=2022.7
+    - tzdata>=2023.3

ci/deps/actions-311-downstream_compat.yaml

@@ -73,4 +73,4 @@ dependencies:
   - pip:
     - adbc-driver-postgresql>=0.10.0
     - adbc-driver-sqlite>=0.8.0
-    - tzdata>=2022.7
+    - tzdata>=2023.3

ci/deps/actions-311-numpydev.yaml

@@ -24,4 +24,4 @@ dependencies:
     - "--extra-index-url https://pypi.anaconda.org/scientific-python-nightly-wheels/simple"
     - "--pre"
     - "numpy"
-    - "tzdata>=2022.7"
+    - "tzdata>=2023.3"

ci/deps/actions-311-pyarrownightly.yaml

@@ -22,7 +22,7 @@ dependencies:
   - pip
 
   - pip:
-    - "tzdata>=2022.7"
+    - "tzdata>=2023.3"
     - "--extra-index-url https://pypi.anaconda.org/scientific-python-nightly-wheels/simple"
     - "--prefer-binary"
     - "--pre"