feat: support Pandas v3

Author: ninoseki

.github/workflows/python.yml

@@ -18,6 +18,7 @@ jobs:
     strategy:
       matrix:
         python-version: ["3.10", 3.11, 3.12, 3.13]
+        pandas-version: [">2.2,<3.0", "3.0,<4.0"]
     defaults:
       run:
         working-directory: ./python-client
@@ -31,8 +32,10 @@ jobs:
       - uses: actions/cache@v4
         with:
           path: ~/.cache/uv
-          key: uv-${{ matrix.python-version }}-${{ hashFiles('uv.lock') }}
-      - run: uv sync --frozen --all-extras
+          key: uv-${{ matrix.python-version }}-${{ matrix.pandas-version }}-${{ hashFiles('uv.lock') }}
+      - run: |
+          uv sync --frozen --all-extras
+          uv pip install "pandas${{ matrix.pandas-version }}"
       - run: uv run pytest
   lint:
     runs-on: ubuntu-latest

python-client/pyproject.toml

@@ -16,7 +16,7 @@ dependencies = [
   "loguru~=0.7",
   "micawber~=0.5",
   "multimethod~=2.0",
-  "pandas~=2.2",
+  "pandas>=2.2,<4.0",
   "pyarrow~=21.0.0",
   "pydantic>2.0.1,!=2.1.0,<3.0.0",                 # same as FastAPI
   "pyhumps~=3.8",

python-client/report.html

@@ -1 +1 @@
-uv==0.9.18
+uv==0.9.26

python-client/requirements.txt

@@ -18,6 +18,9 @@
 )
 from arakawa.types import SList
 
+IS_PANDAS_V2 = pd.__version__.startswith("2.")
+IS_PANDAS_V3 = pd.__version__.startswith("3.")
+
 
 def _check_categories_parsed(df: pd.DataFrame, categorical_columns: SList):
     assert set(df.select_dtypes("category").columns) == set(categorical_columns)
@@ -153,7 +156,12 @@ def test_timedelta_to_str():
         }
     )
     timedelta_to_str(df)
-    assert [str(x) for x in df.dtypes] == ["object", "int64", "object"]
+
+    if IS_PANDAS_V2:
+        assert [str(x) for x in df.dtypes] == ["object", "int64", "object"]
+    if IS_PANDAS_V3:
+        assert [str(x) for x in df.dtypes] == ["str", "int64", "str"]
+
     df1 = df.convert_dtypes()
     assert [str(x) for x in df1.dtypes] == ["string", "Int64", "string"]
 
@@ -211,7 +219,7 @@ def _test_order(df: pd.DataFrame):
 def test_e2e_df_processing(tmp_path: Path):
     def _test_df(
         df: pd.DataFrame,
-        expected_types_pd13: SList,
+        expected_types: SList,
     ):
         df_conv = df.convert_dtypes()
         df_proc = process_df(df, copy=True)
@@ -235,26 +243,40 @@ def _test_df(
             check_column_type=False,
         )
 
-        expected_types = expected_types_pd13
-
         assert [str(x) for x in df2.dtypes] == expected_types
 
     # DF 1
     df = vd.data.cars()
-    _test_df(
-        df,
-        [
-            "string",
-            "Float64",
-            "UInt8",
-            "Float64",
-            "UInt8",
-            "UInt16",
-            "Float64",
-            "datetime64[ns]",
-            "category",
-        ],
-    )
+    if IS_PANDAS_V2:
+        _test_df(
+            df,
+            [
+                "string",
+                "Float64",
+                "UInt8",
+                "Float64",
+                "UInt8",
+                "UInt16",
+                "Float64",
+                "datetime64[ns]",
+                "category",
+            ],
+        )
+    if IS_PANDAS_V3:
+        _test_df(
+            df,
+            [
+                "string",
+                "Float64",
+                "UInt8",
+                "Float64",
+                "UInt8",
+                "UInt16",
+                "Float64",
+                "datetime64[us]",
+                "category",
+            ],
+        )
 
     # DF 2 - float64/int64 downcasting for older pandas versions
     df = pd.DataFrame(
@@ -296,20 +318,36 @@ def _test_df(
             "cat_obj_col": [("a", "b") for x in range(30)],
         }
     )
-    _test_df(
-        df,
-        [
-            "string",
-            "category",
-            "UInt8",
-            "Int64",
-            "Float64",
-            "string",
-            "datetime64[ns]",
-            "string",
-            "category",
-        ],
-    )
+    if IS_PANDAS_V2:
+        _test_df(
+            df,
+            [
+                "string",
+                "category",
+                "UInt8",
+                "Int64",
+                "Float64",
+                "string",
+                "datetime64[ns]",
+                "string",
+                "category",
+            ],
+        )
+    if IS_PANDAS_V3:
+        _test_df(
+            df,
+            [
+                "string",
+                "category",
+                "UInt8",
+                "Int64",
+                "Float64",
+                "string",
+                "datetime64[us]",
+                "string",
+                "category",
+            ],
+        )
 
     # DF 4 - nullable
     df = pd.DataFrame(
@@ -323,4 +361,27 @@ def _test_df(
             "date_col": [datetime.utcnow() for x in range(30)] + [pd.NaT, pd.NaT],
         }
     )
-    _test_df(df, ["string", "category", "UInt8", "Float64", "string", "datetime64[ns]"])
+    if IS_PANDAS_V2:
+        _test_df(
+            df,
+            [
+                "string",
+                "category",
+                "UInt8",
+                "Float64",
+                "string",
+                "datetime64[ns]",
+            ],
+        )
+    if IS_PANDAS_V3:
+        _test_df(
+            df,
+            [
+                "string",
+                "category",
+                "UInt8",
+                "Float64",
+                "string",
+                "datetime64[us]",
+            ],
+        )