Add support for running pandas queries with cudf.pandas enabled

Author: vyasr

Makefile

@@ -92,6 +92,20 @@ run-duckdb: .venv data/tables/ ## Run DuckDB benchmarks
 run-pandas: .venv data/tables/ ## Run pandas benchmarks
 	$(VENV_BIN)/python -m queries.pandas
 
+.PHONY: run-pandas-gpu
+run-pandas-gpu: .venv ## Run cudf.pandas benchmarks
+	# TODO: Change this to use $(PYTHON) once
+	# https://github.com/pola-rs/polars-benchmark/pull/146 is merged
+	RUN_PANDAS_GPU=true $(VENV_BIN)/python -m queries.pandas
+
+.PHONY: run-pandas-no-env
+run-pandas-no-env: ## Run pandas benchmarks
+	python -m queries.pandas
+
+.PHONY: run-pandas-gpu-no-env
+run-pandas-no-env: ## Run pandas benchmarks
+	RUN_PANDAS_GPU=true python -m queries.pandas
+
 .PHONY: run-pyspark
 run-pyspark: .venv data/tables/ ## Run PySpark benchmarks
 	$(VENV_BIN)/python -m queries.pyspark

queries/pandas/utils.py

@@ -80,6 +80,12 @@ def get_part_supp_ds() -> pd.DataFrame:
 
 
 def run_query(query_number: int, query: Callable[..., Any]) -> None:
+    if settings.run.pandas_gpu:
+        # Note that this has a global effect. Using it in this way should be fine though
+        # given that execute_all launches a separate Python process for each query.
+        import cudf.pandas
+        cudf.pandas.install()
+
     run_query_generic(
         query, query_number, "pandas", query_checker=check_query_result_pd
     )

requirements.in

@@ -6,6 +6,8 @@ pandas>=2.0
 polars
 polars_cloud
 pyspark
+cudf-cu12
+cudf-polars-cu12
 
 pyarrow  # Required by duckdb/pandas
 fastparquet  # Required by pandas

requirements.txt

@@ -8,6 +8,8 @@ attrs==25.3.0
     # via
     #   jsonschema
     #   referencing
+cachetools==5.5.2
+    # via cudf-cu12
 certifi==2025.4.26
     # via requests
 charset-normalizer==3.4.1
@@ -22,6 +24,19 @@ contourpy==1.3.2
     # via matplotlib
 cramjam==2.10.0
     # via fastparquet
+cuda-bindings==12.8.0
+    # via cuda-python
+cuda-python==12.8.0
+    # via
+    #   cudf-cu12
+    #   pylibcudf-cu12
+    #   rmm-cu12
+cudf-cu12==25.4.0
+    # via -r requirements.in
+cudf-polars-cu12==25.4.0
+    # via -r requirements.in
+cupy-cuda12x==13.4.1
+    # via cudf-cu12
 cycler==0.12.1
     # via matplotlib
 dask==2025.4.1
@@ -34,6 +49,8 @@ duckdb==1.2.2
     # via -r requirements.in
 fastparquet==2024.11.0
     # via -r requirements.in
+fastrlock==0.8.3
+    # via cupy-cuda12x
 filelock==3.18.0
     # via ray
 fonttools==4.57.0
@@ -44,58 +61,92 @@ frozenlist==1.6.0
     #   ray
 fsspec==2025.3.2
     # via
+    #   cudf-cu12
     #   dask
     #   fastparquet
     #   modin
 idna==3.10
     # via requests
-importlib-metadata==8.6.1
-    # via dask
 jsonschema==4.23.0
     # via ray
 jsonschema-specifications==2025.4.1
     # via jsonschema
 kiwisolver==1.4.8
     # via matplotlib
+libcudf-cu12==25.4.0
+    # via
+    #   cudf-cu12
+    #   pylibcudf-cu12
+libkvikio-cu12==25.4.0
+    # via libcudf-cu12
+librmm-cu12==25.4.0
+    # via
+    #   libcudf-cu12
+    #   rmm-cu12
 linetimer==0.1.5
     # via -r requirements.in
+llvmlite==0.44.0
+    # via numba
 locket==1.0.0
     # via partd
+markdown-it-py==3.0.0
+    # via rich
 matplotlib==3.10.1
     # via plotnine
+mdurl==0.1.2
+    # via markdown-it-py
 mizani==0.13.3
     # via plotnine
 modin==0.32.0
     # via -r requirements.in
 msgpack==1.1.0
     # via ray
-narwhals==1.36.0
+narwhals==1.37.0
     # via plotly
+numba==0.61.2
+    # via
+    #   cudf-cu12
+    #   numba-cuda
+numba-cuda==0.4.0
+    # via cudf-cu12
 numpy==2.2.5
     # via
     #   contourpy
+    #   cudf-cu12
+    #   cupy-cuda12x
     #   dask
     #   fastparquet
     #   matplotlib
     #   mizani
     #   modin
+    #   numba
     #   pandas
     #   patsy
     #   plotnine
+    #   rmm-cu12
     #   scipy
     #   statsmodels
+nvidia-nvcomp-cu12==4.2.0.11
+    # via libcudf-cu12
+nvtx==0.2.11
+    # via
+    #   cudf-cu12
+    #   pylibcudf-cu12
 packaging==25.0
     # via
+    #   cudf-cu12
     #   dask
     #   fastparquet
     #   matplotlib
     #   modin
     #   plotly
+    #   pylibcudf-cu12
     #   ray
     #   statsmodels
 pandas==2.2.3
     # via
     #   -r requirements.in
+    #   cudf-cu12
     #   dask
     #   fastparquet
     #   mizani
@@ -112,11 +163,12 @@ plotly==6.0.1
     # via -r requirements.in
 plotnine==0.14.5
     # via -r requirements.in
-polars==1.28.0
+polars==1.25.2
     # via
     #   -r requirements.in
+    #   cudf-polars-cu12
     #   polars-cloud
-polars-cloud==0.0.4
+polars-cloud==0.0.5
     # via -r requirements.in
 protobuf==6.30.2
     # via ray
@@ -127,8 +179,10 @@ py4j==0.10.9.7
 pyarrow==19.0.1
     # via
     #   -r requirements.in
+    #   cudf-cu12
     #   dask
     #   modin
+    #   pylibcudf-cu12
 pydantic==2.11.3
     # via
     #   -r requirements.in
@@ -137,6 +191,14 @@ pydantic-core==2.33.1
     # via pydantic
 pydantic-settings==2.9.1
     # via -r requirements.in
+pygments==2.19.1
+    # via rich
+pylibcudf-cu12==25.4.0
+    # via
+    #   cudf-cu12
+    #   cudf-polars-cu12
+pynvjitlink-cu12==0.5.2
+    # via cudf-cu12
 pyparsing==3.2.3
     # via matplotlib
 pyspark==3.5.5
@@ -153,6 +215,10 @@ pyyaml==6.0.2
     # via
     #   dask
     #   ray
+rapids-logger==0.1.1
+    # via
+    #   libcudf-cu12
+    #   librmm-cu12
 ray==2.44.1
     # via modin
 referencing==0.36.2
@@ -161,6 +227,12 @@ referencing==0.36.2
     #   jsonschema-specifications
 requests==2.32.3
     # via ray
+rich==14.0.0
+    # via cudf-cu12
+rmm-cu12==25.4.0
+    # via
+    #   cudf-cu12
+    #   pylibcudf-cu12
 rpds-py==0.24.0
     # via
     #   jsonschema
@@ -170,7 +242,7 @@ scipy==1.15.2
     #   mizani
     #   plotnine
     #   statsmodels
-setuptools==79.0.1
+setuptools==80.0.0
     # via -r requirements.in
 six==1.17.0
     # via python-dateutil
@@ -182,9 +254,10 @@ toolz==1.0.0
     #   partd
 typing-extensions==4.13.2
     # via
-    #   polars-cloud
+    #   cudf-cu12
     #   pydantic
     #   pydantic-core
+    #   pylibcudf-cu12
     #   referencing
     #   typing-inspection
 typing-inspection==0.4.0
@@ -195,5 +268,3 @@ tzdata==2025.2
     # via pandas
 urllib3==2.4.0
     # via requests
-zipp==3.21.0
-    # via importlib-metadata

settings.py

@@ -55,6 +55,8 @@ class Run(BaseSettings):
     spark_executor_memory: str = "1g"  # Tune as needed for optimal performance
     spark_log_level: str = "ERROR"
 
+    pandas_gpu: bool = False  # Use cudf.pandas to run pandas benchmarks
+
     @computed_field  # type: ignore[prop-decorator]
     @property
     def include_io(self) -> bool: