refactor: Small changes for cloud benches

.github/workflows/lint.yml

@@ -49,7 +49,7 @@ jobs:
       # Code in scripts/aws_glue.py requires PySpark<4
       - name: Run mypy (PySpark from requirements.txt)
         run: |
-          mypy --exclude scripts/aws_glue.py
+          mypy
 
       - name: Run mypy (PySpark<4)
         run: |

pyproject.toml

@@ -71,6 +71,7 @@ enable_error_code = [
   "ignore-without-code",
 ]
 plugins = ["pydantic.mypy"]
+exclude = ["scripts.aws_emr", "scripts.aws_glue"]
 
 
 [[tool.mypy.overrides]]
@@ -83,7 +84,8 @@ module = [
   "rmm.*",
   "pylibcudf.*",
   "awsglue.*",
-  "scripts/aws_glue.py"
+  "scripts/aws_glue.py",
+  "scripts/aws_emr.py"
 ]
 ignore_missing_imports = true
 

queries/polars/utils.py

@@ -168,13 +168,17 @@ def run_query(query_number: int, lf: pl.LazyFrame) -> None:
         class PatchedComputeContext(pc.ComputeContext):
             def __init__(self, *args, **kwargs) -> None:  # type: ignore[no-untyped-def]
                 self._interactive = True
-                self._compute_address = "localhost:5051"
-                self._compute_public_key = b""
+                compute_address = "localhost:5051"
+                client_options = pc.polars_cloud.ClientOptions()
+                client_options.insecure = True
                 self._compute_id = "1"  # type: ignore[assignment]
+                self._interactive_client = pc.polars_cloud.SchedulerClient(  # type: ignore[attr-defined]
+                    compute_address, client_options
+                )
 
-            def get_status(self: pc.ComputeContext) -> pc.ComputeContextStatus:
-                """Get the status of the compute cluster."""
-                return pc.ComputeContextStatus.RUNNING
+        def get_status(self: pc.ComputeContext) -> pc.ComputeContextStatus:
+            """Get the status of the compute cluster."""
+            return pc.ComputeContextStatus.RUNNING
 
         pc.ComputeContext.__init__ = PatchedComputeContext.__init__  # type: ignore[assignment]
         pc.ComputeContext.get_status = PatchedComputeContext.get_status  # type: ignore[method-assign]

requirements-dev.txt

@@ -1,16 +1,18 @@
 # This file was autogenerated by uv via the following command:
 #    uv pip compile requirements-dev.in
-mypy==1.15.0
+mypy==1.17.1
     # via -r requirements-dev.in
 mypy-extensions==1.1.0
     # via mypy
-numpy==2.2.6
+numpy==2.3.2
     # via pandas-stubs
-pandas-stubs==2.2.3.250308
+pandas-stubs==2.3.2.250827
     # via -r requirements-dev.in
-ruff==0.11.11
+pathspec==0.12.1
+    # via mypy
+ruff==0.12.11
     # via -r requirements-dev.in
-types-pytz==2025.2.0.20250516
+types-pytz==2025.2.0.20250809
     # via pandas-stubs
-typing-extensions==4.13.2
+typing-extensions==4.15.0
     # via mypy

requirements.txt

@@ -6,64 +6,64 @@ attrs==25.3.0
     # via
     #   jsonschema
     #   referencing
-certifi==2025.4.26
+certifi==2025.8.3
     # via requests
-charset-normalizer==3.4.2
+charset-normalizer==3.4.3
     # via requests
 click==8.2.1
     # via
     #   dask
     #   ray
 cloudpickle==3.1.1
     # via dask
-contourpy==1.3.2
+contourpy==1.3.3
     # via matplotlib
-cramjam==2.10.0
+cramjam==2.11.0
     # via fastparquet
 cycler==0.12.1
     # via matplotlib
-dask==2025.5.1
+dask==2025.7.0
     # via
     #   -r requirements.in
     #   dask-expr
 dask-expr==2.0.0
     # via -r requirements.in
-duckdb==1.3.0
+duckdb==1.3.2
     # via -r requirements.in
 fastparquet==2024.11.0
     # via -r requirements.in
-filelock==3.18.0
+filelock==3.19.1
     # via ray
-fonttools==4.58.0
+fonttools==4.59.2
     # via matplotlib
-fsspec==2025.5.1
+fsspec==2025.7.0
     # via
     #   dask
     #   fastparquet
     #   modin
 idna==3.10
     # via requests
-jsonschema==4.23.0
+jsonschema==4.25.1
     # via ray
 jsonschema-specifications==2025.4.1
     # via jsonschema
-kiwisolver==1.4.8
+kiwisolver==1.4.9
     # via matplotlib
 linetimer==0.1.5
     # via -r requirements.in
 locket==1.0.0
     # via partd
-matplotlib==3.10.3
+matplotlib==3.10.6
     # via plotnine
-mizani==0.13.5
+mizani==0.14.2
     # via plotnine
-modin==0.32.0
+modin==0.35.0
     # via -r requirements.in
-msgpack==1.1.0
+msgpack==1.1.1
     # via ray
-narwhals==1.40.0
+narwhals==2.3.0
     # via plotly
-numpy==2.2.6
+numpy==2.3.2
     # via
     #   contourpy
     #   dask
@@ -85,7 +85,7 @@ packaging==25.0
     #   plotly
     #   ray
     #   statsmodels
-pandas==2.2.3
+pandas==2.3.2
     # via
     #   -r requirements.in
     #   dask
@@ -98,36 +98,36 @@ partd==1.4.2
     # via dask
 patsy==1.0.1
     # via statsmodels
-pillow==11.2.1
+pillow==11.3.0
     # via matplotlib
-plotly==6.1.1
+plotly==6.3.0
     # via -r requirements.in
-plotnine==0.14.5
+plotnine==0.15.0
     # via -r requirements.in
-polars==1.30.0
+polars==1.32.0
     # via
     #   -r requirements.in
     #   polars-cloud
-polars-cloud==0.0.9
+polars-cloud==0.0.11
     # via -r requirements.in
-protobuf==6.31.0
+protobuf==6.32.0
     # via ray
 psutil==7.0.0
     # via modin
 py4j==0.10.9.9
     # via pyspark
-pyarrow==20.0.0
+pyarrow==21.0.0
     # via
     #   -r requirements.in
     #   dask
     #   modin
-pydantic==2.11.5
+pydantic==2.11.7
     # via
     #   -r requirements.in
     #   pydantic-settings
 pydantic-core==2.33.2
     # via pydantic
-pydantic-settings==2.9.1
+pydantic-settings==2.10.1
     # via -r requirements.in
 pyparsing==3.2.3
     # via matplotlib
@@ -137,45 +137,46 @@ python-dateutil==2.9.0.post0
     # via
     #   matplotlib
     #   pandas
-python-dotenv==1.1.0
+python-dotenv==1.1.1
     # via pydantic-settings
 pytz==2025.2
     # via pandas
 pyyaml==6.0.2
     # via
     #   dask
     #   ray
-ray==2.46.0
+ray==2.49.0
     # via modin
 referencing==0.36.2
     # via
     #   jsonschema
     #   jsonschema-specifications
-requests==2.32.3
+requests==2.32.5
     # via ray
-rpds-py==0.25.1
+rpds-py==0.27.1
     # via
     #   jsonschema
     #   referencing
-scipy==1.15.3
+scipy==1.16.1
     # via
     #   mizani
     #   plotnine
     #   statsmodels
-setuptools==80.8.0
+setuptools==80.9.0
     # via -r requirements.in
 six==1.17.0
     # via python-dateutil
-statsmodels==0.14.4
+statsmodels==0.14.5
     # via plotnine
 toolz==1.0.0
     # via
     #   dask
     #   partd
-tpchgen-cli==1.1.0
+tpchgen-cli==2.0.0
     # via -r requirements.in
-typing-extensions==4.13.2
+typing-extensions==4.15.0
     # via
+    #   polars-cloud
     #   pydantic
     #   pydantic-core
     #   referencing
@@ -186,5 +187,5 @@ typing-inspection==0.4.1
     #   pydantic-settings
 tzdata==2025.2
     # via pandas
-urllib3==2.4.0
+urllib3==2.5.0
     # via requests