[backport][dask] Workaround the tokenizer. (dmlc#10419) (dmlc#10423)

Co-authored-by: Richard (Rick) Zamora <[email protected]>

Author: trivialfis

python-package/xgboost/dask/__init__.py

@@ -1237,10 +1237,12 @@ def _infer_predict_output(
 async def _get_model_future(
     client: "distributed.Client", model: Union[Booster, Dict, "distributed.Future"]
 ) -> "distributed.Future":
+    # See https://github.com/dask/dask/issues/11179#issuecomment-2168094529 for
+    # the use of hash.
     if isinstance(model, Booster):
-        booster = await client.scatter(model, broadcast=True)
+        booster = await client.scatter(model, broadcast=True, hash=False)
     elif isinstance(model, dict):
-        booster = await client.scatter(model["booster"], broadcast=True)
+        booster = await client.scatter(model["booster"], broadcast=True, hash=False)
     elif isinstance(model, distributed.Future):
         booster = model
         t = booster.type

tests/ci_build/Dockerfile.gpu

@@ -25,7 +25,7 @@ RUN \
     mamba create -y -n gpu_test -c rapidsai -c nvidia -c conda-forge \
         python=3.10 cudf=$RAPIDS_VERSION_ARG* rmm=$RAPIDS_VERSION_ARG* cudatoolkit=$CUDA_VERSION_ARG \
         "nccl>=${NCCL_SHORT_VER}" \
-        dask=2024.1.1 \
+        dask \
         dask-cuda=$RAPIDS_VERSION_ARG* dask-cudf=$RAPIDS_VERSION_ARG* cupy \
         numpy pytest pytest-timeout scipy scikit-learn pandas matplotlib wheel python-kubernetes urllib3 graphviz hypothesis \
         "pyspark>=3.4.0" cloudpickle cuda-python && \

tests/ci_build/conda_env/linux_cpu_test.yml

@@ -17,8 +17,8 @@ dependencies:
 - scikit-learn
 - pandas
 - matplotlib
-- dask>=2022.6
-- distributed>=2022.6
+- dask
+- distributed
 - python-graphviz
 - hypothesis>=6.46
 - astroid

tests/test_distributed/test_gpu_with_dask/test_gpu_with_dask.py

@@ -248,10 +248,10 @@ def test_categorical(self, local_cuda_client: Client) -> None:
         import dask_cudf
 
         X, y = make_categorical(local_cuda_client, 10000, 30, 13)
-        X = dask_cudf.from_dask_dataframe(X)
+        X = X.to_backend("cudf")
 
         X_onehot, _ = make_categorical(local_cuda_client, 10000, 30, 13, True)
-        X_onehot = dask_cudf.from_dask_dataframe(X_onehot)
+        X_onehot = X_onehot.to_backend("cudf")
         run_categorical(local_cuda_client, "hist", "cuda", X, X_onehot, y)
 
     @given(
@@ -383,9 +383,9 @@ def test_dask_classifier(self, model: str, local_cuda_client: Client) -> None:
 
         X_, y_, w_ = generate_array(with_weights=True)
         y_ = (y_ * 10).astype(np.int32)
-        X = dask_cudf.from_dask_dataframe(dd.from_dask_array(X_))
-        y = dask_cudf.from_dask_dataframe(dd.from_dask_array(y_))
-        w = dask_cudf.from_dask_dataframe(dd.from_dask_array(w_))
+        X = dd.from_dask_array(X_).to_backend("cudf")
+        y = dd.from_dask_array(y_).to_backend("cudf")
+        w = dd.from_dask_array(w_).to_backend("cudf")
         run_dask_classifier(X, y, w, model, "gpu_hist", local_cuda_client, 10)
 
     def test_empty_dmatrix(self, local_cuda_client: Client) -> None: