Add checks for invalid inputs for cv. (dmlc#11255)

Author: trivialfis

python-package/xgboost/callback.py

@@ -325,7 +325,9 @@ class EarlyStopping(TrainingCallback):
     maximize :
         Whether to maximize evaluation metric.  None means auto (discouraged).
     save_best :
-        Whether training should return the best model or the last model.
+        Whether training should return the best model or the last model. This is only
+        supported with tree methods. Also, the `cv` function doesn't return a model, the
+        parameter is not applicable.
     min_delta :
 
         .. versionadded:: 1.5.0
@@ -380,6 +382,11 @@ def __init__(
 
     def before_training(self, model: _Model) -> _Model:
         self.starting_round = model.num_boosted_rounds()
+        if not isinstance(model, Booster) and self.save_best:
+            raise ValueError(
+                "`save_best` is not applicable to the `cv` function as it doesn't return"
+                " a model."
+            )
         return model
 
     def _update_rounds(
@@ -428,7 +435,7 @@ def minimize(new: _Score, best: _Score) -> bool:
             self.stopping_history[name][metric] = cast(_ScoreList, [score])
             self.best_scores[name] = {}
             self.best_scores[name][metric] = [score]
-            model.set_attr(best_score=str(score), best_iteration=str(epoch))
+            model.set_attr(best_score=str(get_s(score)), best_iteration=str(epoch))
         elif not improve_op(score, self.best_scores[name][metric][-1]):
             # Not improved
             self.stopping_history[name][metric].append(score)  # type: ignore
@@ -437,7 +444,7 @@ def minimize(new: _Score, best: _Score) -> bool:
             self.stopping_history[name][metric].append(score)  # type: ignore
             self.best_scores[name][metric].append(score)
             record = self.stopping_history[name][metric][-1]
-            model.set_attr(best_score=str(record), best_iteration=str(epoch))
+            model.set_attr(best_score=str(get_s(record)), best_iteration=str(epoch))
             self.current_rounds = 0  # reset
 
         if self.current_rounds >= self.rounds:

python-package/xgboost/training.py

@@ -23,6 +23,7 @@
     Objective,
     XGBoostError,
     _deprecate_positional_args,
+    _RefMixIn,
 )
 
 _CVFolds = Sequence["CVPack"]
@@ -153,7 +154,7 @@ def train(
             raise TypeError("Invalid type for the `evals`.")
 
         if (
-            hasattr(va, "ref")
+            isinstance(va, _RefMixIn)
             and va.ref is not weakref.ref(dtrain)
             and va is not dtrain
         ):
@@ -442,9 +443,10 @@ def cv(
     ----------
     params : dict
         Booster params.
-    dtrain : DMatrix
-        Data to be trained.
-    num_boost_round : int
+    dtrain :
+        Data to be trained. Only the :py:class:`DMatrix` without external memory is
+        supported.
+    num_boost_round :
         Number of boosting iterations.
     nfold : int
         Number of folds in CV.
@@ -525,9 +527,10 @@ def cv(
         raise XGBoostError(
             "sklearn needs to be installed in order to use stratified cv"
         )
-
     if isinstance(metrics, str):
         metrics = [metrics]
+    if isinstance(dtrain, _RefMixIn):
+        raise ValueError("`QuantileDMatrix` is not yet supported.")
 
     params = params.copy()
     if isinstance(params, list):

tests/python/test_early_stopping.py

@@ -107,6 +107,16 @@ def evalerror(preds: np.ndarray, dtrain: xgb.DMatrix) -> Tuple[str, float]:
         )
         self.assert_metrics_length(cv, 1)
 
+        with pytest.raises(ValueError, match="`save_best`"):
+            cv = xgb.cv(
+                params,
+                dm,
+                num_boost_round=10,
+                nfold=10,
+                early_stopping_rounds=1,
+                callbacks=[xgb.callback.EarlyStopping(3, save_best=True)],
+            )
+
     @pytest.mark.skipif(**tm.no_sklearn())
     @pytest.mark.skipif(**tm.no_pandas())
     def test_cv_early_stopping_with_multiple_eval_sets_and_metrics(self):

tests/python/test_quantile_dmatrix.py

@@ -395,3 +395,9 @@ def test_sparse_predict(self) -> None:
         Xy = xgb.DMatrix(X, y, enable_categorical=True)
         p1 = booster.predict(Xy)
         np.testing.assert_allclose(p0, p1)
+
+    def test_cv_error(self) -> None:
+        X, y = make_sparse_regression(8, 2, sparsity=0.2, as_dense=False)
+        Xy = xgb.QuantileDMatrix(X, y)
+        with pytest.raises(ValueError, match=""):
+            cv = xgb.cv({}, Xy, 10, nfold=10, early_stopping_rounds=10)