Provide error message when a pool has taken all memory. (dmlc#11173)

Author: trivialfis

src/common/error_msg.h

@@ -118,5 +118,11 @@ constexpr StringView InconsistentFeatureTypes() {
 }
 
 void CheckOldNccl(std::int32_t major, std::int32_t minor, std::int32_t patch);
+
+constexpr StringView ZeroCudaMemory() {
+  return "No GPU memory is left, are you using RMM? If so, please install XGBoost with RMM "
+         "support. If you are using other types of memory pool, please consider reserving a "
+         "portion of the GPU memory for XGBoost.";
+}
 }  // namespace xgboost::error
 #endif  // XGBOOST_COMMON_ERROR_MSG_H_

src/common/hist_util.cu

@@ -98,12 +98,12 @@ bst_idx_t SketchBatchNumElements(bst_idx_t sketch_batch_num_elements, SketchShap
   return std::min(static_cast<bst_idx_t>(n_max_used_f32), shape.nnz);
 #endif  // defined(XGBOOST_USE_RMM) && XGBOOST_USE_RMM == 1
   (void)container_bytes;  // We known the remaining size when RMM is not used.
-
   if (sketch_batch_num_elements == detail::UnknownSketchNumElements()) {
     auto required_memory =
         RequiredMemory(shape.n_samples, shape.n_features, shape.nnz, num_cuts, has_weight);
     // use up to 80% of available space
     auto avail = dh::AvailableMemory(device) * 0.8;
+    CHECK_GT(avail, 0) << error::ZeroCudaMemory();
     if (required_memory > avail) {
       sketch_batch_num_elements = avail / BytesPerElement(has_weight);
     } else {