Merge PR ceph#54067 into main

* refs/pull/54067/head:
	mds: add dispatch killpoint and delay configs

Reviewed-by: Leonid Usov <[email protected]>

Author: batrick

src/common/options/mds.yaml.in

@@ -1680,3 +1680,19 @@ options:
   - mds
   flags:
   - runtime
+- name: mds_server_dispatch_killpoint_random
+  type: float
+  level: dev
+  default: 0.0
+  services:
+  - mds
+  flags:
+  - runtime
+- name: mds_server_dispatch_client_request_delay
+  type: millisecs
+  level: dev
+  default: 0
+  services:
+  - mds
+  flags:
+  - runtime

src/mds/MDSRank.cc

@@ -4029,6 +4029,8 @@ const char** MDSRankDispatcher::get_tracked_conf_keys() const
     "mds_debug_subtrees",
     "mds_dir_max_entries",
     "mds_dump_cache_threshold_file",
+    "mds_server_dispatch_client_request_delay",
+    "mds_server_dispatch_killpoint_random",
     "mds_dump_cache_threshold_formatter",
     "mds_enable_op_tracker",
     "mds_export_ephemeral_distributed",

src/mds/Server.cc

@@ -273,6 +273,8 @@ Server::Server(MDSRank *m, MetricsHandler *metrics_handler) :
   caps_throttle_retry_request_timeout = g_conf().get_val<double>("mds_cap_acquisition_throttle_retry_request_timeout");
   dir_max_entries = g_conf().get_val<uint64_t>("mds_dir_max_entries");
   bal_fragment_size_max = g_conf().get_val<int64_t>("mds_bal_fragment_size_max");
+  dispatch_client_request_delay = g_conf().get_val<std::chrono::milliseconds>("mds_server_dispatch_client_request_delay");
+  dispatch_killpoint_random = g_conf().get_val<double>("mds_server_dispatch_killpoint_random");
   supported_features = feature_bitset_t(CEPHFS_FEATURES_MDS_SUPPORTED);
   supported_metric_spec = feature_bitset_t(CEPHFS_METRIC_FEATURES_ALL);
 }
@@ -1375,6 +1377,16 @@ void Server::handle_conf_change(const std::set<std::string>& changed) {
   if (changed.count("mds_inject_rename_corrupt_dentry_first")) {
     inject_rename_corrupt_dentry_first = g_conf().get_val<double>("mds_inject_rename_corrupt_dentry_first");
   }
+  if (changed.count("mds_server_dispatch_client_request_delay")) {
+    dispatch_client_request_delay = g_conf().get_val<std::chrono::milliseconds>("mds_server_dispatch_client_request_delay");
+    dout(20) << __func__ << " mds_server_dispatch_client_request_delay now "
+            << dispatch_client_request_delay << dendl;
+  }
+  if (changed.count("mds_server_dispatch_killpoint_random")) {
+    dispatch_killpoint_random = g_conf().get_val<double>("mds_server_dispatch_killpoint_random");
+    dout(20) << __func__ << " mds_server_dispatch_killpoint_random now "
+            << dispatch_killpoint_random << dendl;
+  }
 }
 
 /*
@@ -2667,6 +2679,14 @@ void Server::dispatch_client_request(const MDRequestRef& mdr)
 
   dout(7) << "dispatch_client_request " << *req << dendl;
 
+  auto zeroms = std::chrono::milliseconds::zero();
+  if (unlikely(dispatch_client_request_delay > zeroms)) {
+    std::this_thread::sleep_for(dispatch_client_request_delay);
+  }
+  if (unlikely(dispatch_killpoint_random > 0.0) && dispatch_killpoint_random >= ceph::util::generate_random_number(0.0, 1.0)) {
+    ceph_abort("dispatch_killpoint_random");
+  }
+
   if (req->may_write() && mdcache->is_readonly()) {
     dout(10) << " read-only FS" << dendl;
     respond_to_request(mdr, -CEPHFS_EROFS);

src/mds/Server.h

@@ -566,6 +566,9 @@ class Server {
   double max_caps_throttle_ratio;
   double caps_throttle_retry_request_timeout;
 
+  std::chrono::milliseconds dispatch_client_request_delay{0};
+  double dispatch_killpoint_random{0.0};
+
   size_t alternate_name_max = g_conf().get_val<Option::size_t>("mds_alternate_name_max");
   size_t fscrypt_last_block_max_size = g_conf().get_val<Option::size_t>("mds_fscrypt_last_block_max_size");