Merge pull request ceph#59832 from NitzanMordhai/wip-nitzan-crimson-enable-gate-assertions

common/gated: enable ceph_assert on shard id

Author: NitzanMordhai

src/crimson/common/gated.h

@@ -31,13 +31,13 @@ class Gated {
 
   template <typename Func, typename T>
   inline void dispatch_in_background(const char* what, T& who, Func&& func) {
-    //ceph_assert(seastar::this_shard_id() == sid);
+    ceph_assert(seastar::this_shard_id() == sid);
     (void) dispatch(what, who, std::forward<Func>(func));
   }
 
   template <typename Func, typename T>
   inline seastar::future<> dispatch(const char* what, T& who, Func&& func) {
-    //ceph_assert(seastar::this_shard_id() == sid);
+    ceph_assert(seastar::this_shard_id() == sid);
     return seastar::with_gate(pending_dispatch, std::forward<Func>(func)
     ).handle_exception([what, &who] (std::exception_ptr eptr) {
       if (*eptr.__cxa_exception_type() == typeid(system_shutdown_exception)) {
@@ -58,7 +58,7 @@ class Gated {
 
   template <typename Func>
   auto simple_dispatch(const char* what, Func&& func) {
-    //ceph_assert(seastar::this_shard_id() == sid);
+    ceph_assert(seastar::this_shard_id() == sid);
     return seastar::with_gate(pending_dispatch, std::forward<Func>(func));
   }
 

src/crimson/mgr/client.cc

@@ -41,7 +41,7 @@ seastar::future<> Client::stop()
 {
   logger().info("{}", __func__);
   report_timer.cancel();
-  auto fut = gate.close();
+  auto fut = gates.close_all();
   if (conn) {
     conn->mark_down();
   }
@@ -52,7 +52,7 @@ std::optional<seastar::future<>>
 Client::ms_dispatch(crimson::net::ConnectionRef conn, MessageRef m)
 {
   bool dispatched = true;
-  gate.dispatch_in_background(__func__, *this, [this, conn, &m, &dispatched] {
+  gates.dispatch_in_background(__func__, *this, [this, conn, &m, &dispatched] {
     switch(m->get_type()) {
     case MSG_MGR_MAP:
       return handle_mgr_map(conn, boost::static_pointer_cast<MMgrMap>(m));
@@ -71,7 +71,7 @@ void Client::ms_handle_connect(
     seastar::shard_id prv_shard)
 {
   ceph_assert_always(prv_shard == seastar::this_shard_id());
-  gate.dispatch_in_background(__func__, *this, [this, c] {
+  gates.dispatch_in_background(__func__, *this, [this, c] {
     if (conn == c) {
       // ask for the mgrconfigure message
       auto m = crimson::make_message<MMgrOpen>();
@@ -87,7 +87,7 @@ void Client::ms_handle_connect(
 
 void Client::ms_handle_reset(crimson::net::ConnectionRef c, bool /* is_replace */)
 {
-  gate.dispatch_in_background(__func__, *this, [this, c] {
+  gates.dispatch_in_background(__func__, *this, [this, c] {
     if (conn == c) {
       report_timer.cancel();
       return reconnect();
@@ -158,7 +158,7 @@ seastar::future<> Client::handle_mgr_conf(crimson::net::ConnectionRef,
 void Client::report()
 {
   _send_report();
-  gate.dispatch_in_background(__func__, *this, [this] {
+  gates.dispatch_in_background(__func__, *this, [this] {
     if (!conn) {
       logger().warn("report: no conn available; report skipped");
       return seastar::now();
@@ -178,7 +178,7 @@ void Client::_send_report()
 {
   // TODO: implement daemon_health_metrics support
   // https://tracker.ceph.com/issues/63766
-  gate.dispatch_in_background(__func__, *this, [this] {
+  gates.dispatch_in_background(__func__, *this, [this] {
     if (!conn) {
       logger().warn("cannot send report; no conn available");
       return seastar::now();

src/crimson/mgr/client.h

@@ -55,7 +55,7 @@ class Client : public crimson::net::Dispatcher {
   WithStats& with_stats;
   crimson::net::ConnectionRef conn;
   seastar::timer<seastar::lowres_clock> report_timer;
-  crimson::common::Gated gate;
+  crimson::common::gate_per_shard gates;
   uint64_t last_config_bl_version = 0;
   std::string service_name, daemon_name;
 

src/crimson/mon/MonClient.cc

@@ -464,7 +464,7 @@ seastar::future<> Client::load_keyring()
 
 void Client::tick()
 {
-  gate.dispatch_in_background(__func__, *this, [this] {
+  gates.dispatch_in_background(__func__, *this, [this] {
     if (active_con) {
       return seastar::when_all_succeed(wait_for_send_log(),
                                        active_con->get_conn()->send_keepalive(),
@@ -505,7 +505,7 @@ std::optional<seastar::future<>>
 Client::ms_dispatch(crimson::net::ConnectionRef conn, MessageRef m)
 {
   bool dispatched = true;
-  gate.dispatch_in_background(__func__, *this, [this, conn, &m, &dispatched] {
+  gates.dispatch_in_background(__func__, *this, [this, conn, &m, &dispatched] {
     // we only care about these message types
     switch (m->get_type()) {
     case CEPH_MSG_MON_MAP:
@@ -538,7 +538,7 @@ Client::ms_dispatch(crimson::net::ConnectionRef conn, MessageRef m)
 
 void Client::ms_handle_reset(crimson::net::ConnectionRef conn, bool /* is_replace */)
 {
-  gate.dispatch_in_background(__func__, *this, [this, conn] {
+  gates.dispatch_in_background(__func__, *this, [this, conn] {
     auto found = std::find_if(pending_conns.begin(), pending_conns.end(),
 			      [peer_addr = conn->get_peer_addr()](auto& mc) {
 				return mc->is_my_peer(peer_addr);
@@ -941,7 +941,7 @@ seastar::future<> Client::authenticate()
 seastar::future<> Client::stop()
 {
   logger().info("{}", __func__);
-  auto fut = gate.close();
+  auto fut = gates.close_all();
   timer.cancel();
   ready_to_send = false;
   for (auto& pending_con : pending_conns) {

src/crimson/mon/MonClient.h

@@ -194,7 +194,7 @@ class Client : public crimson::net::Dispatcher,
   std::vector<unsigned> get_random_mons(unsigned n) const;
   seastar::future<> _add_conn(unsigned rank, uint64_t global_id);
   void _finish_auth(const entity_addr_t& peer);
-  crimson::common::Gated gate;
+  crimson::common::gate_per_shard gates;
 
   // messages that are waiting for the active_con to be available
   struct pending_msg_t {