Merge pull request ceph#55641 from Matan-B/wip-matanb-crimson-alien-buf-v2

crimson/osd/main: Enable multicore client messenger

Reviewed-by: Samuel Just <[email protected]>
Reviewed-by: chunmei-liu <[email protected]>

Author: Matan-B

src/crimson/CMakeLists.txt

@@ -24,6 +24,7 @@ set(crimson_common_srcs
   common/throttle.cc
   common/tmap_helpers.cc
   common/tri_mutex.cc
+  common/buffer_seastar.cc
   crush/CrushLocation.cc)
 
 # the specialized version of ceph-common, where
@@ -35,7 +36,6 @@ add_library(crimson-common STATIC
   ${PROJECT_SOURCE_DIR}/src/common/bit_str.cc
   ${PROJECT_SOURCE_DIR}/src/common/bloom_filter.cc
   ${PROJECT_SOURCE_DIR}/src/common/buffer.cc
-  ${PROJECT_SOURCE_DIR}/src/common/buffer_seastar.cc
   ${PROJECT_SOURCE_DIR}/src/common/ceph_argparse.cc
   ${PROJECT_SOURCE_DIR}/src/common/ceph_context.cc
   ${PROJECT_SOURCE_DIR}/src/common/ceph_crypto.cc

src/crimson/auth/KeyRing.cc

@@ -10,7 +10,7 @@
 #include <seastar/core/future-util.hh>
 #include <seastar/core/reactor.hh>
 
-#include "common/buffer_seastar.h"
+#include "crimson/common/buffer_seastar.h"
 #include "auth/KeyRing.h"
 #include "include/denc.h"
 #include "crimson/common/buffer_io.h"

src/common/buffer_seastar.cc src/crimson/common/buffer_seastar.ccsrc/common/buffer_seastar.cc renamed to src/crimson/common/buffer_seastar.cc

@@ -14,6 +14,8 @@
 
 #include <seastar/core/sharded.hh>
 #include <seastar/net/packet.hh>
+#include <seastar/core/reactor.hh>
+#include <seastar/core/alien.hh>
 
 #include "include/buffer_raw.h"
 #include "buffer_seastar.h"
@@ -24,9 +26,21 @@ namespace ceph::buffer {
 
 class raw_seastar_foreign_ptr : public raw {
   seastar::foreign_ptr<temporary_buffer> ptr;
+  seastar::alien::instance& alien;
  public:
   raw_seastar_foreign_ptr(temporary_buffer&& buf)
-    : raw(buf.get_write(), buf.size()), ptr(std::move(buf)) {}
+    : raw(buf.get_write(), buf.size()), ptr(std::move(buf)),
+      alien(seastar::engine().alien()) {}
+
+  ~raw_seastar_foreign_ptr() {
+    if (!seastar::engine_is_ready()) {
+      // we should let a seastar reactor destroy this memory, we are alien.
+      seastar::alien::run_on(alien, ptr.get_owner_shard(),
+      [_ptr = std::move(ptr)]() mutable noexcept {
+        _ptr.reset();
+      });
+    }
+  }
 };
 
 class raw_seastar_local_ptr : public raw {

src/common/buffer_seastar.h src/crimson/common/buffer_seastar.hsrc/common/buffer_seastar.h renamed to src/crimson/common/buffer_seastar.h

@@ -186,8 +186,13 @@ int main(int argc, const char* argv[])
           const auto nonce = crimson::osd::get_nonce();
           crimson::net::MessengerRef cluster_msgr, client_msgr;
           crimson::net::MessengerRef hb_front_msgr, hb_back_msgr;
+          for (auto [msgr, name] : {make_pair(std::ref(client_msgr), "client"s)}) {
+            msgr = crimson::net::Messenger::create(entity_name_t::OSD(whoami),
+                                                   name,
+                                                   nonce,
+                                                   false);
+          }
           for (auto [msgr, name] : {make_pair(std::ref(cluster_msgr), "cluster"s),
-                                    make_pair(std::ref(client_msgr), "client"s),
                                     make_pair(std::ref(hb_front_msgr), "hb_front"s),
                                     make_pair(std::ref(hb_back_msgr), "hb_back"s)}) {
             msgr = crimson::net::Messenger::create(entity_name_t::OSD(whoami),

src/crimson/osd/main.cc

@@ -2,7 +2,7 @@
 #include <seastar/core/temporary_buffer.hh>
 #include <gtest/gtest.h>
 #include "include/denc.h"
-#include "common/buffer_seastar.h"
+#include "crimson/common/buffer_seastar.h"
 
 using temporary_buffer = seastar::temporary_buffer<char>;
 using buffer_iterator = seastar_buffer_iterator;