Skip to content

Commit b7328fe

Browse files
authored
Merge pull request ceph#55592 from cbodley/wip-boost-asio-spawn
rgw: switch back to boost::asio for spawn() and yield_context Reviewed-by: Adam Emerson <[email protected]> Reviewed-by: Yuval Lifshitz <[email protected]>
2 parents 26e6bb0 + 79a6459 commit b7328fe

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

51 files changed

+679
-641
lines changed

.gitmodules

Lines changed: 0 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -50,9 +50,6 @@
5050
[submodule "src/c-ares"]
5151
path = src/c-ares
5252
url = https://github.com/ceph/c-ares.git
53-
[submodule "src/spawn"]
54-
path = src/spawn
55-
url = https://github.com/ceph/spawn.git
5653
[submodule "src/pybind/mgr/rook/rook-client-python"]
5754
path = src/pybind/mgr/rook/rook-client-python
5855
url = https://github.com/ceph/rook-client-python.git

src/CMakeLists.txt

Lines changed: 0 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -896,10 +896,6 @@ if(WITH_RBD)
896896
add_subdirectory(rbd_replay)
897897
endif(WITH_RBD)
898898

899-
set(SPAWN_BUILD_TESTS OFF CACHE INTERNAL "disable building of spawn unit tests")
900-
set(SPAWN_INSTALL OFF CACHE INTERNAL "disable installation of spawn headers")
901-
add_subdirectory(spawn)
902-
903899
# RadosGW
904900
if(WITH_KVS)
905901
add_subdirectory(key_value_store)

src/cls/CMakeLists.txt

Lines changed: 5 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -76,8 +76,7 @@ if (WITH_RADOSGW)
7676
target_link_libraries(cls_otp OATH::OATH)
7777
target_include_directories(cls_otp
7878
PUBLIC "${CMAKE_SOURCE_DIR}/src/rgw/driver/rados"
79-
PUBLIC "${CMAKE_SOURCE_DIR}/src/rgw"
80-
PUBLIC "${CMAKE_SOURCE_DIR}/src/spawn/include")
79+
PUBLIC "${CMAKE_SOURCE_DIR}/src/rgw")
8180
set_target_properties(cls_otp PROPERTIES
8281
VERSION "1.0.0"
8382
SOVERSION "1"
@@ -204,8 +203,7 @@ if (WITH_RADOSGW)
204203
target_link_libraries(cls_rgw ${FMT_LIB} json_spirit)
205204
target_include_directories(cls_rgw
206205
PUBLIC "${CMAKE_SOURCE_DIR}/src/rgw/driver/rados"
207-
PUBLIC "${CMAKE_SOURCE_DIR}/src/rgw"
208-
PUBLIC "${CMAKE_SOURCE_DIR}/src/spawn/include")
206+
PUBLIC "${CMAKE_SOURCE_DIR}/src/rgw")
209207
set_target_properties(cls_rgw PROPERTIES
210208
VERSION "1.0.0"
211209
SOVERSION "1"
@@ -220,8 +218,7 @@ if (WITH_RADOSGW)
220218
add_library(cls_rgw_client STATIC ${cls_rgw_client_srcs})
221219
target_include_directories(cls_rgw_client
222220
PUBLIC "${CMAKE_SOURCE_DIR}/src/rgw/driver/rados"
223-
PUBLIC "${CMAKE_SOURCE_DIR}/src/rgw"
224-
PUBLIC "${CMAKE_SOURCE_DIR}/src/spawn/include")
221+
PUBLIC "${CMAKE_SOURCE_DIR}/src/rgw")
225222

226223
endif (WITH_RADOSGW)
227224

@@ -313,8 +310,7 @@ if (WITH_RADOSGW)
313310
add_library(cls_rgw_gc SHARED ${cls_rgw_gc_srcs})
314311
target_include_directories(cls_rgw_gc
315312
PUBLIC "${CMAKE_SOURCE_DIR}/src/rgw/driver/rados"
316-
PUBLIC "${CMAKE_SOURCE_DIR}/src/rgw"
317-
PUBLIC "${CMAKE_SOURCE_DIR}/src/spawn/include")
313+
PUBLIC "${CMAKE_SOURCE_DIR}/src/rgw")
318314
set_target_properties(cls_rgw_gc PROPERTIES
319315
VERSION "1.0.0"
320316
SOVERSION "1"
@@ -328,8 +324,7 @@ if (WITH_RADOSGW)
328324
add_library(cls_rgw_gc_client STATIC ${cls_rgw_gc_client_srcs})
329325
target_include_directories(cls_rgw_gc_client
330326
PUBLIC "${CMAKE_SOURCE_DIR}/src/rgw/driver/rados"
331-
PUBLIC "${CMAKE_SOURCE_DIR}/src/rgw"
332-
PUBLIC "${CMAKE_SOURCE_DIR}/src/spawn/include")
327+
PUBLIC "${CMAKE_SOURCE_DIR}/src/rgw")
333328
endif (WITH_RADOSGW)
334329

335330

src/common/async/context_pool.h

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -106,6 +106,7 @@ class io_context_pool {
106106
operator boost::asio::io_context&() {
107107
return ioctx;
108108
}
109+
using executor_type = boost::asio::io_context::executor_type;
109110
boost::asio::io_context::executor_type get_executor() {
110111
return ioctx.get_executor();
111112
}

src/common/async/detail/shared_mutex.h

Lines changed: 42 additions & 44 deletions
Original file line numberDiff line numberDiff line change
@@ -123,29 +123,28 @@ auto SharedMutexImpl::async_lock(Mutex& mtx, CompletionToken&& token)
123123
{
124124
using Request = AsyncRequest<Mutex, std::unique_lock>;
125125
using Signature = typename Request::Signature;
126-
boost::asio::async_completion<CompletionToken, Signature> init(token);
127-
auto& handler = init.completion_handler;
128-
auto ex1 = mtx.get_executor();
129-
{
130-
std::lock_guard lock{mutex};
131-
132-
boost::system::error_code ec;
133-
if (state == Unlocked) {
134-
state = Exclusive;
135-
136-
// post a successful completion
137-
auto ex2 = boost::asio::get_associated_executor(handler, ex1);
138-
auto h = boost::asio::bind_executor(ex2, std::move(handler));
139-
boost::asio::post(bind_handler(std::move(h), ec,
140-
std::unique_lock{mtx, std::adopt_lock}));
141-
} else {
142-
// create a request and add it to the exclusive list
143-
using LockCompletion = typename Request::LockCompletion;
144-
auto request = LockCompletion::create(ex1, std::move(handler), mtx);
145-
exclusive_queue.push_back(*request.release());
146-
}
147-
}
148-
return init.result.get();
126+
return boost::asio::async_initiate<CompletionToken, Signature>(
127+
[this] (auto handler, Mutex& mtx) {
128+
auto ex1 = mtx.get_executor();
129+
130+
std::lock_guard lock{mutex};
131+
132+
boost::system::error_code ec;
133+
if (state == Unlocked) {
134+
state = Exclusive;
135+
136+
// post a successful completion
137+
auto ex2 = boost::asio::get_associated_executor(handler, ex1);
138+
auto h = boost::asio::bind_executor(ex2, std::move(handler));
139+
boost::asio::post(bind_handler(std::move(h), ec,
140+
std::unique_lock{mtx, std::adopt_lock}));
141+
} else {
142+
// create a request and add it to the exclusive list
143+
using LockCompletion = typename Request::LockCompletion;
144+
auto request = LockCompletion::create(ex1, std::move(handler), mtx);
145+
exclusive_queue.push_back(*request.release());
146+
}
147+
}, token, mtx);
149148
}
150149

151150
inline void SharedMutexImpl::lock()
@@ -215,27 +214,26 @@ auto SharedMutexImpl::async_lock_shared(Mutex& mtx, CompletionToken&& token)
215214
{
216215
using Request = AsyncRequest<Mutex, std::shared_lock>;
217216
using Signature = typename Request::Signature;
218-
boost::asio::async_completion<CompletionToken, Signature> init(token);
219-
auto& handler = init.completion_handler;
220-
auto ex1 = mtx.get_executor();
221-
{
222-
std::lock_guard lock{mutex};
223-
224-
boost::system::error_code ec;
225-
if (exclusive_queue.empty() && state < MaxShared) {
226-
state++;
227-
228-
auto ex2 = boost::asio::get_associated_executor(handler, ex1);
229-
auto h = boost::asio::bind_executor(ex2, std::move(handler));
230-
boost::asio::post(bind_handler(std::move(h), ec,
231-
std::shared_lock{mtx, std::adopt_lock}));
232-
} else {
233-
using LockCompletion = typename Request::LockCompletion;
234-
auto request = LockCompletion::create(ex1, std::move(handler), mtx);
235-
shared_queue.push_back(*request.release());
236-
}
237-
}
238-
return init.result.get();
217+
return boost::asio::async_initiate<CompletionToken, Signature>(
218+
[this] (auto handler, Mutex& mtx) {
219+
auto ex1 = mtx.get_executor();
220+
221+
std::lock_guard lock{mutex};
222+
223+
boost::system::error_code ec;
224+
if (exclusive_queue.empty() && state < MaxShared) {
225+
state++;
226+
227+
auto ex2 = boost::asio::get_associated_executor(handler, ex1);
228+
auto h = boost::asio::bind_executor(ex2, std::move(handler));
229+
boost::asio::post(bind_handler(std::move(h), ec,
230+
std::shared_lock{mtx, std::adopt_lock}));
231+
} else {
232+
using LockCompletion = typename Request::LockCompletion;
233+
auto request = LockCompletion::create(ex1, std::move(handler), mtx);
234+
shared_queue.push_back(*request.release());
235+
}
236+
}, token, mtx);
239237
}
240238

241239
inline void SharedMutexImpl::lock_shared()

src/common/async/yield_context.h

Lines changed: 7 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -17,23 +17,18 @@
1717
#include <boost/range/begin.hpp>
1818
#include <boost/range/end.hpp>
1919
#include <boost/asio/io_context.hpp>
20+
#include <boost/asio/spawn.hpp>
2021

2122
#include "acconfig.h"
2223

23-
#include <spawn/spawn.hpp>
24-
25-
/// optional-like wrapper for a spawn::yield_context and its associated
26-
/// boost::asio::io_context. operations that take an optional_yield argument
27-
/// will, when passed a non-empty yield context, suspend this coroutine instead
28-
/// of the blocking the thread of execution
24+
/// optional-like wrapper for a boost::asio::yield_context. operations that take
25+
/// an optional_yield argument will, when passed a non-empty yield context,
26+
/// suspend this coroutine instead of the blocking the thread of execution
2927
class optional_yield {
30-
boost::asio::io_context *c = nullptr;
31-
spawn::yield_context *y = nullptr;
28+
boost::asio::yield_context *y = nullptr;
3229
public:
3330
/// construct with a valid io and yield_context
34-
explicit optional_yield(boost::asio::io_context& c,
35-
spawn::yield_context& y) noexcept
36-
: c(&c), y(&y) {}
31+
optional_yield(boost::asio::yield_context& y) noexcept : y(&y) {}
3732

3833
/// type tag to construct an empty object
3934
struct empty_t {};
@@ -42,11 +37,8 @@ class optional_yield {
4237
/// implicit conversion to bool, returns true if non-empty
4338
operator bool() const noexcept { return y; }
4439

45-
/// return a reference to the associated io_context. only valid if non-empty
46-
boost::asio::io_context& get_io_context() const noexcept { return *c; }
47-
4840
/// return a reference to the yield_context. only valid if non-empty
49-
spawn::yield_context& get_yield_context() const noexcept { return *y; }
41+
boost::asio::yield_context& get_yield_context() const noexcept { return *y; }
5042
};
5143

5244
// type tag object to construct an empty optional_yield

src/crypto/isa-l/CMakeLists.txt

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -30,7 +30,7 @@ endif(HAVE_NASM_X64)
3030
add_library(ceph_crypto_isal SHARED ${isal_crypto_plugin_srcs})
3131
target_include_directories(ceph_crypto_isal PRIVATE ${isal_dir}/include)
3232

33-
target_link_libraries(ceph_crypto_isal PRIVATE spawn)
33+
target_link_libraries(ceph_crypto_isal PRIVATE Boost::context)
3434

3535
set_target_properties(ceph_crypto_isal PROPERTIES
3636
VERSION 1.0.0

src/crypto/openssl/CMakeLists.txt

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -8,7 +8,7 @@ add_library(ceph_crypto_openssl SHARED ${openssl_crypto_plugin_srcs})
88
target_link_libraries(ceph_crypto_openssl
99
PRIVATE OpenSSL::Crypto
1010
$<$<PLATFORM_ID:Windows>:ceph-common>
11-
spawn)
11+
Boost::context)
1212
target_include_directories(ceph_crypto_openssl PRIVATE ${OPENSSL_INCLUDE_DIR})
1313
add_dependencies(crypto_plugins ceph_crypto_openssl)
1414
set_target_properties(ceph_crypto_openssl PROPERTIES INSTALL_RPATH "")

src/crypto/qat/CMakeLists.txt

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -14,7 +14,7 @@ add_dependencies(crypto_plugins ceph_crypto_qat)
1414
target_link_libraries(ceph_crypto_qat PRIVATE
1515
QAT::qat
1616
QAT::usdm
17-
spawn)
17+
Boost::context)
1818

1919
add_dependencies(crypto_plugins ceph_crypto_qat)
2020
set_target_properties(ceph_crypto_qat PROPERTIES VERSION 1.0.0 SOVERSION 1)

src/crypto/qat/qcccrypto.cc

Lines changed: 45 additions & 43 deletions
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,8 @@
1212
#include <future>
1313
#include <chrono>
1414

15+
#include <boost/asio/append.hpp>
16+
#include <boost/asio/async_result.hpp>
1517
#include "boost/container/static_vector.hpp"
1618

1719
// -----------------------------------------------------------------------------
@@ -49,36 +51,31 @@ static std::condition_variable poll_inst_cv;
4951

5052
template <typename CompletionToken>
5153
auto QccCrypto::async_get_instance(CompletionToken&& token) {
52-
using boost::asio::async_completion;
5354
using Signature = void(int);
54-
async_completion<CompletionToken, Signature> init(token);
55-
56-
auto ex = boost::asio::get_associated_executor(init.completion_handler);
57-
58-
boost::asio::post(my_pool, [this, ex, handler = std::move(init.completion_handler)]()mutable{
59-
auto handler1 = std::move(handler);
60-
if (!open_instances.empty()) {
61-
int avail_inst = open_instances.front();
62-
open_instances.pop_front();
63-
boost::asio::post(ex, std::bind(handler1, avail_inst));
64-
} else if (!instance_completions.full()) {
65-
// keep a few objects to wait QAT instance to make sure qat full utilization as much as possible,
66-
// that is, QAT don't need to wait for new objects to ensure
67-
// that QAT will not be in a free state as much as possible
68-
instance_completions.push_back([ex, handler2 = std::move(handler1)](int inst)mutable{
69-
boost::asio::post(ex, std::bind(handler2, inst));
70-
});
71-
} else {
72-
boost::asio::post(ex, std::bind(handler1, NON_INSTANCE));
73-
}
74-
});
75-
return init.result.get();
55+
return boost::asio::async_initiate<CompletionToken, Signature>(
56+
[this] (auto handler) {
57+
boost::asio::post(my_pool, [this, handler = std::move(handler)]()mutable{
58+
if (!open_instances.empty()) {
59+
int avail_inst = open_instances.front();
60+
open_instances.pop_front();
61+
boost::asio::post(boost::asio::append(std::move(handler), avail_inst));
62+
} else if (!instance_completions.full()) {
63+
// keep a few objects to wait QAT instance to make sure qat full utilization as much as possible,
64+
// that is, QAT don't need to wait for new objects to ensure
65+
// that QAT will not be in a free state as much as possible
66+
instance_completions.push_back(std::move(handler));
67+
} else {
68+
boost::asio::post(boost::asio::append(std::move(handler), NON_INSTANCE));
69+
}
70+
});
71+
}, token);
7672
}
7773

7874
void QccCrypto::QccFreeInstance(int entry) {
7975
boost::asio::post(my_pool, [this, entry]()mutable{
8076
if (!instance_completions.empty()) {
81-
instance_completions.front()(entry);
77+
boost::asio::dispatch(boost::asio::append(
78+
std::move(instance_completions.front()), entry));
8279
instance_completions.pop_front();
8380
} else {
8481
open_instances.push_back(entry);
@@ -334,7 +331,7 @@ bool QccCrypto::perform_op_batch(unsigned char* out, const unsigned char* in, si
334331
int avail_inst = NON_INSTANCE;
335332

336333
if (y) {
337-
spawn::yield_context yield = y.get_yield_context();
334+
boost::asio::yield_context yield = y.get_yield_context();
338335
avail_inst = async_get_instance(yield);
339336
} else {
340337
auto result = async_get_instance(boost::asio::use_future);
@@ -477,24 +474,29 @@ CpaStatus QccCrypto::initSession(CpaInstanceHandle cyInstHandle,
477474
}
478475

479476
template <typename CompletionToken>
480-
auto QatCrypto::async_perform_op(int avail_inst, std::span<CpaCySymDpOpData*> pOpDataVec, CompletionToken&& token) {
481-
CpaStatus status = CPA_STATUS_SUCCESS;
482-
using boost::asio::async_completion;
477+
auto QatCrypto::async_perform_op(std::span<CpaCySymDpOpData*> pOpDataVec, CompletionToken&& token) {
483478
using Signature = void(CpaStatus);
484-
async_completion<CompletionToken, Signature> init(token);
485-
auto ex = boost::asio::get_associated_executor(init.completion_handler);
486-
completion_handler = [ex, handler = init.completion_handler](CpaStatus stat) {
487-
boost::asio::post(ex, std::bind(handler, stat));
488-
};
479+
return boost::asio::async_initiate<CompletionToken, Signature>(
480+
[this] (auto handler, std::span<CpaCySymDpOpData*> pOpDataVec) {
481+
completion_handler = std::move(handler);
482+
483+
count = pOpDataVec.size();
484+
poll_inst_cv.notify_one();
485+
CpaStatus status = cpaCySymDpEnqueueOpBatch(pOpDataVec.size(), pOpDataVec.data(), CPA_TRUE);
489486

490-
count = pOpDataVec.size();
491-
poll_inst_cv.notify_one();
492-
status = cpaCySymDpEnqueueOpBatch(pOpDataVec.size(), pOpDataVec.data(), CPA_TRUE);
487+
if (status != CPA_STATUS_SUCCESS) {
488+
boost::asio::post(bind_executor(ex,
489+
boost::asio::append(std::move(completion_handler), status)));
490+
}
491+
}, token, pOpDataVec);
492+
}
493493

494-
if (status != CPA_STATUS_SUCCESS) {
495-
completion_handler(status);
494+
void QatCrypto::complete() {
495+
if (--count == 0) {
496+
boost::asio::post(bind_executor(ex,
497+
boost::asio::append(std::move(completion_handler), CPA_STATUS_SUCCESS)));
496498
}
497-
return init.result.get();
499+
return;
498500
}
499501

500502
bool QccCrypto::symPerformOp(int avail_inst,
@@ -510,7 +512,7 @@ bool QccCrypto::symPerformOp(int avail_inst,
510512
Cpa32U iv_index = 0;
511513
size_t perform_retry_num = 0;
512514
for (Cpa32U off = 0; off < size; off += one_batch_size) {
513-
QatCrypto helper;
515+
QatCrypto helper{my_pool.get_executor()};
514516
boost::container::static_vector<CpaCySymDpOpData*, MAX_NUM_SYM_REQ_BATCH> pOpDataVec;
515517
for (Cpa32U offset = off, i = 0; offset < size && i < MAX_NUM_SYM_REQ_BATCH; offset += chunk_size, i++) {
516518
CpaCySymDpOpData *pOpData = qcc_op_mem[avail_inst].sym_op_data[i];
@@ -544,10 +546,10 @@ bool QccCrypto::symPerformOp(int avail_inst,
544546
do {
545547
poll_retry_num = RETRY_MAX_NUM;
546548
if (y) {
547-
spawn::yield_context yield = y.get_yield_context();
548-
status = helper.async_perform_op(avail_inst, std::span<CpaCySymDpOpData*>(pOpDataVec), yield);
549+
boost::asio::yield_context yield = y.get_yield_context();
550+
status = helper.async_perform_op(std::span<CpaCySymDpOpData*>(pOpDataVec), yield);
549551
} else {
550-
auto result = helper.async_perform_op(avail_inst, std::span<CpaCySymDpOpData*>(pOpDataVec), boost::asio::use_future);
552+
auto result = helper.async_perform_op(std::span<CpaCySymDpOpData*>(pOpDataVec), boost::asio::use_future);
551553
status = result.get();
552554
}
553555
if (status == CPA_STATUS_RETRY) {

0 commit comments

Comments
 (0)