Merge pull request ceph#63762 from cbodley/wip-rgw-lc-wp-coro

rgw/lc: replace WorkPool with ceph::async::spawn_throttle

Reviewed-by: Adam Emerson <[email protected]>
Reviewed-by: Matt Benjamin <[email protected]>

Author: cbodley

PendingReleaseNotes

@@ -166,6 +166,9 @@
   Tentacle or earlier release, the value of this field will be 'N/A'. Regular
   subvolumes don't have a source subvolume and therefore the output for them
   won't contain a "source" field regardless of the release.
+* RGW: The default value of `rgw_lc_max_wp_worker` has been raised from 3 to 128
+  to accelerate lifecycle processing. These workers now use coroutines instead of
+  separate threads.
 
 * RGW: Replication policies now validate permissions using `s3:ReplicateObject`,
   `s3:ReplicateDelete`, and `s3:ReplicateTags` for destination buckets. For source

src/common/options/rgw.yaml.in

@@ -441,12 +441,12 @@ options:
 - name: rgw_lc_max_wp_worker
   type: int
   level: advanced
-  desc: Number of workpool threads per LCWorker
-  long_desc: Number of threads in per-LCWorker workpools--used to accelerate per-bucket
+  desc: Number of workpool coroutines per LCWorker
+  long_desc: Number of coroutines in per-LCWorker workpools--used to accelerate per-bucket
     processing
-  fmt_desc: This option specifies the number of threads in each lifecycle
+  fmt_desc: This option specifies the number of coroutines in each lifecycle
     workers work pool. This option can help accelerate processing each bucket.
-  default: 3
+  default: 128
   services:
   - rgw
   with_legacy: true

src/rgw/rgw_asio_thread.h

@@ -15,6 +15,8 @@
 
 #pragma once
 
+#include <assert.h>
+
 class DoutPrefixProvider;
 
 /// indicates whether the current thread is in boost::asio::io_context::run(),
@@ -24,3 +26,14 @@ extern thread_local bool is_asio_thread;
 /// call when an operation will block the calling thread due to an empty
 /// optional_yield. a warning is logged when is_asio_thread is true
 void maybe_warn_about_blocking(const DoutPrefixProvider* dpp);
+
+/// enables warnings while in scope. these scopes must not be nested
+struct warn_about_blocking_in_scope {
+  warn_about_blocking_in_scope() {
+    assert(!is_asio_thread);
+    is_asio_thread = true;
+  }
+  ~warn_about_blocking_in_scope() {
+    is_asio_thread = false;
+  }
+};