rgw: RGWSI_Notify uses Finisher instead of RGWSI_Finisher

RGWSI_Notify is the only user of RGWSI_Finisher, but doesn't "own" it.
this means RGWSI_Notify can't drain/stop the Finisher on shutdown to
guarantee the lifetime of contexts like C_ReinitWatch that it
schedules there

replace the use of RGWSI_Finisher with direct ownership of Finisher, and
drain/stop it at the end of RGWSI_Notify::shutdown()

Signed-off-by: Casey Bodley <[email protected]>

Author: cbodley

src/rgw/driver/rados/rgw_service.cc

@@ -97,8 +97,7 @@ int RGWServices_Def::init(CephContext *cct,
   config_key_rados->init(driver->getRados()->get_rados_handle());
   mdlog->init(driver->getRados()->get_rados_handle(), zone.get(), sysobj.get(),
 	      cls.get(), async_processor.get());
-  notify->init(zone.get(), driver->getRados()->get_rados_handle(),
-	       finisher.get());
+  notify->init(zone.get(), driver->getRados()->get_rados_handle());
   zone->init(sysobj.get(), driver->getRados()->get_rados_handle(),
 	     sync_modules.get(), bucket_sync_sobj.get());
   zone_utils->init(driver->getRados()->get_rados_handle(), zone.get());

src/rgw/services/svc_notify.cc

@@ -9,7 +9,6 @@
 
 #include "rgw_cache.h"
 #include "svc_notify.h"
-#include "svc_finisher.h"
 #include "svc_zone.h"
 
 #include "rgw_zone.h"
@@ -134,22 +133,15 @@ class RGWWatcher : public DoutPrefixProvider , public librados::WatchCtx2 {
   }
 };
 
-RGWSI_Notify::RGWSI_Notify(CephContext *cct) : RGWServiceInstance(cct) {}
+RGWSI_Notify::RGWSI_Notify(CephContext *cct)
+  : RGWServiceInstance(cct), finisher(cct)
+{
+}
 RGWSI_Notify::~RGWSI_Notify()
 {
   shutdown();
 }
 
-class RGWSI_Notify_ShutdownCB : public RGWSI_Finisher::ShutdownCB
-{
-  RGWSI_Notify *svc;
-public:
-  RGWSI_Notify_ShutdownCB(RGWSI_Notify *_svc) : svc(_svc) {}
-  void call() override {
-    svc->shutdown();
-  }
-};
-
 string RGWSI_Notify::get_control_oid(int i)
 {
   char buf[notify_oid_prefix.size() + 16];
@@ -258,10 +250,7 @@ int RGWSI_Notify::do_start(optional_yield y, const DoutPrefixProvider *dpp)
 
   assert(zone_svc->is_started()); /* otherwise there's an ordering problem */
 
-  r = finisher_svc->start(y, dpp);
-  if (r < 0) {
-    return r;
-  }
+  finisher.start();
 
   inject_notify_timeout_probability =
     cct->_conf.get_val<double>("rgw_inject_notify_timeout_probability");
@@ -294,11 +283,6 @@ int RGWSI_Notify::do_start(optional_yield y, const DoutPrefixProvider *dpp)
     return ret;
   }
 
-  shutdown_cb = new RGWSI_Notify_ShutdownCB(this);
-  int handle;
-  finisher_svc->register_caller(shutdown_cb, &handle);
-  finisher_handle = handle;
-
   return 0;
 }
 
@@ -308,10 +292,6 @@ void RGWSI_Notify::shutdown()
     return;
   }
 
-  if (finisher_handle) {
-    finisher_svc->unregister_caller(*finisher_handle);
-  }
-
   // we're not running in a coroutine, so spawn one
   boost::asio::io_context context;
   boost::asio::spawn(context,
@@ -323,7 +303,8 @@ void RGWSI_Notify::shutdown()
       });
   context.run();
 
-  delete shutdown_cb;
+  finisher.wait_for_empty();
+  finisher.stop();
 
   finalized = true;
 }
@@ -525,5 +506,5 @@ void RGWSI_Notify::register_watch_cb(CB *_cb)
 
 void RGWSI_Notify::schedule_context(Context *c)
 {
-  finisher_svc->schedule_context(c);
+  finisher.queue(c);
 }

src/rgw/services/svc_notify.h

@@ -3,24 +3,20 @@
 
 #pragma once
 
+#include "common/Finisher.h"
 #include "rgw_service.h"
 
 #include "rgw_tools.h"
 
 
-class Context;
-
 class RGWSI_Zone;
-class RGWSI_Finisher;
 
 class RGWWatcher;
-class RGWSI_Notify_ShutdownCB;
 struct RGWCacheNotifyInfo;
 
 class RGWSI_Notify : public RGWServiceInstance
 {
   friend class RGWWatcher;
-  friend class RGWSI_Notify_ShutdownCB;
   friend struct RGWServices_Def;
 
 public:
@@ -29,7 +25,7 @@ class RGWSI_Notify : public RGWServiceInstance
 private:
   RGWSI_Zone *zone_svc{nullptr};
   librados::Rados *rados{nullptr};
-  RGWSI_Finisher *finisher_svc{nullptr};
+  Finisher finisher;
 
   ceph::shared_mutex watchers_lock = ceph::make_shared_mutex("watchers_lock");
   rgw_pool control_pool;
@@ -48,21 +44,16 @@ class RGWSI_Notify : public RGWServiceInstance
 
   CB *cb{nullptr};
 
-  std::optional<int> finisher_handle;
-  RGWSI_Notify_ShutdownCB *shutdown_cb{nullptr};
-
   bool finalized{false};
 
   int init_watch(const DoutPrefixProvider *dpp,
                  boost::asio::yield_context yield);
   void finalize_watch(boost::asio::yield_context yield);
 
   void init(RGWSI_Zone *_zone_svc,
-            librados::Rados* rados_,
-            RGWSI_Finisher *_finisher_svc) {
+            librados::Rados* rados_) {
     zone_svc = _zone_svc;
     rados = rados_;
-    finisher_svc = _finisher_svc;
   }
   int do_start(optional_yield, const DoutPrefixProvider *dpp) override;
   void shutdown() override;