rbd-mirror: add cluster fsid to remote meta cache key

Using just pool id is not enough when mirroring several clusters.

Fixes: https://tracker.ceph.com/issues/69492
Signed-off-by: Mykola Golub <[email protected]>

Author: trociny

src/test/rbd_mirror/image_replayer/snapshot/test_mock_Replayer.cc

@@ -144,8 +144,13 @@ class TestImageReplayer : public TestFixture {
                           cls::rbd::MIRROR_PEER_DIRECTION_RX_TX,
                           "siteA", "client", m_local_mirror_uuid}));
 
+      std::string remote_fsid;
+      librados::Rados remote_rados(m_remote_ioctx);
+      EXPECT_EQ(0, remote_rados.cluster_fsid(&remote_fsid));
+
       m_pool_meta_cache.set_remote_pool_meta(
-        m_remote_ioctx.get_id(), {m_remote_mirror_uuid, remote_peer_uuid});
+        remote_fsid, m_remote_ioctx.get_id(),
+        {m_remote_mirror_uuid, remote_peer_uuid});
     }
 
     EXPECT_EQ(0, librbd::api::Mirror<>::uuid_get(m_remote_ioctx,

src/test/rbd_mirror/test_ImageReplayer.cc

@@ -48,12 +48,12 @@ void PoolMetaCache::remove_local_pool_meta(int64_t pool_id) {
 }
 
 int PoolMetaCache::get_remote_pool_meta(
-    int64_t pool_id,
+    const std::string& fsid, int64_t pool_id,
     RemotePoolMeta* remote_pool_meta) const {
-  dout(15) << "pool_id=" << pool_id << dendl;
+  dout(15) << "fsid=" << fsid << ", pool_id=" << pool_id << dendl;
 
   std::shared_lock locker{m_lock};
-  auto it = m_remote_pool_metas.find(pool_id);
+  auto it = m_remote_pool_metas.find({fsid, pool_id});
   if (it == m_remote_pool_metas.end()) {
     return -ENOENT;
   }
@@ -63,20 +63,21 @@ int PoolMetaCache::get_remote_pool_meta(
 }
 
 void PoolMetaCache::set_remote_pool_meta(
-    int64_t pool_id,
+    const std::string& fsid, int64_t pool_id,
     const RemotePoolMeta& remote_pool_meta) {
-  dout(15) << "pool_id=" << pool_id << ", "
-           << "remote_pool_meta=" << remote_pool_meta << dendl;
+  dout(15) << "fsid=" << fsid << ", pool_id=" << pool_id
+           << ", remote_pool_meta=" << remote_pool_meta << dendl;
 
   std::unique_lock locker(m_lock);
-  m_remote_pool_metas[pool_id] = remote_pool_meta;
+  m_remote_pool_metas[{fsid, pool_id}] = remote_pool_meta;
 }
 
-void PoolMetaCache::remove_remote_pool_meta(int64_t pool_id) {
-  dout(15) << "pool_id=" << pool_id << dendl;
+void PoolMetaCache::remove_remote_pool_meta(
+    const std::string& fsid, int64_t pool_id) {
+  dout(15) << "fsid=" << fsid << ", pool_id=" << pool_id << dendl;
 
   std::unique_lock locker(m_lock);
-  m_remote_pool_metas.erase(pool_id);
+  m_remote_pool_metas.erase({fsid, pool_id});
 }
 
 } // namespace mirror

src/tools/rbd_mirror/PoolMetaCache.cc

@@ -26,19 +26,19 @@ class PoolMetaCache {
                            const LocalPoolMeta& local_pool_meta);
   void remove_local_pool_meta(int64_t pool_id);
 
-  int get_remote_pool_meta(int64_t pool_id,
+  int get_remote_pool_meta(const std::string& fsid, int64_t pool_id,
                            RemotePoolMeta* remote_pool_meta) const;
-  void set_remote_pool_meta(int64_t pool_id,
+  void set_remote_pool_meta(const std::string& fsid, int64_t pool_id,
                             const RemotePoolMeta& remote_pool_meta);
-  void remove_remote_pool_meta(int64_t pool_id);
+  void remove_remote_pool_meta(const std::string& fsid, int64_t pool_id);
 
 private:
   CephContext* m_cct;
 
   mutable ceph::shared_mutex m_lock =
     ceph::make_shared_mutex("rbd::mirror::PoolMetaCache::m_lock");
   std::map<int64_t, LocalPoolMeta> m_local_pool_metas;
-  std::map<int64_t, RemotePoolMeta> m_remote_pool_metas;
+  std::map<std::pair<std::string, int64_t>, RemotePoolMeta> m_remote_pool_metas;
 };
 
 } // namespace mirror

src/tools/rbd_mirror/PoolMetaCache.h

@@ -326,6 +326,13 @@ void PoolReplayer<I>::init(const std::string& site_name) {
     return;
   }
 
+  r = m_remote_rados->cluster_fsid(&m_remote_fsid);
+  if (r < 0) {
+    derr << "failed to retrieve remote cluster fsid: " << cpp_strerror(r)
+	 << dendl;
+    return;
+  }
+
   r = m_remote_rados->ioctx_create(m_local_io_ctx.get_pool_name().c_str(),
                                    m_remote_io_ctx);
   if (r < 0) {
@@ -362,9 +369,10 @@ void PoolReplayer<I>::init(const std::string& site_name) {
     m_remote_pool_poller.reset();
     return;
   }
+
   ceph_assert(!m_remote_pool_meta.mirror_uuid.empty());
   m_pool_meta_cache->set_remote_pool_meta(
-    m_remote_io_ctx.get_id(), m_remote_pool_meta);
+    m_remote_fsid, m_remote_io_ctx.get_id(), m_remote_pool_meta);
   m_pool_meta_cache->set_local_pool_meta(
     m_local_io_ctx.get_id(), {m_local_mirror_uuid});
 
@@ -414,7 +422,8 @@ void PoolReplayer<I>::shut_down() {
     m_remote_pool_poller->shut_down(&ctx);
     ctx.wait();
 
-    m_pool_meta_cache->remove_remote_pool_meta(m_remote_io_ctx.get_id());
+    m_pool_meta_cache->remove_remote_pool_meta(m_remote_fsid,
+                                               m_remote_io_ctx.get_id());
     m_pool_meta_cache->remove_local_pool_meta(m_local_io_ctx.get_id());
   }
   m_remote_pool_poller.reset();

src/tools/rbd_mirror/PoolReplayer.cc

@@ -214,6 +214,7 @@ class PoolReplayer {
   librados::IoCtx m_remote_io_ctx;
 
   std::string m_local_mirror_uuid;
+  std::string m_remote_fsid;
 
   RemotePoolMeta m_remote_pool_meta;
   std::unique_ptr<remote_pool_poller::Listener> m_remote_pool_poller_listener;

src/tools/rbd_mirror/PoolReplayer.h

@@ -150,9 +150,19 @@ void Replayer<I>::init(Context* on_finish) {
 
   ceph_assert(m_state == STATE_INIT);
 
+  std::string remote_fsid;
+  librados::Rados remote_rados(m_state_builder->remote_image_ctx->md_ctx);
+  int r = remote_rados.cluster_fsid(&remote_fsid);
+  if (r < 0) {
+    derr << "failed to retrieve remote cluster fsid: " << cpp_strerror(r)
+	 << dendl;
+    return;
+  }
+
   RemotePoolMeta remote_pool_meta;
-  int r = m_pool_meta_cache->get_remote_pool_meta(
-    m_state_builder->remote_image_ctx->md_ctx.get_id(), &remote_pool_meta);
+  r = m_pool_meta_cache->get_remote_pool_meta(
+    remote_fsid, m_state_builder->remote_image_ctx->md_ctx.get_id(),
+    &remote_pool_meta);
   if (r < 0 || remote_pool_meta.mirror_peer_uuid.empty()) {
     derr << "failed to retrieve mirror peer uuid from remote pool" << dendl;
     m_state = STATE_COMPLETE;