Merge pull request ceph#58952 from YiteGu/add-perfcounter-for-blk-discard

blk/kerneldevice: add perfcounter for block async discard

Reviewed-by: Igor Fedotov <[email protected]>

Author: ifed01

src/blk/BlockDevice.cc

@@ -140,13 +140,13 @@ BlockDevice::device_type_from_name(const std::string& blk_dev_name)
 
 BlockDevice* BlockDevice::create_with_type(block_device_t device_type,
   CephContext* cct, const std::string& path, aio_callback_t cb,
-  void *cbpriv, aio_callback_t d_cb, void *d_cbpriv)
+  void *cbpriv, aio_callback_t d_cb, void *d_cbpriv, const char* dev_name)
 {
 
   switch (device_type) {
 #if defined(HAVE_LIBAIO) || defined(HAVE_POSIXAIO)
   case block_device_t::aio:
-    return new KernelDevice(cct, cb, cbpriv, d_cb, d_cbpriv);
+    return new KernelDevice(cct, cb, cbpriv, d_cb, d_cbpriv, dev_name);
 #endif
 #if defined(HAVE_SPDK)
   case block_device_t::spdk:
@@ -164,7 +164,7 @@ BlockDevice* BlockDevice::create_with_type(block_device_t device_type,
 
 BlockDevice *BlockDevice::create(
     CephContext* cct, const string& path, aio_callback_t cb,
-    void *cbpriv, aio_callback_t d_cb, void *d_cbpriv)
+    void *cbpriv, aio_callback_t d_cb, void *d_cbpriv, const char* dev_name)
 {
   const string blk_dev_name = cct->_conf.get_val<string>("bdev_type");
   block_device_t device_type = block_device_t::unknown;
@@ -173,7 +173,7 @@ BlockDevice *BlockDevice::create(
   } else {
     device_type = device_type_from_name(blk_dev_name);
   }
-  return create_with_type(device_type, cct, path, cb, cbpriv, d_cb, d_cbpriv);
+  return create_with_type(device_type, cct, path, cb, cbpriv, d_cb, d_cbpriv, dev_name);
 }
 
 bool BlockDevice::is_valid_io(uint64_t off, uint64_t len) const {

src/blk/BlockDevice.h

@@ -175,7 +175,8 @@ class BlockDevice {
   static block_device_t device_type_from_name(const std::string& blk_dev_name);
   static BlockDevice *create_with_type(block_device_t device_type,
     CephContext* cct, const std::string& path, aio_callback_t cb,
-    void *cbpriv, aio_callback_t d_cb, void *d_cbpriv);
+    void *cbpriv, aio_callback_t d_cb, void *d_cbpriv, const char* dev_name);
+
 protected:
   uint64_t size = 0;
   uint64_t block_size = 0;
@@ -206,7 +207,8 @@ class BlockDevice {
   virtual ~BlockDevice() = default;
 
   static BlockDevice *create(
-    CephContext* cct, const std::string& path, aio_callback_t cb, void *cbpriv, aio_callback_t d_cb, void *d_cbpriv);
+    CephContext* cct, const std::string& path, aio_callback_t cb, void *cbpriv, aio_callback_t d_cb,
+    void *d_cbpriv, const char* dev_name = "");
   virtual bool supported_bdev_label() { return true; }
   virtual bool is_rotational() { return rotational; }
 

src/blk/kernel/KernelDevice.cc

@@ -59,7 +59,7 @@ using ceph::make_timespan;
 using ceph::mono_clock;
 using ceph::operator <<;
 
-KernelDevice::KernelDevice(CephContext* cct, aio_callback_t cb, void *cbpriv, aio_callback_t d_cb, void *d_cbpriv)
+KernelDevice::KernelDevice(CephContext* cct, aio_callback_t cb, void *cbpriv, aio_callback_t d_cb, void *d_cbpriv, const char* dev_name)
   : BlockDevice(cct, cb, cbpriv),
     aio(false), dio(false),
     discard_callback(d_cb),
@@ -89,10 +89,25 @@ KernelDevice::KernelDevice(CephContext* cct, aio_callback_t cb, void *cbpriv, ai
     }
     io_queue = std::make_unique<aio_queue_t>(iodepth);
   }
+
+  char name[128];
+  sprintf(name, "blk-kernel-device-%s", dev_name);
+  PerfCountersBuilder b(cct, name,
+                       l_blk_kernel_device_first, l_blk_kernel_device_last);
+  b.set_prio_default(PerfCountersBuilder::PRIO_USEFUL);
+  b.add_u64_counter(l_blk_kernel_device_discard_op, "discard_op",
+            "Number of discard ops issued to kernel device");
+
+  logger.reset(b.create_perf_counters());
+  cct->get_perfcounters_collection()->add(logger.get());
 }
 
 KernelDevice::~KernelDevice()
 {
+  if (logger) {
+    cct->get_perfcounters_collection()->remove(logger.get());
+    logger.reset();
+  }
   cct->_conf.remove_observer(this);
 }
 
@@ -784,6 +799,7 @@ void KernelDevice::_discard_thread(uint64_t tid)
       discard_running ++;
       l.unlock();
       dout(20) << __func__ << " finishing" << dendl;
+      logger->inc(l_blk_kernel_device_discard_op, discard_processing.size());
       for (auto p = discard_processing.begin(); p != discard_processing.end(); ++p) {
         _discard(p.get_start(), p.get_len());
       }

src/blk/kernel/KernelDevice.h

@@ -29,6 +29,12 @@
 
 #define RW_IO_MAX (INT_MAX & CEPH_PAGE_MASK)
 
+enum {
+  l_blk_kernel_device_first = 1000,
+  l_blk_kernel_device_discard_op,
+  l_blk_kernel_device_last,
+};
+
 class KernelDevice : public BlockDevice,
                      public md_config_obs_t {
 protected:
@@ -53,6 +59,7 @@ class KernelDevice : public BlockDevice,
   void *discard_callback_priv;
   bool aio_stop;
   bool discard_stop;
+  std::unique_ptr<PerfCounters> logger;
 
   ceph::mutex discard_lock = ceph::make_mutex("KernelDevice::discard_lock");
   ceph::condition_variable discard_cond;
@@ -119,7 +126,8 @@ class KernelDevice : public BlockDevice,
   ceph::unique_leakable_ptr<buffer::raw> create_custom_aligned(size_t len, IOContext* ioc) const;
 
 public:
-  KernelDevice(CephContext* cct, aio_callback_t cb, void *cbpriv, aio_callback_t d_cb, void *d_cbpriv);
+  KernelDevice(CephContext* cct, aio_callback_t cb, void *cbpriv, aio_callback_t d_cb,
+    void *d_cbpriv, const char* dev_name = "");
   ~KernelDevice();
 
   void aio_submit(IOContext *ioc) override;

src/os/bluestore/BlueFS.cc

@@ -483,17 +483,21 @@ int BlueFS::add_block_device(unsigned id, const string& path, bool trim,
                              bluefs_shared_alloc_context_t* _shared_alloc)
 {
   uint64_t reserved;
+  string dev_name;
   switch(id) {
     case BDEV_WAL:
     case BDEV_NEWWAL:
       reserved = BDEV_LABEL_BLOCK_SIZE;
+      dev_name = "wal";
       break;
     case BDEV_DB:
     case BDEV_NEWDB:
       reserved = SUPER_RESERVED;
+      dev_name = "db";
       break;
     case BDEV_SLOW:
       reserved = 0;
+      dev_name = "slow";
       break;
     default:
       ceph_assert(false);
@@ -503,7 +507,7 @@ int BlueFS::add_block_device(unsigned id, const string& path, bool trim,
   ceph_assert(id < bdev.size());
   ceph_assert(bdev[id] == NULL);
   BlockDevice *b = BlockDevice::create(cct, path, NULL, NULL,
-				       discard_cb[id], static_cast<void*>(this));
+				       discard_cb[id], static_cast<void*>(this), dev_name.c_str());
   block_reserved[id] = reserved;
   if (_shared_alloc) {
     b->set_no_exclusive_lock();

src/os/bluestore/BlueFS.h

@@ -737,7 +737,7 @@ class BlueFS {
   }
 
   int add_block_device(unsigned bdev, const std::string& path, bool trim,
-		       bluefs_shared_alloc_context_t* _shared_alloc = nullptr);
+                       bluefs_shared_alloc_context_t* _shared_alloc = nullptr);
   bool bdev_support_label(unsigned id);
   uint64_t get_block_device_size(unsigned bdev) const;
   BlockDevice* get_block_device(unsigned bdev) const;

src/os/bluestore/BlueStore.cc

@@ -6994,7 +6994,7 @@ int BlueStore::_open_bdev(bool create)
 {
   ceph_assert(bdev == NULL);
   string p = path + "/block";
-  bdev = BlockDevice::create(cct, p, aio_cb, static_cast<void*>(this), discard_cb, static_cast<void*>(this));
+  bdev = BlockDevice::create(cct, p, aio_cb, static_cast<void*>(this), discard_cb, static_cast<void*>(this), "bluestore");
   int r = bdev->open(p);
   if (r < 0)
     goto fail;