Merge pull request ceph#47974 from zhscn/wip-evict

crimson/os/seastore: evict cold data to slower devices

Reviewed-by: Yingxin Cheng <yingxin.cheng@intel.com>
Reviewed-by: Samuel Just <sjust@redhat.com>

Author: cyx1231st

doc/dev/crimson/crimson.rst

@@ -154,16 +154,16 @@ To facilitate the development of crimson, following options would be handy when
 using ``vstart.sh``,
 
 ``--crimson``
-    start ``crimson-osd`` instead of ``ceph-osd``
+    Start ``crimson-osd`` instead of ``ceph-osd``.
 
 ``--nodaemon``
-    do not daemonize the service
+    Do not daemonize the service.
 
 ``--redirect-output``
-    redirect the stdout and stderr of service to ``out/$type.$num.stdout``.
+    Tedirect the stdout and stderr of service to ``out/$type.$num.stdout``.
 
 ``--osd-args``
-    pass extra command line options to crimson-osd or ceph-osd. It's quite
+    Pass extra command line options to crimson-osd or ceph-osd. It's quite
     useful for passing Seastar options to crimson-osd. For instance, you could
     use ``--osd-args "--memory 2G"`` to set the memory to use. Please refer
     the output of::
@@ -173,14 +173,31 @@ using ``vstart.sh``,
     for more Seastar specific command line options.
 
 ``--cyanstore``
-    use CyanStore as the object store backend.
+    Use CyanStore as the object store backend.
 
 ``--bluestore``
-    use the alienized BlueStore as the object store backend. This is the default
+    Use the alienized BlueStore as the object store backend. This is the default
     setting, if not specified otherwise.
 
 ``--memstore``
-    use the alienized MemStore as the object store backend.
+    Use the alienized MemStore as the object store backend.
+
+``--seastore``
+    Use SeaStore as the back end object store.
+
+``--seastore-devs``
+    Specify the block device used by SeaStore.
+
+``--seastore-secondary-devs``
+    Optional.  SeaStore supports multiple devices.  Enable this feature by
+    passing the block device to this option.
+
+``--seastore-secondary-devs-type``
+    Optional.  Specify device type of secondary devices.  When the secondary
+    device is slower than main device passed to ``--seastore-devs``, the cold
+    data in faster device will be evicted to the slower devices over time.
+    Valid types include ``HDD``, ``SSD``(default), ``ZNS``, and ``RANDOM_BLOCK_SSD``
+    Note secondary devices should not be faster than the main device.
 
 ``--seastore``
     use SeaStore as the object store backend.
@@ -194,6 +211,15 @@ So, a typical command to start a single-crimson-node cluster is::
 
 Where we assign 4 GiB memory, a single thread running on core-0 to crimson-osd.
 
+Another SeaStore example::
+
+  $  MGR=1 MON=1 OSD=1 MDS=0 RGW=0 ../src/vstart.sh -n -x \
+    --without-dashboard --seastore \
+    --crimson --redirect-output \
+    --seastore-devs /dev/sda \
+    --seastore-secondary-devs /dev/sdb \
+    --seastore-secondary-devs-type HDD
+
 You could stop the vstart cluster using::
 
   $ ../src/stop.sh --crimson

src/common/options/crimson.yaml.in

@@ -102,3 +102,18 @@ options:
   level: dev
   desc: Total size to use for CircularBoundedJournal if created, it is valid only if seastore_main_device_type is RANDOM_BLOCK
   default: 5_G
+- name: seastore_multiple_tiers_stop_evict_ratio
+  type: float
+  level: advanced
+  desc: When the used ratio of main tier is less than this value, then stop evict cold data to the cold tier.
+  default: 0.5
+- name: seastore_multiple_tiers_default_evict_ratio
+  type: float
+  level: advanced
+  desc: Begin evicting cold data to the cold tier when the used ratio of the main tier reaches this value.
+  default: 0.6
+- name: seastore_multiple_tiers_fast_evict_ratio
+  type: float
+  level: advanced
+  desc: Begin fast eviction when the used ratio of the main tier reaches this value.
+  default: 0.7

src/crimson/os/seastore/async_cleaner.cc

@@ -830,13 +830,15 @@ SegmentCleaner::SegmentCleaner(
   config_t config,
   SegmentManagerGroupRef&& sm_group,
   BackrefManager &backref_manager,
-  bool detailed)
+  SegmentSeqAllocator &segment_seq_allocator,
+  bool detailed,
+  bool is_cold)
   : detailed(detailed),
+    is_cold(is_cold),
     config(config),
     sm_group(std::move(sm_group)),
     backref_manager(backref_manager),
-    ool_segment_seq_allocator(
-      new SegmentSeqAllocator(segment_type_t::OOL))
+    ool_segment_seq_allocator(segment_seq_allocator)
 {
   config.validate();
 }
@@ -854,7 +856,13 @@ void SegmentCleaner::register_metrics()
   i = get_bucket_index(UTIL_STATE_EMPTY);
   stats.segment_util.buckets[i].count = segments.get_num_segments();
 
-  metrics.add_group("segment_cleaner", {
+  std::string prefix;
+  if (is_cold) {
+    prefix.append("cold_");
+  }
+  prefix.append("segment_cleaner");
+
+  metrics.add_group(prefix, {
     sm::make_counter("segments_number",
 		     [this] { return segments.get_num_segments(); },
 		     sm::description("the number of segments")),
@@ -1053,8 +1061,12 @@ SegmentCleaner::do_reclaim_space(
                         &pin_list, &reclaimed, &runs] {
     reclaimed = 0;
     runs++;
+    auto src = Transaction::src_t::CLEANER_MAIN;
+    if (is_cold) {
+      src = Transaction::src_t::CLEANER_COLD;
+    }
     return extent_callback->with_transaction_intr(
-      Transaction::src_t::CLEANER,
+      src,
       "clean_reclaim_space",
       [this, &backref_extents, &pin_list, &reclaimed](auto &t)
     {
@@ -1137,6 +1149,7 @@ SegmentCleaner::clean_space_ret SegmentCleaner::clean_space()
 {
   LOG_PREFIX(SegmentCleaner::clean_space);
   assert(background_callback->is_ready());
+  ceph_assert(can_clean_space());
   if (!reclaim_state) {
     segment_id_t seg_id = get_next_reclaim_segment();
     auto &segment_info = segments[seg_id];

src/crimson/os/seastore/async_cleaner.h

@@ -544,14 +544,6 @@ class JournalTrimmerImpl : public JournalTrimmer {
     journal_alloc_tail = JOURNAL_SEQ_NULL;
   }
 
-  bool should_trim_dirty() const {
-    return get_dirty_tail_target() > journal_dirty_tail;
-  }
-
-  bool should_trim_alloc() const {
-    return get_alloc_tail_target() > journal_alloc_tail;
-  }
-
   bool should_trim() const {
     return should_trim_alloc() || should_trim_dirty();
   }
@@ -596,6 +588,14 @@ class JournalTrimmerImpl : public JournalTrimmer {
   friend std::ostream &operator<<(std::ostream &, const stat_printer_t &);
 
 private:
+  bool should_trim_dirty() const {
+    return get_dirty_tail_target() > journal_dirty_tail;
+  }
+
+  bool should_trim_alloc() const {
+    return get_alloc_tail_target() > journal_alloc_tail;
+  }
+
   using trim_ertr = crimson::errorator<
     crimson::ct_error::input_output_error>;
   trim_ertr::future<> trim_dirty();
@@ -1144,12 +1144,18 @@ class AsyncCleaner {
 
   virtual bool should_block_io_on_clean() const = 0;
 
+  virtual bool can_clean_space() const = 0;
+
   virtual bool should_clean_space() const = 0;
 
   using clean_space_ertr = base_ertr;
   using clean_space_ret = clean_space_ertr::future<>;
   virtual clean_space_ret clean_space() = 0;
 
+  virtual const std::set<device_id_t>& get_device_ids() const = 0;
+
+  virtual std::size_t get_reclaim_size_per_cycle() const = 0;
+
   // test only
   virtual bool check_usage() = 0;
 
@@ -1210,11 +1216,9 @@ class SegmentCleaner : public SegmentProvider, public AsyncCleaner {
     config_t config,
     SegmentManagerGroupRef&& sm_group,
     BackrefManager &backref_manager,
-    bool detailed);
-
-  SegmentSeqAllocator& get_ool_segment_seq_allocator() {
-    return *ool_segment_seq_allocator;
-  }
+    SegmentSeqAllocator &segment_seq_allocator,
+    bool detailed,
+    bool is_cold);
 
   void set_journal_trimmer(JournalTrimmer &_trimmer) {
     trimmer = &_trimmer;
@@ -1224,9 +1228,12 @@ class SegmentCleaner : public SegmentProvider, public AsyncCleaner {
       config_t config,
       SegmentManagerGroupRef&& sm_group,
       BackrefManager &backref_manager,
-      bool detailed) {
+      SegmentSeqAllocator &ool_seq_allocator,
+      bool detailed,
+      bool is_cold = false) {
     return std::make_unique<SegmentCleaner>(
-        config, std::move(sm_group), backref_manager, detailed);
+        config, std::move(sm_group), backref_manager,
+        ool_seq_allocator, detailed, is_cold);
   }
 
   /*
@@ -1312,6 +1319,11 @@ class SegmentCleaner : public SegmentProvider, public AsyncCleaner {
     return aratio < config.available_ratio_hard_limit;
   }
 
+  bool can_clean_space() const final {
+    assert(background_callback->is_ready());
+    return get_segments_reclaimable() > 0;
+  }
+
   bool should_clean_space() const final {
     assert(background_callback->is_ready());
     if (get_segments_reclaimable() == 0) {
@@ -1328,6 +1340,14 @@ class SegmentCleaner : public SegmentProvider, public AsyncCleaner {
 
   clean_space_ret clean_space() final;
 
+  const std::set<device_id_t>& get_device_ids() const final {
+    return sm_group->get_device_ids();
+  }
+
+  std::size_t get_reclaim_size_per_cycle() const final {
+    return config.reclaim_bytes_per_cycle;
+  }
+
   // Testing interfaces
 
   bool check_usage() final;
@@ -1521,11 +1541,12 @@ class SegmentCleaner : public SegmentProvider, public AsyncCleaner {
     auto new_usage = calc_utilization(segment);
     adjust_segment_util(old_usage, new_usage);
     if (s_type == segment_type_t::OOL) {
-      ool_segment_seq_allocator->set_next_segment_seq(seq);
+      ool_segment_seq_allocator.set_next_segment_seq(seq);
     }
   }
 
   const bool detailed;
+  const bool is_cold;
   const config_t config;
 
   SegmentManagerGroupRef sm_group;
@@ -1574,7 +1595,7 @@ class SegmentCleaner : public SegmentProvider, public AsyncCleaner {
   BackgroundListener *background_callback = nullptr;
 
   // TODO: drop once paddr->journal_seq_t is introduced
-  SegmentSeqAllocatorRef ool_segment_seq_allocator;
+  SegmentSeqAllocator &ool_segment_seq_allocator;
 };
 
 class RBMCleaner;
@@ -1635,12 +1656,24 @@ class RBMCleaner : public AsyncCleaner {
     return false;
   }
 
+  bool can_clean_space() const final {
+    return false;
+  }
+
   bool should_clean_space() const final {
     return false;
   }
 
   clean_space_ret clean_space() final;
 
+  const std::set<device_id_t>& get_device_ids() const final {
+    return rb_group->get_device_ids();
+  }
+
+  std::size_t get_reclaim_size_per_cycle() const final {
+    return 0;
+  }
+
   RandomBlockManager* get_rbm(paddr_t paddr) {
     auto rbs = rb_group->get_rb_managers();
     for (auto p : rbs) {

src/crimson/os/seastore/cache.cc

@@ -46,6 +46,7 @@ Cache::Cache(
   LOG_PREFIX(Cache::Cache);
   INFO("created, lru_size={}", lru.get_capacity());
   register_metrics();
+  segment_providers_by_device_id.resize(DEVICE_ID_MAX, nullptr);
 }
 
 Cache::~Cache()
@@ -135,7 +136,8 @@ void Cache::register_metrics()
     {src_t::READ, sm::label_instance("src", "READ")},
     {src_t::TRIM_DIRTY, sm::label_instance("src", "TRIM_DIRTY")},
     {src_t::TRIM_ALLOC, sm::label_instance("src", "TRIM_ALLOC")},
-    {src_t::CLEANER, sm::label_instance("src", "CLEANER")},
+    {src_t::CLEANER_MAIN, sm::label_instance("src", "CLEANER_MAIN")},
+    {src_t::CLEANER_COLD, sm::label_instance("src", "CLEANER_COLD")},
   };
   assert(labels_by_src.size() == (std::size_t)src_t::MAX);
 
@@ -624,8 +626,10 @@ void Cache::register_metrics()
            src2 == Transaction::src_t::READ) ||
           (src1 == Transaction::src_t::TRIM_DIRTY &&
            src2 == Transaction::src_t::TRIM_DIRTY) ||
-          (src1 == Transaction::src_t::CLEANER &&
-           src2 == Transaction::src_t::CLEANER) ||
+          (src1 == Transaction::src_t::CLEANER_MAIN &&
+           src2 == Transaction::src_t::CLEANER_MAIN) ||
+          (src1 == Transaction::src_t::CLEANER_COLD &&
+           src2 == Transaction::src_t::CLEANER_COLD) ||
           (src1 == Transaction::src_t::TRIM_ALLOC &&
            src2 == Transaction::src_t::TRIM_ALLOC)) {
         continue;
@@ -1113,12 +1117,13 @@ record_t Cache::prepare_record(
       auto stype = segment_type_t::NULL_SEG;
 
       // FIXME: This is specific to the segmented implementation
-      if (segment_provider != nullptr &&
-          i->get_paddr().get_addr_type() == paddr_types_t::SEGMENT) {
+      if (i->get_paddr().get_addr_type() == paddr_types_t::SEGMENT) {
         auto sid = i->get_paddr().as_seg_paddr().get_segment_id();
-        auto &sinfo = segment_provider->get_seg_info(sid);
-        sseq = sinfo.seq;
-        stype = sinfo.type;
+        auto sinfo = get_segment_info(sid);
+        if (sinfo) {
+          sseq = sinfo->seq;
+          stype = sinfo->type;
+        }
       }
 
       record.push_back(
@@ -1389,7 +1394,8 @@ record_t Cache::prepare_record(
   auto &rewrite_version_stats = t.get_rewrite_version_stats();
   if (trans_src == Transaction::src_t::TRIM_DIRTY) {
     stats.committed_dirty_version.increment_stat(rewrite_version_stats);
-  } else if (trans_src == Transaction::src_t::CLEANER) {
+  } else if (trans_src == Transaction::src_t::CLEANER_MAIN ||
+             trans_src == Transaction::src_t::CLEANER_COLD) {
     stats.committed_reclaim_version.increment_stat(rewrite_version_stats);
   } else {
     assert(rewrite_version_stats.is_clear());
@@ -1656,21 +1662,22 @@ Cache::replay_delta(
    * safetly skip these deltas because the extent must already
    * have been rewritten.
    */
-  if (segment_provider != nullptr &&
-      delta.paddr != P_ADDR_NULL &&
+  if (delta.paddr != P_ADDR_NULL &&
       delta.paddr.get_addr_type() == paddr_types_t::SEGMENT) {
     auto& seg_addr = delta.paddr.as_seg_paddr();
-    auto& seg_info = segment_provider->get_seg_info(seg_addr.get_segment_id());
-    auto delta_paddr_segment_seq = seg_info.seq;
-    auto delta_paddr_segment_type = seg_info.type;
-    if (delta_paddr_segment_seq != delta.ext_seq ||
-        delta_paddr_segment_type != delta.seg_type) {
-      DEBUG("delta is obsolete, delta_paddr_segment_seq={},"
-            " delta_paddr_segment_type={} -- {}",
-            segment_seq_printer_t{delta_paddr_segment_seq},
-            delta_paddr_segment_type,
-            delta);
-      return replay_delta_ertr::make_ready_future<bool>(false);
+    auto seg_info = get_segment_info(seg_addr.get_segment_id());
+    if (seg_info) {
+      auto delta_paddr_segment_seq = seg_info->seq;
+      auto delta_paddr_segment_type = seg_info->type;
+      if (delta_paddr_segment_seq != delta.ext_seq ||
+          delta_paddr_segment_type != delta.seg_type) {
+        DEBUG("delta is obsolete, delta_paddr_segment_seq={},"
+              " delta_paddr_segment_type={} -- {}",
+              segment_seq_printer_t{delta_paddr_segment_seq},
+              delta_paddr_segment_type,
+              delta);
+        return replay_delta_ertr::make_ready_future<bool>(false);
+      }
     }
   }