osd/scrub: remove the deep-scrubs deadline attribute

As it is no longer meaningful in the context of the new
scrub scheduling design.

The change mandates fixes to the way 'schedule-[deeps]crub'
commands are implemented. The offset to use when forcing the
last-scrub timestamp to a new value in now calculated in
ScrubJob::guaranteed_offset(), as ScrubJob is where all
schedule adjustments (which employ the same logic) are
implemented.

Signed-off-by: Ronen Friedman <[email protected]>
(cherry picked from commit 4e6323f)

Author: ronen-fr

src/osd/scrubber/pg_scrubber.cc

@@ -687,14 +687,15 @@ Scrub::sched_conf_t PgScrubber::populate_config_params() const
       deep_pool > 0.0 ? deep_pool : conf->osd_deep_scrub_interval;
 
   /**
-   * 'max_deep' and 'max_shallow' are set to the maximum allowed delay between
-   * scrubs. These deadlines have almost no effect on scrub scheduling
+   * 'max_shallow' is set to the maximum allowed delay between
+   * scrubs. This deadline has almost no effect on scrub scheduling
    * (the only minor exception: when sorting two scrub jobs that are
-   * equivalent in all but the deadline).
+   * equivalent in all but the deadline). It will be removed in
+   * the next version.
    *
    * 'max_shallow' is controlled by a pool option and a configuration
    * parameter. Note that if the value configured is less than the
-   * shallow interval, the max_shallow is disabled.
+   * shallow interval (plus expenses), the max_shallow is disabled.
    */
   auto max_shallow = pool_conf.value_or(pool_opts_t::SCRUB_MAX_INTERVAL, 0.0);
   if (max_shallow <= 0.0) {
@@ -721,10 +722,6 @@ Scrub::sched_conf_t PgScrubber::populate_config_params() const
     }
   }
 
-  // There are no comparable options for max_deep. We set it here to
-  // 4X the deep interval, as a reasonable default.
-  configs.max_deep = 4 * configs.deep_interval;
-
   configs.interval_randomize_ratio = conf->osd_scrub_interval_randomize_ratio;
   configs.deep_randomize_ratio =
       conf.get_val<double>("osd_deep_scrub_interval_cv");
@@ -762,33 +759,29 @@ void PgScrubber::on_operator_periodic_cmd(
     scrub_level_t scrub_level,
     int64_t offset)
 {
-  const auto cnf = populate_config_params();
-  dout(10) << fmt::format(
-		  "{}: {} (cmd offset:{}) conf:{}", __func__,
-		  (scrub_level == scrub_level_t::deep ? "deep" : "shallow"), offset,
-		  cnf)
-	   << dendl;
-
-  // move the relevant time-stamp backwards - enough to trigger a scrub
-  utime_t stamp = ceph_clock_now();
-  if (offset > 0) {
-    stamp -= offset;
+  // if 'offset' wasn't specified - find a value that guarantees the scrub
+  // target will appear ready for scrubbing (even after random adjustments)
+  if (offset == 0) {
+    const auto cnf = populate_config_params();
+    offset = m_scrub_job->guaranteed_offset(scrub_level, cnf);
+    dout(15) << fmt::format(
+		    "{}: {} (calculated offset:{}) conf:{}", __func__,
+		    (scrub_level == scrub_level_t::deep ? "deep" : "shallow"),
+		    offset, cnf)
+	     << dendl;
   } else {
-    double max_iv =
-	(scrub_level == scrub_level_t::deep)
-	    ? 2 * cnf.max_deep
-	    : (cnf.max_shallow ? *cnf.max_shallow : cnf.shallow_interval);
-    dout(20) << fmt::format(
-		    "{}: stamp:{:s} ms:{}/{}/{}", __func__, stamp,
-		    (cnf.max_shallow ? "ms+" : "ms-"),
-		    (cnf.max_shallow ? *cnf.max_shallow : -999.99),
-		    cnf.shallow_interval)
+    dout(15) << fmt::format(
+		    "{}: {} command offset:{}", __func__,
+		    (scrub_level == scrub_level_t::deep ? "deep" : "shallow"),
+		    offset)
 	     << dendl;
-    stamp -= max_iv;
   }
-  stamp -= 100.0;  // for good measure
+  // move the relevant time-stamp backwards - enough to trigger a scrub
+  utime_t stamp = ceph_clock_now();
+  stamp -= offset;  // can't combine with prev line
 
-  dout(10) << fmt::format("{}: stamp:{:s} ", __func__, stamp) << dendl;
+  dout(10) << fmt::format("{}: calculated stamp:{:s}", __func__, stamp)
+	   << dendl;
   asok_response_section(f, true, scrub_level, stamp);
 
   if (scrub_level == scrub_level_t::deep) {

src/osd/scrubber/scrub_job.cc

@@ -146,6 +146,22 @@ void ScrubJob::adjust_shallow_schedule(
 }
 
 
+double ScrubJob::guaranteed_offset(
+    scrub_level_t s_or_d,
+    const Scrub::sched_conf_t& app_conf)
+{
+  if (s_or_d == scrub_level_t::deep) {
+    // use the sdv of the deep scrub distribution, times 3 (3-sigma...)
+    const double sdv = app_conf.deep_interval * app_conf.deep_randomize_ratio;
+  // note: the '+10.0' is there just to guarantee inequality if '._ratio' is 0
+    return app_conf.deep_interval + abs(3 * sdv) + 10.0;
+  }
+
+  // shallow scrub
+  return app_conf.shallow_interval * (2.0 + app_conf.interval_randomize_ratio);
+}
+
+
 void ScrubJob::operator_forced(scrub_level_t s_or_d, scrub_type_t scrub_type)
 {
   auto& trgt = get_target(s_or_d);
@@ -237,44 +253,35 @@ void ScrubJob::adjust_deep_schedule(
 	   << dendl;
 
   auto& dp_times = deep_target.sched_info.schedule;  // shorthand
+  dp_times.deadline = utime_t::max(); // no 'max' for deep scrubs
 
   if (ScrubJob::requires_randomization(deep_target.urgency())) {
-    utime_t adj_not_before = last_deep;
     utime_t adj_target = last_deep;
-    dp_times.deadline = adj_target;
 
     // add a random delay to the proposed scheduled time
     const double sdv = app_conf.deep_interval * app_conf.deep_randomize_ratio;
     std::normal_distribution<double> normal_dist{app_conf.deep_interval, sdv};
-    auto next_delay =
-        std::clamp(normal_dist(random_gen), app_conf.deep_interval - 2 * sdv,
-                   app_conf.deep_interval + 2 * sdv);
+    auto next_delay = std::clamp(
+	normal_dist(random_gen), app_conf.deep_interval - 2 * sdv,
+	app_conf.deep_interval + 2 * sdv);
     adj_target += next_delay;
     dout(20) << fmt::format(
-                    "deep scrubbing: next_delay={:.0f} (interval={:.0f}, "
-                    "ratio={:.3f}), adjusted:{:s}",
-                    next_delay, app_conf.deep_interval,
-                    app_conf.deep_randomize_ratio, adj_target)
-             << dendl;
-
-    dp_times.deadline += app_conf.max_deep;
+		    "deep scrubbing: next_delay={:.0f} (interval={:.0f}, "
+		    "ratio={:.3f}), adjusted:{:s}",
+		    next_delay, app_conf.deep_interval,
+		    app_conf.deep_randomize_ratio, adj_target)
+	     << dendl;
 
-    if (adj_not_before < adj_target) {
-      adj_not_before = adj_target;
-    }
     dp_times.scheduled_at = adj_target;
-    dp_times.not_before = adj_not_before;
+    dp_times.not_before = adj_target;
   } else {
-    // the target time is already set. Make sure to reset the n.b. and
-    // the (irrelevant) deadline
+    // the target time is already set. The n.b. is set to same
     dp_times.not_before = dp_times.scheduled_at;
-    dp_times.deadline = utime_t::max();
   }
 
   dout(10) << fmt::format(
-		  "adjusted: nb:{:s} target:{:s} deadline:{:s} ({})",
-		  dp_times.not_before, dp_times.scheduled_at, dp_times.deadline,
-		  state_desc())
+		  "adjusted: nb:{:s} target:{:s} ({})", dp_times.not_before,
+		  dp_times.scheduled_at, state_desc())
 	   << dendl;
 }
 

src/osd/scrubber/scrub_job.h

@@ -38,19 +38,12 @@ struct sched_conf_t {
   double deep_interval{0.0};
 
   /**
-   * the maximum interval between shallow scrubs, as determined by either the
-   * OSD or the pool configuration. Empty if no limit is configured.
-   */
-  std::optional<double> max_shallow;
-
-  /**
-   * the maximum interval between deep scrubs, after which the
+   * the maximum interval between shallow scrubs, after which the
    * (info-only) "overdue" field in the scheduler dump is set.
-   * There is no specific configuration parameter to control the
-   * deep scrubs max. Instead - we set it to 4 times the average
-   * interval.
+   * Determined by either the pool or the cluster configuration.
+   * Empty if no limit is configured.
    */
-  double max_deep{std::numeric_limits<double>::max()};
+  std::optional<double> max_shallow;
 
   /**
    * interval_randomize_ratio
@@ -274,6 +267,17 @@ class ScrubJob {
    */
   void operator_forced(scrub_level_t s_or_d, scrub_type_t scrub_type);
 
+  /**
+   * calculate a time offset large enough, so that once the relevant
+   * last-scrub timestamp is forced back by this amount, the PG is
+   * eligible for a periodic scrub of the specified level.
+   * Used by the scrubber upon receiving a 'fake a scheduled scrub' request
+   * from the operator.
+   */
+  double guaranteed_offset(
+      scrub_level_t s_or_d,
+      const Scrub::sched_conf_t& app_conf);
+
   void dump(ceph::Formatter* f) const;
 
   bool is_registered() const { return registered; }
@@ -430,9 +434,9 @@ struct formatter<Scrub::sched_conf_t> {
   {
     return fmt::format_to(
 	ctx.out(),
-	"periods:s:{}/{},d:{}/{},iv-ratio:{},deep-rand:{},on-inv:{}",
+	"periods:s:{}/{},d:{},iv-ratio:{},deep-rand:{},on-inv:{}",
 	cf.shallow_interval, cf.max_shallow.value_or(-1.0), cf.deep_interval,
-	cf.max_deep, cf.interval_randomize_ratio, cf.deep_randomize_ratio,
+	cf.interval_randomize_ratio, cf.deep_randomize_ratio,
 	cf.mandatory_on_invalid);
   }
 };