fix(tiering): Cooldown for DiskStorage::Grow() retries (#5803)

Cooldown for DiskStorage::Grow() retries

Author: dranikpg

src/server/tiering/disk_storage.cc

@@ -204,6 +204,11 @@ error_code DiskStorage::TryGrow(off_t grow_size) {
   if (alloc_.capacity() + ExternalAllocator::kExtAlignment >= static_cast<size_t>(max_size_))
     return make_error_code(errc::file_too_large);
 
+  // Don't try again immediately, most likely it won't succeed ever.
+  const uint64_t kCooldownTime = 100'000'000;  // 100ms
+  if (grow_.last_err && (ProactorBase::GetMonotonicTimeNs() - grow_.timestamp_ns) < kCooldownTime)
+    return make_error_code(errc::operation_canceled);
+
   if (std::exchange(grow_.pending, true)) {
     LOG_EVERY_T(WARNING, 1) << "Blocked on concurrent grow";
     return make_error_code(errc::operation_in_progress);
@@ -212,6 +217,7 @@ error_code DiskStorage::TryGrow(off_t grow_size) {
   off_t end = alloc_.capacity();
   grow_.last_err = DoFiberCall(&SubmitEntry::PrepFallocate, backing_file_->fd(), 0, end, grow_size);
   grow_.pending = false;
+  grow_.timestamp_ns = ProactorBase::GetMonotonicTimeNs();
   grow_.ev.notifyAll();
 
   if (!grow_.last_err)

src/server/tiering/disk_storage.h

@@ -64,10 +64,11 @@ class DiskStorage {
   uint64_t heap_buf_alloc_cnt_ = 0, reg_buf_alloc_cnt_ = 0;
 
   struct {
-    bool pending = false;
+    bool pending = false;  // whether currently in progress
     std::error_code last_err;
-    util::fb2::EventCount ev;
-  } grow_;
+    uint64_t timestamp_ns;     // when last os finished (with any status)
+    util::fb2::EventCount ev;  // woken up when in-progress op finishes
+  } grow_;                     // status of latest blocking Grow() operation
 
   std::unique_ptr<util::fb2::LinuxFile> backing_file_;
   ExternalAllocator alloc_;