test/store_test: Expose race in BlueFS truncate / remove

Created test that exposes race between BlueFS::truncate and BlueFS::unlink.
Test requires injection of 1ms sleep to BlueFS::truncate.
Therefore, in this form, it is unsuitable for merge.

Signed-off-by: Adam Kupczyk <[email protected]>

Author: aclamk

src/os/bluestore/BlueFS.cc

@@ -3861,7 +3861,7 @@ int BlueFS::truncate(FileWriter *h, uint64_t offset)/*_WF_L*/
   if (offset > fnode.size) {
     ceph_abort_msg("truncate up not supported");
   }
-
+  unittest_inject_delay();
   _flush_bdev(h);
   {
     std::lock_guard ll(log.lock);

src/os/bluestore/BlueFS.h

@@ -237,6 +237,7 @@ class debug_point_t {
   : m_func(func) {}
   template<typename... Arg>
   void operator()(Arg... arg) { if (m_func) m_func(std::forward<Arg...>(arg...)); }
+  void operator()() { if (m_func) m_func(); }
   void operator=(T&& func) { m_func = std::move(func);}
   void operator=(T& func) { m_func = func;}
 private:
@@ -813,6 +814,7 @@ class BlueFS {
   bool debug_get_is_dev_dirty(FileWriter *h, uint8_t dev);
   debug_point_t<std::function<void(uint32_t)>> tracepoint_async_compact;
   void trim_free_space(const std::string& type, std::ostream& outss);
+  debug_point_t<std::function<void()>> unittest_inject_delay;
 
 private:
   // Wrappers for BlockDevice::read(...) and BlockDevice::read_random(...)

src/test/objectstore/store_test.cc

@@ -11996,6 +11996,53 @@ TEST_P(StoreTestSpecificAUSize, BlueFSReservedTest) {
             g_conf()->bluefs_alloc_size + wal_extra);
 }
 
+
+TEST_P(StoreTest, BlueFS_truncate_remove_race) {
+  if (string(GetParam()) != "bluestore")
+    GTEST_SKIP();
+
+  BlueStore* bstore = dynamic_cast<BlueStore*> (store.get());
+  ceph_assert(bstore);
+  BlueFS& fs = *bstore->get_bluefs();
+  fs.unittest_inject_delay = []() { usleep(1000); };
+  static constexpr uint32_t batch_size = 20;
+  std::binary_semaphore go_remove(0);
+  std::binary_semaphore done_remove(0);
+
+  auto files_remover = [&]() {
+    for (uint32_t cnt = 0; cnt < batch_size; cnt++) {
+      go_remove.acquire();
+      std::string name = "test-file-" + std::to_string(cnt);
+      fs.unlink("dir", name);
+      fs.sync_metadata(false);
+      done_remove.release();
+    }
+  };
+
+  ASSERT_EQ(0, fs.mkdir("dir"));
+  std::thread remover_thread(files_remover);
+
+  for (uint32_t cnt = 0; cnt < batch_size; cnt++) {
+    std::string name = "test-file-" + std::to_string(cnt);
+    BlueFS::FileWriter *f = nullptr;
+    ASSERT_EQ(0, fs.open_for_write("dir", name, &f, false));
+    fs.preallocate(f->file, 0, 100000);
+    for (uint32_t i = 0; i < 10; i++) {
+      fs.append_try_flush(f, "x", 1);
+      fs.fsync(f);
+    }
+    go_remove.release();
+    fs.truncate(f, 10);
+    fs.close_writer(f);
+    done_remove.acquire();
+  }
+
+  remover_thread.join();
+  fs.unittest_inject_delay = nullptr;
+  EXPECT_EQ(store->umount(), 0);
+  EXPECT_EQ(store->mount(), 0);
+}
+
 #endif  // WITH_BLUESTORE
 
 int main(int argc, char **argv) {