[fix][mds] Fixup known issues.

rock-git · chuandew · commit 88274f16b6b0 · 2025-10-13T06:50:06.000Z
diff --git a/src/mds/background/gc.cc b/src/mds/background/gc.cc
@@ -91,7 +91,7 @@ static IntRange CalBlockIndex(uint64_t block_size, uint64_t chunk_offset, const
 void CleanDelSliceTask::Run() {
   auto status = CleanDelSlice();
   if (!status.ok()) {
-    DINGO_LOG(ERROR) << fmt::format("[gc.delslice] clean deleted slice fail, {}", status.error_str());
+    DINGO_LOG(ERROR) << fmt::format("[gc.delslice.{}] clean deleted slice fail, {}", ino_, status.error_str());
   }
 
   // forget task
@@ -112,7 +112,7 @@ Status CleanDelSliceTask::CleanDelSlice() {
         cache::BlockKey block_key(slice.fs_id(), slice.ino(), slice.slice_id(), block_index,
                                   slice_range.compaction_version());
 
-        DINGO_LOG(INFO) << fmt::format("[gc.delslice] delete block key({}).", block_key.StoreKey());
+        DINGO_LOG(INFO) << fmt::format("[gc.delslice.{}] delete block key({}).", ino_, block_key.StoreKey());
         keys.push_back(block_key.StoreKey());
       }
     }
@@ -140,15 +140,15 @@ Status CleanDelSliceTask::CleanDelSlice() {
     return status;
   }
 
-  DINGO_LOG(INFO) << fmt::format("[gc.delslice] clean slice finish, slice({}).", slice_id_trace);
+  DINGO_LOG(INFO) << fmt::format("[gc.delslice.{}] clean slice finish, slice({}).", ino_, slice_id_trace);
 
   return Status::OK();
 }
 
 void CleanDelFileTask::Run() {
   auto status = CleanDelFile(attr_);
   if (!status.ok()) {
-    DINGO_LOG(ERROR) << fmt::format("[gc.delfile] clean delfile fail, status({}).", status.error_str());
+    DINGO_LOG(ERROR) << fmt::format("[gc.delfile.{}] clean delfile fail, status({}).", attr_.ino(), status.error_str());
   }
 
   // forget task
@@ -176,7 +176,7 @@ static Status GetChunks(OperationProcessorSPtr operation_processor, uint32_t fs_
 }
 
 Status CleanDelFileTask::CleanDelFile(const AttrEntry& attr) {
-  DINGO_LOG(INFO) << fmt::format("[gc.delfile] clean delfile, ino({}) nlink({}) len({}) version({}).", attr.ino(),
+  DINGO_LOG(INFO) << fmt::format("[gc.delfile.{}] clean delfile, nlink({}) len({}) version({}).", attr.ino(),
                                  attr.nlink(), attr.length(), attr.version());
   // get file chunks
   std::vector<ChunkEntry> chunks;
@@ -194,7 +194,7 @@ Status CleanDelFileTask::CleanDelFile(const AttrEntry& attr) {
       for (uint32_t block_index = range.start; block_index < range.end; ++block_index) {
         cache::BlockKey block_key(attr.fs_id(), attr.ino(), slice.id(), block_index, slice.compaction_version());
 
-        DINGO_LOG(INFO) << fmt::format("[gc.delfile] delete block key({}).", block_key.StoreKey());
+        DINGO_LOG(INFO) << fmt::format("[gc.delfile.{}] delete block key({}).", attr.ino(), block_key.StoreKey());
         keys.push_back(block_key.StoreKey());
       }
     }
@@ -216,7 +216,7 @@ Status CleanDelFileTask::CleanDelFile(const AttrEntry& attr) {
     return status;
   }
 
-  DINGO_LOG(INFO) << fmt::format("[gc.delfile] clean file({}/{}) finish.", attr.fs_id(), attr.ino());
+  DINGO_LOG(INFO) << fmt::format("[gc.delfile.{}] clean file({}/{}) finish.", attr.ino(), attr.fs_id(), attr.ino());
 
   return Status::OK();
 }
@@ -335,7 +335,7 @@ Status GcProcessor::ManualCleanDelSlice(Trace& trace, uint32_t fs_id, Ino ino, u
 
   ScanDelSliceOperation operation(
       trace, fs_id, ino, chunk_index, [&](const std::string& key, const std::string& value) -> bool {
-        auto task = CleanDelSliceTask::New(operation_processor_, block_accessor, nullptr, key, value);
+        auto task = CleanDelSliceTask::New(operation_processor_, block_accessor, nullptr, ino, key, value);
         auto status = task->CleanDelSlice();
         if (!status.ok()) {
           LOG(ERROR) << fmt::format("[gc.delslice] clean delfile fail, status({}).", status.error_str());
@@ -525,7 +525,7 @@ void GcProcessor::ScanDelSlice(const FsInfoEntry& fs_info) {
     }
 
     task_memo_->Remember(key);
-    if (!Execute(ino, CleanDelSliceTask::New(operation_processor_, block_accessor, task_memo_, key, value))) {
+    if (!Execute(ino, CleanDelSliceTask::New(operation_processor_, block_accessor, task_memo_, ino, key, value))) {
       task_memo_->Forget(key);
       return false;
     }
diff --git a/src/mds/background/gc.h b/src/mds/background/gc.h
@@ -97,18 +97,19 @@ class TaskMemo {
 class CleanDelSliceTask : public TaskRunnable {
  public:
   CleanDelSliceTask(OperationProcessorSPtr operation_processor, blockaccess::BlockAccesserSPtr block_accessor,
-                    TaskMemoSPtr task_memo, const std::string& key, const std::string& value)
+                    TaskMemoSPtr task_memo, Ino ino, const std::string& key, const std::string& value)
       : operation_processor_(operation_processor),
         data_accessor_(block_accessor),
+        ino_(ino),
         key_(key),
         value_(value),
         task_memo_(task_memo) {}
   ~CleanDelSliceTask() override = default;
 
   static CleanDelSliceTaskSPtr New(OperationProcessorSPtr operation_processor,
-                                   blockaccess::BlockAccesserSPtr block_accessor, TaskMemoSPtr task_memo,
+                                   blockaccess::BlockAccesserSPtr block_accessor, TaskMemoSPtr task_memo, Ino ino,
                                    const std::string& key, const std::string& value) {
-    return std::make_shared<CleanDelSliceTask>(operation_processor, block_accessor, task_memo, key, value);
+    return std::make_shared<CleanDelSliceTask>(operation_processor, block_accessor, task_memo, ino, key, value);
   }
   std::string Type() override { return "CLEAN_DELETED_SLICE"; }
 
@@ -119,6 +120,8 @@ class CleanDelSliceTask : public TaskRunnable {
 
   Status CleanDelSlice();
 
+  const Ino ino_;
+
   const std::string key_;
   const std::string value_;
 
diff --git a/src/mds/filesystem/filesystem.cc b/src/mds/filesystem/filesystem.cc
@@ -1944,12 +1944,11 @@ Status FileSystem::WriteSlice(Context& ctx, Ino, Ino ino, const std::vector<Delt
       if (CompactChunkOperation::MaybeCompact(fs_info, ino, attr.length(), chunk)) {
         DINGO_LOG(INFO) << fmt::format("[fs.{}] trigger compact chunk({}) for ino({}).", fs_id_, chunk.index(), ino);
 
-        auto post_handler = [&](OperationSPtr operation) {
+        auto post_handler = [this](OperationSPtr operation) {
           auto origin_operation = std::dynamic_pointer_cast<CompactChunkOperation>(operation);
           auto& result = origin_operation->GetResult();
-          auto& attr = result.attr;
           // update chunk cache
-          chunk_cache_.PutIf(attr.ino(), std::move(result.effected_chunk));
+          chunk_cache_.PutIf(origin_operation->GetIno(), std::move(result.effected_chunk));
         };
         operation_processor_->AsyncRun(CompactChunkOperation::New(fs_info, ino, chunk.index(), attr.length()),
                                        post_handler);
diff --git a/src/mds/filesystem/store_operation.cc b/src/mds/filesystem/store_operation.cc
@@ -893,6 +893,8 @@ std::vector<std::string> UpsertChunkOperation::PrefetchKey() {
 }
 
 Status UpsertChunkOperation::RunInBatch(TxnUPtr& txn, AttrEntry& attr, const std::vector<KeyValue>& prefetch_kvs) {
+  result_.effected_chunks.clear();
+
   uint64_t prev_length = attr.length();
   for (const auto& delta_slices : delta_slices_) {
     ChunkEntry chunk;
@@ -1580,17 +1582,13 @@ TrashSliceList CompactChunkOperation::GenTrashSlices(const FsInfoEntry& fs_info,
   }
 
   size_t out_of_length_count = trash_slices.slices_size();
-  DINGO_LOG(INFO) << fmt::format("[operation.{}.{}] out-of-file-length trash slices count({}).", fs_id, ino,
-                                 out_of_length_count);
 
   // 2. complete overlapped slices
   //     |______4________|      slices
   // |__1__|___2___|____3_____| slices
   // |________________________| chunk
   // slice-2 is complete overlapped by slice-4
   // sort by offset
-  std::stable_sort(chunk_copy.slices.begin(), chunk_copy.slices.end(),  // NOLINT
-                   [](const Slice& a, const Slice& b) { return a.offset < b.offset; });
 
   // get offset ranges
   std::set<uint64_t> offsets;
@@ -1653,8 +1651,6 @@ TrashSliceList CompactChunkOperation::GenTrashSlices(const FsInfoEntry& fs_info,
   }
 
   size_t complete_overlapped_count = trash_slices.slices_size() - out_of_length_count;
-  DINGO_LOG(INFO) << fmt::format("[operation.{}.{}] complete overlapped trash slice count({}).", fs_id, ino,
-                                 complete_overlapped_count);
 
   // 3. partial overlapped slices
   //     |______4________|      slices
@@ -1720,16 +1716,18 @@ TrashSliceList CompactChunkOperation::GenTrashSlices(const FsInfoEntry& fs_info,
   // }
 
   size_t partial_overlapped_count = trash_slices.slices_size() - out_of_length_count - complete_overlapped_count;
-  DINGO_LOG(INFO) << fmt::format("[operation.{}.{}] partial overlapped trash slice count({}) total({})", fs_id, ino,
-                                 partial_overlapped_count, trash_slices.slices_size());
 
-  if (FLAGS_mds_compact_chunk_detail_log_enable) {
-    for (const auto& slice : trash_slices.slices()) {
-      DINGO_LOG(INFO) << fmt::format("[operation.{}.{}] trash slice, chunk_index({}) slice_id({}), is_partial({}).",
-                                     fs_id, ino, slice.chunk_index(), slice.slice_id(), slice.is_partial());
-    }
+  std::string slice_id_str;
+  slice_id_str.reserve(trash_slices.slices_size() * 9);
+  for (const auto& slice : trash_slices.slices()) {
+    slice_id_str += std::to_string(slice.slice_id());
+    slice_id_str += ",";
   }
 
+  DINGO_LOG(INFO) << fmt::format("[operation.{}.{}.{}] trash slice, count({}/{}/{}/{}) slice_ids({}).", fs_id, ino,
+                                 chunk.index(), out_of_length_count, complete_overlapped_count,
+                                 partial_overlapped_count, trash_slices.slices_size(), slice_id_str);
+
   return trash_slices;
 }
 
@@ -1812,14 +1810,14 @@ Status CompactChunkOperation::Run(TxnUPtr& txn) {
   // reduce compact frequency
   if (!is_force_ && chunk.last_compaction_time_ms() + FLAGS_mds_compact_chunk_interval_ms >
                         static_cast<uint64_t>(Helper::TimestampMs())) {
-    return Status::OK();
+    return Status(pb::error::ENOT_MATCH, "not match compact condition");
   }
 
   auto trash_slice_list = CompactChunk(txn, fs_id, ino_, file_length_, chunk);
   if (!trash_slice_list.slices().empty()) {
     chunk.set_version(chunk.version() + 1);
     chunk.set_last_compaction_time_ms(Helper::TimestampMs());
-    txn->Put(MetaCodec::EncodeChunkKey(fs_id, ino_, chunk.index()), MetaCodec::EncodeChunkValue(chunk));
+    txn->Put(key, MetaCodec::EncodeChunkValue(chunk));
 
     result_.trash_slice_list = std::move(trash_slice_list);
     result_.effected_chunk = std::move(chunk);
diff --git a/src/mds/filesystem/store_operation.h b/src/mds/filesystem/store_operation.h
@@ -1010,6 +1010,11 @@ class CompactChunkOperation : public Operation {
 
   static bool MaybeCompact(const FsInfoEntry& fs_info, Ino ino, uint64_t file_length, const ChunkEntry& chunk);
 
+  static TrashSliceList TestGenTrashSlices(const FsInfoEntry& fs_info, Ino ino, uint64_t file_length,
+                                           const ChunkEntry& chunk) {
+    return GenTrashSlices(fs_info, ino, file_length, chunk);
+  }
+
  private:
   static TrashSliceList GenTrashSlices(const FsInfoEntry& fs_info, Ino ino, uint64_t file_length,
                                        const ChunkEntry& chunk);
diff --git a/test/mds/filesystem/test_store_operation.cc b/test/mds/filesystem/test_store_operation.cc
diff --git a/test/mds/main.cc b/test/mds/main.cc

Original file line number	Diff line number	Diff line change
`@@ -91,7 +91,7 @@ static IntRange CalBlockIndex(uint64_t block_size, uint64_t chunk_offset, const`
`91`	`91`	`void CleanDelSliceTask::Run() {`
`92`	`92`	`auto status = CleanDelSlice();`
`93`	`93`	`if (!status.ok()) {`
`94`		`- DINGO_LOG(ERROR) << fmt::format("[gc.delslice] clean deleted slice fail, {}", status.error_str());`
	`94`	`+ DINGO_LOG(ERROR) << fmt::format("[gc.delslice.{}] clean deleted slice fail, {}", ino_, status.error_str());`
`95`	`95`	`}`
`96`	`96`
`97`	`97`	`// forget task`
`@@ -112,7 +112,7 @@ Status CleanDelSliceTask::CleanDelSlice() {`
`112`	`112`	`cache::BlockKey block_key(slice.fs_id(), slice.ino(), slice.slice_id(), block_index,`
`113`	`113`	`slice_range.compaction_version());`
`114`	`114`
`115`		`- DINGO_LOG(INFO) << fmt::format("[gc.delslice] delete block key({}).", block_key.StoreKey());`
	`115`	`+ DINGO_LOG(INFO) << fmt::format("[gc.delslice.{}] delete block key({}).", ino_, block_key.StoreKey());`
`116`	`116`	`keys.push_back(block_key.StoreKey());`
`117`	`117`	`}`
`118`	`118`	`}`
`@@ -140,15 +140,15 @@ Status CleanDelSliceTask::CleanDelSlice() {`
`140`	`140`	`return status;`
`141`	`141`	`}`
`142`	`142`
`143`		`- DINGO_LOG(INFO) << fmt::format("[gc.delslice] clean slice finish, slice({}).", slice_id_trace);`
	`143`	`+ DINGO_LOG(INFO) << fmt::format("[gc.delslice.{}] clean slice finish, slice({}).", ino_, slice_id_trace);`
`144`	`144`
`145`	`145`	`return Status::OK();`
`146`	`146`	`}`
`147`	`147`
`148`	`148`	`void CleanDelFileTask::Run() {`
`149`	`149`	`auto status = CleanDelFile(attr_);`
`150`	`150`	`if (!status.ok()) {`
`151`		`- DINGO_LOG(ERROR) << fmt::format("[gc.delfile] clean delfile fail, status({}).", status.error_str());`
	`151`	`+ DINGO_LOG(ERROR) << fmt::format("[gc.delfile.{}] clean delfile fail, status({}).", attr_.ino(), status.error_str());`
`152`	`152`	`}`
`153`	`153`
`154`	`154`	`// forget task`
`@@ -176,7 +176,7 @@ static Status GetChunks(OperationProcessorSPtr operation_processor, uint32_t fs_`
`176`	`176`	`}`
`177`	`177`
`178`	`178`	`Status CleanDelFileTask::CleanDelFile(const AttrEntry& attr) {`
`179`		`- DINGO_LOG(INFO) << fmt::format("[gc.delfile] clean delfile, ino({}) nlink({}) len({}) version({}).", attr.ino(),`
	`179`	`+ DINGO_LOG(INFO) << fmt::format("[gc.delfile.{}] clean delfile, nlink({}) len({}) version({}).", attr.ino(),`
`180`	`180`	`attr.nlink(), attr.length(), attr.version());`
`181`	`181`	`// get file chunks`
`182`	`182`	`std::vector<ChunkEntry> chunks;`
`@@ -194,7 +194,7 @@ Status CleanDelFileTask::CleanDelFile(const AttrEntry& attr) {`
`194`	`194`	`for (uint32_t block_index = range.start; block_index < range.end; ++block_index) {`
`195`	`195`	`cache::BlockKey block_key(attr.fs_id(), attr.ino(), slice.id(), block_index, slice.compaction_version());`
`196`	`196`
`197`		`- DINGO_LOG(INFO) << fmt::format("[gc.delfile] delete block key({}).", block_key.StoreKey());`
	`197`	`+ DINGO_LOG(INFO) << fmt::format("[gc.delfile.{}] delete block key({}).", attr.ino(), block_key.StoreKey());`
`198`	`198`	`keys.push_back(block_key.StoreKey());`
`199`	`199`	`}`
`200`	`200`	`}`
`@@ -216,7 +216,7 @@ Status CleanDelFileTask::CleanDelFile(const AttrEntry& attr) {`
`216`	`216`	`return status;`
`217`	`217`	`}`
`218`	`218`
`219`		`- DINGO_LOG(INFO) << fmt::format("[gc.delfile] clean file({}/{}) finish.", attr.fs_id(), attr.ino());`
	`219`	`+ DINGO_LOG(INFO) << fmt::format("[gc.delfile.{}] clean file({}/{}) finish.", attr.ino(), attr.fs_id(), attr.ino());`
`220`	`220`
`221`	`221`	`return Status::OK();`
`222`	`222`	`}`
`@@ -335,7 +335,7 @@ Status GcProcessor::ManualCleanDelSlice(Trace& trace, uint32_t fs_id, Ino ino, u`
`335`	`335`
`336`	`336`	`ScanDelSliceOperation operation(`
`337`	`337`	`trace, fs_id, ino, chunk_index, [&](const std::string& key, const std::string& value) -> bool {`
`338`		`- auto task = CleanDelSliceTask::New(operation_processor_, block_accessor, nullptr, key, value);`
	`338`	`+ auto task = CleanDelSliceTask::New(operation_processor_, block_accessor, nullptr, ino, key, value);`
`339`	`339`	`auto status = task->CleanDelSlice();`
`340`	`340`	`if (!status.ok()) {`
`341`	`341`	`LOG(ERROR) << fmt::format("[gc.delslice] clean delfile fail, status({}).", status.error_str());`
`@@ -525,7 +525,7 @@ void GcProcessor::ScanDelSlice(const FsInfoEntry& fs_info) {`
`525`	`525`	`}`
`526`	`526`
`527`	`527`	`task_memo_->Remember(key);`
`528`		`- if (!Execute(ino, CleanDelSliceTask::New(operation_processor_, block_accessor, task_memo_, key, value))) {`
	`528`	`+ if (!Execute(ino, CleanDelSliceTask::New(operation_processor_, block_accessor, task_memo_, ino, key, value))) {`
`529`	`529`	`task_memo_->Forget(key);`
`530`	`530`	`return false;`
`531`	`531`	`}`