perf(chunkserver): Protect IO performance during massive delete

This PR further reduces trash-GC impact on foreground I/O
by dynamically throttling expired-file deletion based on observed
disk throughput, and improves delete wall-clock time
by running removals per-disk in parallel.

Key changes:
- I/O-aware GC throttling (with floor guard)
  - Adds GC-specific I/O counters in HddStats
 and increments them on every read/write accounting path.
  - In ChunkTrashManagerImpl::collectGarbage():
    - Samples and resets GC I/O counters each cycle (exchange(0)).
    - Normalizes by disk count (gDisks.size() under gDisksMutex)
    to estimate per-disk pressure.
    - Maintains max observed per-disk bytes (read/write) to create
    a stable baseline.
    - Computes totalIOPercentage and feeds it through
    an inverted sigmoid (steepness=12, center=0.3) to scale down
    trashGarbageCollectorBulkSize under load.
    - Adds a minimum threshold: only perform expired-file deletion
    when the scaled bulk size is >= 5, preventing “death by a thousand
    tiny deletes” during heavy I/O.

- Parallel trash removal across disks
  -removeTrashFiles() now spawns a std::jthread per disk and
  delegates actual deletion/index updates to removeTrashFilesFromDisk()
  allowing independent disks to progress concurrently.

- Behavioral defaults
  - CHUNK_TRASH_ENABLED default flips 0 → 1 (trash manager enabled
  unless explicitly disabled).
  - Default retention (kDefaultTrashTimeLimitSeconds) flips 259200 → 0
  (immediate expiry unless configured).
  - Changed documentation accordingly.

- Resource thread pacing
  - hddFreeResourcesThread() loop period increases 2s → 3s,
  replacing sleep() with Timeout + usleep(timeout.remaining_us())
  for more precise loop cadence.

- Test dependency: disabled GC for rebalancing test,
mainly because chunks in trash affect disk usage statistics.

Rationale:

Large delete storms can saturate disks and degrade chunkserver latency.
This patch makes the garbage collector self-throttling using real I/O
signals and avoids thrashing by skipping deletion work when
the system is already busy, while parallelizing deletes per disk
to reclaim space faster when bandwidth is available.

Signed-off-by: GigaCronos <jorge.cabrera@leil.io>

Author: GigaCronos

doc/sfschunkserver.cfg.5.adoc

@@ -186,7 +186,7 @@ defined at build time), /usr/local/lib/saunafs/plugins/chunkserver, and
 
 *CHUNK_TRASH_ENABLED (EXPERIMENTAL)*:: enables or disables the chunk trash
 feature. When enabled, deleted chunks are moved to a trash directory instead of
-being immediately removed. (Default: 0)
+being immediately removed. (Default: 1)
 
 *CHUNK_TRASH_EXPIRATION_SECONDS (EXPERIMENTAL)*:: specifies the timeout in
 seconds for chunks to remain in the trash before being permanently deleted.

src/admin/dump_config_command.cc

@@ -176,7 +176,7 @@ const static std::unordered_map<std::string, std::string> defaultOptionsCS = {
     {"REPLICATION_CONNECTION_TIMEOUT_MS", "1000"},
     {"REPLICATION_WAVE_TIMEOUT_MS", "500"},
     {"PLUGINS_DIR", ""},
-    {"CHUNK_TRASH_ENABLED", "0"},
+    {"CHUNK_TRASH_ENABLED", "1"},
     {"CHUNK_TRASH_EXPIRATION_SECONDS", "259200"},
     {"CHUNK_TRASH_FREE_SPACE_THRESHOLD_GB", "0"},
     {"CHUNK_TRASH_GC_BATCH_SIZE", "1000"},

src/chunkserver/chunkserver-common/chunk_trash_index.cc

@@ -80,10 +80,8 @@ ChunkTrashIndex::TrashIndexDiskEntries ChunkTrashIndex::getExpiredFilesInternal(
     const time_t &timeLimit, size_t bulkSize) {
 	TrashIndexDiskEntries expiredFiles;
 
-	size_t count = 0;
 	for (const auto &diskEntry : trashIndex) {
-		if (bulkSize != 0 && count >= bulkSize) { break; }
-		count += getExpiredFilesInternal(diskEntry.first, timeLimit, expiredFiles, bulkSize);
+		getExpiredFilesInternal(diskEntry.first, timeLimit, expiredFiles, bulkSize);
 	}
 
 	return expiredFiles;

src/chunkserver/chunkserver-common/chunk_trash_manager.cc

@@ -106,7 +106,7 @@ void ChunkTrashManager::reloadConfig() {
 		return;
 	}
 
-	isEnabled = cfg_get("CHUNK_TRASH_ENABLED", static_cast<u_short>(0));
+	isEnabled = cfg_get("CHUNK_TRASH_ENABLED", static_cast<u_short>(1));
 	safs::log_info("Chunk trash manager is {}", isEnabled ? "enabled" : "disabled");
 	impl->reloadConfig();
 }

src/chunkserver/chunkserver-common/chunk_trash_manager_impl.cc

@@ -20,11 +20,14 @@
 
 #include <sys/statvfs.h>
 #include <algorithm>
+#include <cstdint>
 #include <ctime>
 
 #include "chunkserver-common/chunk_trash_manager_impl.h"
 #include "config/cfg.h"
 #include "errors/saunafs_error_codes.h"
+#include "global_shared_resources.h"
+#include "hdd_stats.h"
 #include "slogger/slogger.h"
 
 namespace fs = std::filesystem;
@@ -35,6 +38,9 @@ size_t ChunkTrashManagerImpl::trashGarbageCollectorBulkSize = kDefaultTrashGarba
 size_t ChunkTrashManagerImpl::garbageCollectorSpaceRecoveryStep =
     kDefaultGarbageCollectorSpaceRecoveryStep;
 
+uint64_t ChunkTrashManagerImpl::maxBytesReadPerDisk = 1;
+uint64_t ChunkTrashManagerImpl::maxBytesWritePerDisk = 1;	
+
 const std::string ChunkTrashManagerImpl::kTrashGuardString =
     std::string("/") + ChunkTrashManager::kTrashDirname + "/";
 
@@ -158,11 +164,18 @@ int ChunkTrashManagerImpl::moveToTrash(const fs::path &filePath, const fs::path
 
 void ChunkTrashManagerImpl::removeTrashFiles(
     const ChunkTrashIndex::TrashIndexDiskEntries &filesToRemove) const {
+	std::vector<std::jthread> paralellRemovers;
+
 	for (const auto &[diskPath, fileEntries] : filesToRemove) {
-		for (const auto &fileEntry : fileEntries) {
-			if (removeFileFromTrash(fileEntry.second) != SAUNAFS_STATUS_OK) { continue; }
-			getTrashIndex().remove(fileEntry.first, fileEntry.second, diskPath);
-		}
+		paralellRemovers.emplace_back(removeTrashFilesFromDisk, fileEntries, diskPath);
+	}
+}
+
+void ChunkTrashManagerImpl::removeTrashFilesFromDisk(
+    const ChunkTrashIndex::TrashIndexFileEntries &filesToRemove, const std::string &diskPath) {
+	for (const auto &fileEntry : filesToRemove) {
+		if (removeFileFromTrash(fileEntry.second) != SAUNAFS_STATUS_OK) { continue; }
+		getTrashIndex().remove(fileEntry.first, fileEntry.second, diskPath);
 	}
 }
 
@@ -254,7 +267,36 @@ void ChunkTrashManagerImpl::collectGarbage() {
 	if (!ChunkTrashManager::isEnabled) { return; }
 	std::time_t const currentTime = std::time(nullptr);
 	std::time_t const expirationTime = currentTime - trashTimeLimitSeconds;
-	removeExpiredFiles(expirationTime, trashGarbageCollectorBulkSize);
+	uint64_t currentBytesWrite = HddStats::gBytesWrittenSinceLastGCSweep.exchange(0);
+	uint64_t currentBytesRead = HddStats::gBytesReadSinceLastGCSweep.exchange(0);
+	uint64_t currentDiskCount = 1;
+	{
+		std::lock_guard disksLockGuard(gDisksMutex);
+		currentDiskCount = std::max(currentDiskCount, gDisks.size());
+	}
+	currentBytesRead /= currentDiskCount;
+	currentBytesWrite /= currentDiskCount;
+	maxBytesReadPerDisk = std::max(maxBytesReadPerDisk, currentBytesRead);
+	maxBytesWritePerDisk = std::max(maxBytesWritePerDisk, currentBytesWrite);
+
+	uint64_t totalIOPercentage = currentBytesRead * 100 / maxBytesReadPerDisk +
+	                             currentBytesWrite * 100 / maxBytesWritePerDisk;
+	totalIOPercentage = std::min(totalIOPercentage, uint64_t(100));
+
+	auto invertedSigmoid = [](double val) -> double {
+		const double steepness = 12.0;       // steepness
+		const double center = 0.3;           // center in [0,1]
+		const double valnorm = val / 100.0;  // normalize to [0,1]
+
+		const double res = 1.0 / (1.0 + std::exp(-steepness * (valnorm - center)));
+		return 1.0 - res;
+	};
+
+	uint64_t bulksizeScaled = trashGarbageCollectorBulkSize * invertedSigmoid(totalIOPercentage);
+
+	static constexpr uint64_t kMinGCBulkSizeForActivation = 5;
+
+	if (bulksizeScaled >= kMinGCBulkSizeForActivation) { removeExpiredFiles(expirationTime, bulksizeScaled); }
 	makeSpace(availableThresholdGB, garbageCollectorSpaceRecoveryStep);
 }
 

src/chunkserver/chunkserver-common/chunk_trash_manager_impl.h

@@ -80,6 +80,14 @@ class ChunkTrashManagerImpl : public IChunkTrashManagerImpl {
 	 */
 	void removeTrashFiles(const ChunkTrashIndex::TrashIndexDiskEntries &filesToRemove) const;
 
+	/**
+	 * @brief Removes a set of specified files from the trash for an specific FileEntries.
+	 * @param filesToRemove The list of files to be permanently deleted.
+	 * @param diskPath The path of the disk of the files.
+	 */
+	static void removeTrashFilesFromDisk(
+	    const ChunkTrashIndex::TrashIndexFileEntries &filesToRemove, const std::string &diskPath);
+
 	/**
 	 * @brief Checks if a given timestamp string matches the expected format.
 	 * @param timestamp The timestamp string to validate.
@@ -137,6 +145,11 @@ class ChunkTrashManagerImpl : public IChunkTrashManagerImpl {
 	void reloadConfig() override;
 
 private:
+
+	/// Maximum record of BytesReadPerDisk and BytesWritePerDisk in a GC cycle	
+	static uint64_t maxBytesReadPerDisk;
+	static uint64_t maxBytesWritePerDisk;	
+	
 	/// Minimum available space threshold (in GB) before triggering garbage
 	/// collection.
 	static size_t availableThresholdGB;
@@ -145,7 +158,7 @@ class ChunkTrashManagerImpl : public IChunkTrashManagerImpl {
 	/// Time limit (in seconds) for files to be considered eligible for
 	/// deletion.
 	static size_t trashTimeLimitSeconds;
-	static constexpr size_t kDefaultTrashTimeLimitSeconds = 259200;
+	static constexpr size_t kDefaultTrashTimeLimitSeconds = 0;
 
 	/// Number of files processed in each bulk operation during garbage
 	/// collection.

src/chunkserver/chunkserver-common/hdd_stats.cc

@@ -41,6 +41,7 @@ static inline void totalRead(IDisk *disk, uint64_t size, MicroSeconds duration)
 
 	gStatsTotalOperationsRead++;
 	gStatsTotalBytesRead += size;
+	gBytesReadSinceLastGCSweep += size;
 	gStatsTotalTimeRead += duration;
 
 	auto &diskStats = disk->getCurrentStats();
@@ -60,6 +61,7 @@ static inline void totalWrite(IDisk *disk, uint64_t size,
 
 	gStatsTotalOperationsWrite++;
 	gStatsTotalBytesWrite += size;
+	gBytesWrittenSinceLastGCSweep += size;
 	gStatsTotalTimeWrite += duration;
 
 	auto &diskStats = disk->getCurrentStats();

src/chunkserver/chunkserver-common/hdd_stats.h

@@ -56,6 +56,12 @@ inline std::atomic<uint32_t> gStatsOperationsDuplicate(0);
 inline std::atomic<uint32_t> gStatsOperationsTruncate(0);
 inline std::atomic<uint32_t> gStatsOperationsDupTrunc(0);
 
+
+// This is for internal use of GarbageCollector
+inline std::atomic<uint64_t> gBytesWrittenSinceLastGCSweep(0);
+inline std::atomic<uint64_t> gBytesReadSinceLastGCSweep(0);
+
+
 struct statsReport {
 	statsReport(uint64_t *overBytesRead, uint64_t *overBytesWrite,
 	            uint32_t *overOpsRead, uint32_t *overOpsWrite,

src/chunkserver/hddspacemgr.cc

@@ -2524,22 +2524,22 @@ void hddDisksThread() {
 }
 
 void hddFreeResourcesThread() {
-	static const int kDelayedStep = 2;
+	static const int kDelayedStep = 3;
 	static const uint32_t kOldIoBuffersExpirationTimeMs = kDelayedStep * 1000;
 	static const int kMaxFreeUnused = 1024;
 	TRACETHIS();
 
 	pthread_setname_np(pthread_self(), "freeResThread");
 
 	while (!gTerminate) {
+		Timeout timeout(std::chrono::microseconds(kDelayedStep * 1'000'000));
 		gOpenChunks.freeUnused(eventloop_time(), gChunksMapMutex,
 		                       kMaxFreeUnused);
 		ChunkTrashManager::collectGarbage();
 		hddReleaseDisksToBeDeleted();
 		/// Release buffers older than kDelayedStep seconds
 		releaseOldIoBuffers(kOldIoBuffersExpirationTimeMs);
-
-		sleep(kDelayedStep);
+		usleep(timeout.remaining_us());
 	}
 }
 

tests/test_suites/RebalancingTests/test_chunk_rebalancing.sh

@@ -4,7 +4,7 @@ rebalancing_timeout=90
 CHUNKSERVERS=5 \
 	USE_LOOP_DISKS=YES \
 	MOUNT_EXTRA_CONFIG="sfscachemode=NEVER" \
-	CHUNKSERVER_EXTRA_CONFIG="HDD_TEST_FREQ = 10000|HDD_LEAVE_SPACE_DEFAULT = 0MiB" \
+	CHUNKSERVER_EXTRA_CONFIG="HDD_TEST_FREQ = 10000|HDD_LEAVE_SPACE_DEFAULT = 0MiB|CHUNK_TRASH_ENABLED=0" \
 	MASTER_EXTRA_CONFIG="ACCEPTABLE_DIFFERENCE = 0.0015`
 			`|CHUNKS_LOOP_MAX_CPS = 2`
 			`|CHUNKS_LOOP_MAX_CPU = 90`