diff --git a/doc/sfschunkserver.cfg.5.adoc b/doc/sfschunkserver.cfg.5.adoc index ce7383f45..cf43f2392 100644 --- a/doc/sfschunkserver.cfg.5.adoc +++ b/doc/sfschunkserver.cfg.5.adoc @@ -186,11 +186,11 @@ defined at build time), /usr/local/lib/saunafs/plugins/chunkserver, and *CHUNK_TRASH_ENABLED (EXPERIMENTAL)*:: enables or disables the chunk trash feature. When enabled, deleted chunks are moved to a trash directory instead of -being immediately removed. (Default: 0) +being immediately removed. (Default: 1) *CHUNK_TRASH_EXPIRATION_SECONDS (EXPERIMENTAL)*:: specifies the timeout in seconds for chunks to remain in the trash before being permanently deleted. -(Default: 259200) +(Default: 0) *CHUNK_TRASH_FREE_SPACE_THRESHOLD_GB (EXPERIMENTAL)*:: sets the available space threshold in gigabytes. If the available space on the disk falls below this @@ -198,9 +198,11 @@ threshold, the system will start deleting older chunks from the trash to free up space. A suggested value is about the 15% of the capacity of the smaller configured disk. (Default: 0) -*CHUNK_TRASH_GC_BATCH_SIZE (EXPERIMENTAL)*:: defines the bulk size for the -garbage collector when processing chunks in the trash. This determines how many -files are processed in each garbage collection cycle. (Default: 1000) +*CHUNK_TRASH_GC_BATCH_SIZE (EXPERIMENTAL)*:: defines the maximum bulk size +per disk for the garbage collector when processing chunks in the trash. +This value is used when no I/O pressure is detected and is dynamically +scaled down based on current disk load. +(Default: 500) *CHUNK_TRASH_GC_SPACE_RECOVERY_BATCH_SIZE (EXPERIMENTAL)*:: [ADVANCED] The number of files to remove from the trash in a single GC cycle, in case the disk diff --git a/src/admin/dump_config_command.cc b/src/admin/dump_config_command.cc index a1c94c3b0..3a377a3c7 100644 --- a/src/admin/dump_config_command.cc +++ b/src/admin/dump_config_command.cc @@ -183,10 +183,10 @@ const static std::unordered_map defaultOptionsCS = { {"REPLICATION_CONNECTION_TIMEOUT_MS", "1000"}, {"REPLICATION_WAVE_TIMEOUT_MS", "500"}, {"PLUGINS_DIR", ""}, - {"CHUNK_TRASH_ENABLED", "0"}, - {"CHUNK_TRASH_EXPIRATION_SECONDS", "259200"}, + {"CHUNK_TRASH_ENABLED", "1"}, + {"CHUNK_TRASH_EXPIRATION_SECONDS", "0"}, {"CHUNK_TRASH_FREE_SPACE_THRESHOLD_GB", "0"}, - {"CHUNK_TRASH_GC_BATCH_SIZE", "1000"}, + {"CHUNK_TRASH_GC_BATCH_SIZE", "500"}, {"CHUNK_TRASH_GC_SPACE_RECOVERY_BATCH_SIZE", "100"}, }; diff --git a/src/chunkserver/chunkserver-common/chunk_trash_index.cc b/src/chunkserver/chunkserver-common/chunk_trash_index.cc index faf7ff669..f35d9dcf5 100644 --- a/src/chunkserver/chunkserver-common/chunk_trash_index.cc +++ b/src/chunkserver/chunkserver-common/chunk_trash_index.cc @@ -80,10 +80,8 @@ ChunkTrashIndex::TrashIndexDiskEntries ChunkTrashIndex::getExpiredFilesInternal( const time_t &timeLimit, size_t bulkSize) { TrashIndexDiskEntries expiredFiles; - size_t count = 0; for (const auto &diskEntry : trashIndex) { - if (bulkSize != 0 && count >= bulkSize) { break; } - count += getExpiredFilesInternal(diskEntry.first, timeLimit, expiredFiles, bulkSize); + getExpiredFilesInternal(diskEntry.first, timeLimit, expiredFiles, bulkSize); } return expiredFiles; diff --git a/src/chunkserver/chunkserver-common/chunk_trash_manager.cc b/src/chunkserver/chunkserver-common/chunk_trash_manager.cc index 57a028e7e..8f0dbc066 100644 --- a/src/chunkserver/chunkserver-common/chunk_trash_manager.cc +++ b/src/chunkserver/chunkserver-common/chunk_trash_manager.cc @@ -54,9 +54,11 @@ int ChunkTrashManager::moveToTrash(const std::filesystem::path &filePath, if (!isEnabled) { return 0; } // Protect against concurrent access - std::lock_guard lock(implMutex); - - auto &impl = getImpl(); + ImplementationPtr impl; + { + std::lock_guard lock(implMutex); + impl = getImpl(); + } if (!impl) { safs::log_error_code(SAUNAFS_ERROR_EINVAL, "ChunkTrashManager implementation not initialized"); @@ -69,9 +71,11 @@ int ChunkTrashManager::init(const std::string &diskPath) { reloadConfig(); // Protect against concurrent access - std::lock_guard lock(implMutex); - - auto &impl = getImpl(); + ImplementationPtr impl; + { + std::lock_guard lock(implMutex); + impl = getImpl(); + } if (!impl) { safs::log_error_code(SAUNAFS_ERROR_EINVAL, "ChunkTrashManager implementation not initialized"); @@ -80,13 +84,33 @@ int ChunkTrashManager::init(const std::string &diskPath) { return impl->init(diskPath); } +void ChunkTrashManager::terminate(){ + if (!isEnabled) { return; } + + // Protect against concurrent access + ImplementationPtr impl; + { + std::lock_guard lock(implMutex); + impl = getImpl(); + } + if (!impl) { + safs::log_error_code(SAUNAFS_ERROR_EINVAL, + "ChunkTrashManager implementation not initialized"); + return; + } + impl->terminate(); + +} + void ChunkTrashManager::collectGarbage() { if (!isEnabled) { return; } // Protect against concurrent access - std::lock_guard lock(implMutex); - - auto &impl = getImpl(); + ImplementationPtr impl; + { + std::lock_guard lock(implMutex); + impl = getImpl(); + } if (!impl) { safs::log_error_code(SAUNAFS_ERROR_EINVAL, "ChunkTrashManager implementation not initialized"); @@ -96,17 +120,19 @@ void ChunkTrashManager::collectGarbage() { } void ChunkTrashManager::reloadConfig() { - // Protect against concurrent access - std::lock_guard lock(implMutex); - - auto &impl = getImpl(); + // Protect against concurrent access + ImplementationPtr impl; + { + std::lock_guard lock(implMutex); + impl = getImpl(); + } if (!impl) { safs::log_error_code(SAUNAFS_ERROR_EINVAL, "ChunkTrashManager implementation not initialized"); return; } - isEnabled = cfg_get("CHUNK_TRASH_ENABLED", static_cast(0)); + isEnabled = cfg_get("CHUNK_TRASH_ENABLED", static_cast(1)); safs::log_info("Chunk trash manager is {}", isEnabled ? "enabled" : "disabled"); impl->reloadConfig(); } diff --git a/src/chunkserver/chunkserver-common/chunk_trash_manager.h b/src/chunkserver/chunkserver-common/chunk_trash_manager.h index 5958ee06d..c0a097790 100644 --- a/src/chunkserver/chunkserver-common/chunk_trash_manager.h +++ b/src/chunkserver/chunkserver-common/chunk_trash_manager.h @@ -34,6 +34,8 @@ class IChunkTrashManagerImpl { virtual int init(const std::string &) = 0; + virtual void terminate() = 0; + virtual void collectGarbage() = 0; virtual void reloadConfig() = 0; @@ -73,6 +75,11 @@ class ChunkTrashManager { */ static int init(const std::string &diskPath); + /** + * @brief Join the background threads. + */ + static void terminate(); + /** * @brief Moves a file to the trash directory. * diff --git a/src/chunkserver/chunkserver-common/chunk_trash_manager_impl.cc b/src/chunkserver/chunkserver-common/chunk_trash_manager_impl.cc index 01da88053..dcbdb03de 100644 --- a/src/chunkserver/chunkserver-common/chunk_trash_manager_impl.cc +++ b/src/chunkserver/chunkserver-common/chunk_trash_manager_impl.cc @@ -20,12 +20,21 @@ #include #include +#include +#include +#include #include +#include +#include +#include +#include #include "chunkserver-common/chunk_trash_manager_impl.h" #include "config/cfg.h" #include "errors/saunafs_error_codes.h" +#include "global_shared_resources.h" #include "hdd_stats.h" +#include "hdd_utils.h" #include "slogger/slogger.h" namespace fs = std::filesystem; @@ -36,9 +45,25 @@ size_t ChunkTrashManagerImpl::trashGarbageCollectorBulkSize = kDefaultTrashGarba size_t ChunkTrashManagerImpl::garbageCollectorSpaceRecoveryStep = kDefaultGarbageCollectorSpaceRecoveryStep; +uint64_t ChunkTrashManagerImpl::maxBytesReadPerDisk = 1024 * 1024; //1MiB +uint64_t ChunkTrashManagerImpl::maxBytesWritePerDisk = 1024 * 1024; //1MiB + +uint64_t ChunkTrashManagerImpl::previousBytesReadPerDisk = 1024 * 1024; //1MiB +uint64_t ChunkTrashManagerImpl::previousBytesWritePerDisk = 1024 * 1024; //1MiB + const std::string ChunkTrashManagerImpl::kTrashGuardString = std::string("/") + ChunkTrashManager::kTrashDirname + "/"; +std::vector ChunkTrashManagerImpl::removeFromTrashThreads{}; +std::unique_ptr ChunkTrashManagerImpl::removeFromTrashJobQueue = + std::make_unique(); + +std::atomic ChunkTrashManagerImpl::toDoJobsCount{0}; + +std::mutex ChunkTrashManagerImpl::mutex_{}; + +std::condition_variable ChunkTrashManagerImpl::trashRemoverCV{}; + void ChunkTrashManagerImpl::reloadConfig() { availableThresholdGB = cfg_get("CHUNK_TRASH_FREE_SPACE_THRESHOLD_GB", kDefaultAvailableThresholdGB); @@ -160,12 +185,51 @@ int ChunkTrashManagerImpl::moveToTrash(const fs::path &filePath, const fs::path void ChunkTrashManagerImpl::removeTrashFiles( const ChunkTrashIndex::TrashIndexDiskEntries &filesToRemove) const { for (const auto &[diskPath, fileEntries] : filesToRemove) { - for (const auto &fileEntry : fileEntries) { + toDoJobsCount++; + removeFromTrashJobQueue->put( + 0, 1, + reinterpret_cast( + new std::pair(fileEntries, + diskPath)), + 1); + } + std::unique_lock trashRemoverlock(mutex_); + trashRemoverCV.wait(trashRemoverlock,[&] { return toDoJobsCount==0 && removeFromTrashJobQueue->isEmpty(); }); + +} + +void ChunkTrashManagerImpl::removeTrashFilesFromDiskThread(uint8_t workerId) { + std::string threadName ="removeTrashFilesFromDisk_worker_" + std::to_string(workerId); + pthread_setname_np(pthread_self(), threadName.c_str()); + + std::cout<<"GigaCronos: ThreadOpened"<get(&jobId, &operation, &jobPtrArg, nullptr); + + if(operation == 0){ + break; + } + auto tempTuple = reinterpret_cast *>(jobPtrArg); + + ChunkTrashIndex::TrashIndexFileEntries filesToRemove = tempTuple->first; + std::string diskPath = tempTuple->second; + for (const auto &fileEntry : filesToRemove) { if (removeFileFromTrash(fileEntry.second) != SAUNAFS_STATUS_OK) { continue; } HddStats::gStatsOperationsGCPurge++; getTrashIndex().remove(fileEntry.first, fileEntry.second, diskPath); } + + delete tempTuple; + toDoJobsCount --; + std::unique_lock trashRemoverlock(mutex_); + trashRemoverCV.notify_one(); } + + std::cout<<"GigaCronos: ThreadClosed"<put(0, 0, nullptr, 1); + } + + + for (auto &thread : ChunkTrashManagerImpl::removeFromTrashThreads) { + if (thread.joinable()) { thread.join(); } + } + + ChunkTrashManagerImpl::removeFromTrashThreads.clear(); + +} + + bool ChunkTrashManagerImpl::isValidTimestampFormat(const std::string ×tamp) { return timestamp.size() == kTimeStampLength && std::ranges::all_of(timestamp, ::isdigit); } @@ -256,8 +341,51 @@ void ChunkTrashManagerImpl::collectGarbage() { if (!ChunkTrashManager::isEnabled) { return; } std::time_t const currentTime = std::time(nullptr); std::time_t const expirationTime = currentTime - trashTimeLimitSeconds; - removeExpiredFiles(expirationTime, trashGarbageCollectorBulkSize); + + uint64_t currentBytesWrite = HddStats::gBytesWrittenSinceLastGCSweep.exchange(0); + uint64_t currentBytesRead = HddStats::gBytesReadSinceLastGCSweep.exchange(0); + uint64_t currentDiskCount = 1; + { + std::lock_guard disksLockGuard(gDisksMutex); + currentDiskCount = std::max(currentDiskCount, gDisks.size()); + } + currentBytesRead /= currentDiskCount; + currentBytesWrite /= currentDiskCount; + + // 0.99997 ^ (30 cycles/min * 60 minutes an hour * 72 hours a day) = 0.02 (2%) + maxBytesReadPerDisk = + std::max({uint64_t(maxBytesReadPerDisk * 0.99997), currentBytesRead, uint64_t(1'000'000)}); + maxBytesWritePerDisk = std::max( + {uint64_t(maxBytesWritePerDisk * 0.99997), currentBytesWrite, uint64_t(1'000'000)}); + + double totalIOPercentage = + static_cast(currentBytesRead) * 100.0 / maxBytesReadPerDisk + + static_cast(currentBytesWrite) * 100.0 / maxBytesWritePerDisk; + + auto invertedSigmoid = [](double val) -> double { + const double steepness = 10.0; // steepness + const double center = 0.15; // center in [0,1] + const double valnorm = val / 100.0; // normalize so that 100% total I/O maps to 1.0 + + const double res = 1.0 / (1.0 + std::exp(-steepness * (valnorm - center))); + return 1.0 - res; + }; + + uint64_t bulksizeScaled = trashGarbageCollectorBulkSize * invertedSigmoid(totalIOPercentage); + if ((currentBytesRead + 1) / (previousBytesReadPerDisk + 1) + + (currentBytesWrite + 1) / (previousBytesWritePerDisk + 1) >= + 10) { + bulksizeScaled = 0; + } + + static constexpr uint64_t kMinGCBulkSizeForActivation = 5; + + if (bulksizeScaled >= kMinGCBulkSizeForActivation) { + removeExpiredFiles(expirationTime, bulksizeScaled); + } makeSpace(availableThresholdGB, garbageCollectorSpaceRecoveryStep); + previousBytesReadPerDisk = currentBytesRead; + previousBytesWritePerDisk = currentBytesWrite; } bool ChunkTrashManagerImpl::isTrashPath(const std::string &filePath) { diff --git a/src/chunkserver/chunkserver-common/chunk_trash_manager_impl.h b/src/chunkserver/chunkserver-common/chunk_trash_manager_impl.h index 063a314c5..e28830331 100644 --- a/src/chunkserver/chunkserver-common/chunk_trash_manager_impl.h +++ b/src/chunkserver/chunkserver-common/chunk_trash_manager_impl.h @@ -20,10 +20,15 @@ #include "common/platform.h" +#include +#include #include +#include + #include "chunkserver-common/chunk_trash_index.h" #include "chunkserver-common/chunk_trash_manager.h" +#include "common/pcqueue.h" #include "errors/saunafs_error_codes.h" class ChunkTrashManagerImpl : public IChunkTrashManagerImpl { @@ -49,6 +54,9 @@ class ChunkTrashManagerImpl : public IChunkTrashManagerImpl { */ int init(const std::string &diskPath) override; + + void terminate() override; + /** * @brief Moves a specified file to the trash directory. * @param filePath The path of the file to be moved. @@ -80,6 +88,13 @@ class ChunkTrashManagerImpl : public IChunkTrashManagerImpl { */ void removeTrashFiles(const ChunkTrashIndex::TrashIndexDiskEntries &filesToRemove) const; + /** + * @brief Removes a set of specified files from the trash for an specific FileEntries. + * @param filesToRemove The list of files to be permanently deleted. + * @param diskPath The path of the disk of the files. + */ + static void removeTrashFilesFromDiskThread(uint8_t workerId); + /** * @brief Checks if a given timestamp string matches the expected format. * @param timestamp The timestamp string to validate. @@ -137,6 +152,14 @@ class ChunkTrashManagerImpl : public IChunkTrashManagerImpl { void reloadConfig() override; private: + /// Maximum recorded BytesReadPerDisk and BytesWritePerDisk in a GC cycle + static uint64_t maxBytesReadPerDisk; + static uint64_t maxBytesWritePerDisk; + + /// Previous recorded BytesReadPerDisk and BytesWritePerDisk in a GC cycle + static uint64_t previousBytesReadPerDisk; //1MiB + static uint64_t previousBytesWritePerDisk; //1MiB + /// Minimum available space threshold (in GB) before triggering garbage /// collection. static size_t availableThresholdGB; @@ -145,17 +168,24 @@ class ChunkTrashManagerImpl : public IChunkTrashManagerImpl { /// Time limit (in seconds) for files to be considered eligible for /// deletion. static size_t trashTimeLimitSeconds; - static constexpr size_t kDefaultTrashTimeLimitSeconds = 259200; + static constexpr size_t kDefaultTrashTimeLimitSeconds = 0; /// Number of files processed in each bulk operation during garbage /// collection. static size_t trashGarbageCollectorBulkSize; - static constexpr size_t kDefaultTrashGarbageCollectorBulkSize = 1000; + static constexpr size_t kDefaultTrashGarbageCollectorBulkSize = 500; /// Number of files to remove in each step to free up space when required. static size_t garbageCollectorSpaceRecoveryStep; static constexpr size_t kDefaultGarbageCollectorSpaceRecoveryStep = 100; + static std::vector removeFromTrashThreads; /// Vector of worker threads. + static std::unique_ptr removeFromTrashJobQueue; /// Queue for jobs. + + static std::atomic toDoJobsCount; + + static std::mutex mutex_; + static std::condition_variable trashRemoverCV; // Use a function to safely get the ChunkTrashIndex instance // This prevents issues during program shutdown /** diff --git a/src/chunkserver/chunkserver-common/chunk_trash_manager_impl_unittest.cc b/src/chunkserver/chunkserver-common/chunk_trash_manager_impl_unittest.cc index 94a71696c..9cef919f0 100644 --- a/src/chunkserver/chunkserver-common/chunk_trash_manager_impl_unittest.cc +++ b/src/chunkserver/chunkserver-common/chunk_trash_manager_impl_unittest.cc @@ -55,7 +55,7 @@ class ChunkTrashManagerImplTest : public ::testing::Test { chunkTrashManagerImpl.init(testDir.string()); } - void TearDown() override { fs::remove_all(testDir); } + void TearDown() override { fs::remove_all(testDir);chunkTrashManagerImpl.terminate(); } }; TEST_F(ChunkTrashManagerImplTest, MoveToTrashValidFile) { diff --git a/src/chunkserver/chunkserver-common/chunk_trash_manager_unittest.cc b/src/chunkserver/chunkserver-common/chunk_trash_manager_unittest.cc index ba3715e2f..258265a48 100644 --- a/src/chunkserver/chunkserver-common/chunk_trash_manager_unittest.cc +++ b/src/chunkserver/chunkserver-common/chunk_trash_manager_unittest.cc @@ -29,6 +29,7 @@ class MockChunkTrashManagerImpl : public IChunkTrashManagerImpl { (const std::filesystem::path &, const std::filesystem::path &, const std::time_t &), ()); MOCK_METHOD(int, init, (const std::string &), ()); + MOCK_METHOD(void, terminate, (), ()); MOCK_METHOD(void, collectGarbage, (), ()); MOCK_METHOD(void, reloadConfig, (), ()); }; @@ -44,7 +45,10 @@ class ChunkTrashManagerTest : public ::testing::Test { ChunkTrashManager::isEnabled = 1; } - void TearDown() override { mockImpl.reset(); } + void TearDown() override { + ChunkTrashManager::terminate(); + mockImpl.reset(); + } }; TEST_F(ChunkTrashManagerTest, MoveToTrashForwardsCall) { @@ -81,6 +85,12 @@ TEST_F(ChunkTrashManagerTest, ReloadConfigForwardsCall) { ChunkTrashManager::reloadConfig(); // Call the method. } +TEST_F(ChunkTrashManagerTest, TerminateForwardsCall) { + EXPECT_CALL(*mockImpl, terminate()).Times(::testing::AnyNumber()); + + ChunkTrashManager::terminate(); +} + TEST_F(ChunkTrashManagerTest, DisabledMoveToTrashDoesNotForwardsCall) { std::filesystem::path filePath = "example.txt"; std::filesystem::path diskPath = "/disk/"; diff --git a/src/chunkserver/chunkserver-common/hdd_stats.cc b/src/chunkserver/chunkserver-common/hdd_stats.cc index 4e862bada..55cfc2e29 100644 --- a/src/chunkserver/chunkserver-common/hdd_stats.cc +++ b/src/chunkserver/chunkserver-common/hdd_stats.cc @@ -42,6 +42,7 @@ static inline void totalRead(IDisk *disk, uint64_t size, MicroSeconds duration) gStatsTotalOperationsRead++; gStatsTotalBytesRead += size; + gBytesReadSinceLastGCSweep += size; gStatsTotalTimeRead += duration; auto &diskStats = disk->getCurrentStats(); @@ -61,6 +62,7 @@ static inline void totalWrite(IDisk *disk, uint64_t size, gStatsTotalOperationsWrite++; gStatsTotalBytesWrite += size; + gBytesWrittenSinceLastGCSweep += size; gStatsTotalTimeWrite += duration; auto &diskStats = disk->getCurrentStats(); diff --git a/src/chunkserver/chunkserver-common/hdd_stats.h b/src/chunkserver/chunkserver-common/hdd_stats.h index 9d91f574a..9871c2d2e 100644 --- a/src/chunkserver/chunkserver-common/hdd_stats.h +++ b/src/chunkserver/chunkserver-common/hdd_stats.h @@ -58,6 +58,12 @@ inline std::atomic gStatsOperationsDupTrunc(0); inline std::atomic gStatsOperationsGCPurge(0); + +// This is for internal use of GarbageCollector +inline std::atomic gBytesWrittenSinceLastGCSweep(0); +inline std::atomic gBytesReadSinceLastGCSweep(0); + + struct statsReport { statsReport(uint64_t *overBytesRead, uint64_t *overBytesWrite, uint32_t *overOpsRead, uint32_t *overOpsWrite, diff --git a/src/chunkserver/hddspacemgr.cc b/src/chunkserver/hddspacemgr.cc index a364dfb74..b5bd0b5cb 100644 --- a/src/chunkserver/hddspacemgr.cc +++ b/src/chunkserver/hddspacemgr.cc @@ -2532,15 +2532,19 @@ void hddFreeResourcesThread() { pthread_setname_np(pthread_self(), "freeResThread"); while (!gTerminate) { + Timeout timeout(std::chrono::microseconds(kDelayedStep * 1'000'000)); gOpenChunks.freeUnused(eventloop_time(), gChunksMapMutex, kMaxFreeUnused); ChunkTrashManager::collectGarbage(); hddReleaseDisksToBeDeleted(); /// Release buffers older than kDelayedStep seconds releaseOldIoBuffers(kOldIoBuffersExpirationTimeMs); - - sleep(kDelayedStep); + usleep(timeout.remaining_us()); } + + + ChunkTrashManager::terminate(); + } void hddTerminate(void) { diff --git a/src/data/sfschunkserver.cfg.in b/src/data/sfschunkserver.cfg.in index 28b690cf1..a39d16447 100644 --- a/src/data/sfschunkserver.cfg.in +++ b/src/data/sfschunkserver.cfg.in @@ -244,8 +244,8 @@ # CHUNK_TRASH_ENABLED = 0 ## Specifies the timeout in seconds for chunks to remain in the trash before -## being permanently deleted. (Default: 259200) -# CHUNK_TRASH_EXPIRATION_SECONDS = 259200 +## being permanently deleted. (Default: 0) +# CHUNK_TRASH_EXPIRATION_SECONDS = 0 ## Sets the available space threshold in gigabytes. If the available space on ## the disk falls below this threshold, the system will start deleting older @@ -253,10 +253,11 @@ ## the capacity of the smaller configured disk. (Default: 0) # CHUNK_TRASH_FREE_SPACE_THRESHOLD_GB = 0 -## Defines the bulk size for the garbage collector when processing chunks in the -## trash. This determines how many files are processed in each garbage -## collection cycle. (Default: 1000) -# CHUNK_TRASH_GC_BATCH_SIZE = 1000 +## Defines the maximum bulk size per disk for the garbage +## collector when processing chunks in the trash. +## This value is used when no I/O pressure is detected and is dynamically +## scaled down based on current disk load. (Default: 1000) +# CHUNK_TRASH_GC_BATCH_SIZE = 500 ## [ADVANCED] The number of files to remove from the trash in a single GC cycle, ## in case the disk is full and space needs to be recovered. (Default: 100) diff --git a/tests/test_suites/RebalancingTests/test_chunk_rebalancing.sh b/tests/test_suites/RebalancingTests/test_chunk_rebalancing.sh index 51c1cd1e0..552918fc5 100644 --- a/tests/test_suites/RebalancingTests/test_chunk_rebalancing.sh +++ b/tests/test_suites/RebalancingTests/test_chunk_rebalancing.sh @@ -4,7 +4,7 @@ rebalancing_timeout=90 CHUNKSERVERS=5 \ USE_LOOP_DISKS=YES \ MOUNT_EXTRA_CONFIG="sfscachemode=NEVER" \ - CHUNKSERVER_EXTRA_CONFIG="HDD_TEST_FREQ = 10000|HDD_LEAVE_SPACE_DEFAULT = 0MiB" \ + CHUNKSERVER_EXTRA_CONFIG="HDD_TEST_FREQ = 10000|HDD_LEAVE_SPACE_DEFAULT = 0MiB|CHUNK_TRASH_ENABLED=0" \ MASTER_EXTRA_CONFIG="ACCEPTABLE_DIFFERENCE = 0.0015` `|CHUNKS_LOOP_MAX_CPS = 2` `|CHUNKS_LOOP_MAX_CPU = 90` diff --git a/tests/test_suites/RebalancingTests/test_custom_goals_generate_no_accidental_rebalancing.sh b/tests/test_suites/RebalancingTests/test_custom_goals_generate_no_accidental_rebalancing.sh index 307f9f5df..23118b0c1 100644 --- a/tests/test_suites/RebalancingTests/test_custom_goals_generate_no_accidental_rebalancing.sh +++ b/tests/test_suites/RebalancingTests/test_custom_goals_generate_no_accidental_rebalancing.sh @@ -19,6 +19,7 @@ CHUNKSERVERS=6 \ MASTER_CUSTOM_GOALS="1 default: eu _" \ CHUNKSERVER_EXTRA_CONFIG="HDD_LEAVE_SPACE_DEFAULT = 0MiB` `|HDD_TEST_FREQ = 10000` + `|CHUNK_TRASH_ENABLED=0` `|MAGIC_DEBUG_LOG = $TEMP_DIR/log" \ MASTER_EXTRA_CONFIG="ACCEPTABLE_DIFFERENCE = 0.03` `|CHUNKS_LOOP_MAX_CPU = 90` diff --git a/tests/test_suites/RebalancingTests/test_custom_goals_rebalancing_case_1.sh b/tests/test_suites/RebalancingTests/test_custom_goals_rebalancing_case_1.sh index 19dca3860..c70e0134c 100644 --- a/tests/test_suites/RebalancingTests/test_custom_goals_rebalancing_case_1.sh +++ b/tests/test_suites/RebalancingTests/test_custom_goals_rebalancing_case_1.sh @@ -4,7 +4,7 @@ CHUNKSERVERS=5 \ USE_LOOP_DISKS=YES \ CHUNKSERVER_LABELS="0,1,2:ssd|3,4:hdd" \ MASTER_CUSTOM_GOALS="1 default: ssd _" \ - CHUNKSERVER_EXTRA_CONFIG="PERFORM_FSYNC = 1|HDD_TEST_FREQ = 10000|HDD_LEAVE_SPACE_DEFAULT = 0MiB" \ + CHUNKSERVER_EXTRA_CONFIG="PERFORM_FSYNC = 1|HDD_TEST_FREQ = 10000|HDD_LEAVE_SPACE_DEFAULT = 0MiB|CHUNK_TRASH_ENABLED=0" \ MASTER_EXTRA_CONFIG="CHUNKS_LOOP_MIN_TIME = 10` `|CHUNKS_LOOP_MAX_CPU = 90` `|CHUNKS_WRITE_REP_LIMIT = 2` diff --git a/tests/test_suites/RebalancingTests/test_custom_goals_rebalancing_case_2.sh b/tests/test_suites/RebalancingTests/test_custom_goals_rebalancing_case_2.sh index a578b9572..e53f9933b 100644 --- a/tests/test_suites/RebalancingTests/test_custom_goals_rebalancing_case_2.sh +++ b/tests/test_suites/RebalancingTests/test_custom_goals_rebalancing_case_2.sh @@ -4,7 +4,7 @@ CHUNKSERVERS=5 \ USE_LOOP_DISKS=YES \ CHUNKSERVER_LABELS="0:A|1:B|2:C|3:D|4:E" \ MASTER_CUSTOM_GOALS="1 default: _ _" \ - CHUNKSERVER_EXTRA_CONFIG="PERFORM_FSYNC = 1|HDD_TEST_FREQ = 10000|HDD_LEAVE_SPACE_DEFAULT = 0MiB" \ + CHUNKSERVER_EXTRA_CONFIG="PERFORM_FSYNC = 1|HDD_TEST_FREQ = 10000|HDD_LEAVE_SPACE_DEFAULT = 0MiB|CHUNK_TRASH_ENABLED=0" \ MASTER_EXTRA_CONFIG="CHUNKS_LOOP_MIN_TIME = 10` `|CHUNKS_LOOP_MAX_CPU = 90` `|CHUNKS_WRITE_REP_LIMIT = 2` diff --git a/tests/test_suites/RebalancingTests/test_custom_goals_rebalancing_case_3.sh b/tests/test_suites/RebalancingTests/test_custom_goals_rebalancing_case_3.sh index ed12c72d1..aa39106c3 100644 --- a/tests/test_suites/RebalancingTests/test_custom_goals_rebalancing_case_3.sh +++ b/tests/test_suites/RebalancingTests/test_custom_goals_rebalancing_case_3.sh @@ -4,7 +4,7 @@ CHUNKSERVERS=6 \ USE_LOOP_DISKS=YES \ CHUNKSERVER_LABELS="0,1,2:eu|3,4,5:us" \ MASTER_CUSTOM_GOALS="5 eu_eu: eu eu" \ - CHUNKSERVER_EXTRA_CONFIG="PERFORM_FSYNC = 1|HDD_TEST_FREQ = 10000|HDD_LEAVE_SPACE_DEFAULT = 0MiB" \ + CHUNKSERVER_EXTRA_CONFIG="PERFORM_FSYNC = 1|HDD_TEST_FREQ = 10000|HDD_LEAVE_SPACE_DEFAULT = 0MiB|CHUNK_TRASH_ENABLED=0" \ MASTER_EXTRA_CONFIG="CHUNKS_LOOP_MIN_TIME = 10` `|CHUNKS_LOOP_MAX_CPU = 90` `|CHUNKS_WRITE_REP_LIMIT = 2` diff --git a/tests/tools/saunafs.sh b/tests/tools/saunafs.sh index 5b92ed143..e51f50fbd 100644 --- a/tests/tools/saunafs.sh +++ b/tests/tools/saunafs.sh @@ -1130,9 +1130,9 @@ sfschunkserver_check_no_buffer_in_use() { saunafs_chunkserver_daemon ${i} reload done - sleep 5 + sleep 10 - # Assert that the last 5 seconds of log contains chunkserver_count lines with: + # Assert that the last 10 seconds of log contains chunkserver_count lines with: # "Current total buffer blocks per operation: read 0, write 0, replicate 0" # Plus, the last chunkserver_count lines should be unique disregarding the timestamp. # The difference should be the CS pid. This means that the buffers were released correctly.