From 3a78802cb6bce26bf5eb88a0035db327491f3aa6 Mon Sep 17 00:00:00 2001 From: guillex Date: Tue, 7 Apr 2026 17:45:00 +0200 Subject: [PATCH] feat(chunkserver): add HDD_CHUNK_BULK_SIZE option Introduce a reloadable HDD_CHUNK_BULK_SIZE option for chunkserver. Use it for chunk registration and damaged/lost/new report bulks. Add default (1000) to config template, admin defaults, and docs. Signed-off-by: guillex --- doc/sfschunkserver.cfg.5.adoc | 5 ++++ src/admin/dump_config_command.cc | 1 + src/chunkserver/hddspacemgr.cc | 19 ++++++++++++ src/chunkserver/hddspacemgr.h | 14 ++++++--- src/chunkserver/master_connection.cc | 8 +++-- src/chunkserver/masterconn.cc | 8 +++-- src/data/sfschunkserver.cfg.in | 6 ++++ .../test_hdd_chunk_bulk_size_hot_reload.sh | 30 +++++++++++++++++++ 8 files changed, 81 insertions(+), 10 deletions(-) create mode 100644 tests/test_suites/ShortSystemTests/test_hdd_chunk_bulk_size_hot_reload.sh diff --git a/doc/sfschunkserver.cfg.5.adoc b/doc/sfschunkserver.cfg.5.adoc index a029d1cbb..cb9287008 100644 --- a/doc/sfschunkserver.cfg.5.adoc +++ b/doc/sfschunkserver.cfg.5.adoc @@ -102,6 +102,11 @@ maximum throughput. *HDD_ADVISE_NO_CACHE*:: whether to remove each chunk from page when closing it to reduce cache pressure generated by chunkserver (default is 0, i.e. no) +*HDD_CHUNK_BULK_SIZE*:: *ADVANCED*:: number of chunks to process in a single +bulk operation when reporting chunks to the master (e.g., when registering chunks, +reporting new chunks, or reporting damaged or lost chunks). +Reloadable. (default is 1000, minimum is 1, maximum is 100000) + *HDD_PUNCH_HOLES*:: if enabled then chunkserver detects zero values in chunk data and frees corresponding file blocks (decreasing file system usage). This option works only on Linux with file systems supporting punching holes (XFS, diff --git a/src/admin/dump_config_command.cc b/src/admin/dump_config_command.cc index a7cb5deab..8b9c07d5f 100644 --- a/src/admin/dump_config_command.cc +++ b/src/admin/dump_config_command.cc @@ -171,6 +171,7 @@ const static std::unordered_map defaultOptionsCS = { {"HDD_CHECK_CRC_WHEN_READING", "1"}, {"HDD_CHECK_CRC_WHEN_WRITING", "1"}, {"HDD_ADVISE_NO_CACHE", "0"}, + {"HDD_CHUNK_BULK_SIZE", "1000"}, {"HDD_PUNCH_HOLES", "0"}, {"ENABLE_LOAD_FACTOR", "0"}, {"REPLICATION_BANDWIDTH_LIMIT_KBPS", "0"}, diff --git a/src/chunkserver/hddspacemgr.cc b/src/chunkserver/hddspacemgr.cc index dd95567d8..37bbc1f9f 100644 --- a/src/chunkserver/hddspacemgr.cc +++ b/src/chunkserver/hddspacemgr.cc @@ -176,6 +176,21 @@ int hddGetAndResetSpaceChanged() { return gHddSpaceChanged.exchange(false); } +static void hddReloadChunkBulkSize(bool isReload) { + const auto previousChunkBulkSize = gChunkBulkSize.load(std::memory_order_relaxed); + const auto configuredChunkBulkSize = cfg_get_minmaxvalue( + "HDD_CHUNK_BULK_SIZE", kDefaultChunkBulkSize, kMinChunkBulkSize, kMaxChunkBulkSize); + + gChunkBulkSize.store(configuredChunkBulkSize, std::memory_order_relaxed); + + if (isReload && previousChunkBulkSize != configuredChunkBulkSize) { + safs::log_info("hdd space manager: HDD_CHUNK_BULK_SIZE changed from {} to {}", + previousChunkBulkSize, configuredChunkBulkSize); + } + + safs::log_info("hdd space manager: Effective HDD_CHUNK_BULK_SIZE: {}", configuredChunkBulkSize); +} + uint32_t hddGetSerializedSizeOfAllDiskInfosV2() { TRACETHIS(); uint32_t serializedSizeOfAllDisks = 0; @@ -2799,6 +2814,8 @@ void hddReload(void) { gCheckCrcWhenWriting = cfg_getuint8("HDD_CHECK_CRC_WHEN_WRITING", 1) != 0U; gPunchHolesInFiles = cfg_getuint32("HDD_PUNCH_HOLES", 0); + hddReloadChunkBulkSize(true); + char *leaveFreeStr = cfg_getstr("HDD_LEAVE_SPACE_DEFAULT", disk::gLeaveSpaceDefaultDefaultStrValue); auto parsedLeaveFree = cfg_parse_size(leaveFreeStr); @@ -2968,6 +2985,8 @@ int hddInit() { gPunchHolesInFiles = cfg_getuint32("HDD_PUNCH_HOLES", 0); + hddReloadChunkBulkSize(false); + eventloop_reloadregister(hddReload); eventloop_timeregister(TIMEMODE_RUN_LATE, SECONDS_IN_ONE_MINUTE, 0, hddDiskInfoRotateStats); diff --git a/src/chunkserver/hddspacemgr.h b/src/chunkserver/hddspacemgr.h index 58ae8831a..f2fed7ad4 100644 --- a/src/chunkserver/hddspacemgr.h +++ b/src/chunkserver/hddspacemgr.h @@ -22,6 +22,7 @@ #include "common/platform.h" +#include #include #include "chunkserver-common/chunk_interface.h" @@ -44,11 +45,16 @@ void hddSerializeAllDiskInfosV2(uint8_t *buff); std::string hddGetDiskGroups(); -const std::size_t kChunkBulkSize = 1000; +// Controls the number of chunks processed per bulk operation when reporting +// chunks to the master (e.g. registration, damaged, lost, new chunks). +inline constexpr uint32_t kDefaultChunkBulkSize = 1'000; +inline constexpr uint32_t kMinChunkBulkSize = 1; +inline constexpr uint32_t kMaxChunkBulkSize = 100'000; +inline std::atomic_uint32_t gChunkBulkSize{kDefaultChunkBulkSize}; + using BulkFunction = std::function&)>; -/// Executes the callback for each bulk of at most \p kChunkBulkSize chunks. -void hddForeachChunkInBulks(BulkFunction bulkCallback, - std::size_t bulkSize = kChunkBulkSize); +/// Executes the callback for each bulk of at most \p bulkSize chunks. +void hddForeachChunkInBulks(BulkFunction bulkCallback, std::size_t bulkSize); int hddGetAndResetSpaceChanged(); void hddGetTotalSpace(uint64_t *usedSpace, uint64_t *totalSpace, diff --git a/src/chunkserver/master_connection.cc b/src/chunkserver/master_connection.cc index da09edbee..ed9f64cf9 100644 --- a/src/chunkserver/master_connection.cc +++ b/src/chunkserver/master_connection.cc @@ -184,9 +184,11 @@ void MasterConn::onRegistered(const std::vector &data) { isVersionLessThan5_ = false; registrationAttempts_ = 0; - hddForeachChunkInBulks([this](const std::vector &chunksBulk) { - createAttachedPacket(cstoma::registerChunks::build(chunksBulk)); - }); + hddForeachChunkInBulks( + [this](const std::vector &chunksBulk) { + createAttachedPacket(cstoma::registerChunks::build(chunksBulk)); + }, + gChunkBulkSize.load(std::memory_order_relaxed)); uint64_t usedSpace; uint64_t totalSpace; diff --git a/src/chunkserver/masterconn.cc b/src/chunkserver/masterconn.cc index 0f1fd9f6f..9d2541b2a 100644 --- a/src/chunkserver/masterconn.cc +++ b/src/chunkserver/masterconn.cc @@ -117,19 +117,21 @@ void masterconn_check_hdd_reports() { errorcounter--; } + const auto chunkBulkSize = gChunkBulkSize.load(std::memory_order_relaxed); + std::vector chunks_with_type; - hddGetDamagedChunks(chunks_with_type, 1000); + hddGetDamagedChunks(chunks_with_type, chunkBulkSize); if (!chunks_with_type.empty()) { eptr->createAttachedPacket(cstoma::chunkDamaged::build(chunks_with_type)); } - hddGetLostChunks(chunks_with_type, 1000); + hddGetLostChunks(chunks_with_type, chunkBulkSize); if (!chunks_with_type.empty()) { eptr->createAttachedPacket(cstoma::chunkLost::build(chunks_with_type)); } std::vector chunks_with_version; - hddGetNewChunks(chunks_with_version, 1000); + hddGetNewChunks(chunks_with_version, chunkBulkSize); if (!chunks_with_version.empty()) { eptr->createAttachedPacket(cstoma::chunkNew::build(chunks_with_version)); } diff --git a/src/data/sfschunkserver.cfg.in b/src/data/sfschunkserver.cfg.in index 0f511683a..d7912718a 100644 --- a/src/data/sfschunkserver.cfg.in +++ b/src/data/sfschunkserver.cfg.in @@ -113,6 +113,12 @@ ## (Default: 0) # HDD_ADVISE_NO_CACHE = 0 +## [ADVANCED] Number of chunks to process in a single bulk operation when reporting +## chunks to the master (e.g., when registering chunks, reporting new +## chunks, or reporting damaged or lost chunks). +## Reloadable. (Default: 1000, Minimum: 1, Maximum: 100000) +# HDD_CHUNK_BULK_SIZE = 1000 + ## If enabled then chunkserver detects zero values in chunk data and frees ## corresponding file blocks (decreasing file system usage). ## This option works only on Linux diff --git a/tests/test_suites/ShortSystemTests/test_hdd_chunk_bulk_size_hot_reload.sh b/tests/test_suites/ShortSystemTests/test_hdd_chunk_bulk_size_hot_reload.sh new file mode 100644 index 000000000..fa1d3b8ec --- /dev/null +++ b/tests/test_suites/ShortSystemTests/test_hdd_chunk_bulk_size_hot_reload.sh @@ -0,0 +1,30 @@ +timeout_set 1 minute + +CHUNKSERVERS=1 \ + USE_RAMDISK=YES \ + CHUNKSERVER_EXTRA_CONFIG="HDD_CHUNK_BULK_SIZE = 1|\ + MAGIC_DEBUG_LOG = ${TEMP_DIR}/log|\ + LOG_FLUSH_ON=INFO" \ + setup_local_empty_saunafs info + +apply_chunkserver_config() { + local parameter=$1 + local value=$2 + for i in $(seq 0 $(( ${info[chunkserver_count]} - 1 ))); do + sed -i -re "s/^$parameter = .*/$parameter = $value/" "${info[chunkserver${i}_cfg]}" + done +} + +# Verify the effective value logged at startup. +assert_eventually_prints 1 \ + "grep -m1 'Effective HDD_CHUNK_BULK_SIZE: 1' '${TEMP_DIR}/log' | wc -l" + +# Change the value and hot-reload chunkserver config. +apply_chunkserver_config HDD_CHUNK_BULK_SIZE 1000 +saunafs_chunkserver_daemon 0 reload + +# Verify reload logs include change notification and the new effective value. +assert_eventually_prints 1 \ + "grep -m1 'HDD_CHUNK_BULK_SIZE changed from 1 to 1000' '${TEMP_DIR}/log' | wc -l" +assert_eventually_prints 1 \ + "grep -m1 'Effective HDD_CHUNK_BULK_SIZE: 1000' '${TEMP_DIR}/log' | wc -l"