Skip to content

Commit 3a78802

Browse files
committed
feat(chunkserver): add HDD_CHUNK_BULK_SIZE option
Introduce a reloadable HDD_CHUNK_BULK_SIZE option for chunkserver. Use it for chunk registration and damaged/lost/new report bulks. Add default (1000) to config template, admin defaults, and docs. Signed-off-by: guillex <guillex@leil.io>
1 parent 3d3c47f commit 3a78802

File tree

8 files changed

+81
-10
lines changed

8 files changed

+81
-10
lines changed

doc/sfschunkserver.cfg.5.adoc

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -102,6 +102,11 @@ maximum throughput.
102102
*HDD_ADVISE_NO_CACHE*:: whether to remove each chunk from page when closing it
103103
to reduce cache pressure generated by chunkserver (default is 0, i.e. no)
104104

105+
*HDD_CHUNK_BULK_SIZE*:: *ADVANCED*:: number of chunks to process in a single
106+
bulk operation when reporting chunks to the master (e.g., when registering chunks,
107+
reporting new chunks, or reporting damaged or lost chunks).
108+
Reloadable. (default is 1000, minimum is 1, maximum is 100000)
109+
105110
*HDD_PUNCH_HOLES*:: if enabled then chunkserver detects zero values in chunk
106111
data and frees corresponding file blocks (decreasing file system usage). This
107112
option works only on Linux with file systems supporting punching holes (XFS,

src/admin/dump_config_command.cc

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -171,6 +171,7 @@ const static std::unordered_map<std::string, std::string> defaultOptionsCS = {
171171
{"HDD_CHECK_CRC_WHEN_READING", "1"},
172172
{"HDD_CHECK_CRC_WHEN_WRITING", "1"},
173173
{"HDD_ADVISE_NO_CACHE", "0"},
174+
{"HDD_CHUNK_BULK_SIZE", "1000"},
174175
{"HDD_PUNCH_HOLES", "0"},
175176
{"ENABLE_LOAD_FACTOR", "0"},
176177
{"REPLICATION_BANDWIDTH_LIMIT_KBPS", "0"},

src/chunkserver/hddspacemgr.cc

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -176,6 +176,21 @@ int hddGetAndResetSpaceChanged() {
176176
return gHddSpaceChanged.exchange(false);
177177
}
178178

179+
static void hddReloadChunkBulkSize(bool isReload) {
180+
const auto previousChunkBulkSize = gChunkBulkSize.load(std::memory_order_relaxed);
181+
const auto configuredChunkBulkSize = cfg_get_minmaxvalue<uint32_t>(
182+
"HDD_CHUNK_BULK_SIZE", kDefaultChunkBulkSize, kMinChunkBulkSize, kMaxChunkBulkSize);
183+
184+
gChunkBulkSize.store(configuredChunkBulkSize, std::memory_order_relaxed);
185+
186+
if (isReload && previousChunkBulkSize != configuredChunkBulkSize) {
187+
safs::log_info("hdd space manager: HDD_CHUNK_BULK_SIZE changed from {} to {}",
188+
previousChunkBulkSize, configuredChunkBulkSize);
189+
}
190+
191+
safs::log_info("hdd space manager: Effective HDD_CHUNK_BULK_SIZE: {}", configuredChunkBulkSize);
192+
}
193+
179194
uint32_t hddGetSerializedSizeOfAllDiskInfosV2() {
180195
TRACETHIS();
181196
uint32_t serializedSizeOfAllDisks = 0;
@@ -2799,6 +2814,8 @@ void hddReload(void) {
27992814
gCheckCrcWhenWriting = cfg_getuint8("HDD_CHECK_CRC_WHEN_WRITING", 1) != 0U;
28002815
gPunchHolesInFiles = cfg_getuint32("HDD_PUNCH_HOLES", 0);
28012816

2817+
hddReloadChunkBulkSize(true);
2818+
28022819
char *leaveFreeStr = cfg_getstr("HDD_LEAVE_SPACE_DEFAULT",
28032820
disk::gLeaveSpaceDefaultDefaultStrValue);
28042821
auto parsedLeaveFree = cfg_parse_size(leaveFreeStr);
@@ -2968,6 +2985,8 @@ int hddInit() {
29682985

29692986
gPunchHolesInFiles = cfg_getuint32("HDD_PUNCH_HOLES", 0);
29702987

2988+
hddReloadChunkBulkSize(false);
2989+
29712990
eventloop_reloadregister(hddReload);
29722991
eventloop_timeregister(TIMEMODE_RUN_LATE, SECONDS_IN_ONE_MINUTE, 0,
29732992
hddDiskInfoRotateStats);

src/chunkserver/hddspacemgr.h

Lines changed: 10 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,7 @@
2222

2323
#include "common/platform.h"
2424

25+
#include <atomic>
2526
#include <vector>
2627

2728
#include "chunkserver-common/chunk_interface.h"
@@ -44,11 +45,16 @@ void hddSerializeAllDiskInfosV2(uint8_t *buff);
4445

4546
std::string hddGetDiskGroups();
4647

47-
const std::size_t kChunkBulkSize = 1000;
48+
// Controls the number of chunks processed per bulk operation when reporting
49+
// chunks to the master (e.g. registration, damaged, lost, new chunks).
50+
inline constexpr uint32_t kDefaultChunkBulkSize = 1'000;
51+
inline constexpr uint32_t kMinChunkBulkSize = 1;
52+
inline constexpr uint32_t kMaxChunkBulkSize = 100'000;
53+
inline std::atomic_uint32_t gChunkBulkSize{kDefaultChunkBulkSize};
54+
4855
using BulkFunction = std::function<void(std::vector<ChunkWithVersionAndType>&)>;
49-
/// Executes the callback for each bulk of at most \p kChunkBulkSize chunks.
50-
void hddForeachChunkInBulks(BulkFunction bulkCallback,
51-
std::size_t bulkSize = kChunkBulkSize);
56+
/// Executes the callback for each bulk of at most \p bulkSize chunks.
57+
void hddForeachChunkInBulks(BulkFunction bulkCallback, std::size_t bulkSize);
5258

5359
int hddGetAndResetSpaceChanged();
5460
void hddGetTotalSpace(uint64_t *usedSpace, uint64_t *totalSpace,

src/chunkserver/master_connection.cc

Lines changed: 5 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -184,9 +184,11 @@ void MasterConn::onRegistered(const std::vector<uint8_t> &data) {
184184
isVersionLessThan5_ = false;
185185
registrationAttempts_ = 0;
186186

187-
hddForeachChunkInBulks([this](const std::vector<ChunkWithVersionAndType> &chunksBulk) {
188-
createAttachedPacket(cstoma::registerChunks::build(chunksBulk));
189-
});
187+
hddForeachChunkInBulks(
188+
[this](const std::vector<ChunkWithVersionAndType> &chunksBulk) {
189+
createAttachedPacket(cstoma::registerChunks::build(chunksBulk));
190+
},
191+
gChunkBulkSize.load(std::memory_order_relaxed));
190192

191193
uint64_t usedSpace;
192194
uint64_t totalSpace;

src/chunkserver/masterconn.cc

Lines changed: 5 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -117,19 +117,21 @@ void masterconn_check_hdd_reports() {
117117
errorcounter--;
118118
}
119119

120+
const auto chunkBulkSize = gChunkBulkSize.load(std::memory_order_relaxed);
121+
120122
std::vector<ChunkWithType> chunks_with_type;
121-
hddGetDamagedChunks(chunks_with_type, 1000);
123+
hddGetDamagedChunks(chunks_with_type, chunkBulkSize);
122124
if (!chunks_with_type.empty()) {
123125
eptr->createAttachedPacket(cstoma::chunkDamaged::build(chunks_with_type));
124126
}
125127

126-
hddGetLostChunks(chunks_with_type, 1000);
128+
hddGetLostChunks(chunks_with_type, chunkBulkSize);
127129
if (!chunks_with_type.empty()) {
128130
eptr->createAttachedPacket(cstoma::chunkLost::build(chunks_with_type));
129131
}
130132

131133
std::vector<ChunkWithVersionAndType> chunks_with_version;
132-
hddGetNewChunks(chunks_with_version, 1000);
134+
hddGetNewChunks(chunks_with_version, chunkBulkSize);
133135
if (!chunks_with_version.empty()) {
134136
eptr->createAttachedPacket(cstoma::chunkNew::build(chunks_with_version));
135137
}

src/data/sfschunkserver.cfg.in

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -113,6 +113,12 @@
113113
## (Default: 0)
114114
# HDD_ADVISE_NO_CACHE = 0
115115

116+
## [ADVANCED] Number of chunks to process in a single bulk operation when reporting
117+
## chunks to the master (e.g., when registering chunks, reporting new
118+
## chunks, or reporting damaged or lost chunks).
119+
## Reloadable. (Default: 1000, Minimum: 1, Maximum: 100000)
120+
# HDD_CHUNK_BULK_SIZE = 1000
121+
116122
## If enabled then chunkserver detects zero values in chunk data and frees
117123
## corresponding file blocks (decreasing file system usage).
118124
## This option works only on Linux
Lines changed: 30 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,30 @@
1+
timeout_set 1 minute
2+
3+
CHUNKSERVERS=1 \
4+
USE_RAMDISK=YES \
5+
CHUNKSERVER_EXTRA_CONFIG="HDD_CHUNK_BULK_SIZE = 1|\
6+
MAGIC_DEBUG_LOG = ${TEMP_DIR}/log|\
7+
LOG_FLUSH_ON=INFO" \
8+
setup_local_empty_saunafs info
9+
10+
apply_chunkserver_config() {
11+
local parameter=$1
12+
local value=$2
13+
for i in $(seq 0 $(( ${info[chunkserver_count]} - 1 ))); do
14+
sed -i -re "s/^$parameter = .*/$parameter = $value/" "${info[chunkserver${i}_cfg]}"
15+
done
16+
}
17+
18+
# Verify the effective value logged at startup.
19+
assert_eventually_prints 1 \
20+
"grep -m1 'Effective HDD_CHUNK_BULK_SIZE: 1' '${TEMP_DIR}/log' | wc -l"
21+
22+
# Change the value and hot-reload chunkserver config.
23+
apply_chunkserver_config HDD_CHUNK_BULK_SIZE 1000
24+
saunafs_chunkserver_daemon 0 reload
25+
26+
# Verify reload logs include change notification and the new effective value.
27+
assert_eventually_prints 1 \
28+
"grep -m1 'HDD_CHUNK_BULK_SIZE changed from 1 to 1000' '${TEMP_DIR}/log' | wc -l"
29+
assert_eventually_prints 1 \
30+
"grep -m1 'Effective HDD_CHUNK_BULK_SIZE: 1000' '${TEMP_DIR}/log' | wc -l"

0 commit comments

Comments
 (0)