fix(master): avoid rebalancing to disconnected CS

At the instant of looking for a server to replicate a part when
rebalancing, the current implementation uses the sortedServers_ or
labeledSortedServers_ variables. These containers may have stale
chunkserver entries that have been recently switched to KILL mode.
Those entries must not be used as a destination for the replicate
parts. These commit fixes that issue.

The test test_kill_cs_while_writing_small_files should stop being flaky
after the merge.

Signed-off-by: Dave <dave@leil.io>

Author: dmga44

src/master/chunks.cc

@@ -2712,6 +2712,8 @@ bool ChunkWorker::rebalanceChunkParts(Chunk *c, ChunkCopiesCalculator &calc, boo
 		                              : labeledSortedServers_[current_copy_label];
 
 		for (const auto &empty_server : sorted_servers) {
+			if (matocsserv_is_killed(empty_server.server)) { continue; }
+
 			if (!only_todel && gAvoidSameIpChunkservers) {
 				auto empty_server_ip = matocsserv_get_servip(empty_server.server);
 				auto it = ip_counter.find(empty_server_ip);
@@ -2784,6 +2786,8 @@ bool ChunkWorker::rebalanceChunkPartsWithSameIp(Chunk *c, ChunkCopiesCalculator
 			           });
 
 		for (const auto &empty_server : sorted_by_ip_count) {
+			if (matocsserv_is_killed(empty_server.server)) { continue; }
+
 			auto empty_server_ip = matocsserv_get_servip(empty_server.server);
 			auto it = ip_counter.find(empty_server_ip);
 			auto empty_server_ip_count = it != ip_counter.end() ? it->second : 0;
@@ -2942,7 +2946,6 @@ void ChunkWorker::doChunkJobs(Chunk *c, uint16_t serverCount) {
 	if (rebalanceChunkParts(c, calc, false, ip_occurrence)) {
 		return;
 	}
-
 }
 
 bool ChunkWorker::deleteUnusedChunks() {

src/master/matocsserv.cc

@@ -1655,6 +1655,10 @@ void matocsserv_reload() {
 	lsock = newlsock;
 }
 
+bool matocsserv_is_killed(matocsserventry* eptr) {
+	return eptr != nullptr && eptr->mode == ChunkserverConnectionMode::KILL;
+}
+
 uint32_t matocsserv_get_version(matocsserventry *eptr) {
 	return eptr->version;
 }

src/master/matocsserv.h

@@ -86,6 +86,9 @@ double matocsserv_get_usage(matocsserventry* eptr);
 /*! \brief Get chunkservers ordered by disk usage. */
 std::vector<ServerWithUsage> matocsserv_getservers_sorted();
 
+/*! \brief Check if chunkserver is killed. */
+bool matocsserv_is_killed(matocsserventry* eptr);
+
 uint32_t matocsserv_get_version(matocsserventry* eptr);
 void matocsserv_usagedifference(double *minusage, double *maxusage, uint16_t *usablescount,
                                 uint16_t *totalscount);