fix: make NOSPACE error code more accurate

Current implementation has a critical error when the following
conditions are met:
- clients are creating files in goal ec(D, P).
- more than one minute has passed.
- all the chunkservers restart.

In such case, it is very likely that master replies to some of the
chunk write (creations) requests SAUNAFS_ERROR_NOSPACE. This happens
when not being able to create the chunk due to not having at least D
chunkservers available and having at least one available.

The error code SAUNAFS_ERROR_NOSPACE is not retryable by the client,
which is correct, but makes the write operations to completely fail.
The incorrect behavior is the one from the master, which must reply
SAUNAFS_ERROR_NOCHUNKSERVERS in such cases and only leave the NOSPACE
error code for the case when at least D chunkservers are available.

Signed-off-by: Dave <dave@leil.io>

Author: dmga44

src/master/chunks.cc

@@ -1379,15 +1379,18 @@ uint8_t chunk_create(bool quotaExceeded, uint8_t goal, uint8_t *operation, uint6
 	if (quotaExceeded) { return SAUNAFS_ERROR_QUOTA; }
 
 	// Next check availability of chunkservers for the given goal
-	auto serversWithChunkTypes = matocsserv_getservers_for_new_chunk(goal, minServerVersion);
+	uint16_t minServerCount = 0;
+	auto serversWithChunkTypes =
+	    matocsserv_getservers_for_new_chunk(goal, minServerCount, minServerVersion);
 	if (serversWithChunkTypes.empty()) {
 		uint16_t usableChunkservers, totalChunkservers;
 		double minUsage, maxUsage;
 		matocsserv_usagedifference(&minUsage, &maxUsage, &usableChunkservers, &totalChunkservers);
 
-		if (usableChunkservers > 0 && eventloop_time() > starttime + kStartupGracePeriodSeconds) {
-			// if there are chunkservers and it's at least one minute after start then it means that
-			// there is no space left
+		if (usableChunkservers >= minServerCount &&
+		    eventloop_time() > starttime + kStartupGracePeriodSeconds) {
+			// if there are enough chunkservers and it's at least one minute after start then it
+			// means that there is no space left
 			return SAUNAFS_ERROR_NOSPACE;
 		}
 

src/master/matocsserv.cc

@@ -352,11 +352,13 @@ std::vector<ServerWithUsage> matocsserv_getservers_sorted() {
 }
 
 std::vector<std::pair<matocsserventry *, ChunkPartType>> matocsserv_getservers_for_new_chunk(
-		uint8_t goal_id, uint32_t min_server_version) {
+    uint8_t goal_id, uint16_t &min_server_count, uint32_t min_server_version) {
 	static std::array<ChunkCreationHistory, GoalId::kMax + 1> history;
 	GetServersForNewChunk getter;
 	const Goal &goal(gFSOperations->getGoalDefinition(goal_id));
 
+	min_server_count = std::numeric_limits<uint16_t>::max();
+
 	for (const auto &eptr : matocsservList) {
 		if (eptr->mode != ChunkserverConnectionMode::KILL && eptr->totalspace > 0 &&
 		    eptr->usedspace <= eptr->totalspace &&
@@ -386,6 +388,9 @@ std::vector<std::pair<matocsserventry *, ChunkPartType>> matocsserv_getservers_f
 	// we don't use any chunkserver twice if not necessary.
 	for (const auto &slice : goal) {
 		std::vector<std::pair<matocsserventry *, ChunkPartType>> slice_ret;
+		min_server_count =
+		    std::min(min_server_count,
+		             static_cast<uint16_t>(slice_traits::getNumberOfDataParts(slice.getType())));
 
 		// add parts index permutation here
 		std::array<int, Goal::Slice::kMaxPartsCount> shuffle;

src/master/matocsserv.h

@@ -89,8 +89,22 @@ std::vector<ServerWithUsage> matocsserv_getservers_sorted();
 uint32_t matocsserv_get_version(matocsserventry* eptr);
 void matocsserv_usagedifference(double *minusage, double *maxusage, uint16_t *usablescount,
                                 uint16_t *totalscount);
-std::vector<std::pair<matocsserventry*, ChunkPartType>> matocsserv_getservers_for_new_chunk(
-		uint8_t goalId, uint32_t min_server_version = 0);
+
+/*! \brief Get chunkservers for a new chunk.
+ *
+ * This function returns a list of chunkservers that can be used for a new chunk creation. The
+ * returned servers are randomly shuffled and sorted by disk usage, so the least used and least
+ * loaded servers are returned first.
+ *
+ * \param goalId - the id of the goal for which the chunk is being created.
+ * \param min_server_count[out] - the minimum number of servers that should be returned for each
+ *                                slice in the goal. This is needed to determine if there are
+ *                                enough servers to create a chunk with the given goal.
+ * \param min_server_version - return only chunkservers with higher (or equal) version.
+ * \return List of chunkservers for new chunk creation with their types.
+ */
+std::vector<std::pair<matocsserventry *, ChunkPartType>> matocsserv_getservers_for_new_chunk(
+    uint8_t goalId, uint16_t &min_server_count, uint32_t min_server_version = 0);
 void matocsserv_getspace(uint64_t* totalspace, uint64_t* availspace);
 const char* matocsserv_getstrip(matocsserventry* eptr);
 uint32_t matocsserv_get_servip(matocsserventry *eptr);