DAOS-18355 chk: check leader waits all check engines before exited (#17315)

In old implementation, when the PS leader notifies the check leader
that related pool has been checked, the check leader will mark such
pool as 'done'. If all required pools have been marked as 'done',
then the check leader will exit. But at that time, the check engine
on related PS leader may not complete yet. There are something to
be processed (such as restart pool server) after the checking the
pool. The check engine will notify the check leader via CHK IV when
exit. But the check leader does not wait such notification. Under
such case, if someone tries to trigger new check instance, it will
create new IV namespace. That will cause some check engines and
the check leader to use different IV namespace, as to the CHK IV
logic cannot recognize the leadership correctly.

The patch adjust the leader exit logic: the leader scheduler needs
to wait all check engines' notification before exit.

Signed-off-by: Fan Yong <[email protected]>

Author: Nasf-Fan

src/chk/chk_engine.c

@@ -1,6 +1,6 @@
 /**
  * (C) Copyright 2022-2024 Intel Corporation.
- * (C) Copyright 2025 Hewlett Packard Enterprise Development LP
+ * (C) Copyright 2025-2026 Hewlett Packard Enterprise Development LP
  *
  * SPDX-License-Identifier: BSD-2-Clause-Patent
  */
@@ -1981,7 +1981,7 @@ chk_engine_sched(void *args)
 			D_GOTO(out, rc);
 		}
 
-		if (ins_phase > cbk->cb_phase) {
+		if (ins_phase != CHK_INVAL_PHASE && ins_phase > cbk->cb_phase) {
 			D_INFO(DF_ENGINE" on rank %u moves from phase %u to phase %u\n",
 			       DP_ENGINE(ins), myrank, cbk->cb_phase, ins_phase);
 

src/chk/chk_internal.h

@@ -1,6 +1,6 @@
 /**
  * (C) Copyright 2022-2024 Intel Corporation.
- * (C) Copyright 2025 Hewlett Packard Enterprise Development LP
+ * (C) Copyright 2025-2026 Hewlett Packard Enterprise Development LP
  *
  * SPDX-License-Identifier: BSD-2-Clause-Patent
  */
@@ -1173,6 +1173,14 @@ chk_pools_find_slowest(struct chk_instance *ins, int *done)
 			phase = cpr->cpr_bk.cb_phase;
 	}
 
+	/* All pools have been done, some check engines are still running, leader needs to wait. */
+	if (ins->ci_orphan_done && *done > 0 && !d_list_empty(&ins->ci_rank_list)) {
+		D_ASSERT(ins->ci_is_leader);
+
+		phase = CHK_INVAL_PHASE;
+		*done = 0;
+	}
+
 	return phase;
 }
 

src/chk/chk_leader.c

@@ -1,6 +1,6 @@
 /**
  * (C) Copyright 2022-2024 Intel Corporation.
- * (C) Copyright 2025 Hewlett Packard Enterprise Development LP
+ * (C) Copyright 2025-2026 Hewlett Packard Enterprise Development LP
  *
  * SPDX-License-Identifier: BSD-2-Clause-Patent
  */
@@ -226,9 +226,10 @@ static void
 chk_leader_exit(struct chk_instance *ins, uint32_t ins_phase, uint32_t ins_status,
 		uint32_t pool_status, bool bcast)
 {
-	struct chk_bookmark	*cbk = &ins->ci_bk;
-	struct chk_iv		 iv = { 0 };
-	int			 rc = 0;
+	struct chk_dead_rank *cdr;
+	struct chk_bookmark  *cbk = &ins->ci_bk;
+	struct chk_iv         iv  = {0};
+	int                   rc  = 0;
 
 	ins->ci_sched_exiting = 1;
 
@@ -237,8 +238,7 @@ chk_leader_exit(struct chk_instance *ins, uint32_t ins_phase, uint32_t ins_statu
 	chk_pool_stop_all(ins, pool_status, NULL);
 
 	if ((bcast && ins_status == CHK__CHECK_INST_STATUS__CIS_FAILED) ||
-	    ins_status == CHK__CHECK_INST_STATUS__CIS_IMPLICATED ||
-	    unlikely(ins_status == CHK__CHECK_INST_STATUS__CIS_COMPLETED && !ins->ci_orphan_done)) {
+	    ins_status == CHK__CHECK_INST_STATUS__CIS_IMPLICATED || !ins->ci_orphan_done) {
 		iv.ci_gen = cbk->cb_gen;
 		iv.ci_phase = ins_phase != CHK_INVAL_PHASE ? ins_phase : cbk->cb_phase;
 		iv.ci_ins_status = ins_status;
@@ -264,6 +264,10 @@ chk_leader_exit(struct chk_instance *ins, uint32_t ins_phase, uint32_t ins_statu
 				DP_LEADER(ins), ins_status, DP_RC(rc));
 	}
 
+	while ((cdr = d_list_pop_entry(&ins->ci_dead_ranks, struct chk_dead_rank, cdr_link)) !=
+	       NULL)
+		D_FREE(cdr);
+
 	ins->ci_sched_exiting = 0;
 }
 
@@ -305,31 +309,24 @@ chk_leader_post_repair(struct chk_instance *ins, struct chk_pool_rec *cpr,
 			       DP_UUID(cpr->cpr_uuid), rc);
 	}
 
-	/*
-	 * If the operation failed and 'failout' is set, then do nothing here.
-	 * chk_leader_exit will handle all the IV and bookmark related things.
-	 */
-	if (*result == 0 || !(ins->ci_prop.cp_flags & CHK__CHECK_FLAG__CF_FAILOUT)) {
-		if (notify) {
-			iv.ci_gen = cbk->cb_gen;
-			uuid_copy(iv.ci_uuid, cpr->cpr_uuid);
-			iv.ci_ins_status = ins->ci_bk.cb_ins_status;
-			iv.ci_phase = cbk->cb_phase;
-			iv.ci_pool_status = cbk->cb_pool_status;
-
-			/* Synchronously notify the engines that check on the pool got failure. */
-			rc = chk_iv_update(ins->ci_iv_ns, &iv, CRT_IV_SHORTCUT_NONE,
-					   CRT_IV_SYNC_EAGER, true);
-			D_CDEBUG(rc != 0, DLOG_ERR, DLOG_INFO,
-				 DF_LEADER" notify the engines that the check for pool "
-				 DF_UUIDF" is done with status %u: rc = %d\n",
-				 DP_LEADER(ins), DP_UUID(cpr->cpr_uuid), iv.ci_pool_status, rc);
-			if (rc == 0)
-				cpr->cpr_notified_exit = 1;
-		}
+	if (notify) {
+		uuid_copy(iv.ci_uuid, cpr->cpr_uuid);
+		iv.ci_gen         = cbk->cb_gen;
+		iv.ci_ins_status  = ins->ci_bk.cb_ins_status;
+		iv.ci_phase       = cbk->cb_phase;
+		iv.ci_pool_status = cbk->cb_pool_status;
+
+		rc = chk_iv_update(ins->ci_iv_ns, &iv, CRT_IV_SHORTCUT_NONE, CRT_IV_SYNC_EAGER,
+				   true);
+		DL_CDEBUG(rc != 0, DLOG_WARN, DLOG_INFO, rc,
+			  DF_LEADER " notify engines that check pool " DF_UUIDF " done, status %u",
+			  DP_LEADER(ins), DP_UUID(cpr->cpr_uuid), iv.ci_pool_status);
+		if (rc == 0)
+			cpr->cpr_notified_exit = 1;
+	}
 
+	if (!(ins->ci_prop.cp_flags & CHK__CHECK_FLAG__CF_FAILOUT))
 		*result = 0;
-	}
 
 	if (update) {
 		rc = chk_bk_update_leader(&ins->ci_bk);
@@ -2284,7 +2281,8 @@ chk_leader_sched(void *args)
 
 		ins_phase = chk_pools_find_slowest(ins, &done);
 
-		if (ins_phase >= CHK__CHECK_SCAN_PHASE__CSP_POOL_MBS && !ins->ci_orphan_done &&
+		if (ins_phase != CHK_INVAL_PHASE &&
+		    ins_phase >= CHK__CHECK_SCAN_PHASE__CSP_POOL_MBS && !ins->ci_orphan_done &&
 		    !DAOS_FAIL_CHECK(DAOS_CHK_SYNC_ORPHAN_PROCESS)) {
 			iv.ci_gen = cbk->cb_gen;
 			iv.ci_phase = ins_phase;
@@ -2316,7 +2314,7 @@ chk_leader_sched(void *args)
 			D_GOTO(out, rc);
 		}
 
-		if (cbk->cb_phase == CHK_INVAL_PHASE || cbk->cb_phase < ins_phase) {
+		if (ins_phase != CHK_INVAL_PHASE && ins_phase > cbk->cb_phase) {
 			D_INFO(DF_LEADER" moves from phase %u to phase %u\n",
 			       DP_LEADER(ins), cbk->cb_phase, ins_phase);
 

src/tests/suite/daos_cr.c

@@ -1,6 +1,6 @@
 /**
  * (C) Copyright 2023-2024 Intel Corporation.
- * (C) Copyright 2025 Hewlett Packard Enterprise Development LP
+ * (C) Copyright 2025-2026 Hewlett Packard Enterprise Development LP
  *
  * SPDX-License-Identifier: BSD-2-Clause-Patent
  */
@@ -3427,12 +3427,6 @@ cr_fail_sync_orphan(void **state)
 	rc = cr_pool_verify(&dci, pool.pool_uuid, TCPS_CHECKED, 0, NULL, NULL, NULL);
 	assert_rc_equal(rc, 0);
 
-	/* Check leader may be completed earlier than check engines in this case, double check. */
-	cr_ins_wait(0, NULL, &dci);
-
-	rc = cr_ins_verify(&dci, TCIS_COMPLETED);
-	assert_rc_equal(rc, 0);
-
 	cr_debug_set_params(arg, 0);
 
 	rc = cr_mode_switch(false);