DAOS-17535 chk: misc improvements for CR logic - b26

Include the followings:

1. When create CHK IV namespace, make the secondary group to be same as
   the primary group. Otherwise, CHK logic may hit DER_NONEXIST trouble
   when communicate via IV.

2. Integrate CHK IV namespace create and destroy API, cleanup related
   logic, redefine the version.

3. Get ranks list and IV namespace version from CHK leader when rejoin.
   Adjust CHK_REJOIN RPC for related changes.

4. Remove unsupported functionality for checking the specified 'phase'.

5. Add new test for case of lost some engine(s) before start checker.

6. Dedicated ULT to handle dead rank event, that will not be affected
   by checker start or stop. Then even if check scheduler exited, the
   subsequent check query still can work against the latest rank list.

Test-tag: recovery

Signed-off-by: Fan Yong <fan.yong@hpe.com>

Author: Nasf-Fan

src/chk/chk_common.c

@@ -1019,8 +1019,7 @@ chk_pending_destroy(struct chk_pending_rec *cpr)
 }
 
 int
-chk_prop_prepare(d_rank_t leader, uint32_t flags, int phase,
-		 uint32_t policy_nr, struct chk_policy *policies,
+chk_prop_prepare(d_rank_t leader, uint32_t flags, uint32_t policy_nr, struct chk_policy *policies,
 		 d_rank_list_t *ranks, struct chk_property *prop)
 {
 	int	rc = 0;
@@ -1033,11 +1032,8 @@ chk_prop_prepare(d_rank_t leader, uint32_t flags, int phase,
 		prop->cp_flags &= ~CHK__CHECK_FLAG__CF_FAILOUT;
 	if (flags & CHK__CHECK_FLAG__CF_NO_AUTO)
 		prop->cp_flags &= ~CHK__CHECK_FLAG__CF_AUTO;
-	prop->cp_flags |= flags & ~(CHK__CHECK_FLAG__CF_RESET |
-				    CHK__CHECK_FLAG__CF_ORPHAN_POOL |
-				    CHK__CHECK_FLAG__CF_NO_FAILOUT |
-				    CHK__CHECK_FLAG__CF_NO_AUTO);
-	prop->cp_phase = phase;
+	prop->cp_flags |= flags & ~(CHK__CHECK_FLAG__CF_RESET | CHK__CHECK_FLAG__CF_ORPHAN_POOL |
+				    CHK__CHECK_FLAG__CF_NO_FAILOUT | CHK__CHECK_FLAG__CF_NO_AUTO);
 	if (ranks != NULL)
 		prop->cp_rank_nr = ranks->rl_nr;
 
@@ -1195,12 +1191,7 @@ chk_ins_cleanup(struct chk_instance *ins)
 	chk_stop_sched(ins);
 	ins->ci_inited = 0;
 
-	chk_iv_ns_cleanup(&ins->ci_iv_ns);
-
-	if (ins->ci_iv_group != NULL) {
-		crt_group_secondary_destroy(ins->ci_iv_group);
-		ins->ci_iv_group = NULL;
-	}
+	chk_iv_ns_destroy(ins);
 }
 
 int
@@ -1215,7 +1206,8 @@ chk_ins_init(struct chk_instance **p_ins)
 	if (ins == NULL)
 		D_GOTO(out_init, rc = -DER_NOMEM);
 
-	ins->ci_sched = ABT_THREAD_NULL;
+	ins->ci_sched         = ABT_THREAD_NULL;
+	ins->ci_dead_rank_ult = ABT_THREAD_NULL;
 
 	ins->ci_rank_hdl = DAOS_HDL_INVAL;
 	D_INIT_LIST_HEAD(&ins->ci_rank_list);
@@ -1281,6 +1273,8 @@ chk_ins_fini(struct chk_instance **p_ins)
 	D_ASSERT(daos_handle_is_inval(ins->ci_pending_hdl));
 	D_ASSERT(d_list_empty(&ins->ci_pool_shutdown_list));
 
+	D_ASSERT(ins->ci_dead_rank_ult == ABT_THREAD_NULL);
+
 	if (ins->ci_sched != ABT_THREAD_NULL)
 		ABT_thread_free(&ins->ci_sched);