DAOS-17422 chk: enhance dmg check repair API

When -f option is specified for "dmg check repair" command, we allows the
user/admin to handle the same type of inconsistencies with the same action
subsequently. Under such scenario, using the inconsistency-class is more
clear instead of the seq-num. Then the SYNOPSIS will be as following:

dmg [OPTIONS] check repair [repair-OPTIONS] <seq-num|inconsistency-class>
	<interact-opt|action>
...
[repair command options]
	-f, --for-all	Take the same action for all (potential) inconsistencies
			with the same class. If this option is specified, then
			"inconsistency-class" and "action" will be accepted for
			subsequent parameters; otherwise, "seq-num" and "interact-opt"
			will be used.

Enhance test logic to make it to be workable after landing c9745d8.

Test-tag: test_daos_cat_recov_core

Signed-off-by: Fan Yong <[email protected]>

Author: Nasf-Fan

src/chk/chk_common.c

@@ -1,5 +1,6 @@
 /**
  * (C) Copyright 2022-2024 Intel Corporation.
+ * (C) Copyright 2025 Hewlett Packard Enterprise Development LP
  *
  * SPDX-License-Identifier: BSD-2-Clause-Patent
  */
@@ -183,9 +184,7 @@ btr_ops_t chk_pool_ops = {
 struct chk_pending_bundle {
 	d_list_t		*cpb_pool_head;
 	d_list_t		*cpb_rank_head;
-	d_rank_t		 cpb_rank;
-	uuid_t			 cpb_uuid;
-	uint32_t		 cpb_class;
+	struct chk_report_unit  *cpb_cru;
 	uint64_t		 cpb_seq;
 };
 
@@ -210,11 +209,14 @@ chk_pending_alloc(struct btr_instance *tins, d_iov_t *key_iov, d_iov_t *val_iov,
 	struct chk_pending_bundle	*cpb = val_iov->iov_buf;
 	struct chk_pending_rec		*cpr = NULL;
 	int				 rc = 0;
+	int                              i;
+	size_t                           size;
 
 	D_ASSERT(cpb != NULL);
 	D_ASSERT(val_out != NULL);
 
-	D_ALLOC_PTR(cpr);
+	size = sizeof(*cpr) + sizeof(uint32_t) * cpb->cpb_cru->cru_option_nr;
+	D_ALLOC(cpr, size);
 	if (cpr == NULL)
 		D_GOTO(out, rc = -DER_NOMEM);
 
@@ -226,11 +228,14 @@ chk_pending_alloc(struct btr_instance *tins, d_iov_t *key_iov, d_iov_t *val_iov,
 	if (rc != 0)
 		D_GOTO(out, rc = dss_abterr2der(rc));
 
-	uuid_copy(cpr->cpr_uuid, cpb->cpb_uuid);
-	cpr->cpr_seq = cpb->cpb_seq;
-	cpr->cpr_rank = cpb->cpb_rank;
-	cpr->cpr_class = cpb->cpb_class;
-	cpr->cpr_action = CHK__CHECK_INCONSIST_ACTION__CIA_INTERACT;
+	uuid_copy(cpr->cpr_uuid, *(cpb->cpb_cru->cru_pool));
+	cpr->cpr_seq       = cpb->cpb_seq;
+	cpr->cpr_rank      = cpb->cpb_cru->cru_rank;
+	cpr->cpr_class     = cpb->cpb_cru->cru_cla;
+	cpr->cpr_action    = CHK__CHECK_INCONSIST_ACTION__CIA_INTERACT;
+	cpr->cpr_option_nr = cpb->cpb_cru->cru_option_nr;
+	for (i = 0; i < cpr->cpr_option_nr; i++)
+		cpr->cpr_options[i] = cpb->cpb_cru->cru_options[i];
 
 	if (cpb->cpb_rank_head != NULL)
 		d_list_add_tail(&cpr->cpr_rank_link, cpb->cpb_rank_head);
@@ -332,20 +337,15 @@ chk_ranks_dump(uint32_t rank_nr, d_rank_t *ranks)
 	D_INFO("Ranks List:\n");
 
 	while (rank_nr >= 8) {
-		D_INFO("%8u %8u %8u %8u %8u %8u %8u %8u\n",
-		       ranks[0], ranks[1], ranks[2], ranks[3],
+		D_INFO("%8u%8u%8u%8u%8u%8u%8u%8u\n", ranks[0], ranks[1], ranks[2], ranks[3],
 		       ranks[4], ranks[5], ranks[6], ranks[7]);
 		rank_nr -= 8;
 		ranks += 8;
 	}
 
 	if (rank_nr > 0) {
-		rc = snprintf(ptr, 79, "%8u", ranks[0]);
-		D_ASSERT(rc > 0);
-		ptr += rc;
-
-		for (i = 1; i < rank_nr; i++) {
-			rc = snprintf(ptr, 79 - 8 * i, " %8u", ranks[i]);
+		for (i = 0; i < rank_nr; i++) {
+			rc = snprintf(ptr, 79 - 8 * i, "%8u", ranks[i]);
 			D_ASSERT(rc > 0);
 			ptr += rc;
 		}
@@ -889,8 +889,8 @@ chk_pool_shard_cleanup(struct chk_instance *ins)
 }
 
 int
-chk_pending_add(struct chk_instance *ins, d_list_t *pool_head, d_list_t *rank_head, uuid_t uuid,
-		uint64_t seq, uint32_t rank, uint32_t cla, struct chk_pending_rec **cpr)
+chk_pending_add(struct chk_instance *ins, d_list_t *pool_head, d_list_t *rank_head,
+		struct chk_report_unit *cru, uint64_t seq, struct chk_pending_rec **cpr)
 {
 	struct chk_pending_bundle	rbund;
 	d_iov_t				kiov;
@@ -900,12 +900,10 @@ chk_pending_add(struct chk_instance *ins, d_list_t *pool_head, d_list_t *rank_he
 
 	D_ASSERT(cpr != NULL);
 
-	uuid_copy(rbund.cpb_uuid, uuid);
 	rbund.cpb_pool_head = pool_head;
 	rbund.cpb_rank_head = rank_head;
-	rbund.cpb_seq = seq;
-	rbund.cpb_rank = rank;
-	rbund.cpb_class = cla;
+	rbund.cpb_seq       = seq;
+	rbund.cpb_cru       = cru;
 
 	d_iov_set(&viov, NULL, 0);
 	d_iov_set(&riov, &rbund, sizeof(rbund));
@@ -922,36 +920,55 @@ chk_pending_add(struct chk_instance *ins, d_list_t *pool_head, d_list_t *rank_he
 	ABT_rwlock_unlock(ins->ci_abt_lock);
 
 	D_CDEBUG(rc != 0, DLOG_ERR, DLOG_DBG,
-		 "Add pending record with gen "DF_X64", seq "DF_X64", rank %u, class %u: "DF_RC"\n",
-		 ins->ci_bk.cb_gen, seq, rank, cla, DP_RC(rc));
+		 "Add pending record, gen " DF_X64 ", seq " DF_X64 ", rank %u, cla %u: " DF_RC "\n",
+		 ins->ci_bk.cb_gen, seq, cru->cru_rank, cru->cru_cla, DP_RC(rc));
 
 	return rc;
 }
 
 int
-chk_pending_del(struct chk_instance *ins, uint64_t seq, bool locked, struct chk_pending_rec **cpr)
+chk_pending_del(struct chk_instance *ins, uint64_t seq, uint32_t act, bool locked,
+		struct chk_pending_rec **cpr)
 {
 	d_iov_t		kiov;
 	d_iov_t		riov;
 	int		rc;
+	int             i;
+	bool            matched = false;
 
 	d_iov_set(&riov, NULL, 0);
 	d_iov_set(&kiov, &seq, sizeof(seq));
 
 	if (!locked)
 		ABT_rwlock_wrlock(ins->ci_abt_lock);
-	rc = dbtree_delete(ins->ci_pending_hdl, BTR_PROBE_EQ, &kiov, &riov);
-	if (!locked)
-		ABT_rwlock_unlock(ins->ci_abt_lock);
+	rc = dbtree_lookup(ins->ci_pending_hdl, &kiov, &riov);
+	if (rc != 0)
+		goto out;
 
-	if (rc == 0)
-		*cpr = (struct chk_pending_rec *)riov.iov_buf;
+	*cpr = (struct chk_pending_rec *)riov.iov_buf;
+	for (i = 0; i < (*cpr)->cpr_option_nr; i++) {
+		if ((*cpr)->cpr_options[i] == act) {
+			matched = true;
+			break;
+		}
+	}
+
+	if (matched)
+		rc = dbtree_delete(ins->ci_pending_hdl, BTR_PROBE_EQ | BTR_PROBE_BYPASS, &kiov,
+				   &riov);
 	else
-		*cpr = NULL;
+		rc = -DER_MISMATCH;
+
+out:
+	if (!locked)
+		ABT_rwlock_unlock(ins->ci_abt_lock);
 
 	D_CDEBUG(rc != 0, DLOG_ERR, DLOG_DBG,
-		 "Del pending record with gen "DF_X64", seq "DF_X64": "DF_RC"\n",
-		 ins->ci_bk.cb_gen, seq, DP_RC(rc));
+		 "Del pending record, gen " DF_X64 ", seq " DF_X64 ", act %u: " DF_RC "\n",
+		 ins->ci_bk.cb_gen, seq, act, DP_RC(rc));
+
+	if (rc != 0)
+		*cpr = NULL;
 
 	return rc;
 }
@@ -1032,9 +1049,9 @@ chk_prop_prepare(d_rank_t leader, uint32_t flags, int phase,
 
 	/* Reuse former policies if "policy_nr == 0". */
 	if (policy_nr > 0) {
-		memset(prop->cp_policies, 0, sizeof(Chk__CheckInconsistAction) * CHK_POLICY_MAX);
+		memset(prop->cp_policies, 0, sizeof(Chk__CheckInconsistAction) * CHK_CLASS_MAX);
 		for (i = 0; i < policy_nr; i++) {
-			if (unlikely(policies[i].cp_class >= CHK_POLICY_MAX)) {
+			if (unlikely(policies[i].cp_class >= CHK_CLASS_MAX)) {
 				D_ERROR("Invalid DAOS inconsistency class %u\n",
 					policies[i].cp_class);
 				D_GOTO(out, rc = -DER_INVAL);

src/chk/chk_engine.c

@@ -1,5 +1,6 @@
 /**
  * (C) Copyright 2022-2024 Intel Corporation.
+ * (C) Copyright 2025 Hewlett Packard Enterprise Development LP
  *
  * SPDX-License-Identifier: BSD-2-Clause-Patent
  */
@@ -2642,7 +2643,7 @@ chk_engine_act_internal(struct chk_instance *ins, uint64_t seq, uint32_t act, bo
 	struct chk_pending_rec	*cpr = NULL;
 	int			 rc;
 
-	rc = chk_pending_del(ins, seq, locked, &cpr);
+	rc = chk_pending_del(ins, seq, act, locked, &cpr);
 	if (rc == 0) {
 		/* The cpr will be destroyed by the waiter via chk_engine_report(). */
 		D_ASSERT(cpr->cpr_busy);
@@ -2674,12 +2675,13 @@ chk_engine_act(uint64_t gen, uint64_t seq, uint32_t cla, uint32_t act, uint32_t
 	struct chk_pool_rec	*pool_tmp = NULL;
 	struct chk_pending_rec	*cpr = NULL;
 	struct chk_pending_rec	*cpr_tmp = NULL;
-	int			 rc;
+	int                      rc       = 0;
+	int                      rc1;
 
 	if (ins->ci_bk.cb_gen != gen)
 		D_GOTO(out, rc = -DER_NOTAPPLICABLE);
 
-	if (unlikely(cla >= CHK_POLICY_MAX)) {
+	if (unlikely(cla >= CHK_CLASS_MAX)) {
 		D_ERROR("Invalid DAOS inconsistency class %u\n", cla);
 		D_GOTO(out, rc = -DER_INVAL);
 	}
@@ -2690,12 +2692,12 @@ chk_engine_act(uint64_t gen, uint64_t seq, uint32_t cla, uint32_t act, uint32_t
 		D_GOTO(out, rc = -DER_INVAL);
 	}
 
-	rc = chk_engine_act_internal(ins, seq, act, false);
-	if (rc == -DER_NONEXIST || rc == -DER_NO_HDL)
-		rc = 0;
-
-	if (rc != 0 || !(flags & CAF_FOR_ALL))
+	if (!(flags & CAF_FOR_ALL)) {
+		rc = chk_engine_act_internal(ins, seq, act, false);
+		if (rc == -DER_NONEXIST || rc == -DER_NO_HDL)
+			rc = 0;
 		goto out;
+	}
 
 	if (likely(prop->cp_policies[cla] != act)) {
 		prop->cp_policies[cla] = act;
@@ -2709,23 +2711,20 @@ chk_engine_act(uint64_t gen, uint64_t seq, uint32_t cla, uint32_t act, uint32_t
 	d_list_for_each_entry(pool, &ins->ci_pool_list, cpr_link)
 		chk_pool_get(pool);
 
+	ABT_rwlock_wrlock(ins->ci_abt_lock);
 	d_list_for_each_entry_safe(pool, pool_tmp, &ins->ci_pool_list, cpr_link) {
-		if (rc == 0) {
-			ABT_rwlock_wrlock(ins->ci_abt_lock);
-			d_list_for_each_entry_safe(cpr, cpr_tmp, &pool->cpr_pending_list,
-						   cpr_pool_link) {
-				if (cpr->cpr_class != cla ||
-				    cpr->cpr_action != CHK__CHECK_INCONSIST_ACTION__CIA_INTERACT)
-					continue;
-
-				rc = chk_engine_act_internal(ins, cpr->cpr_seq, act, true);
-				if (rc != 0)
-					break;
-			}
-			ABT_rwlock_unlock(ins->ci_abt_lock);
+		d_list_for_each_entry_safe(cpr, cpr_tmp, &pool->cpr_pending_list, cpr_pool_link) {
+			if (cpr->cpr_class != cla ||
+			    cpr->cpr_action != CHK__CHECK_INCONSIST_ACTION__CIA_INTERACT)
+				continue;
+
+			rc1 = chk_engine_act_internal(ins, cpr->cpr_seq, act, true);
+			if (rc1 != 0 && rc == 0)
+				rc = rc1;
 		}
 		chk_pool_put(pool);
 	}
+	ABT_rwlock_unlock(ins->ci_abt_lock);
 
 out:
 	D_CDEBUG(rc != 0, DLOG_ERR, DLOG_INFO,
@@ -3148,8 +3147,7 @@ chk_engine_report(struct chk_report_unit *cru, uint64_t *seq, int *decision)
 
 		pool = (struct chk_pool_rec *)riov.iov_buf;
 
-		rc = chk_pending_add(ins, &pool->cpr_pending_list, NULL, *cru->cru_pool, *seq,
-				     cru->cru_rank, cru->cru_cla, &cpr);
+		rc = chk_pending_add(ins, &pool->cpr_pending_list, NULL, cru, *seq, &cpr);
 		if (unlikely(rc == -DER_AGAIN))
 			goto new_seq;
 
@@ -3166,9 +3164,9 @@ chk_engine_report(struct chk_report_unit *cru, uint64_t *seq, int *decision)
 	if (unlikely(rc == -DER_AGAIN)) {
 		D_ASSERT(cru->cru_act == CHK__CHECK_INCONSIST_ACTION__CIA_INTERACT);
 
-		rc = chk_pending_del(ins, *seq, false, &tmp);
+		rc = chk_pending_del(ins, *seq, cru->cru_options[0], false, &tmp);
 		if (rc == 0)
-			D_ASSERT(tmp == NULL);
+			D_ASSERT(tmp == cpr);
 		else if (rc != -DER_NONEXIST)
 			goto log;
 

src/chk/chk_internal.h

@@ -1,5 +1,6 @@
 /**
  * (C) Copyright 2022-2024 Intel Corporation.
+ * (C) Copyright 2025 Hewlett Packard Enterprise Development LP
  *
  * SPDX-License-Identifier: BSD-2-Clause-Patent
  */
@@ -312,7 +313,7 @@ CRT_RPC_DECLARE(chk_rejoin, DAOS_ISEQ_CHK_REJOIN, DAOS_OSEQ_CHK_REJOIN);
  * NOTE: Please be careful when change CHK__CHECK_INCONSIST_CLASS__CIC_UNKNOWN
  *	 to avoid hole is the struct chk_property.
  */
-#define CHK_POLICY_MAX		(CHK__CHECK_INCONSIST_CLASS__CIC_UNKNOWN + 1)
+#define CHK_CLASS_MAX           (CHK__CHECK_INCONSIST_CLASS__CIC_UNKNOWN + 1)
 
 struct chk_co_rpc_cb_args {
 	void		*cb_priv;
@@ -486,7 +487,7 @@ struct chk_bookmark {
 struct chk_property {
 	d_rank_t			cp_leader;
 	Chk__CheckFlag			cp_flags;
-	Chk__CheckInconsistAction	cp_policies[CHK_POLICY_MAX];
+	Chk__CheckInconsistAction       cp_policies[CHK_CLASS_MAX];
 	/*
 	 * NOTE: Preserve for supporting to continue the check until the specified phase in the
 	 *	 future. -1 means to check all phases.
@@ -621,23 +622,6 @@ struct chk_pool_rec {
 	ABT_cond		 cpr_cond;
 };
 
-struct chk_pending_rec {
-	/* Link into chk_pool_rec::cpr_pending_list. */
-	d_list_t		 cpr_pool_link;
-	/* Link into chk_rank_rec::crr_pending_list. */
-	d_list_t		 cpr_rank_link;
-	uuid_t			 cpr_uuid;
-	uint64_t		 cpr_seq;
-	d_rank_t		 cpr_rank;
-	uint32_t		 cpr_class;
-	uint32_t		 cpr_action;
-	uint32_t		 cpr_busy:1,
-				 cpr_exiting:1,
-				 cpr_on_leader:1;
-	ABT_mutex		 cpr_mutex;
-	ABT_cond		 cpr_cond;
-};
-
 struct chk_report_unit {
 	uint64_t		 cru_gen;
 	uint32_t		 cru_cla;
@@ -660,6 +644,23 @@ struct chk_report_unit {
 	uint32_t		 cru_result;
 };
 
+struct chk_pending_rec {
+	/* Link into chk_pool_rec::cpr_pending_list. */
+	d_list_t  cpr_pool_link;
+	/* Link into chk_rank_rec::crr_pending_list. */
+	d_list_t  cpr_rank_link;
+	uuid_t    cpr_uuid;
+	uint64_t  cpr_seq;
+	uint32_t  cpr_rank;
+	uint32_t  cpr_class;
+	uint32_t  cpr_action;
+	uint32_t  cpr_busy : 1, cpr_exiting : 1, cpr_on_leader : 1;
+	ABT_mutex cpr_mutex;
+	ABT_cond  cpr_cond;
+	uint32_t  cpr_option_nr;
+	uint32_t  cpr_options[0];
+};
+
 struct chk_traverse_pools_args {
 	uint64_t		 ctpa_gen;
 	struct chk_instance	*ctpa_ins;
@@ -725,11 +726,13 @@ int chk_pool_add_shard(daos_handle_t hdl, d_list_t *head, uuid_t uuid, d_rank_t
 
 void chk_pool_shard_cleanup(struct chk_instance *ins);
 
-int chk_pending_add(struct chk_instance *ins, d_list_t *pool_head, d_list_t *rank_head, uuid_t uuid,
-		    uint64_t seq, uint32_t rank, uint32_t cla, struct chk_pending_rec **cpr);
+/* clang-format off */
+int chk_pending_add(struct chk_instance *ins, d_list_t *pool_head, d_list_t *rank_head,
+		    struct chk_report_unit *cru, uint64_t seq, struct chk_pending_rec **cpr);
 
-int chk_pending_del(struct chk_instance *ins, uint64_t seq, bool locked,
+int chk_pending_del(struct chk_instance *ins, uint64_t seq, uint32_t act, bool locked,
 		    struct chk_pending_rec **cpr);
+/* clang-format on */
 
 int chk_pending_wakeup(struct chk_instance *ins, struct chk_pending_rec *cpr);