DAOS-17353 object: reduce noisy warning for DTX refresh (#16184)

When process IO on DTX non-leader target, if hit prepared DTX,
related retry logic may generate warning message for reminder
before refresh related DTX. That is not fatal but some noisy.

This patch controls such warning message rate to avoid flood
in log: one warning message per about 10 seconds.

Signed-off-by: Fan Yong <fan.yong@hpe.com>

Author: Nasf-Fan

src/object/cli_obj.c

@@ -1,6 +1,7 @@
 /**
  * (C) Copyright 2016-2024 Intel Corporation.
  * (C) Copyright 2025 Google LLC
+ * (C) Copyright 2025 Hewlett Packard Enterprise Development LP
  *
  * SPDX-License-Identifier: BSD-2-Clause-Patent
  */
@@ -1775,8 +1776,18 @@ obj_retry_cb(tse_task_t *task, struct dc_object *obj,
 		}
 
 		if (!pmap_stale) {
+			uint32_t now = daos_gettime_coarse();
+
 			delay = dc_obj_retry_delay(task, result, &obj_auxi->retry_cnt,
 						   &obj_auxi->inprogress_cnt, obj_auxi->max_delay);
+			if (result == -DER_INPROGRESS &&
+			    ((obj_auxi->retry_warn_ts == 0 && obj_auxi->inprogress_cnt >= 10) ||
+			     (obj_auxi->retry_warn_ts > 0 && obj_auxi->retry_warn_ts + 10 < now))) {
+				obj_auxi->retry_warn_ts = now;
+				obj_auxi->flags |= ORF_MAYBE_STARVE;
+				D_WARN("The task %p has been retried for %u times, maybe starve\n",
+				       task, obj_auxi->inprogress_cnt);
+			}
 		}
 
 		rc = tse_task_reinit_with_delay(task, delay);
@@ -4698,6 +4709,7 @@ obj_comp_cb(tse_task_t *task, void *data)
 	obj_auxi->csum_retry	= 0;
 	obj_auxi->tx_uncertain	= 0;
 	obj_auxi->nvme_io_err	= 0;
+	obj_auxi->flags &= ~ORF_MAYBE_STARVE;
 
 	rc = obj_comp_cb_internal(obj_auxi);
 	if (rc != 0 || obj_auxi->result) {

src/object/obj_internal.h

@@ -486,6 +486,8 @@ struct obj_auxi_args {
 	uint32_t			 flags;
 	uint16_t			 retry_cnt;
 	uint16_t			 inprogress_cnt;
+	/* Last timestamp (in second) when report retry warning message. */
+	uint32_t                         retry_warn_ts;
 	struct obj_req_tgts		 req_tgts;
 	d_sg_list_t			*sgls_dup;
 	crt_bulk_t			*bulks;

src/object/obj_rpc.h

@@ -1,5 +1,6 @@
 /**
  * (C) Copyright 2016-2024 Intel Corporation.
+ * (C) Copyright 2025 Hewlett Packard Enterprise Development LP
  *
  * SPDX-License-Identifier: BSD-2-Clause-Patent
  */
@@ -131,6 +132,7 @@ obj_opc_to_str(crt_opcode_t opc)
 	return "unknown";
 }
 
+/* clang-format off */
 enum obj_rpc_flags {
 	ORF_BULK_BIND		= (1 << 0),
 	/** It is a resent RPC. */
@@ -167,8 +169,8 @@ enum obj_rpc_flags {
 	ORF_CREATE_MAP_DETAIL	= (1 << 13),
 	/* For data migration. */
 	ORF_FOR_MIGRATION	= (1 << 14),
-	/* Force DTX refresh if hit non-committed DTX on non-leader. Out-of-date DAOS-7878. */
-	ORF_DTX_REFRESH		= (1 << 15),
+	/* Retried IO for too many times, maybe starve. */
+	ORF_MAYBE_STARVE	= (1 << 15),
 	/* for EC aggregate (to bypass read perm check related with RF) */
 	ORF_FOR_EC_AGG		= (1 << 16),
 	/* for EC data recovery */
@@ -190,6 +192,7 @@ enum obj_rpc_flags {
 	/* The CPD RPC only contains read-only transaction. */
 	ORF_CPD_RDONLY		= (1 << 25),
 };
+/* clang-format on */
 
 /* common for update/fetch */
 #define DAOS_ISEQ_OBJ_RW	/* input fields */		 \

src/object/obj_tx.c

@@ -1,5 +1,6 @@
 /**
  * (C) Copyright 2020-2024 Intel Corporation.
+ * (C) Copyright 2025 Hewlett Packard Enterprise Development LP
  *
  * SPDX-License-Identifier: BSD-2-Clause-Patent
  */
@@ -80,14 +81,10 @@ struct dc_tx {
 	tse_task_t		*tx_orig_task;
 	/** Transaction flags (DAOS_TF_RDONLY, DAOS_TF_ZERO_COPY, etc.) */
 	uint64_t		 tx_flags;
-	uint32_t		 tx_fixed_epoch:1, /** epoch is specified. */
-				 tx_retry:1, /** Retry the commit RPC. */
-				 tx_set_resend:1, /** Set 'resend' flag. */
-				 tx_for_convert:1,
-				 tx_has_cond:1,
-				 tx_renew:1,
-				 tx_closed:1,
-				 tx_reintegrating:1;
+	uint32_t                 tx_fixed_epoch : 1,                      /** epoch is specified. */
+	    tx_retry : 1, /** Retry the commit RPC. */ tx_set_resend : 1, /** Set 'resend' flag. */
+	    tx_for_convert : 1, tx_has_cond : 1, tx_renew : 1, tx_closed : 1, tx_reintegrating : 1,
+	    tx_maybe_starve : 1;
 	/** Transaction status (OPEN, COMMITTED, etc.), see dc_tx_status. */
 	enum dc_tx_status	 tx_status;
 	/** The rank for the server on which the TX leader resides. */
@@ -106,6 +103,8 @@ struct dc_tx {
 
 	uint16_t		 tx_retry_cnt;
 	uint16_t		 tx_inprogress_cnt;
+	/* Last timestamp (in second) when report retry warning message. */
+	uint32_t                 tx_retry_warn_ts;
 	/** Pool map version when trigger first IO. */
 	uint32_t		 tx_pm_ver;
 	/** Reference the pool. */
@@ -994,6 +993,8 @@ dc_tx_commit_cb(tse_task_t *task, void *data)
 
 	D_MUTEX_LOCK(&tx->tx_lock);
 
+	tx->tx_maybe_starve = 0;
+
 	if (rc == 0) {
 		uint64_t	*sub_epochs = oco->oco_sub_epochs.ca_arrays;
 
@@ -1091,7 +1092,18 @@ dc_tx_commit_cb(tse_task_t *task, void *data)
 	locked = false;
 
 	if (rc != -DER_TX_RESTART) {
+		uint32_t now = daos_gettime_coarse();
+
 		delay = dc_obj_retry_delay(task, rc, &tx->tx_retry_cnt, &tx->tx_inprogress_cnt, 0);
+		if (rc == -DER_INPROGRESS &&
+		    ((tx->tx_retry_warn_ts == 0 && tx->tx_inprogress_cnt >= 10) ||
+		     (tx->tx_retry_warn_ts > 0 && tx->tx_retry_warn_ts + 10 < now))) {
+			tx->tx_retry_warn_ts = now;
+			tx->tx_maybe_starve  = 1;
+			D_WARN("The dist TX task %p has been retried for %u times, maybe starve\n",
+			       task, tx->tx_inprogress_cnt);
+		}
+
 		rc1 = tse_task_reinit_with_delay(task, delay);
 		if (rc1 != 0) {
 			D_ERROR("Failed to reinit task %p: %d, %d\n", task, rc1, rc);
@@ -2308,6 +2320,8 @@ dc_tx_commit_trigger(tse_task_t *task, struct dc_tx *tx, daos_tx_commit_t *args)
 	oci->oci_flags = ORF_LEADER;
 	if (tx->tx_set_resend && !tx->tx_renew)
 		oci->oci_flags |= ORF_RESEND;
+	if (tx->tx_maybe_starve)
+		oci->oci_flags |= ORF_MAYBE_STARVE;
 	tx->tx_renew = 0;
 	if (tx->tx_reintegrating)
 		oci->oci_flags |= ORF_REINTEGRATING_IO;

src/object/srv_obj.c

@@ -2031,25 +2031,24 @@ obj_local_rw_internal_wrap(crt_rpc_t *rpc, struct obj_io_context *ioc, struct dt
 static int
 obj_local_rw(crt_rpc_t *rpc, struct obj_io_context *ioc, struct dtx_handle *dth)
 {
+	struct obj_rw_in        *orw = crt_req_get(rpc);
 	struct dtx_share_peer	*dsp;
 	uint32_t		 retry = 0;
 	int			 rc;
 
 again:
 	rc = obj_local_rw_internal_wrap(rpc, ioc, dth);
 	if (dth != NULL && obj_dtx_need_refresh(dth, rc)) {
-		if (unlikely(++retry % 10 == 9)) {
-			dsp = d_list_entry(dth->dth_share_tbd_list.next, struct dtx_share_peer,
-					   dsp_link);
-			D_WARN("DTX refresh for "DF_DTI" because of "DF_DTI" (%d) for %d times, "
-			       "maybe starve\n", DP_DTI(&dth->dth_xid), DP_DTI(&dsp->dsp_xid),
-			       dth->dth_share_tbd_count, retry);
-		}
-
-		if (!obj_rpc_is_fetch(rpc) || retry < 30) {
+		if (++retry < 3) {
 			rc = dtx_refresh(dth, ioc->ioc_coc);
 			if (rc == -DER_AGAIN)
 				goto again;
+		} else if (orw->orw_flags & ORF_MAYBE_STARVE) {
+			dsp = d_list_entry(dth->dth_share_tbd_list.next, struct dtx_share_peer,
+					   dsp_link);
+			D_WARN(
+			    "DTX refresh for " DF_DTI " because of " DF_DTI " (%d), maybe starve\n",
+			    DP_DTI(&dth->dth_xid), DP_DTI(&dsp->dsp_xid), dth->dth_share_tbd_count);
 		}
 	}
 
@@ -3565,12 +3564,16 @@ obj_local_punch(struct obj_punch_in *opi, crt_opcode_t opc, uint32_t shard_nr, u
 	}
 
 	if (obj_dtx_need_refresh(dth, rc)) {
-		if (unlikely(++retry % 10 == 9)) {
-			dsp = d_list_entry(dth->dth_share_tbd_list.next,
-					   struct dtx_share_peer, dsp_link);
-			D_WARN("DTX refresh for "DF_DTI" because of "DF_DTI" (%d) for %d "
-			       "times, maybe starve\n", DP_DTI(&dth->dth_xid),
-			       DP_DTI(&dsp->dsp_xid), dth->dth_share_tbd_count, retry);
+		if (++retry >= 3) {
+			if (opi->opi_flags & ORF_MAYBE_STARVE) {
+				dsp = d_list_entry(dth->dth_share_tbd_list.next,
+						   struct dtx_share_peer, dsp_link);
+				D_WARN("DTX refresh for " DF_DTI " because of " DF_DTI
+				       " (%d), maybe starve\n",
+				       DP_DTI(&dth->dth_xid), DP_DTI(&dsp->dsp_xid),
+				       dth->dth_share_tbd_count);
+			}
+			goto out;
 		}
 
 		rc = dtx_refresh(dth, ioc->ioc_coc);
@@ -4955,24 +4958,25 @@ ds_cpd_handle_one_wrap(crt_rpc_t *rpc, struct daos_cpd_sub_head *dcsh,
 		       struct daos_cpd_disp_ent *dcde, struct daos_cpd_sub_req *dcsrs,
 		       struct obj_io_context *ioc, struct dtx_handle *dth)
 {
+	struct obj_cpd_in       *oci = crt_req_get(rpc);
 	struct dtx_share_peer	*dsp;
 	uint32_t		 retry = 0;
 	int			 rc;
 
 again:
 	rc = ds_cpd_handle_one(rpc, dcsh, dcde, dcsrs, ioc, dth);
 	if (obj_dtx_need_refresh(dth, rc)) {
-		if (unlikely(++retry % 10 == 9)) {
+		if (++retry < 3) {
+			rc = dtx_refresh(dth, ioc->ioc_coc);
+			if (rc == -DER_AGAIN)
+				goto again;
+		} else if (oci->oci_flags & ORF_MAYBE_STARVE) {
 			dsp = d_list_entry(dth->dth_share_tbd_list.next,
 					   struct dtx_share_peer, dsp_link);
-			D_WARN("DTX refresh for "DF_DTI" because of "DF_DTI" (%d) for %d "
-			       "times, maybe starve\n", DP_DTI(&dth->dth_xid),
-			       DP_DTI(&dsp->dsp_xid), dth->dth_share_tbd_count, retry);
+			D_WARN(
+			    "DTX refresh for " DF_DTI " because of " DF_DTI " (%d), maybe starve\n",
+			    DP_DTI(&dth->dth_xid), DP_DTI(&dsp->dsp_xid), dth->dth_share_tbd_count);
 		}
-
-		rc = dtx_refresh(dth, ioc->ioc_coc);
-		if (rc == -DER_AGAIN)
-			goto again;
 	}
 
 	return rc;