DAOS-15993 rebuild: for manual rebuilds do not eval self_heal (#17345)

Consider a quick maintenance scenario in which a daos_engine
is stopped briefly, and the administrator does not wish to have
the DAOS automatic recovery / rebuild mechanism occur. That is,
a pool map update (targets from UP_IN to DOWN) is to occur, the
pool to enter a degraded mode (still allowing ongoing I/O), and
NO rebuild to be triggered during this brief time window.

The above can be arranged by modifying the system or pool-specific
self_heal property value (to not set the rebuild bit), and then
stopping the engine.

Now also consider the conclusion of the maintenance that involes
re-starting the engine, and reintegrating that rank back into the pool.
It is most convenient to directly issue a dmg pool reintegrate command
from the maintenance state.

Before this change, manual administration commands such as
dmg pool exclude/reintegrate were prevented from triggering rebuilds
due to the pool self_heal property setting. However, the intention
of the self_heal (aka auto recovery) feature is to only apply
to automatic rebuilds.

With this change, the is_pool_rebuild_allowed() function is updated
to accept an indication of whether the self_heal checks are applicable.
Manual pool map update and rebuild cases supply false for this argument
(allowing those cases to result in a rebuild being scheduled).

Signed-off-by: Kenneth Cain <kenneth.cain@hpe.com>

Author: kccain

src/include/daos_srv/pool.h

@@ -567,18 +567,19 @@ int
 ds_pool_prop_recov_cont_reset(struct rdb_tx *tx, struct ds_rsvc *rsvc);
 
 static inline bool
-is_pool_rebuild_allowed(struct ds_pool *pool, bool check_delayed_rebuild)
+is_pool_rebuild_allowed(struct ds_pool *pool, uint64_t self_heal, bool auto_recovery)
 {
-	uint64_t flags = DAOS_SELF_HEAL_AUTO_REBUILD;
-
-	if (check_delayed_rebuild)
-		flags |= DAOS_SELF_HEAL_DELAY_REBUILD;
+	bool auto_rebuild_enabled  = self_heal & DAOS_SELF_HEAL_AUTO_REBUILD;
+	bool delay_rebuild_enabled = self_heal & DAOS_SELF_HEAL_DELAY_REBUILD;
 
 	if (pool->sp_disable_rebuild)
 		return false;
-	if (!(pool->sp_self_heal & flags))
+
+	/* If auto recovery is requested, only allow if self_heal enables auto or delay_rebuild */
+	if (auto_recovery && !(auto_rebuild_enabled || delay_rebuild_enabled))
 		return false;
 
+	/* Otherwise, rebuild is allowed */
 	return true;
 }
 

src/include/daos_srv/rebuild.h

@@ -1,6 +1,6 @@
 /**
  * (C) Copyright 2017-2023 Intel Corporation.
- * (C) Copyright 2025 Hewlett Packard Enterprise Development LP
+ * (C) Copyright 2025-2026 Hewlett Packard Enterprise Development LP
  *
  * SPDX-License-Identifier: BSD-2-Clause-Patent
  */
@@ -96,7 +96,7 @@ void ds_rebuild_running_query(uuid_t pool_uuid, uint32_t opc, uint32_t *rebuild_
 			      daos_epoch_t *current_eph, uint32_t *rebuild_gen);
 int
      ds_rebuild_regenerate_task(struct ds_pool *pool, daos_prop_t *prop, uint64_t sys_self_heal,
-				uint64_t delay_sec);
+				bool auto_recovery, uint64_t delay_sec);
 void ds_rebuild_leader_stop_all(void);
 void ds_rebuild_abort(uuid_t pool_uuid, unsigned int version, uint32_t rebuild_gen,
 		      uint64_t term);

src/pool/srv_pool.c

@@ -1,6 +1,6 @@
 /*
  * (C) Copyright 2016-2024 Intel Corporation.
- * (C) Copyright 2025 Hewlett Packard Enterprise Development LP
+ * (C) Copyright 2025-2026 Hewlett Packard Enterprise Development LP
  * (C) Copyright 2025 Google LLC
  *
  * SPDX-License-Identifier: BSD-2-Clause-Patent
@@ -2577,7 +2577,8 @@ pool_svc_step_up_cb(struct ds_rsvc *rsvc)
 	if (rc != 0)
 		goto out;
 
-	rc = ds_rebuild_regenerate_task(svc->ps_pool, prop, sys_self_heal, 0);
+	rc = ds_rebuild_regenerate_task(svc->ps_pool, prop, sys_self_heal, true /* auto_recovery */,
+					0 /* delay_sec */);
 	if (rc != 0)
 		goto out;
 
@@ -7746,23 +7747,24 @@ pool_svc_update_map(struct pool_svc *svc, crt_opcode_t opc, bool exclude_rank,
 		    struct pool_target_addr_list *inval_list_out, uint32_t *map_version,
 		    struct rsvc_hint *hint, enum map_update_source src, uint32_t flags)
 {
-	struct pool_target_id_list       target_list = {0};
-	uint32_t                         tgt_map_ver = 0;
-	bool				updated;
-	int				rc;
-	char				*env;
-	daos_epoch_t			rebuild_eph = d_hlc_get();
-	uint64_t			delay = 2;
-	bool                             sys_self_heal_applicable;
-	uint64_t                         sys_self_heal = 0;
+	struct pool_target_id_list target_list = {0};
+	uint32_t                   tgt_map_ver = 0;
+	bool                       updated;
+	int                        rc;
+	char                      *env;
+	daos_epoch_t               rebuild_eph = d_hlc_get();
+	uint64_t                   delay       = 2;
+	bool                       auto_recovery;
+	uint64_t                   sys_self_heal = 0;
 
 	/*
-	 * The system self-heal policy only applies to automatic pool exclude
+	 * The pool and system self-heal policies only apply to automatic pool exclude
 	 * and rebuild operations.
 	 */
-	sys_self_heal_applicable = (opc == MAP_EXCLUDE && src == MUS_SWIM);
+	auto_recovery = (opc == MAP_EXCLUDE && src == MUS_SWIM);
 
-	if (sys_self_heal_applicable) {
+	/* If applicable, check system self-heal policy. */
+	if (auto_recovery) {
 		rc = ds_mgmt_get_self_heal_policy(pool_svc_abort_gshp, svc, &sys_self_heal);
 		if (rc != 0) {
 			DL_ERROR(rc, DF_UUID ": failed to get self-heal policy",
@@ -7784,6 +7786,7 @@ pool_svc_update_map(struct pool_svc *svc, crt_opcode_t opc, bool exclude_rank,
 		}
 	}
 
+	/* Pool self-heal policy is checked in this call. */
 	rc = pool_svc_update_map_internal(svc, opc, exclude_rank, extend_rank_list,
 					  extend_domains_nr, extend_domains, &target_list, list,
 					  hint, &updated, map_version, &tgt_map_ver, inval_list_out,
@@ -7804,14 +7807,14 @@ pool_svc_update_map(struct pool_svc *svc, crt_opcode_t opc, bool exclude_rank,
 	}
 	d_freeenv_str(&env);
 
-	if (sys_self_heal_applicable && !(sys_self_heal & DS_MGMT_SELF_HEAL_POOL_REBUILD)) {
+	if (auto_recovery && !(sys_self_heal & DS_MGMT_SELF_HEAL_POOL_REBUILD)) {
 		D_DEBUG(DB_MD, DF_UUID ": pool_rebuild disabled in system property self_heal\n",
 			DP_UUID(svc->ps_uuid));
 		rc = 0;
 		goto out;
 	}
 
-	if (!is_pool_rebuild_allowed(svc->ps_pool, true)) {
+	if (!is_pool_rebuild_allowed(svc->ps_pool, svc->ps_pool->sp_self_heal, auto_recovery)) {
 		D_DEBUG(DB_MD, DF_UUID ": rebuild disabled for pool\n",
 			DP_UUID(svc->ps_pool->sp_uuid));
 		D_GOTO(out, rc);

src/rebuild/srv.c

@@ -2614,19 +2614,20 @@ regenerate_task_of_type(struct ds_pool *pool, pool_comp_state_t match_states, ui
 	return rc;
 }
 
-
-/* Regenerate the rebuild tasks when changing the leader. */
+/* Regenerate rebuild tasks when changing the leader, or manually starting rebuilds.
+ * auto_recovery (true for leader change, false for manual) applies to both sys_self_heal and prop.
+ */
 int
 ds_rebuild_regenerate_task(struct ds_pool *pool, daos_prop_t *prop, uint64_t sys_self_heal,
-			   uint64_t delay_sec)
+			   bool auto_recovery, uint64_t delay_sec)
 {
 	struct daos_prop_entry *entry;
 	char                   *env;
 	int                     rc = 0;
 
 	rebuild_gst.rg_abort = 0;
 
-	if (!(sys_self_heal & DS_MGMT_SELF_HEAL_POOL_REBUILD)) {
+	if (auto_recovery && !(sys_self_heal & DS_MGMT_SELF_HEAL_POOL_REBUILD)) {
 		D_DEBUG(DB_REBUILD, DF_UUID ": pool_rebuild disabled in sys_self_heal\n",
 			DP_UUID(pool->sp_uuid));
 		return DER_SUCCESS;
@@ -2648,10 +2649,8 @@ ds_rebuild_regenerate_task(struct ds_pool *pool, daos_prop_t *prop, uint64_t sys
 	}
 
 	entry = daos_prop_entry_get(prop, DAOS_PROP_PO_SELF_HEAL);
-
 	D_ASSERT(entry != NULL);
-	if (entry->dpe_val & (DAOS_SELF_HEAL_AUTO_REBUILD | DAOS_SELF_HEAL_DELAY_REBUILD) &&
-	    !pool->sp_disable_rebuild) {
+	if (is_pool_rebuild_allowed(pool, entry->dpe_val /* self_heal */, auto_recovery)) {
 		rc = regenerate_task_of_type(
 		    pool, PO_COMP_ST_DOWN,
 		    entry->dpe_val & DAOS_SELF_HEAL_DELAY_REBUILD ? -1 : delay_sec);
@@ -2662,7 +2661,7 @@ ds_rebuild_regenerate_task(struct ds_pool *pool, daos_prop_t *prop, uint64_t sys
 		if (rc != 0)
 			return rc;
 	} else {
-		D_DEBUG(DB_REBUILD, DF_UUID" self healing is disabled\n",
+		D_DEBUG(DB_REBUILD, "Pool " DF_UUID " self healing is disabled\n",
 			DP_UUID(pool->sp_uuid));
 	}
 
@@ -2699,7 +2698,8 @@ ds_rebuild_admin_start(struct ds_pool *pool)
 		goto out;
 	}
 
-	rc = ds_rebuild_regenerate_task(pool, &prop, DS_MGMT_SELF_HEAL_ALL, 0);
+	rc = ds_rebuild_regenerate_task(pool, &prop, DS_MGMT_SELF_HEAL_ALL /* sys_self_heal */,
+					false /* auto_recovery */, 0 /* delay_sec */);
 	daos_prop_fini(&prop);
 	if (rc)
 		DL_ERROR(rc, DF_UUID ": regenerate rebuild task failed", DP_UUID(pool->sp_uuid));

src/tests/suite/daos_degrade_ec.c

@@ -1,5 +1,6 @@
 /**
  * (C) Copyright 2016-2023 Intel Corporation.
+ * (C) Copyright 2026 Hewlett Packard Enterprise Development LP
  *
  * SPDX-License-Identifier: BSD-2-Clause-Patent
  */
@@ -38,6 +39,11 @@ degrade_small_sub_setup(void **state)
 
 	arg = *state;
 	arg->no_rebuild = 1;
+
+	/* Disable manual rebuilds */
+	test_set_engine_fail_loc(arg, CRT_NO_RANK, DAOS_REBUILD_DISABLE | DAOS_FAIL_ALWAYS);
+
+	/* Disable automatic rebuilds */
 	rc = daos_pool_set_prop(arg->pool.pool_uuid, "self_heal",
 				"exclude");
 	return rc;
@@ -56,6 +62,11 @@ degrade_sub_setup(void **state)
 
 	arg = *state;
 	arg->no_rebuild = 1;
+
+	/* Disable manual rebuilds */
+	test_set_engine_fail_loc(arg, CRT_NO_RANK, DAOS_REBUILD_DISABLE | DAOS_FAIL_ALWAYS);
+
+	/* Disable automatic rebuilds */
 	rc = daos_pool_set_prop(arg->pool.pool_uuid, "self_heal",
 				"exclude");
 	return rc;
@@ -74,6 +85,11 @@ degrade_sub_rf1_setup(void **state)
 
 	arg = *state;
 	arg->no_rebuild = 1;
+
+	/* Disable manual rebuilds */
+	test_set_engine_fail_loc(arg, CRT_NO_RANK, DAOS_REBUILD_DISABLE | DAOS_FAIL_ALWAYS);
+
+	/* Disable automatic rebuilds */
 	rc = daos_pool_set_prop(arg->pool.pool_uuid, "self_heal",
 				"exclude");
 	return rc;

src/tests/suite/daos_degraded.c

@@ -1,5 +1,6 @@
 /**
  * (C) Copyright 2016-2023 Intel Corporation.
+ * (C) Copyright 2026 Hewlett Packard Enterprise Development LP
  *
  * SPDX-License-Identifier: BSD-2-Clause-Patent
  */
@@ -258,6 +259,11 @@ degraded_setup(void **state)
 
 	arg = *state;
 	arg->no_rebuild = 1;
+
+	/* Disable manual rebuilds */
+	test_set_engine_fail_loc(arg, CRT_NO_RANK, DAOS_REBUILD_DISABLE | DAOS_FAIL_ALWAYS);
+
+	/* Disable automatic rebuilds */
 	rc = daos_pool_set_prop(arg->pool.pool_uuid, "self_heal",
 				"exclude");
 	return rc;