Merge remote-tracking branch 'origin/master' into ckochhof/dev/master/daos-16501-part1

Author: kanard38

src/chk/chk_engine.c

@@ -60,6 +60,7 @@ struct chk_cont_rec {
 	d_iov_t				 ccr_label_cs;
 	uint32_t			 ccr_label_checked:1,
 					 ccr_skip:1;
+	uint32_t ccr_tgt_nr;
 };
 
 struct chk_cont_bundle {
@@ -114,6 +115,7 @@ chk_cont_alloc(struct btr_instance *tins, d_iov_t *key_iov, d_iov_t *val_iov,
 	if (ccr == NULL)
 		D_GOTO(out, rc = -DER_NOMEM);
 
+	ccr->ccr_tgt_nr = 1;
 	uuid_copy(ccr->ccr_uuid, ccb->ccb_uuid);
 	ccr->ccr_aggregator = ccb->ccb_aggregator;
 	d_list_add_tail(&ccr->ccr_link, &ccb->ccb_aggregator->ccla_list);
@@ -159,6 +161,10 @@ static int
 chk_cont_update(struct btr_instance *tins, struct btr_record *rec,
 		d_iov_t *key, d_iov_t *val, d_iov_t *val_out)
 {
+	struct chk_cont_rec *ccr = umem_off2ptr(&tins->ti_umm, rec->rec_off);
+
+	ccr->ccr_tgt_nr++;
+
 	return 0;
 }
 
@@ -1591,12 +1597,19 @@ static int
 chk_engine_cont_cleanup(struct chk_pool_rec *cpr, struct ds_pool_svc *ds_svc,
 			struct chk_cont_list_aggregator *aggregator)
 {
-	struct chk_instance		*ins = cpr->cpr_ins;
-	struct cont_svc			*svc;
-	struct chk_cont_rec		*ccr;
-	struct chk_cont_label_cb_args	 cclca = { 0 };
-	int				 rc = 0;
-	bool				 failout;
+	struct chk_instance          *ins = cpr->cpr_ins;
+	struct cont_svc              *svc;
+	struct chk_cont_rec          *ccr;
+	char                          msg[CHK_MSG_BUFLEN];
+	struct chk_cont_label_cb_args cclca = {0};
+	struct chk_report_unit        cru   = {0};
+	uint64_t                      seq;
+	uint32_t                      exp_tgt_nr = 0;
+	int                           rc         = 0;
+	bool                          failout;
+
+	if (DAOS_FAIL_CHECK(DAOS_CHK_VERIFY_CONT_SHARDS))
+		exp_tgt_nr = daos_fail_value_get();
 
 	if (ins->ci_prop.cp_flags & CHK__CHECK_FLAG__CF_FAILOUT)
 		failout = true;
@@ -1636,11 +1649,36 @@ chk_engine_cont_cleanup(struct chk_pool_rec *cpr, struct ds_pool_svc *ds_svc,
 		goto out;
 
 	d_list_for_each_entry(ccr, &aggregator->ccla_list, ccr_link) {
-		if (!ccr->ccr_skip && !ccr->ccr_label_checked) {
+		if (ccr->ccr_skip)
+			continue;
+
+		if (!ccr->ccr_label_checked) {
 			rc = chk_engine_cont_set_label(cpr, ccr, svc);
 			if (rc != 0)
 				goto out;
 		}
+
+		if (likely(ccr->ccr_tgt_nr >= exp_tgt_nr))
+			continue;
+
+		snprintf(
+		    msg, CHK_MSG_BUFLEN - 1,
+		    "The container " DF_UUID " in the pool " DF_UUID " lost some shards: %u/%u\n",
+		    DP_UUID(ccr->ccr_uuid), DP_UUID(cpr->cpr_uuid), ccr->ccr_tgt_nr, exp_tgt_nr);
+
+		cru.cru_msg        = msg;
+		cru.cru_gen        = cpr->cpr_bk.cb_gen;
+		cru.cru_cla        = CHK__CHECK_INCONSIST_CLASS__CIC_UNKNOWN;
+		cru.cru_act        = CHK__CHECK_INCONSIST_ACTION__CIA_IGNORE;
+		cru.cru_rank       = dss_self_rank();
+		cru.cru_pool       = (uuid_t *)&cpr->cpr_uuid;
+		cru.cru_pool_label = cpr->cpr_label;
+		cru.cru_cont       = (uuid_t *)&ccr->ccr_uuid;
+		if (ccr->ccr_label_prop != NULL && ccr->ccr_label_prop->dpp_entries != NULL)
+			cru.cru_cont_label = ccr->ccr_label_prop->dpp_entries[0].dpe_str;
+
+		seq = 0;
+		chk_engine_report(&cru, &seq, NULL);
 	}
 
 out:
@@ -3206,10 +3244,11 @@ chk_engine_report(struct chk_report_unit *cru, uint64_t *seq, int *decision)
 		pool->cpr_bk.cb_pool_status = CHK__CHECK_POOL_STATUS__CPS_PENDING;
 
 log:
-	D_CDEBUG(rc != 0, DLOG_ERR, DLOG_INFO,
-		 DF_ENGINE" on rank %u report with class %u, action %u, seq "
-		 DF_X64", handle_rc %d, report_rc %d\n", DP_ENGINE(ins),
-		 cru->cru_rank, cru->cru_cla, cru->cru_act, *seq, cru->cru_result, rc);
+	DL_CDEBUG(rc != 0, DLOG_ERR, DLOG_INFO, rc,
+		  DF_ENGINE " on rank %u report with class %u, action %u, seq " DF_X64 ", %s, "
+			    "handle_rc %d, report_rc %d",
+		  DP_ENGINE(ins), cru->cru_rank, cru->cru_cla, cru->cru_act, *seq, cru->cru_msg,
+		  cru->cru_result, rc);
 
 	if (rc != 0 || cpr == NULL)
 		goto out;

src/chk/chk_leader.c

@@ -3631,10 +3631,10 @@ chk_leader_report(struct chk_report_unit *cru, uint64_t *seq, int *decision)
 		*seq = chk_report_seq_gen(ins);
 	}
 
-	D_INFO(DF_LEADER" handle %s report from rank %u with seq "
-	       DF_X64" class %u, action %u, result %d\n", DP_LEADER(ins),
-	       decision != NULL ? "local" : "remote", cru->cru_rank, *seq, cru->cru_cla,
-	       cru->cru_act, cru->cru_result);
+	D_INFO(DF_LEADER " handle %s report from rank %u with seq " DF_X64 " class %u, action %u, "
+			 "%s, result %d\n",
+	       DP_LEADER(ins), decision != NULL ? "local" : "remote", cru->cru_rank, *seq,
+	       cru->cru_cla, cru->cru_act, cru->cru_msg, cru->cru_result);
 
 	if (cru->cru_act == CHK__CHECK_INCONSIST_ACTION__CIA_INTERACT) {
 		if (cru->cru_pool == NULL)

src/client/java/daos-java/src/main/native/io_daos_dfs_DaosFsClient.c

@@ -458,13 +458,6 @@ Java_io_daos_dfs_DaosFsClient_createNewFile(JNIEnv *env,
 	dfs_obj_t *parent = NULL;
 	mode_t tmp_mode;
 
-	if (unlikely(!type_id)) {
-		char *msg = NULL;
-
-		asprintf(&msg, "unsupported object class, %s", object_type);
-		throw_exc(env, msg, CUSTOM_ERR6);
-		goto out;
-	}
 	int rc = dfs_lookup(dfs, parent_path, O_RDWR, &parent,
 			    &tmp_mode, NULL);
 

src/client/java/hadoop-daos/src/main/java/io/daos/fs/hadoop/DaosFileSystem.java

@@ -381,10 +381,10 @@ public FSDataOutputStream create(Path f,
         throw new IOException("failed to delete existing file " + daosFile);
       }
     }
-
+   
     daosFile.createNewFile(
             Constants.DAOS_MODLE,
-            DaosObjectClass.OC_SX,
+            DaosObjectClass.OC_UNKNOWN,
             this.chunkSize,
             true);
 

src/container/rpc.h

@@ -850,7 +850,6 @@ cont_snap_oit_oid_get_in_set_data(crt_rpc_t *rpc, crt_opcode_t opc, int cont_pro
 	((uuid_t)		(tdi_uuid)		CRT_VAR)
 
 #define DAOS_OSEQ_TGT_DESTROY	/* output fields */		 \
-				/* number of errors */		 \
 	((int32_t)		(tdo_rc)		CRT_VAR)
 
 CRT_RPC_DECLARE(cont_tgt_destroy, DAOS_ISEQ_TGT_DESTROY, DAOS_OSEQ_TGT_DESTROY)

src/container/srv_container.c

@@ -102,10 +102,16 @@ cont_svc_init(struct cont_svc *svc, const uuid_t pool_uuid, uint64_t id,
 		D_GOTO(err, rc = dss_abterr2der(rc));
 	}
 
+	rc = ABT_mutex_create(&svc->cs_cont_ephs_mutex);
+	if (rc != ABT_SUCCESS) {
+		D_ERROR("failed to create cs_cont_ephs_mutex: %d\n", rc);
+		D_GOTO(err_rwlock, rc = dss_abterr2der(rc));
+	}
+
 	/* cs_root */
 	rc = rdb_path_init(&svc->cs_root);
 	if (rc != 0)
-		goto err_lock;
+		goto err_mutex;
 	rc = rdb_path_push(&svc->cs_root, &rdb_path_root_key);
 	if (rc != 0)
 		goto err_root;
@@ -144,7 +150,9 @@ cont_svc_init(struct cont_svc *svc, const uuid_t pool_uuid, uint64_t id,
 	rdb_path_fini(&svc->cs_uuids);
 err_root:
 	rdb_path_fini(&svc->cs_root);
-err_lock:
+err_mutex:
+	ABT_mutex_free(&svc->cs_cont_ephs_mutex);
+err_rwlock:
 	ABT_rwlock_free(&svc->cs_lock);
 err:
 	return rc;
@@ -158,6 +166,7 @@ cont_svc_fini(struct cont_svc *svc)
 	rdb_path_fini(&svc->cs_uuids);
 	rdb_path_fini(&svc->cs_root);
 	ABT_rwlock_free(&svc->cs_lock);
+	ABT_mutex_free(&svc->cs_cont_ephs_mutex);
 }
 
 int
@@ -713,7 +722,6 @@ cont_create_prop_prepare(struct ds_pool_hdl *pool_hdl,
 			D_ERROR("container global %u version could be not set\n", entry->dpe_type);
 			return -DER_INVAL;
 		case DAOS_PROP_CO_OBJ_VERSION:
-			/* this is a walkaround for 2.6 only */
 			entry_def->dpe_val = entry->dpe_val;
 			break;
 		default:
@@ -768,10 +776,14 @@ cont_create_prop_prepare(struct ds_pool_hdl *pool_hdl,
 	if (entry_def)
 		entry_def->dpe_val = pool_hdl->sph_global_ver;
 
-	/* inherit object version from pool*/
+	/*
+	 * New container creation by clients will specify the object version.
+	 * If not specified (dpe_val == 0), it indicates a client from before
+	 * DAOS 2.6.4, so use VERSION 1 for backward compatibility.
+	 */
 	entry_def = daos_prop_entry_get(prop_def, DAOS_PROP_CO_OBJ_VERSION);
 	if (entry_def && entry_def->dpe_val == 0)
-		entry_def->dpe_val = pool_hdl->sph_obj_ver;
+		entry_def->dpe_val = DAOS_POOL_OBJ_VERSION_1;
 
 	/* for new container set HEALTHY status with current pm ver */
 	entry_def = daos_prop_entry_get(prop_def, DAOS_PROP_CO_STATUS);
@@ -1046,6 +1058,11 @@ cont_create(struct rdb_tx *tx, struct ds_pool_hdl *pool_hdl, struct cont_svc *sv
 		D_GOTO(out, rc = -DER_NO_PERM);
 	}
 
+	/* Reset recov_cont prop to notify on flight pool_recov_cont to retry. */
+	rc = ds_pool_prop_recov_cont_reset(tx, svc->cs_rsvc);
+	if (rc != 0)
+		goto out;
+
 	cont_create_in_get_data(rpc, CONT_CREATE, cont_proto_ver, &cprop);
 
 	/* Determine if the label property was supplied, and if so,
@@ -1291,10 +1308,18 @@ cont_destroy_bcast(crt_context_t ctx, struct cont_svc *svc,
 
 	out = crt_reply_get(rpc);
 	rc = out->tdo_rc;
-	if (rc != 0) {
-		D_ERROR(DF_CONT": failed to destroy %d targets\n",
-			DP_CONT(svc->cs_pool_uuid, cont_uuid), rc);
-		rc = -DER_IO;
+	if (rc == -DER_BUSY) {
+		D_INFO(DF_CONT ": some target busy\n", DP_CONT(svc->cs_pool_uuid, cont_uuid));
+		/*
+		 * Must return an error that ds_pool_svc_ops_save considers
+		 * retryable. Otherwise, when it is retried, this container
+		 * destroy operation would always get its result from svc_ops
+		 * without being executed.
+		 */
+		rc = -DER_TIMEDOUT;
+	} else if (rc != 0) {
+		DL_ERROR(rc, DF_CONT ": failed to destroy targets",
+			 DP_CONT(svc->cs_pool_uuid, cont_uuid));
 	}
 
 out_rpc:
@@ -1550,6 +1575,11 @@ cont_destroy(struct rdb_tx *tx, struct ds_pool_hdl *pool_hdl, struct cont *cont,
 	D_DEBUG(DB_MD, DF_CONT ": processing rpc: %p force=%u\n",
 		DP_CONT(pool_hdl->sph_pool->sp_uuid, cont->c_uuid), rpc, force);
 
+	/* Reset recov_cont prop to notify on flight pool_recov_cont to retry. */
+	rc = ds_pool_prop_recov_cont_reset(tx, cont->c_svc->cs_rsvc);
+	if (rc != 0)
+		goto out;
+
 	/* Fetch the container props to check access for delete */
 	rc = cont_prop_read(tx, cont,
 			    DAOS_CO_QUERY_PROP_ACL |
@@ -2032,6 +2062,7 @@ cont_agg_eph_sync(struct ds_pool *pool, struct cont_svc *svc)
 		return;
 	}
 
+	ABT_mutex_lock(svc->cs_cont_ephs_mutex);
 	d_list_for_each_entry_safe(eph_ldr, tmp, &svc->cs_cont_ephs_leader_list, cte_list) {
 		if (eph_ldr->cte_deleted) {
 			d_list_del(&eph_ldr->cte_list);
@@ -2135,6 +2166,7 @@ cont_agg_eph_sync(struct ds_pool *pool, struct cont_svc *svc)
 		if (pool->sp_rebuilding)
 			break;
 	}
+	ABT_mutex_unlock(svc->cs_cont_ephs_mutex);
 
 	map_ranks_fini(&fail_ranks);
 }

src/container/srv_internal.h

@@ -95,6 +95,7 @@ struct cont_svc {
 	/* Manage the EC aggregation epoch and stable epoch */
 	struct sched_request   *cs_cont_ephs_leader_req;
 	d_list_t		cs_cont_ephs_leader_list; /* link cont_track_eph_leader */
+	ABT_mutex               cs_cont_ephs_mutex;       /* protect cs_cont_ephs_leader_list */
 };
 
 /* Container descriptor */