perftest: Fix double frees during RDMA CM connection retry

When running perftest with RDMA CM enabled (-R), if the connection
request is rejected (RDMA_CM_EVENT_REJECTED), the client enters a retry
loop in `rdma_cm_client_connection`.
However, the previous retry logic contained multiple flaws causing
segmentation faults, double frees, and heap corruption.

The following specific issues were identified and fixed:

1. Double Free of Event Channel:
   The `rdma_destroy_event_channel()` was called inside `rdma_cm_destroy_cma()`
during internal cleanup, but the pointer was not cleared. The caller function
`create_rdma_cm_connection()` would then attempt to destroy the same channel
again in its error path.
   Fix: Set the channel pointer to NULL after destruction and check for
NULL before attempting to destroy it. The event channel and cm nodes
will be reallocated when entering another retry attempt.

2. Heap Corruption via Index Overflow:
   The `ctx->cma_master.connection_index` was incremented on every
connection attempt but was never reset upon failure. During retries,
this index would exceed the bounds of the `nodes` array, leading to
out-of-bound writes and heap metadata corruption. Similar things would
happen for other fields of cma_master.
   Fix: Complete reset for fields of cma_master in
`rdma_cm_destroy_cma()`.

3. Context Corruption and Leaks:
   `rdma_cm_route_handler()` unconditionally called `ctx_init()` and
`create_qp_main()` on
every retry attempt. This overwrote existing pointers (PD, MR, Buffers)
in the context structure without releasing the old resources, causing
memory leaks and "Bad file descriptor" errors during final cleanup.
Recreating old qp would cause qp creation error causing retry to fail.
   Fix: Add a check in `rdma_cm_route_handler()` to ensure `ctx_init()`
and `create_qp_main` are only called if the context has not been
initialized yet.

4. Similar issues happened on server side retry in
`rdma_cm_connection_request_handler`. Apply same fix as in
`rdma_cm_route_handler()`.

Signed-off-by: Ruizhe Zhou <[email protected]>

Author: Ruizhe Zhou

src/perftest_communication.c

@@ -2416,7 +2416,7 @@ int rdma_cm_route_handler(struct pingpong_context *ctx,
 	connection_index = ctx->cma_master.connection_index;
 
 	// Initialization of client contexts in case of first connection:
-	if (connection_index == 0) {
+	if ((user_param->use_event && !ctx->send_channel)|| !ctx->pd) {
 		rc = ctx_init(ctx, user_param);
 		if (rc) {
 			error_message = "Failed to initialize RDMA contexts.";
@@ -2425,10 +2425,15 @@ int rdma_cm_route_handler(struct pingpong_context *ctx,
 	}
 
 	ctx->cm_id = cma_id;
-	rc = create_qp_main(ctx, user_param, connection_index);
-	if (rc) {
-		error_message = "Failed to create QP.";
-		goto error;
+
+	/* Only create qp when it's not available
+	 * (i.e. avoid recreating qp during retry) */
+	if(!ctx->qp[connection_index]) {
+		rc = create_qp_main(ctx, user_param, connection_index);
+		if (rc) {
+			error_message = "Failed to create QP.";
+			goto error;
+		}
 	}
 
 	memset(&conn_param, 0, sizeof conn_param);
@@ -2446,6 +2451,9 @@ int rdma_cm_route_handler(struct pingpong_context *ctx,
 	rc = rdma_connect(cma_id, &conn_param);
 	if (rc) {
 		error_message = "Failed to connect through RDMA CM.";
+		/* IB core will destroy cm id if failed.
+		 * Set cma_id to NULL to avoid double free.*/
+		cma_id = NULL;
 		goto error;
 	}
 
@@ -2484,7 +2492,7 @@ int rdma_cm_connection_request_handler(struct pingpong_context *ctx,
 
 	ctx->context = cma_id->verbs;
 	// Initialization of server contexts in case of first connection:
-	if (connection_index == 0) {
+	if ((user_param->use_event && !ctx->send_channel)|| !ctx->pd) {
 		rc = ctx_init(ctx, user_param);
 		if (rc) {
 			error_message = "Failed to initialize RDMA contexts.";
@@ -2493,10 +2501,15 @@ int rdma_cm_connection_request_handler(struct pingpong_context *ctx,
 	}
 
 	ctx->cm_id = cm_node->cma_id;
-	rc = create_qp_main(ctx, user_param, connection_index);
-	if (rc) {
-		error_message = "Failed to create QP.";
-		goto error_2;
+
+	/* Only create qp when it's not available
+	 * (i.e. avoid recreating qp during retry) */
+	if(!ctx->qp[connection_index]) {
+		rc = create_qp_main(ctx, user_param, connection_index);
+		if (rc) {
+			error_message = "Failed to create QP.";
+			goto error_2;
+		}
 	}
 
 	memset(&conn_param, 0, sizeof(conn_param));
@@ -2859,6 +2872,20 @@ int rdma_cm_client_connection(struct pingpong_context *ctx,
 	char error_message[ERROR_MSG_SIZE] = "";
 
 	for (i = 0; i < max_retries; i++) {
+		if (i > 0 && ctx->cma_master.channel == NULL) {
+			ctx->cma_master.channel = rdma_create_event_channel();
+			if (!ctx->cma_master.channel) {
+				sprintf(error_message,
+					"Failed to recreate RDMA CM event channel during retry.");
+				goto error;
+			}
+			rc = rdma_cm_allocate_nodes(ctx, user_param, hints);
+			if (rc) {
+				sprintf(error_message,
+					"Failed to reallocate RDMA CM nodes during retry.");
+				goto error;
+			}
+		}
 		rc = _rdma_cm_client_connection(ctx, user_param, hints);
 		if (!rc) {
 			return rc;
@@ -2945,7 +2972,10 @@ int create_rdma_cm_connection(struct pingpong_context *ctx,
 	free(hints.ai_src_addr);
 
 destroy_event_channel:
-	rdma_destroy_event_channel(ctx->cma_master.channel);
+	if (ctx->cma_master.channel) {
+		rdma_destroy_event_channel(ctx->cma_master.channel);
+		ctx->cma_master.channel = NULL;
+	}
 
 error:
 	return error_handler(error_message);

src/perftest_resources.c

@@ -2665,11 +2665,14 @@ xrcd: __attribute__((unused))
 #endif
 
 	ibv_dealloc_pd(ctx->pd);
+	ctx->pd = NULL;
 
 comp_channel:
 	if (user_param->use_event) {
 		ibv_destroy_comp_channel(ctx->send_channel);
 		ibv_destroy_comp_channel(ctx->recv_channel);
+		ctx->send_channel = NULL;
+		ctx->recv_channel = NULL;
 	}
 
 	return FAILURE;
@@ -6154,20 +6157,36 @@ int rdma_cm_destroy_cma(struct pingpong_context *ctx,
 
 	for (i = 0; i < user_param->num_of_qps; i++) {
 		cm_node = &ctx->cma_master.nodes[i];
-		rc = rdma_destroy_id(cm_node->cma_id);
-		if (rc) {
-			sprintf(error_message,
-				"Failed to destroy RDMA CM ID number %d.", i);
-			goto error;
+		if(cm_node && cm_node->cma_id)
+		{
+			rc = rdma_destroy_id(cm_node->cma_id);
+			if (rc) {
+				sprintf(error_message,
+					"Failed to destroy RDMA CM ID number %d.",
+					i);
+				goto error;
+			}
+			cm_node->cma_id = NULL;
 		}
 	}
 
-	rdma_destroy_event_channel(ctx->cma_master.channel);
+	if (ctx->cma_master.channel) {
+		rdma_destroy_event_channel(ctx->cma_master.channel);
+		ctx->cma_master.channel = NULL;
+	}
 	if (ctx->cma_master.rai) {
 		rdma_freeaddrinfo(ctx->cma_master.rai);
+		ctx->cma_master.rai = NULL;
 	}
 
-	free(ctx->cma_master.nodes);
+	ctx->cma_master.connects_left = user_param->num_of_qps;
+	ctx->cma_master.connection_index = 0;
+	ctx->cma_master.disconnects_left = 0;
+
+	if (ctx->cma_master.nodes) {
+		free(ctx->cma_master.nodes);
+		ctx->cma_master.nodes = NULL;
+	}
 	return rc;
 
 error: