From ed58e337d2b9297252946b816b7bea76f35cf5b5 Mon Sep 17 00:00:00 2001 From: Qingmin Liu Date: Mon, 21 May 2018 15:11:22 +0800 Subject: [PATCH] Extend multiple sge support for RDMA read/write. - Generally speaking, there will be hardware SGE limitation. - Useful to verify the functionality of multiple sges with this tool. - Usage: Change NUM_SGE from rdma-common.h rebuild and verify it. --- 02_read-write/rdma-client.c | 6 ++++- 02_read-write/rdma-common.c | 45 ++++++++++++++++++++----------------- 02_read-write/rdma-common.h | 4 +++- 02_read-write/rdma-server.c | 5 ++++- 4 files changed, 37 insertions(+), 23 deletions(-) diff --git a/02_read-write/rdma-client.c b/02_read-write/rdma-client.c index 8950fe1..9ac9904 100644 --- a/02_read-write/rdma-client.c +++ b/02_read-write/rdma-client.c @@ -51,10 +51,14 @@ int main(int argc, char **argv) int on_addr_resolved(struct rdma_cm_id *id) { + int i = 0; + printf("address resolved.\n"); build_connection(id); - sprintf(get_local_message_region(id->context), "message from active/client side with pid %d", getpid()); + for (i=0; icontext, i), "[Client: buf%d] message from active/client side with pid %d", i, getpid()); + } TEST_NZ(rdma_resolve_route(id, TIMEOUT_IN_MS)); return 0; diff --git a/02_read-write/rdma-common.c b/02_read-write/rdma-common.c index 4024b83..574f149 100644 --- a/02_read-write/rdma-common.c +++ b/02_read-write/rdma-common.c @@ -57,7 +57,7 @@ struct connection { static void build_context(struct ibv_context *verbs); static void build_qp_attr(struct ibv_qp_init_attr *qp_attr); -static char * get_peer_message_region(struct connection *conn); +static char * get_peer_message_region(struct connection *conn, int sge); static void on_completion(struct ibv_wc *); static void * poll_cq(void *); static void post_receives(struct connection *conn); @@ -136,8 +136,8 @@ void build_qp_attr(struct ibv_qp_init_attr *qp_attr) qp_attr->cap.max_send_wr = 10; qp_attr->cap.max_recv_wr = 10; - qp_attr->cap.max_send_sge = 1; - qp_attr->cap.max_recv_sge = 1; + qp_attr->cap.max_send_sge = NUM_SGE; + qp_attr->cap.max_recv_sge = NUM_SGE; } void destroy_connection(void *context) @@ -161,25 +161,26 @@ void destroy_connection(void *context) free(conn); } -void * get_local_message_region(void *context) +void * get_local_message_region(void *context, int sge) { if (s_mode == M_WRITE) - return ((struct connection *)context)->rdma_local_region; + return ((struct connection *)context)->rdma_local_region + sge * RDMA_BUFFER_SIZE; else - return ((struct connection *)context)->rdma_remote_region; + return ((struct connection *)context)->rdma_remote_region + sge * RDMA_BUFFER_SIZE; } -char * get_peer_message_region(struct connection *conn) +char * get_peer_message_region(struct connection *conn, int sge) { if (s_mode == M_WRITE) - return conn->rdma_remote_region; + return conn->rdma_remote_region + sge * RDMA_BUFFER_SIZE; else - return conn->rdma_local_region; + return conn->rdma_local_region + sge * RDMA_BUFFER_SIZE; } void on_completion(struct ibv_wc *wc) { struct connection *conn = (struct connection *)(uintptr_t)wc->wr_id; + int i = 0; if (wc->status != IBV_WC_SUCCESS) die("on_completion: status is not IBV_WC_SUCCESS."); @@ -202,7 +203,7 @@ void on_completion(struct ibv_wc *wc) if (conn->send_state == SS_MR_SENT && conn->recv_state == RS_MR_RECV) { struct ibv_send_wr wr, *bad_wr = NULL; - struct ibv_sge sge; + struct ibv_sge sge[NUM_SGE]; if (s_mode == M_WRITE) printf("received MSG_MR. writing message to remote memory...\n"); @@ -213,15 +214,17 @@ void on_completion(struct ibv_wc *wc) wr.wr_id = (uintptr_t)conn; wr.opcode = (s_mode == M_WRITE) ? IBV_WR_RDMA_WRITE : IBV_WR_RDMA_READ; - wr.sg_list = &sge; - wr.num_sge = 1; + wr.sg_list = sge; + wr.num_sge = NUM_SGE; wr.send_flags = IBV_SEND_SIGNALED; wr.wr.rdma.remote_addr = (uintptr_t)conn->peer_mr.addr; wr.wr.rdma.rkey = conn->peer_mr.rkey; - sge.addr = (uintptr_t)conn->rdma_local_region; - sge.length = RDMA_BUFFER_SIZE; - sge.lkey = conn->rdma_local_mr->lkey; + for (i=0; irdma_local_region + i * RDMA_BUFFER_SIZE; + sge[i].length = RDMA_BUFFER_SIZE; + sge[i].lkey = conn->rdma_local_mr->lkey; + } TEST_NZ(ibv_post_send(conn->qp, &wr, &bad_wr)); @@ -229,7 +232,9 @@ void on_completion(struct ibv_wc *wc) send_message(conn); } else if (conn->send_state == SS_DONE_SENT && conn->recv_state == RS_DONE_RECV) { - printf("remote buffer: %s\n", get_peer_message_region(conn)); + for (i=0; iid); } } @@ -278,8 +283,8 @@ void register_memory(struct connection *conn) conn->send_msg = malloc(sizeof(struct message)); conn->recv_msg = malloc(sizeof(struct message)); - conn->rdma_local_region = malloc(RDMA_BUFFER_SIZE); - conn->rdma_remote_region = malloc(RDMA_BUFFER_SIZE); + conn->rdma_local_region = malloc(RDMA_BUFFER_SIZE * NUM_SGE); + conn->rdma_remote_region = malloc(RDMA_BUFFER_SIZE * NUM_SGE); TEST_Z(conn->send_mr = ibv_reg_mr( s_ctx->pd, @@ -296,13 +301,13 @@ void register_memory(struct connection *conn) TEST_Z(conn->rdma_local_mr = ibv_reg_mr( s_ctx->pd, conn->rdma_local_region, - RDMA_BUFFER_SIZE, + RDMA_BUFFER_SIZE * NUM_SGE, IBV_ACCESS_LOCAL_WRITE)); TEST_Z(conn->rdma_remote_mr = ibv_reg_mr( s_ctx->pd, conn->rdma_remote_region, - RDMA_BUFFER_SIZE, + RDMA_BUFFER_SIZE * NUM_SGE, IBV_ACCESS_LOCAL_WRITE | ((s_mode == M_WRITE) ? IBV_ACCESS_REMOTE_WRITE : IBV_ACCESS_REMOTE_READ))); } diff --git a/02_read-write/rdma-common.h b/02_read-write/rdma-common.h index ed3f097..78a8380 100644 --- a/02_read-write/rdma-common.h +++ b/02_read-write/rdma-common.h @@ -8,6 +8,8 @@ #include #include +#define NUM_SGE (2) + #define TEST_NZ(x) do { if ( (x)) die("error: " #x " failed (returned non-zero)." ); } while (0) #define TEST_Z(x) do { if (!(x)) die("error: " #x " failed (returned zero/null)."); } while (0) @@ -21,7 +23,7 @@ void die(const char *reason); void build_connection(struct rdma_cm_id *id); void build_params(struct rdma_conn_param *params); void destroy_connection(void *context); -void * get_local_message_region(void *context); +void * get_local_message_region(void *context, int sge); void on_connect(void *context); void send_mr(void *context); void set_mode(enum mode m); diff --git a/02_read-write/rdma-server.c b/02_read-write/rdma-server.c index aa662b1..d94a7e9 100644 --- a/02_read-write/rdma-server.c +++ b/02_read-write/rdma-server.c @@ -55,11 +55,14 @@ int main(int argc, char **argv) int on_connect_request(struct rdma_cm_id *id) { struct rdma_conn_param cm_params; + int i = 0; printf("received connection request.\n"); build_connection(id); build_params(&cm_params); - sprintf(get_local_message_region(id->context), "message from passive/server side with pid %d", getpid()); + for (i=0; icontext, i), "[Server: buf%d] message from passive/server side with pid %d", i, getpid()); + } TEST_NZ(rdma_accept(id, &cm_params)); return 0;