Skip to content

Extend multiple sge support for RDMA read/write. #4

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wants to merge 2 commits into
base: master
Choose a base branch
from
Open

Extend multiple sge support for RDMA read/write. #4

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
ed58e33
Extend multiple sge support for RDMA read/write.
qingmin-liu May 21, 2018
dc33f0c
Merge branch 'master' into master
qingmin-liu Nov 15, 2019
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 5 additions & 1 deletion 02_read-write/rdma-client.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -51,10 +51,14 @@ int main(int argc, char **argv)

int on_addr_resolved(struct rdma_cm_id *id)
{
int i = 0;

printf("address resolved.\n");

build_connection(id);
sprintf(get_local_message_region(id->context), "message from active/client side with pid %d", getpid());
for (i=0; i<NUM_SGE; i++) {
sprintf(get_local_message_region(id->context, i), "[Client: buf%d] message from active/client side with pid %d", i, getpid());
}
TEST_NZ(rdma_resolve_route(id, TIMEOUT_IN_MS));

return 0;
Expand Down
45 changes: 25 additions & 20 deletions 02_read-write/rdma-common.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -57,7 +57,7 @@ struct connection {

static void build_context(struct ibv_context *verbs);
static void build_qp_attr(struct ibv_qp_init_attr *qp_attr);
static char * get_peer_message_region(struct connection *conn);
static char * get_peer_message_region(struct connection *conn, int sge);
static void on_completion(struct ibv_wc *);
static void * poll_cq(void *);
static void post_receives(struct connection *conn);
Expand Down Expand Up @@ -136,8 +136,8 @@ void build_qp_attr(struct ibv_qp_init_attr *qp_attr)

qp_attr->cap.max_send_wr = 10;
qp_attr->cap.max_recv_wr = 10;
qp_attr->cap.max_send_sge = 1;
qp_attr->cap.max_recv_sge = 1;
qp_attr->cap.max_send_sge = NUM_SGE;
qp_attr->cap.max_recv_sge = NUM_SGE;
}

void destroy_connection(void *context)
Expand All @@ -161,25 +161,26 @@ void destroy_connection(void *context)
free(conn);
}

void * get_local_message_region(void *context)
void * get_local_message_region(void *context, int sge)
{
if (s_mode == M_WRITE)
return ((struct connection *)context)->rdma_local_region;
return ((struct connection *)context)->rdma_local_region + sge * RDMA_BUFFER_SIZE;
else
return ((struct connection *)context)->rdma_remote_region;
return ((struct connection *)context)->rdma_remote_region + sge * RDMA_BUFFER_SIZE;
}

char * get_peer_message_region(struct connection *conn)
char * get_peer_message_region(struct connection *conn, int sge)
{
if (s_mode == M_WRITE)
return conn->rdma_remote_region;
return conn->rdma_remote_region + sge * RDMA_BUFFER_SIZE;
else
return conn->rdma_local_region;
return conn->rdma_local_region + sge * RDMA_BUFFER_SIZE;
}

void on_completion(struct ibv_wc *wc)
{
struct connection *conn = (struct connection *)(uintptr_t)wc->wr_id;
int i = 0;

if (wc->status != IBV_WC_SUCCESS)
die("on_completion: status is not IBV_WC_SUCCESS.");
Expand All @@ -202,7 +203,7 @@ void on_completion(struct ibv_wc *wc)

if (conn->send_state == SS_MR_SENT && conn->recv_state == RS_MR_RECV) {
struct ibv_send_wr wr, *bad_wr = NULL;
struct ibv_sge sge;
struct ibv_sge sge[NUM_SGE];

if (s_mode == M_WRITE)
printf("received MSG_MR. writing message to remote memory...\n");
Expand All @@ -213,23 +214,27 @@ void on_completion(struct ibv_wc *wc)

wr.wr_id = (uintptr_t)conn;
wr.opcode = (s_mode == M_WRITE) ? IBV_WR_RDMA_WRITE : IBV_WR_RDMA_READ;
wr.sg_list = &sge;
wr.num_sge = 1;
wr.sg_list = sge;
wr.num_sge = NUM_SGE;
wr.send_flags = IBV_SEND_SIGNALED;
wr.wr.rdma.remote_addr = (uintptr_t)conn->peer_mr.addr;
wr.wr.rdma.rkey = conn->peer_mr.rkey;

sge.addr = (uintptr_t)conn->rdma_local_region;
sge.length = RDMA_BUFFER_SIZE;
sge.lkey = conn->rdma_local_mr->lkey;
for (i=0; i<NUM_SGE; i++) {
sge[i].addr = (uintptr_t)conn->rdma_local_region + i * RDMA_BUFFER_SIZE;
sge[i].length = RDMA_BUFFER_SIZE;
sge[i].lkey = conn->rdma_local_mr->lkey;
}

TEST_NZ(ibv_post_send(conn->qp, &wr, &bad_wr));

conn->send_msg->type = MSG_DONE;
send_message(conn);

} else if (conn->send_state == SS_DONE_SENT && conn->recv_state == RS_DONE_RECV) {
printf("remote buffer: %s\n", get_peer_message_region(conn));
for (i=0; i<NUM_SGE; i++) {
printf("remote buffer: %s\n", get_peer_message_region(conn, i));
}
rdma_disconnect(conn->id);
}
}
Expand Down Expand Up @@ -278,8 +283,8 @@ void register_memory(struct connection *conn)
conn->send_msg = malloc(sizeof(struct message));
conn->recv_msg = malloc(sizeof(struct message));

conn->rdma_local_region = malloc(RDMA_BUFFER_SIZE);
conn->rdma_remote_region = malloc(RDMA_BUFFER_SIZE);
conn->rdma_local_region = malloc(RDMA_BUFFER_SIZE * NUM_SGE);
conn->rdma_remote_region = malloc(RDMA_BUFFER_SIZE * NUM_SGE);

TEST_Z(conn->send_mr = ibv_reg_mr(
s_ctx->pd,
Expand All @@ -296,13 +301,13 @@ void register_memory(struct connection *conn)
TEST_Z(conn->rdma_local_mr = ibv_reg_mr(
s_ctx->pd,
conn->rdma_local_region,
RDMA_BUFFER_SIZE,
RDMA_BUFFER_SIZE * NUM_SGE,
((s_mode == M_WRITE) ? 0 : IBV_ACCESS_LOCAL_WRITE));

TEST_Z(conn->rdma_remote_mr = ibv_reg_mr(
s_ctx->pd,
conn->rdma_remote_region,
RDMA_BUFFER_SIZE,
RDMA_BUFFER_SIZE * NUM_SGE,
((s_mode == M_WRITE) ? (IBV_ACCESS_LOCAL_WRITE | IBV_ACCESS_REMOTE_WRITE) : IBV_ACCESS_REMOTE_READ)));
}

Expand Down
4 changes: 3 additions & 1 deletion 02_read-write/rdma-common.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,8 @@
#include <unistd.h>
#include <rdma/rdma_cma.h>

#define NUM_SGE (2)

#define TEST_NZ(x) do { if ( (x)) die("error: " #x " failed (returned non-zero)." ); } while (0)
#define TEST_Z(x) do { if (!(x)) die("error: " #x " failed (returned zero/null)."); } while (0)

Expand All @@ -21,7 +23,7 @@ void die(const char *reason);
void build_connection(struct rdma_cm_id *id);
void build_params(struct rdma_conn_param *params);
void destroy_connection(void *context);
void * get_local_message_region(void *context);
void * get_local_message_region(void *context, int sge);
void on_connect(void *context);
void send_mr(void *context);
void set_mode(enum mode m);
Expand Down
5 changes: 4 additions & 1 deletion 02_read-write/rdma-server.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -55,11 +55,14 @@ int main(int argc, char **argv)
int on_connect_request(struct rdma_cm_id *id)
{
struct rdma_conn_param cm_params;
int i = 0;

printf("received connection request.\n");
build_connection(id);
build_params(&cm_params);
sprintf(get_local_message_region(id->context), "message from passive/server side with pid %d", getpid());
for (i=0; i<NUM_SGE; i++) {
sprintf(get_local_message_region(id->context, i), "[Server: buf%d] message from passive/server side with pid %d", i, getpid());
}
TEST_NZ(rdma_accept(id, &cm_params));

return 0;
Expand Down