Skip to content

btl/portals4: Take into account the limitation of portals4 (max_msg_s… #1895

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 1 commit into from
Aug 8, 2016
Merged

btl/portals4: Take into account the limitation of portals4 (max_msg_s… #1895

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
105 changes: 57 additions & 48 deletions opal/mca/btl/portals4/btl_portals4.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -99,7 +99,6 @@ btl_portals4_init_interface(void)

/* Create recv_idx portal table entry */
ret = PtlPTAlloc(portals4_btl->portals_ni_h,
PTL_PT_ONLY_USE_ONCE |
PTL_PT_ONLY_TRUNCATE,
portals4_btl->recv_eq_h,
REQ_BTL_TABLE_ID,
Expand Down Expand Up @@ -429,7 +428,7 @@ mca_btl_portals4_add_procs(struct mca_btl_base_module_t* btl_base,
opal_bitmap_set_bit(reachable, i);

OPAL_OUTPUT_VERBOSE((90, opal_btl_base_framework.framework_output,
"add_procs: rank=%x nid=%x pid=%x for NI %d\n",
"add_procs: rank=%lx nid=%x pid=%x for NI %d",
i,
btl_peer_data[i]->ptl_proc.phys.nid,
btl_peer_data[i]->ptl_proc.phys.pid,
Expand Down Expand Up @@ -591,7 +590,7 @@ mca_btl_portals4_prepare_src(struct mca_btl_base_module_t* btl_base,
ret = opal_convertor_pack(convertor, &iov, &iov_count, &max_data );

*size = max_data;
if ( ret < 0 ) {
if (ret < 0) {
mca_btl_portals4_free(btl_base, (mca_btl_base_descriptor_t *) frag);
return NULL;
}
Expand Down Expand Up @@ -624,65 +623,75 @@ mca_btl_portals4_register_mem(mca_btl_base_module_t *btl_base,
}

handle->key = OPAL_THREAD_ADD64(&(portals4_btl->portals_rdma_key), 1);
handle->remote_offset = 0;

OPAL_OUTPUT_VERBOSE((90, opal_btl_base_framework.framework_output,
"mca_btl_portals4_register_mem NI=%d base=%p size=%ld handle=%p key=%ld\n",
portals4_btl->interface_num, base, size, (void *)handle, handle->key));

if (MCA_BTL_FLAGS_PUT == flags) {
/* create a match entry */
me.start = base;
me.length = size;
me.ct_handle = PTL_CT_NONE;
me.min_free = 0;
me.uid = PTL_UID_ANY;
me.options = PTL_ME_OP_GET | PTL_ME_USE_ONCE |
PTL_ME_EVENT_LINK_DISABLE |
PTL_ME_EVENT_COMM_DISABLE |
PTL_ME_EVENT_UNLINK_DISABLE;

if (mca_btl_portals4_component.use_logical) {
me.match_id.rank = endpoint->ptl_proc.rank;
} else {
me.match_id.phys.nid = endpoint->ptl_proc.phys.nid;
me.match_id.phys.pid = endpoint->ptl_proc.phys.pid;
}
me.match_bits = handle->key;
me.ignore_bits = BTL_PORTALS4_PROTOCOL_MASK |
BTL_PORTALS4_CONTEXT_MASK |
BTL_PORTALS4_SOURCE_MASK;
me.ignore_bits = 0;
"mca_btl_portals4_register_mem NI=%d base=%p size=%ld handle=%p key=%ld flags=%d",
portals4_btl->interface_num, base, size, (void *)handle, handle->key, flags));

/* create a match entry */
me.start = base;
me.length = size;
me.ct_handle = PTL_CT_NONE;
me.min_free = 0;
me.uid = PTL_UID_ANY;
me.options = PTL_ME_OP_GET |
PTL_ME_EVENT_LINK_DISABLE |
PTL_ME_EVENT_COMM_DISABLE |
PTL_ME_EVENT_UNLINK_DISABLE;

if (mca_btl_portals4_component.use_logical) {
me.match_id.rank = endpoint->ptl_proc.rank;
} else {
me.match_id.phys.nid = endpoint->ptl_proc.phys.nid;
me.match_id.phys.pid = endpoint->ptl_proc.phys.pid;
}
me.match_bits = handle->key;
me.ignore_bits = BTL_PORTALS4_PROTOCOL_MASK |
BTL_PORTALS4_CONTEXT_MASK |
BTL_PORTALS4_SOURCE_MASK;
me.ignore_bits = 0;

ret = PtlMEAppend(portals4_btl->portals_ni_h,
portals4_btl->recv_idx,
&me,
PTL_PRIORITY_LIST,
handle,
&(handle->me_h));
if (PTL_OK != ret) {
opal_output_verbose(1, opal_btl_base_framework.framework_output,
"%s:%d: PtlMEAppend failed: %d\n",
__FILE__, __LINE__, ret);
OPAL_THREAD_ADD32(&portals4_btl->portals_outstanding_ops, -1);
return NULL;
}
OPAL_OUTPUT_VERBOSE((90, opal_btl_base_framework.framework_output,
"PtlMEAppend (mca_btl_portals4_register_mem) handle=%p, me_h=%d start=%p length=%ld rank=%x nid=%x pid=%x match_bits=%lx\n",
(void *)handle, handle->me_h, me.start, me.length,
me.match_id.rank, me.match_id.phys.nid, me.match_id.phys.pid, me.match_bits));
ret = PtlMEAppend(portals4_btl->portals_ni_h,
portals4_btl->recv_idx,
&me,
PTL_PRIORITY_LIST,
handle,
&(handle->me_h));
if (PTL_OK != ret) {
opal_output_verbose(1, opal_btl_base_framework.framework_output,
"%s:%d: PtlMEAppend failed: %d\n",
__FILE__, __LINE__, ret);
OPAL_THREAD_ADD32(&portals4_btl->portals_outstanding_ops, -1);
return NULL;
}
OPAL_OUTPUT_VERBOSE((90, opal_btl_base_framework.framework_output,
"PtlMEAppend (mca_btl_portals4_register_mem) handle=%p, me_h=%d start=%p length=%ld rank=%x nid=%x pid=%x match_bits=%lx\n",
(void *)handle, handle->me_h, me.start, me.length,
me.match_id.rank, me.match_id.phys.nid, me.match_id.phys.pid, me.match_bits));
return handle;
}

int
mca_btl_portals4_deregister_mem(mca_btl_base_module_t *btl_base,
mca_btl_base_registration_handle_t *handle)
{
int ret;
struct mca_btl_portals4_module_t *portals4_btl = (struct mca_btl_portals4_module_t*) btl_base;

OPAL_OUTPUT_VERBOSE((90, opal_btl_base_framework.framework_output,
"mca_btl_portals4_deregister_mem NI=%d handle=%p key=%ld\n",
portals4_btl->interface_num, (void *)handle, handle->key));
"mca_btl_portals4_deregister_mem NI=%d handle=%p key=%ld me_h=%d\n",
portals4_btl->interface_num, (void *)handle, handle->key, handle->me_h));

if (!PtlHandleIsEqual(handle->me_h, PTL_INVALID_HANDLE)) {
ret = PtlMEUnlink(handle->me_h);
if (PTL_OK != ret) {
opal_output_verbose(1, opal_btl_base_framework.framework_output,
"%s:%d: PtlMEUnlink failed: %d\n",__FILE__, __LINE__, ret);
return OPAL_ERROR;
}
handle->me_h = PTL_INVALID_HANDLE;
}

free(handle);

Expand Down
5 changes: 5 additions & 0 deletions opal/mca/btl/portals4/btl_portals4.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -79,6 +79,9 @@ struct mca_btl_portals4_component_t {

/** Event queue handles table used in PtlEQPoll */
ptl_handle_eq_t *eqs_h;

/** Upper limit for message sizes */
unsigned long portals_max_msg_size;
};

typedef struct mca_btl_portals4_component_t mca_btl_portals4_component_t;
Expand Down Expand Up @@ -255,6 +258,8 @@ struct mca_btl_base_registration_handle_t {
ptl_match_bits_t key;
/** Portals4 me_h */
ptl_handle_me_t me_h;
/** Remote offset */
ptl_size_t remote_offset;
};

/*
Expand Down
33 changes: 29 additions & 4 deletions opal/mca/btl/portals4/btl_portals4_component.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -201,6 +201,18 @@ mca_btl_portals4_component_register(void)
OPAL_INFO_LVL_5,
MCA_BASE_VAR_SCOPE_READONLY,
&(mca_btl_portals4_component.portals_recv_mds_size));

mca_btl_portals4_component.portals_max_msg_size = PTL_SIZE_MAX;
(void) mca_base_component_var_register(&mca_btl_portals4_component.super.btl_version,
"max_msg_size",
"Max size supported by portals4 (above that, a message is cut into messages less than that size)",
MCA_BASE_VAR_TYPE_UNSIGNED_LONG,
NULL,
0,
0,
OPAL_INFO_LVL_5,
MCA_BASE_VAR_SCOPE_READONLY,
&(mca_btl_portals4_component.portals_max_msg_size));
return OPAL_SUCCESS;
}

Expand All @@ -216,6 +228,8 @@ mca_btl_portals4_component_open(void)
mca_btl_portals4_module.super.btl_eager_limit = 32 * 1024;
mca_btl_portals4_module.super.btl_rndv_eager_limit = 32 * 1024;
mca_btl_portals4_module.super.btl_max_send_size = 64 * 1024;
if (mca_btl_portals4_module.super.btl_max_send_size > mca_btl_portals4_component.portals_max_msg_size)
mca_btl_portals4_module.super.btl_max_send_size = mca_btl_portals4_component.portals_max_msg_size;
mca_btl_portals4_module.super.btl_rdma_pipeline_send_length = 64 * 1024;
mca_btl_portals4_module.super.btl_rdma_pipeline_frag_size = INT_MAX;
mca_btl_portals4_module.super.btl_min_rdma_pipeline_size = 0;
Expand All @@ -227,6 +241,8 @@ mca_btl_portals4_component_open(void)
mca_btl_portals4_module.super.btl_registration_handle_size = sizeof (mca_btl_base_registration_handle_t);

mca_btl_portals4_module.super.btl_get_limit = SIZE_MAX;
if (mca_btl_portals4_module.super.btl_get_limit > mca_btl_portals4_component.portals_max_msg_size)
mca_btl_portals4_module.super.btl_get_limit = mca_btl_portals4_component.portals_max_msg_size;
mca_btl_portals4_module.super.btl_put_limit = 0; /* not implemented */
mca_btl_portals4_module.super.btl_get_alignment = 0;
mca_btl_portals4_module.super.btl_put_alignment = 0;
Expand Down Expand Up @@ -293,6 +309,7 @@ static mca_btl_base_module_t** mca_btl_portals4_component_init(int *num_btls,
mca_btl_base_module_t **btls = NULL;
unsigned int ret, interface;
ptl_handle_ni_t *portals4_nis_h = NULL;
ptl_ni_limits_t portals4_ni_limits ;
ptl_process_t *ptl_process_ids = NULL;

opal_output_verbose(50, opal_btl_base_framework.framework_output, "mca_btl_portals4_component_init\n");
Expand Down Expand Up @@ -325,22 +342,30 @@ static mca_btl_base_module_t** mca_btl_portals4_component_init(int *num_btls,
PTL_NI_LOGICAL | PTL_NI_MATCHING,
PTL_PID_ANY, /* let library assign our pid */
NULL, /* no desired limits */
NULL, /* actual limits */
&portals4_ni_limits, /* actual limits */
&portals4_nis_h[*num_btls] /* our interface handle */
);
else ret = PtlNIInit((1 == mca_btl_portals4_component.max_btls) ? PTL_IFACE_DEFAULT : interface,
PTL_NI_PHYSICAL | PTL_NI_MATCHING,
PTL_PID_ANY, /* let library assign our pid */
NULL, /* no desired limits */
NULL, /* actual limits */
&portals4_ni_limits, /* actual limits */
&portals4_nis_h[*num_btls] /* our interface handle */
);
if (PTL_OK != ret) {
opal_output_verbose(90, opal_btl_base_framework.framework_output,
"%s:%d: PtlNIInit failed for NI %d: %d\n", __FILE__, __LINE__, interface, ret);
}
else {
OPAL_OUTPUT_VERBOSE((90, opal_btl_base_framework.framework_output, "PtlNIInit OK for NI %d\n", *num_btls));
if (mca_btl_portals4_component.portals_max_msg_size > portals4_ni_limits.max_msg_size)
mca_btl_portals4_component.portals_max_msg_size = portals4_ni_limits.max_msg_size;
if (mca_btl_portals4_module.super.btl_max_send_size > portals4_ni_limits.max_msg_size)
mca_btl_portals4_module.super.btl_max_send_size = portals4_ni_limits.max_msg_size;
if (mca_btl_portals4_module.super.btl_get_limit > portals4_ni_limits.max_msg_size)
mca_btl_portals4_module.super.btl_get_limit = portals4_ni_limits.max_msg_size;
OPAL_OUTPUT_VERBOSE((90, opal_btl_base_framework.framework_output, "PtlNIInit OK for NI %d max_msg_size=%ld",
*num_btls, mca_btl_portals4_component.portals_max_msg_size));

(*num_btls)++;
}
}
Expand Down Expand Up @@ -698,7 +723,7 @@ mca_btl_portals4_component_progress(void)
frag->peer_proc,
portals4_btl->recv_idx,
frag->match_bits, /* match bits */
0,
0, // Warning : should be ev.remote_offset but it is not defined,
frag);
if (OPAL_UNLIKELY(PTL_OK != ret)) {
opal_output_verbose(1, opal_btl_base_framework.framework_output,
Expand Down
10 changes: 4 additions & 6 deletions opal/mca/btl/portals4/btl_portals4_rdma.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -50,7 +50,6 @@ mca_btl_portals4_get(struct mca_btl_base_module_t* btl_base,
{
mca_btl_portals4_module_t *portals4_btl = (mca_btl_portals4_module_t *) btl_base;
mca_btl_portals4_frag_t *frag = NULL;
ptl_md_t md;
int ret;

/* reserve space in the event queue for rdma operations immediately */
Expand Down Expand Up @@ -83,25 +82,24 @@ mca_btl_portals4_get(struct mca_btl_base_module_t* btl_base,
frag->length = size;
frag->peer_proc = btl_peer->ptl_proc;

OPAL_OUTPUT_VERBOSE((90, opal_btl_base_framework.framework_output, "PtlGet start=%p length=%ld nid=%x pid=%x match_bits=%lx\n",
md.start, md.length, btl_peer->ptl_proc.phys.nid, btl_peer->ptl_proc.phys.pid, frag->match_bits));
OPAL_OUTPUT_VERBOSE((90, opal_btl_base_framework.framework_output, "PtlGet offset=%p length=%ld remote_offset=%p nid=%x pid=%x match_bits=%lx",
local_address, size, (void*)local_handle->remote_offset, btl_peer->ptl_proc.phys.nid, btl_peer->ptl_proc.phys.pid, frag->match_bits));

ret = PtlGet(portals4_btl->send_md_h,
(ptl_size_t) local_address,
size,
btl_peer->ptl_proc,
portals4_btl->recv_idx,
frag->match_bits, /* match bits */
0,
local_handle->remote_offset,
frag);
if (OPAL_UNLIKELY(PTL_OK != ret)) {
opal_output_verbose(1, opal_btl_base_framework.framework_output,
"%s:%d: PtlGet failed: %d",
__FILE__, __LINE__, ret);
return OPAL_ERROR;
}
OPAL_OUTPUT_VERBOSE((90, opal_btl_base_framework.framework_output, "SUCCESS: PtlGet start=%p length=%ld nid=%x pid=%x match_bits=%lx\n",
md.start, md.length, btl_peer->ptl_proc.phys.nid, btl_peer->ptl_proc.phys.pid, frag->match_bits));
local_handle->remote_offset += size;

return OPAL_SUCCESS;
}