From bed572cd6c78851e30fb7e6bb383d59960cfb83b Mon Sep 17 00:00:00 2001 From: Pascal Deveze Date: Tue, 5 Jul 2016 11:32:53 +0200 Subject: [PATCH 1/5] mtl/portals4: Unlink the ME first, then free the CT and at the end free the PT --- ompi/mca/mtl/portals4/mtl_portals4_flowctl.c | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/ompi/mca/mtl/portals4/mtl_portals4_flowctl.c b/ompi/mca/mtl/portals4/mtl_portals4_flowctl.c index d6ba01918d6..0464015dd2c 100644 --- a/ompi/mca/mtl/portals4/mtl_portals4_flowctl.c +++ b/ompi/mca/mtl/portals4/mtl_portals4_flowctl.c @@ -219,15 +219,16 @@ ompi_mtl_portals4_flowctl_init(void) int ompi_mtl_portals4_flowctl_fini(void) { - PtlPTFree(ompi_mtl_portals4.ni_h, ompi_mtl_portals4.flowctl_idx); - PtlCTFree(ompi_mtl_portals4.flowctl.trigger_ct_h); PtlMEUnlink(ompi_mtl_portals4.flowctl.trigger_me_h); - PtlCTFree(ompi_mtl_portals4.flowctl.alert_ct_h); + PtlCTFree(ompi_mtl_portals4.flowctl.trigger_ct_h); PtlMEUnlink(ompi_mtl_portals4.flowctl.alert_me_h); - PtlCTFree(ompi_mtl_portals4.flowctl.fanin_ct_h); + PtlCTFree(ompi_mtl_portals4.flowctl.alert_ct_h); PtlMEUnlink(ompi_mtl_portals4.flowctl.fanin_me_h); - PtlCTFree(ompi_mtl_portals4.flowctl.fanout_ct_h); + PtlCTFree(ompi_mtl_portals4.flowctl.fanin_ct_h); PtlMEUnlink(ompi_mtl_portals4.flowctl.fanout_me_h); + PtlCTFree(ompi_mtl_portals4.flowctl.fanout_ct_h); + + PtlPTFree(ompi_mtl_portals4.ni_h, ompi_mtl_portals4.flowctl_idx); return OMPI_SUCCESS; } From f666b0d9aab5d7da249573c199285aca6b84fe5c Mon Sep 17 00:00:00 2001 From: Pascal Deveze Date: Tue, 5 Jul 2016 13:00:51 +0200 Subject: [PATCH 2/5] mtl/portals4: Allocate a PT with the PTL_PT_FLOWCTRL flag only if OMPI_MTL_PORTALS4_FLOW_CONTROL is set --- ompi/mca/mtl/portals4/mtl_portals4.c | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/ompi/mca/mtl/portals4/mtl_portals4.c b/ompi/mca/mtl/portals4/mtl_portals4.c index d0cb7f5269e..2d25c8db7dd 100644 --- a/ompi/mca/mtl/portals4/mtl_portals4.c +++ b/ompi/mca/mtl/portals4/mtl_portals4.c @@ -86,9 +86,11 @@ portals4_init_interface(void) /* Create send and long message (read) portal table entries */ ret = PtlPTAlloc(ompi_mtl_portals4.ni_h, +#if OMPI_MTL_PORTALS4_FLOW_CONTROL + PTL_PT_FLOWCTRL | +#endif PTL_PT_ONLY_USE_ONCE | - PTL_PT_ONLY_TRUNCATE | - PTL_PT_FLOWCTRL, + PTL_PT_ONLY_TRUNCATE, ompi_mtl_portals4.recv_eq_h, REQ_RECV_TABLE_ID, &ompi_mtl_portals4.recv_idx); From b87ed1ad4ab601db4e1e97761913c320aa2819e6 Mon Sep 17 00:00:00 2001 From: Pascal Deveze Date: Tue, 5 Jul 2016 13:01:36 +0200 Subject: [PATCH 3/5] mtl/portals4: Display actual limits given by the portals4 PtlNIInit function --- .../mca/mtl/portals4/mtl_portals4_component.c | 24 +++++++++++++++++-- 1 file changed, 22 insertions(+), 2 deletions(-) diff --git a/ompi/mca/mtl/portals4/mtl_portals4_component.c b/ompi/mca/mtl/portals4/mtl_portals4_component.c index b0ee9ae9b59..1c39fc0e034 100644 --- a/ompi/mca/mtl/portals4/mtl_portals4_component.c +++ b/ompi/mca/mtl/portals4/mtl_portals4_component.c @@ -277,6 +277,7 @@ ompi_mtl_portals4_component_init(bool enable_progress_threads, { int ret; ptl_process_t id; + ptl_ni_limits_t actual_limits; if (enable_mpi_threads && ompi_mpi_thread_multiple) { opal_output_verbose(1, ompi_mtl_base_framework.framework_output, @@ -298,13 +299,13 @@ ompi_mtl_portals4_component_init(bool enable_progress_threads, PTL_NI_LOGICAL | PTL_NI_MATCHING, PTL_PID_ANY, NULL, - NULL, + &actual_limits, &ompi_mtl_portals4.ni_h); else ret = PtlNIInit(PTL_IFACE_DEFAULT, PTL_NI_PHYSICAL | PTL_NI_MATCHING, PTL_PID_ANY, NULL, - NULL, + &actual_limits, &ompi_mtl_portals4.ni_h); if (PTL_OK != ret) { opal_output_verbose(1, ompi_mtl_base_framework.framework_output, @@ -313,6 +314,25 @@ ompi_mtl_portals4_component_init(bool enable_progress_threads, goto error; } + if (ompi_comm_rank(MPI_COMM_WORLD) == 0) { + opal_output_verbose(10, ompi_mtl_base_framework.framework_output, "max_entries=%d", actual_limits.max_entries); + opal_output_verbose(10, ompi_mtl_base_framework.framework_output, "max_unexpected_headers=%d", actual_limits.max_unexpected_headers); + opal_output_verbose(10, ompi_mtl_base_framework.framework_output, "max_mds=%d", actual_limits.max_mds); + opal_output_verbose(10, ompi_mtl_base_framework.framework_output, "max_eqs=%d", actual_limits.max_eqs); + opal_output_verbose(10, ompi_mtl_base_framework.framework_output, "max_cts=%d", actual_limits.max_cts); + opal_output_verbose(10, ompi_mtl_base_framework.framework_output, "max_pt_index=%d", actual_limits.max_pt_index); + opal_output_verbose(10, ompi_mtl_base_framework.framework_output, "max_iovecs=%d", actual_limits.max_iovecs); + opal_output_verbose(10, ompi_mtl_base_framework.framework_output, "max_list_size=%d", actual_limits.max_list_size); + opal_output_verbose(10, ompi_mtl_base_framework.framework_output, "max_triggered_ops=%d", actual_limits.max_triggered_ops); + opal_output_verbose(10, ompi_mtl_base_framework.framework_output, "max_msg_size=%ld", actual_limits.max_msg_size); + opal_output_verbose(10, ompi_mtl_base_framework.framework_output, "max_atomic_size=%ld", actual_limits.max_atomic_size); + opal_output_verbose(10, ompi_mtl_base_framework.framework_output, "max_fetch_atomic_size=%ld", actual_limits.max_fetch_atomic_size); + opal_output_verbose(10, ompi_mtl_base_framework.framework_output, "max_waw_ordered_size=%ld", actual_limits.max_waw_ordered_size); + opal_output_verbose(10, ompi_mtl_base_framework.framework_output, "max_war_ordered_size=%ld", actual_limits.max_war_ordered_size); + opal_output_verbose(10, ompi_mtl_base_framework.framework_output, "max_volatile_size=%ld", actual_limits.max_volatile_size); + opal_output_verbose(10, ompi_mtl_base_framework.framework_output, "features=%u", actual_limits.features); + } + ret = PtlGetUid(ompi_mtl_portals4.ni_h, &ompi_mtl_portals4.uid); if (PTL_OK != ret) { opal_output_verbose(1, ompi_mtl_base_framework.framework_output, From 49e993691493b2151ce6f88bbe3396effc0e85d5 Mon Sep 17 00:00:00 2001 From: Pascal Deveze Date: Tue, 5 Jul 2016 13:08:35 +0200 Subject: [PATCH 4/5] mtl/portals4: Some little patches --- ompi/mca/mtl/portals4/mtl_portals4.h | 16 +++++------ .../mca/mtl/portals4/mtl_portals4_component.c | 3 +- ompi/mca/mtl/portals4/mtl_portals4_recv.c | 7 +++-- .../mtl/portals4/mtl_portals4_recv_short.c | 28 +++++++++++-------- 4 files changed, 31 insertions(+), 23 deletions(-) diff --git a/ompi/mca/mtl/portals4/mtl_portals4.h b/ompi/mca/mtl/portals4/mtl_portals4.h index 2cde423a313..e350d571159 100644 --- a/ompi/mca/mtl/portals4/mtl_portals4.h +++ b/ompi/mca/mtl/portals4/mtl_portals4.h @@ -42,23 +42,23 @@ struct mca_mtl_portals4_module_t { mca_mtl_base_module_t base; /* add_procs() can get called multiple times. this prevents multiple calls to portals4_init_interface(). */ - int need_init; + int32_t need_init; /* Use the logical to physical table to accelerate portals4 adressing: 1 (true) : 0 (false) */ - int use_logical; + int32_t use_logical; /* Use flow control: 1 (true) : 0 (false) */ - int use_flowctl; + int32_t use_flowctl; /** Eager limit; messages greater than this use a rendezvous protocol */ - unsigned long long eager_limit; + uint64_t eager_limit; /** Size of short message blocks */ - unsigned long long recv_short_size; + uint64_t recv_short_size; /** Number of short message blocks which should be created during startup */ - int recv_short_num; + uint32_t recv_short_num; /** Length of the send event queues */ - int send_queue_size; + uint32_t send_queue_size; /** Length of the receive event queues */ - int recv_queue_size; + uint32_t recv_queue_size; /** Protocol for long message transfer */ enum { eager, rndv } protocol; diff --git a/ompi/mca/mtl/portals4/mtl_portals4_component.c b/ompi/mca/mtl/portals4/mtl_portals4_component.c index 1c39fc0e034..892bb67b7f5 100644 --- a/ompi/mca/mtl/portals4/mtl_portals4_component.c +++ b/ompi/mca/mtl/portals4/mtl_portals4_component.c @@ -224,7 +224,7 @@ ompi_mtl_portals4_component_open(void) ompi_mtl_portals4.send_eq_h = PTL_INVALID_HANDLE; ompi_mtl_portals4.recv_eq_h = PTL_INVALID_HANDLE; ompi_mtl_portals4.zero_md_h = PTL_INVALID_HANDLE; - + ompi_mtl_portals4.send_md_h = PTL_INVALID_HANDLE; ompi_mtl_portals4.long_overflow_me_h = PTL_INVALID_HANDLE; ompi_mtl_portals4.recv_idx = (ptl_pt_index_t) ~0UL; ompi_mtl_portals4.read_idx = (ptl_pt_index_t) ~0UL; @@ -364,6 +364,7 @@ ompi_mtl_portals4_component_init(bool enable_progress_threads, "My nid,pid = %x,%x", id.phys.nid, id.phys.pid)); + ompi_mtl_portals4.base.mtl_max_tag = MTL_PORTALS4_MAX_TAG; return &ompi_mtl_portals4.base; error: diff --git a/ompi/mca/mtl/portals4/mtl_portals4_recv.c b/ompi/mca/mtl/portals4/mtl_portals4_recv.c index 448372917db..077f1f88661 100644 --- a/ompi/mca/mtl/portals4/mtl_portals4_recv.c +++ b/ompi/mca/mtl/portals4/mtl_portals4_recv.c @@ -91,6 +91,7 @@ ompi_mtl_portals4_recv_progress(ptl_event_t *ev, opal_output_verbose(1, ompi_mtl_base_framework.framework_output, "%s:%d: PTL_EVENT_PUT with ni_fail_type: %d", __FILE__, __LINE__, ev->ni_fail_type); + ret = PTL_FAIL; goto callback_error; } @@ -118,7 +119,7 @@ ompi_mtl_portals4_recv_progress(ptl_event_t *ev, to pull the second part of the message. */ ret = read_msg((char*) ptl_request->delivery_ptr + ompi_mtl_portals4.eager_limit, ((msg_length > ptl_request->delivery_len) ? - ptl_request->delivery_len : msg_length) - ompi_mtl_portals4.eager_limit, + ptl_request->delivery_len : msg_length) - ompi_mtl_portals4.eager_limit, ev->initiator, ev->hdr_data, ompi_mtl_portals4.eager_limit, @@ -159,6 +160,7 @@ ompi_mtl_portals4_recv_progress(ptl_event_t *ev, opal_output_verbose(1, ompi_mtl_base_framework.framework_output, "%s:%d: PTL_EVENT_REPLY with ni_fail_type: %d", __FILE__, __LINE__, ev->ni_fail_type); + ret = PTL_FAIL; goto callback_error; } @@ -204,6 +206,7 @@ ompi_mtl_portals4_recv_progress(ptl_event_t *ev, opal_output_verbose(1, ompi_mtl_base_framework.framework_output, "%s:%d: PTL_EVENT_PUT_OVERFLOW with ni_fail_type: %d", __FILE__, __LINE__, ev->ni_fail_type); + ret = PTL_FAIL; goto callback_error; } @@ -285,7 +288,7 @@ ompi_mtl_portals4_recv_progress(ptl_event_t *ev, ret = read_msg((char*) ptl_request->delivery_ptr + ev->mlength, ((msg_length > ptl_request->delivery_len) ? - ptl_request->delivery_len : msg_length) - ev->mlength, + ptl_request->delivery_len : msg_length) - ev->mlength, ev->initiator, ev->hdr_data, ev->mlength, diff --git a/ompi/mca/mtl/portals4/mtl_portals4_recv_short.c b/ompi/mca/mtl/portals4/mtl_portals4_recv_short.c index cddb6535820..e6d69083015 100644 --- a/ompi/mca/mtl/portals4/mtl_portals4_recv_short.c +++ b/ompi/mca/mtl/portals4/mtl_portals4_recv_short.c @@ -37,6 +37,7 @@ static int ompi_mtl_portals4_recv_block_progress(ptl_event_t *ev, ompi_mtl_portals4_base_request_t* ptl_base_request) { + int ret = OMPI_SUCCESS; ompi_mtl_portals4_recv_short_request_t *ptl_request = (ompi_mtl_portals4_recv_short_request_t*) ptl_base_request; ompi_mtl_portals4_recv_short_block_t *block = ptl_request->block; @@ -58,10 +59,10 @@ ompi_mtl_portals4_recv_block_progress(ptl_event_t *ev, opal_list_remove_item(&ompi_mtl_portals4.recv_short_blocks, &block->base); OPAL_THREAD_UNLOCK(&ompi_mtl_portals4.short_block_mutex); - ompi_mtl_portals4_recv_short_block_free(block); + ret = ompi_mtl_portals4_recv_short_block_free(block); } else { OPAL_THREAD_UNLOCK(&ompi_mtl_portals4.short_block_mutex); - ompi_mtl_portals4_activate_block(block); + ret = ompi_mtl_portals4_activate_block(block); } break; @@ -89,12 +90,12 @@ ompi_mtl_portals4_recv_block_progress(ptl_event_t *ev, opal_list_remove_item(&ompi_mtl_portals4.recv_short_blocks, &block->base); OPAL_THREAD_UNLOCK(&ompi_mtl_portals4.short_block_mutex); - ompi_mtl_portals4_recv_short_block_free(block); + ret = ompi_mtl_portals4_recv_short_block_free(block); } else { OPAL_THREAD_UNLOCK(&ompi_mtl_portals4.short_block_mutex); OPAL_OUTPUT_VERBOSE((10, ompi_mtl_base_framework.framework_output, "mtl:portals4 PTL_EVENT_AUTO_UNLINK received after PTL_EVENT_AUTO_FREE")); - ompi_mtl_portals4_activate_block(block); + ret = ompi_mtl_portals4_activate_block(block); } break; @@ -133,7 +134,7 @@ ompi_mtl_portals4_recv_block_progress(ptl_event_t *ev, break; } - return OMPI_SUCCESS; + return ret; } @@ -227,7 +228,8 @@ ompi_mtl_portals4_activate_block(ompi_mtl_portals4_recv_short_block_t *block) int ompi_mtl_portals4_recv_short_init(void) { - int i; + int ret = OMPI_SUCCESS; + uint32_t i; OBJ_CONSTRUCT(&ompi_mtl_portals4.short_block_mutex, opal_mutex_t); OBJ_CONSTRUCT(&(ompi_mtl_portals4.recv_short_blocks), opal_list_t); @@ -241,10 +243,10 @@ ompi_mtl_portals4_recv_short_init(void) } opal_list_append(&ompi_mtl_portals4.recv_short_blocks, &block->base); - ompi_mtl_portals4_activate_block(block); + ret = ompi_mtl_portals4_activate_block(block); } - return OMPI_SUCCESS; + return ret; } @@ -252,22 +254,24 @@ int ompi_mtl_portals4_recv_short_fini(void) { opal_list_item_t *item; + int ret = OMPI_SUCCESS; OPAL_THREAD_LOCK(&ompi_mtl_portals4.short_block_mutex); while (NULL != (item = opal_list_remove_first(&ompi_mtl_portals4.recv_short_blocks))) { ompi_mtl_portals4_recv_short_block_t *block = (ompi_mtl_portals4_recv_short_block_t*) item; - ompi_mtl_portals4_recv_short_block_free(block); + ret = ompi_mtl_portals4_recv_short_block_free(block); } OPAL_THREAD_UNLOCK(&ompi_mtl_portals4.short_block_mutex); - return OMPI_SUCCESS; + return ret; } int ompi_mtl_portals4_recv_short_link(int count) { + int ret = OMPI_SUCCESS; int active = ompi_mtl_portals4.active_recv_short_blocks; int i; @@ -278,9 +282,9 @@ ompi_mtl_portals4_recv_short_link(int count) if (NULL == block) { return OMPI_ERR_OUT_OF_RESOURCE; } - ompi_mtl_portals4_activate_block(block); + ret = ompi_mtl_portals4_activate_block(block); } } - return OMPI_SUCCESS; + return ret; } From 9cac32ba6a17cff92899bda698eb9a3be34c0197 Mon Sep 17 00:00:00 2001 From: Pascal Deveze Date: Tue, 19 Jul 2016 11:21:50 +0200 Subject: [PATCH 5/5] mtl/portals4: Modifications concerning the short message management --- ompi/mca/mtl/portals4/mtl_portals4_recv_short.c | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/ompi/mca/mtl/portals4/mtl_portals4_recv_short.c b/ompi/mca/mtl/portals4/mtl_portals4_recv_short.c index e6d69083015..936b92e1ec5 100644 --- a/ompi/mca/mtl/portals4/mtl_portals4_recv_short.c +++ b/ompi/mca/mtl/portals4/mtl_portals4_recv_short.c @@ -77,6 +77,7 @@ ompi_mtl_portals4_recv_block_progress(ptl_event_t *ev, break; case PTL_EVENT_AUTO_UNLINK: + block->me_h = PTL_INVALID_HANDLE; OPAL_THREAD_LOCK(&ompi_mtl_portals4.short_block_mutex); switch (block->status) { case BLOCK_STATUS_ACTIVATED: /* Normal case */ @@ -194,6 +195,7 @@ ompi_mtl_portals4_activate_block(ompi_mtl_portals4_recv_short_block_t *block) me.uid = ompi_mtl_portals4.uid; me.options = PTL_ME_OP_PUT | + PTL_ME_EVENT_COMM_DISABLE | PTL_ME_MANAGE_LOCAL | PTL_ME_MAY_ALIGN; if (ompi_mtl_portals4.use_logical) { @@ -261,6 +263,7 @@ ompi_mtl_portals4_recv_short_fini(void) ompi_mtl_portals4_recv_short_block_t *block = (ompi_mtl_portals4_recv_short_block_t*) item; ret = ompi_mtl_portals4_recv_short_block_free(block); + ompi_mtl_portals4.active_recv_short_blocks--; } OPAL_THREAD_UNLOCK(&ompi_mtl_portals4.short_block_mutex); @@ -278,10 +281,17 @@ ompi_mtl_portals4_recv_short_link(int count) if (active < count) { for (i = 0 ; i < (count - active) ; ++i) { ompi_mtl_portals4_recv_short_block_t *block = - ompi_mtl_portals4_recv_short_block_alloc(false); + ompi_mtl_portals4_recv_short_block_alloc(true); if (NULL == block) { return OMPI_ERR_OUT_OF_RESOURCE; } + OPAL_THREAD_LOCK(&ompi_mtl_portals4.short_block_mutex); + opal_list_append(&ompi_mtl_portals4.recv_short_blocks, + &block->base); + OPAL_OUTPUT_VERBOSE((10, ompi_mtl_base_framework.framework_output, + "recv_short_link: total=%d active=%d", + (int) opal_list_get_size(&ompi_mtl_portals4.recv_short_blocks), ompi_mtl_portals4.active_recv_short_blocks)); + OPAL_THREAD_UNLOCK(&ompi_mtl_portals4.short_block_mutex); ret = ompi_mtl_portals4_activate_block(block); } }