osc/portals4: Be sure that the ME are operationnal (wait for the PTL_EVENT_LINK)

Author: PDeveze

ompi/mca/osc/portals4/osc_portals4.h

@@ -54,6 +54,8 @@ struct ompi_osc_portals4_component_t {
     ptl_size_t matching_fetch_atomic_max;
     ptl_size_t matching_atomic_ordered_size;
     ptl_uid_t uid;
+    opal_mutex_t lock;
+    opal_condition_t cond;
 
     opal_free_list_t requests; /* request free list for the r* communication variants */
 };
@@ -83,6 +85,7 @@ struct ompi_osc_portals4_module_t {
     ptl_handle_ni_t ni_h; /* network interface used by this window */
     ptl_pt_index_t pt_idx; /* portal table index used by this window (this will be same across window) */
     ptl_handle_ct_t ct_h; /* Counting event handle used for completion in this window */
+    int ct_link; /* PTL_EVENT_LINK flag */
     ptl_handle_md_t md_h; /* memory descriptor describing all of memory used by this window */
     ptl_handle_md_t req_md_h; /* memory descriptor with event completion used by this window */
     ptl_handle_me_t data_me_h; /* data match list entry (MB are CID | OSC_PORTALS4_MB_DATA) */

ompi/mca/osc/portals4/osc_portals4_component.c

@@ -218,6 +218,13 @@ progress_callback(void)
         count++;
 
         if (NULL != ev.user_ptr) {
+            /* be sure that we receive the PTL_EVENT_LINK */
+            if (ev.type == PTL_EVENT_LINK) {
+              *(int *)ev.user_ptr = *(int *)ev.user_ptr + 1;
+              opal_condition_broadcast(&mca_osc_portals4_component.cond);
+              continue;
+            }
+
             req = (ompi_osc_portals4_request_t*) ev.user_ptr;
             opal_atomic_add_size_t(&req->super.req_status._ucount, ev.mlength);
             ops = opal_atomic_add_32(&req->ops_committed, 1);
@@ -503,7 +510,7 @@ component_select(struct ompi_win_t *win, void **base, size_t size, int disp_unit
                       module->pt_idx,
                       &me,
                       PTL_PRIORITY_LIST,
-                      NULL,
+                      &module->ct_link,
                       &module->data_me_h);
     if (PTL_OK != ret) {
         opal_output_verbose(1, ompi_osc_base_framework.framework_output,
@@ -526,7 +533,7 @@ component_select(struct ompi_win_t *win, void **base, size_t size, int disp_unit
                       module->pt_idx,
                       &me,
                       PTL_PRIORITY_LIST,
-                      NULL,
+                      &module->ct_link,
                       &module->control_me_h);
     if (PTL_OK != ret) {
         opal_output_verbose(1, ompi_osc_base_framework.framework_output,
@@ -574,6 +581,13 @@ component_select(struct ompi_win_t *win, void **base, size_t size, int disp_unit
     PtlAtomicSync();
 
     /* Make sure that everyone's ready to receive. */
+    OPAL_THREAD_LOCK(&mca_osc_portals4_component.lock);
+    while (module->ct_link != 2) {
+        opal_condition_wait(&mca_osc_portals4_component.cond,
+                            &mca_osc_portals4_component.lock);
+    }
+    OPAL_THREAD_UNLOCK(&mca_osc_portals4_component.lock);
+
     module->comm->c_coll.coll_barrier(module->comm,
                                       module->comm->c_coll.coll_barrier_module);
 
@@ -618,6 +632,7 @@ ompi_osc_portals4_free(struct ompi_win_t *win)
                                       module->comm->c_coll.coll_barrier_module);
 
     /* cleanup */
+    PtlMEUnlink(module->control_me_h);
     PtlMEUnlink(module->data_me_h);
     PtlMDRelease(module->md_h);
     PtlMDRelease(module->req_md_h);