Merge pull request #2448 from thananon/remove_request_lock

Ralph Castain · web-flow · commit dadc6fbaf625 · 2017-01-03T19:31:46.000-08:00
Completely removed ompi_request_lock and ompi_request_cond
diff --git a/ompi/debuggers/ompi_mpihandles_dll.c b/ompi/debuggers/ompi_mpihandles_dll.c
@@ -86,10 +86,6 @@ struct mpidbg_name_map_t *mpidbg_status_name_map = NULL;
  * cases like the above with compilers that require the symbol (like
  * Sun Studio) we add in these definitions here.
  */
-size_t ompi_request_completed;
-opal_condition_t ompi_request_cond;
-size_t ompi_request_waiting;
-opal_mutex_t ompi_request_lock;
 opal_mutex_t opal_event_lock;
 int opal_progress_spin_count;
 bool opal_mutex_check_locks;
diff --git a/ompi/debuggers/ompi_msgq_dll.c b/ompi/debuggers/ompi_msgq_dll.c
@@ -1066,10 +1066,6 @@ static void dump_request( mqs_taddr_t current_item, mqs_pending_operation *res )
     printf( "+===============================================+\n\n" );
 }
 
-/**
- * TODO: ompi_request_completed can be used to detect any changes in the request handles.
- */
-
 /**
  * Handle the send queue as well as the receive queue. The unexpected queue
  * is a whole different story ...
diff --git a/ompi/mca/bcol/iboffload/bcol_iboffload_component.c b/ompi/mca/bcol/iboffload/bcol_iboffload_component.c
@@ -931,9 +931,8 @@ static int progress_one_device(mca_bcol_iboffload_device_t *device)
 
             if (coll_request->n_frag_mpi_complete ==
                             coll_request->n_fragments) {
-                coll_request->super.req_complete = true;
-                opal_condition_broadcast(&ompi_request_cond);
-                IBOFFLOAD_VERBOSE(10, ("After opal_condition_broadcast.\n"));
+                OPAL_ATOMIC_SWAP_PTR(&coll_request->super.reg_complete, REQUEST_COMPLETED);
+                IBOFFLOAD_VERBOSE(10, ("After request completion.\n"));
             }
 
             rc = handle_collfrag_done(coll_frag, coll_request, device);
diff --git a/ompi/mca/coll/libnbc/coll_libnbc_component.c b/ompi/mca/coll/libnbc/coll_libnbc_component.c
@@ -286,9 +286,7 @@ ompi_coll_libnbc_progress(void)
             else {
                 request->super.req_status.MPI_ERROR = res;
             }
-            OPAL_THREAD_LOCK(&ompi_request_lock);
             ompi_request_complete(&request->super, true);
-            OPAL_THREAD_UNLOCK(&ompi_request_lock);
         }
         OPAL_THREAD_LOCK(&mca_coll_libnbc_component.lock);
     }
diff --git a/ompi/mca/coll/portals4/coll_portals4_allreduce.c b/ompi/mca/coll/portals4/coll_portals4_allreduce.c
@@ -417,10 +417,7 @@ int
 ompi_coll_portals4_iallreduce_intra_fini(struct ompi_coll_portals4_request_t *request)
 {
     allreduce_kary_tree_bottom(request);
-
-    OPAL_THREAD_LOCK(&ompi_request_lock);
     ompi_request_complete(&request->super, true);
-    OPAL_THREAD_UNLOCK(&ompi_request_lock);
 
     return (OMPI_SUCCESS);
 }
diff --git a/ompi/mca/coll/portals4/coll_portals4_barrier.c b/ompi/mca/coll/portals4/coll_portals4_barrier.c
@@ -311,9 +311,7 @@ ompi_coll_portals4_ibarrier_intra_fini(ompi_coll_portals4_request_t *request)
         return OMPI_ERROR;
     }
 
-    OPAL_THREAD_LOCK(&ompi_request_lock);
     ompi_request_complete(&request->super, true);
-    OPAL_THREAD_UNLOCK(&ompi_request_lock);
 
     return OMPI_SUCCESS;
 }
diff --git a/ompi/mca/coll/portals4/coll_portals4_bcast.c b/ompi/mca/coll/portals4/coll_portals4_bcast.c
@@ -941,9 +941,7 @@ ompi_coll_portals4_ibcast_intra_fini(ompi_coll_portals4_request_t *request)
 
     post_bcast_data(request);
 
-    OPAL_THREAD_LOCK(&ompi_request_lock);
     ompi_request_complete(&request->super, true);
-    OPAL_THREAD_UNLOCK(&ompi_request_lock);
 
     opal_output_verbose(10, ompi_coll_base_framework.framework_output, "ibcast_intra_fini");
     return (OMPI_SUCCESS);
diff --git a/ompi/mca/coll/portals4/coll_portals4_gather.c b/ompi/mca/coll/portals4/coll_portals4_gather.c
@@ -1132,9 +1132,7 @@ ompi_coll_portals4_gather_intra_binomial_bottom(struct ompi_communicator_t *comm
 
     request->super.req_status.MPI_ERROR = OMPI_SUCCESS;
 
-    OPAL_THREAD_LOCK(&ompi_request_lock);
     ompi_request_complete(&request->super, true);
-    OPAL_THREAD_UNLOCK(&ompi_request_lock);
 
     OPAL_OUTPUT_VERBOSE((10, ompi_coll_base_framework.framework_output,
                  "coll:portals4:gather_intra_binomial_bottom exit rank %d", request->u.gather.my_rank));
@@ -1195,9 +1193,7 @@ ompi_coll_portals4_gather_intra_linear_bottom(struct ompi_communicator_t *comm,
 
     request->super.req_status.MPI_ERROR = OMPI_SUCCESS;
 
-    OPAL_THREAD_LOCK(&ompi_request_lock);
     ompi_request_complete(&request->super, true);
-    OPAL_THREAD_UNLOCK(&ompi_request_lock);
 
     OPAL_OUTPUT_VERBOSE((10, ompi_coll_base_framework.framework_output,
                  "coll:portals4:gather_intra_linear_bottom exit rank %d", request->u.gather.my_rank));
diff --git a/ompi/mca/coll/portals4/coll_portals4_reduce.c b/ompi/mca/coll/portals4/coll_portals4_reduce.c
@@ -441,9 +441,7 @@ ompi_coll_portals4_ireduce_intra_fini(ompi_coll_portals4_request_t *request)
     if (OMPI_SUCCESS != ret)
         return ret;
 
-    OPAL_THREAD_LOCK(&ompi_request_lock);
     ompi_request_complete(&request->super, true);
-    OPAL_THREAD_UNLOCK(&ompi_request_lock);
 
     return (OMPI_SUCCESS);
 }
diff --git a/ompi/mca/coll/portals4/coll_portals4_scatter.c b/ompi/mca/coll/portals4/coll_portals4_scatter.c
@@ -637,9 +637,7 @@ ompi_coll_portals4_scatter_intra_linear_bottom(struct ompi_communicator_t *comm,
 
     request->super.req_status.MPI_ERROR = OMPI_SUCCESS;
 
-    OPAL_THREAD_LOCK(&ompi_request_lock);
     ompi_request_complete(&request->super, true);
-    OPAL_THREAD_UNLOCK(&ompi_request_lock);
 
     OPAL_OUTPUT_VERBOSE((10, ompi_coll_base_framework.framework_output,
                  "coll:portals4:scatter_intra_linear_bottom exit rank %d", request->u.scatter.my_rank));
diff --git a/ompi/mca/osc/portals4/osc_portals4_component.c b/ompi/mca/osc/portals4/osc_portals4_component.c
@@ -229,9 +229,7 @@ progress_callback(void)
             opal_atomic_add_size_t(&req->super.req_status._ucount, ev.mlength);
             ops = opal_atomic_add_32(&req->ops_committed, 1);
             if (ops == req->ops_expected) {
-                OPAL_THREAD_LOCK(&ompi_request_lock);
                 ompi_request_complete(&req->super, true);
-                OPAL_THREAD_UNLOCK(&ompi_request_lock);
             }
         }
     }
diff --git a/ompi/mca/pml/cm/pml_cm_recvreq.c b/ompi/mca/pml/cm/pml_cm_recvreq.c
@@ -28,7 +28,6 @@ mca_pml_cm_recv_request_free(struct ompi_request_t** request)
 
     assert( false == recvreq->req_free_called );
 
-    OPAL_THREAD_LOCK(&ompi_request_lock);
     recvreq->req_free_called = true;
     if( true == recvreq->req_pml_complete ) {
         if( MCA_PML_CM_REQUEST_RECV_THIN == recvreq->req_pml_type ) {
@@ -38,8 +37,6 @@ mca_pml_cm_recv_request_free(struct ompi_request_t** request)
         }
     }
 
-    OPAL_THREAD_UNLOCK(&ompi_request_lock);
-
     *request = MPI_REQUEST_NULL;
     return OMPI_SUCCESS;
 }
diff --git a/ompi/mca/pml/cm/pml_cm_sendreq.c b/ompi/mca/pml/cm/pml_cm_sendreq.c
@@ -35,7 +35,6 @@ mca_pml_cm_send_request_free(struct ompi_request_t** request)
     mca_pml_cm_send_request_t* sendreq = *(mca_pml_cm_send_request_t**)request;
     assert( false == sendreq->req_base.req_free_called );
 
-    OPAL_THREAD_LOCK(&ompi_request_lock);
     sendreq->req_base.req_free_called = true;
     if( true == sendreq->req_base.req_pml_complete ) {
         if( MCA_PML_CM_REQUEST_SEND_THIN == sendreq->req_base.req_pml_type ) {
@@ -45,8 +44,6 @@ mca_pml_cm_send_request_free(struct ompi_request_t** request)
         }
     }
 
-    OPAL_THREAD_UNLOCK(&ompi_request_lock);
-
     *request = MPI_REQUEST_NULL;
     return OMPI_SUCCESS;
 }
diff --git a/ompi/request/request.c b/ompi/request/request.c
@@ -33,11 +33,6 @@
 #include "ompi/constants.h"
 
 opal_pointer_array_t             ompi_request_f_to_c_table = {{0}};
-size_t                           ompi_request_waiting = 0;
-size_t                           ompi_request_completed = 0;
-size_t                           ompi_request_failed = 0;
-opal_recursive_mutex_t           ompi_request_lock = {{0}};
-opal_condition_t                 ompi_request_cond = {{0}};
 ompi_predefined_request_t        ompi_request_null = {{{{{0}}}}};
 ompi_predefined_request_t        *ompi_request_null_addr = &ompi_request_null;
 ompi_request_t                   ompi_request_empty = {{{{0}}}};
@@ -109,8 +104,6 @@ OBJ_CLASS_INSTANCE(
 
 int ompi_request_init(void)
 {
-    OBJ_CONSTRUCT(&ompi_request_lock, opal_recursive_mutex_t);
-    OBJ_CONSTRUCT(&ompi_request_cond, opal_condition_t);
 
     OBJ_CONSTRUCT(&ompi_request_null, ompi_request_t);
     OBJ_CONSTRUCT(&ompi_request_f_to_c_table, opal_pointer_array_t);
@@ -186,8 +179,6 @@ int ompi_request_finalize(void)
     OBJ_DESTRUCT( &ompi_request_null.request );
     OMPI_REQUEST_FINI( &ompi_request_empty );
     OBJ_DESTRUCT( &ompi_request_empty );
-    OBJ_DESTRUCT( &ompi_request_cond );
-    OBJ_DESTRUCT( &ompi_request_lock );
     OBJ_DESTRUCT( &ompi_request_f_to_c_table );
     return OMPI_SUCCESS;
 }
diff --git a/ompi/request/request.h b/ompi/request/request.h
@@ -316,12 +316,6 @@ typedef struct ompi_request_fns_t {
  * Globals used for tracking requests and request completion.
  */
 OMPI_DECLSPEC extern opal_pointer_array_t   ompi_request_f_to_c_table;
-OMPI_DECLSPEC extern size_t                 ompi_request_waiting;
-OMPI_DECLSPEC extern size_t                 ompi_request_completed;
-OMPI_DECLSPEC extern size_t                 ompi_request_failed;
-OMPI_DECLSPEC extern int32_t                ompi_request_poll;
-OMPI_DECLSPEC extern opal_recursive_mutex_t ompi_request_lock;
-OMPI_DECLSPEC extern opal_condition_t       ompi_request_cond;
 OMPI_DECLSPEC extern ompi_predefined_request_t        ompi_request_null;
 OMPI_DECLSPEC extern ompi_predefined_request_t        *ompi_request_null_addr;
 OMPI_DECLSPEC extern ompi_request_t         ompi_request_empty;
@@ -433,10 +427,6 @@ static inline int ompi_request_complete(ompi_request_t* request, bool with_signa
             }
         } else
             request->req_complete = REQUEST_COMPLETED;
-
-        if( OPAL_UNLIKELY(MPI_SUCCESS != request->req_status.MPI_ERROR) ) {
-            ompi_request_failed++;
-        }
     }
 
     return OMPI_SUCCESS;

Original file line number	Diff line number	Diff line change
`@@ -286,9 +286,7 @@ ompi_coll_libnbc_progress(void)`
`286`	`286`	`else {`
`287`	`287`	`request->super.req_status.MPI_ERROR = res;`
`288`	`288`	`}`
`289`		`- OPAL_THREAD_LOCK(&ompi_request_lock);`
`290`	`289`	`ompi_request_complete(&request->super, true);`
`291`		`- OPAL_THREAD_UNLOCK(&ompi_request_lock);`
`292`	`290`	`}`
`293`	`291`	`OPAL_THREAD_LOCK(&mca_coll_libnbc_component.lock);`
`294`	`292`	`}`
Original file line number	Diff line number	Diff line change
`@@ -417,10 +417,7 @@ int`
`417`	`417`	`ompi_coll_portals4_iallreduce_intra_fini(struct ompi_coll_portals4_request_t *request)`
`418`	`418`	`{`
`419`	`419`	`allreduce_kary_tree_bottom(request);`
`420`		`-`
`421`		`- OPAL_THREAD_LOCK(&ompi_request_lock);`
`422`	`420`	`ompi_request_complete(&request->super, true);`
`423`		`- OPAL_THREAD_UNLOCK(&ompi_request_lock);`
`424`	`421`
`425`	`422`	`return (OMPI_SUCCESS);`
`426`	`423`	`}`
Original file line number	Diff line number	Diff line change
`@@ -311,9 +311,7 @@ ompi_coll_portals4_ibarrier_intra_fini(ompi_coll_portals4_request_t *request)`
`311`	`311`	`return OMPI_ERROR;`
`312`	`312`	`}`
`313`	`313`
`314`		`- OPAL_THREAD_LOCK(&ompi_request_lock);`
`315`	`314`	`ompi_request_complete(&request->super, true);`
`316`		`- OPAL_THREAD_UNLOCK(&ompi_request_lock);`
`317`	`315`
`318`	`316`	`return OMPI_SUCCESS;`
`319`	`317`	`}`
Original file line number	Diff line number	Diff line change
`@@ -441,9 +441,7 @@ ompi_coll_portals4_ireduce_intra_fini(ompi_coll_portals4_request_t *request)`
`441`	`441`	`if (OMPI_SUCCESS != ret)`
`442`	`442`	`return ret;`
`443`	`443`
`444`		`- OPAL_THREAD_LOCK(&ompi_request_lock);`
`445`	`444`	`ompi_request_complete(&request->super, true);`
`446`		`- OPAL_THREAD_UNLOCK(&ompi_request_lock);`
`447`	`445`
`448`	`446`	`return (OMPI_SUCCESS);`
`449`	`447`	`}`