Skip to content

v4.0.x: pml/ob1: fix deadlock with communicator flag ALLOW_OVERTAKE. #6582

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 1 commit into from
Apr 10, 2019
Merged

v4.0.x: pml/ob1: fix deadlock with communicator flag ALLOW_OVERTAKE. #6582

Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
30 changes: 18 additions & 12 deletions ompi/mca/pml/ob1/pml_ob1_recvfrag.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@
* Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
* University Research and Technology
* Corporation. All rights reserved.
* Copyright (c) 2004-2018 The University of Tennessee and The University
* Copyright (c) 2004-2019 The University of Tennessee and The University
* of Tennessee Research Foundation. All rights
* reserved.
* Copyright (c) 2004-2007 High Performance Computing Center Stuttgart,
Expand Down Expand Up @@ -929,19 +929,21 @@ static int mca_pml_ob1_recv_frag_match( mca_btl_base_module_t *btl,
frag_msg_seq = hdr->hdr_seq;
next_msg_seq_expected = (uint16_t)proc->expected_sequence;

/* If the sequence number is wrong, queue it up for later. */
if(OPAL_UNLIKELY(frag_msg_seq != next_msg_seq_expected)) {
mca_pml_ob1_recv_frag_t* frag;
MCA_PML_OB1_RECV_FRAG_ALLOC(frag);
MCA_PML_OB1_RECV_FRAG_INIT(frag, hdr, segments, num_segments, btl);
append_frag_to_ordered_list(&proc->frags_cant_match, frag, next_msg_seq_expected);
if (!OMPI_COMM_CHECK_ASSERT_ALLOW_OVERTAKE(comm_ptr)) {
/* If the sequence number is wrong, queue it up for later. */
if(OPAL_UNLIKELY(frag_msg_seq != next_msg_seq_expected)) {
mca_pml_ob1_recv_frag_t* frag;
MCA_PML_OB1_RECV_FRAG_ALLOC(frag);
MCA_PML_OB1_RECV_FRAG_INIT(frag, hdr, segments, num_segments, btl);
append_frag_to_ordered_list(&proc->frags_cant_match, frag, next_msg_seq_expected);

SPC_RECORD(OMPI_SPC_OUT_OF_SEQUENCE, 1);
SPC_RECORD(OMPI_SPC_OOS_IN_QUEUE, 1);
SPC_UPDATE_WATERMARK(OMPI_SPC_MAX_OOS_IN_QUEUE, OMPI_SPC_OOS_IN_QUEUE);
SPC_RECORD(OMPI_SPC_OUT_OF_SEQUENCE, 1);
SPC_RECORD(OMPI_SPC_OOS_IN_QUEUE, 1);
SPC_UPDATE_WATERMARK(OMPI_SPC_MAX_OOS_IN_QUEUE, OMPI_SPC_OOS_IN_QUEUE);

OB1_MATCHING_UNLOCK(&comm->matching_lock);
return OMPI_SUCCESS;
OB1_MATCHING_UNLOCK(&comm->matching_lock);
return OMPI_SUCCESS;
}
}

/* mca_pml_ob1_recv_frag_match_proc() will release the lock. */
Expand Down Expand Up @@ -977,6 +979,10 @@ mca_pml_ob1_recv_frag_match_proc( mca_btl_base_module_t *btl,

match_this_frag:
/* We're now expecting the next sequence number. */
/* NOTE: We should have checked for ALLOW_OVERTAKE comm flag here
* but adding a branch in this critical path is not ideal for performance.
* We decided to let it run the sequence number even we are not doing
* anything with it. */
proc->expected_sequence++;

/* We generate the SEARCH_POSTED_QUEUE only when the message is
Expand Down