Skip to content

Commit 2329c3c

Browse files
hzhouhzhou
authored
Merge pull request pmodels#7326 from hzhou/2503_vcis_2
ch4/ofi: refactor vci and nic address exchange Approved-by: Ken Raffenetti
2 parents 1b5bffb + fc11664 commit 2329c3c

File tree

20 files changed

+496
-492
lines changed

20 files changed

+496
-492
lines changed

‎src/mpid/ch4/netmod/ofi/coll/ofi_coll_util.h‎

Lines changed: 29 additions & 28 deletions
Original file line numberDiff line numberDiff line change
@@ -96,9 +96,10 @@ MPL_STATIC_INLINE_PREFIX int MPIDI_OFI_Ibcast_knomial_triggered_tagged(void *buf
9696
/* Post recv for RTR from children */
9797
for (p = (int *) utarray_front(my_tree->children); p != NULL;
9898
p = (int *) utarray_next(my_tree->children, p)) {
99+
MPIDI_av_entry_t *av = MPIDIU_comm_rank_to_av(comm_ptr, *p);
99100
ret = MPIDI_OFI_prepare_tagged_control_cmd(MPIDI_OFI_global.ctx[0].domain,
100101
MPIDI_OFI_global.ctx[0].rx, &((*works)[i + j]),
101-
MPIDI_OFI_comm_to_phys(comm_ptr, *p, 0, 0),
102+
MPIDI_OFI_av_to_phys_root(av),
102103
MPIDI_OFI_TRIGGERED_TAGGED_RECV, NULL, 0,
103104
rtr_tag, comm_ptr, *p, 0, *rcv_cntr, *rcv_cntr,
104105
false);
@@ -114,10 +115,10 @@ MPL_STATIC_INLINE_PREFIX int MPIDI_OFI_Ibcast_knomial_triggered_tagged(void *buf
114115
MPIR_ERR_POP(mpi_errno);
115116

116117
if (!is_root) { /* non-root nodes post recv for data from parents */
118+
MPIDI_av_entry_t *av = MPIDIU_comm_rank_to_av(comm_ptr, parent);
117119
mpi_errno = MPIDI_OFI_prepare_tagged_control_cmd(MPIDI_OFI_global.ctx[0].domain,
118120
MPIDI_OFI_global.ctx[0].rx, &((*works)[i]),
119-
MPIDI_OFI_comm_to_phys(comm_ptr, parent,
120-
0, 0),
121+
MPIDI_OFI_av_to_phys_root(av),
121122
MPIDI_OFI_TRIGGERED_TAGGED_RECV, buffer,
122123
count * data_sz, tag, comm_ptr, parent, 0,
123124
*rcv_cntr, *rcv_cntr, false);
@@ -132,10 +133,10 @@ MPL_STATIC_INLINE_PREFIX int MPIDI_OFI_Ibcast_knomial_triggered_tagged(void *buf
132133
if (!is_root) { /* Non-root nodes send RTR to parents */
133134
uint64_t match_bits =
134135
MPIDI_OFI_init_sendtag(comm_ptr->context_id + context_offset, comm_ptr->rank, rtr_tag);
136+
MPIDI_av_entry_t *av = MPIDIU_comm_rank_to_av(comm_ptr, parent);
135137
MPIDI_OFI_CALL_RETRY(fi_tinject
136138
(MPIDI_OFI_global.ctx[0].tx, NULL, 0,
137-
MPIDI_OFI_comm_to_phys(comm_ptr, parent, 0, 0), match_bits), 0,
138-
tinject);
139+
MPIDI_OFI_av_to_phys_root(av), match_bits), 0, tinject);
139140
}
140141

141142
if (is_root) {
@@ -148,10 +149,11 @@ MPL_STATIC_INLINE_PREFIX int MPIDI_OFI_Ibcast_knomial_triggered_tagged(void *buf
148149
/* Root and intermediate nodes send data to children */
149150
for (p = (int *) utarray_front(my_tree->children); p != NULL;
150151
p = (int *) utarray_next(my_tree->children, p)) {
152+
MPIDI_av_entry_t *av = MPIDIU_comm_rank_to_av(comm_ptr, *p);
151153
mpi_errno = MPIDI_OFI_prepare_tagged_control_cmd(MPIDI_OFI_global.ctx[0].domain,
152154
MPIDI_OFI_global.ctx[0].tx,
153155
&((*works)[index + k]),
154-
MPIDI_OFI_comm_to_phys(comm_ptr, *p, 0, 0),
156+
MPIDI_OFI_av_to_phys_root(av),
155157
MPIDI_OFI_TRIGGERED_TAGGED_SEND, buffer,
156158
count * data_sz, tag, comm_ptr, *p,
157159
threshold, *rcv_cntr, *snd_cntr, false);
@@ -243,12 +245,11 @@ MPL_STATIC_INLINE_PREFIX int MPIDI_OFI_Ibcast_kary_triggered_tagged(void *buffer
243245

244246
/* Post recv for RTR from children */
245247
for (j = 0; j < num_children; j++) {
248+
MPIDI_av_entry_t *av = MPIDIU_comm_rank_to_av(comm_ptr, first_child + j);
246249
mpi_errno = MPIDI_OFI_prepare_tagged_control_cmd(MPIDI_OFI_global.ctx[0].domain,
247250
MPIDI_OFI_global.ctx[0].rx,
248251
&((*works)[i + j]),
249-
MPIDI_OFI_comm_to_phys(comm_ptr,
250-
first_child + j, 0,
251-
0),
252+
MPIDI_OFI_av_to_phys_root(av),
252253
MPIDI_OFI_TRIGGERED_TAGGED_RECV, NULL, 0,
253254
rtr_tag, comm_ptr, first_child + j, 0,
254255
*rcv_cntr, *rcv_cntr, false);
@@ -263,10 +264,10 @@ MPL_STATIC_INLINE_PREFIX int MPIDI_OFI_Ibcast_kary_triggered_tagged(void *buffer
263264
MPIR_ERR_POP(mpi_errno);
264265

265266
if (!is_root) { /* Non-root nodes post recv for data */
267+
MPIDI_av_entry_t *av = MPIDIU_comm_rank_to_av(comm_ptr, parent);
266268
mpi_errno = MPIDI_OFI_prepare_tagged_control_cmd(MPIDI_OFI_global.ctx[0].domain,
267269
MPIDI_OFI_global.ctx[0].rx, &((*works)[i]),
268-
MPIDI_OFI_comm_to_phys(comm_ptr, parent, 0,
269-
0),
270+
MPIDI_OFI_av_to_phys_root(av),
270271
MPIDI_OFI_TRIGGERED_TAGGED_RECV, buffer,
271272
count * data_sz, tag, comm_ptr, parent, 0,
272273
*rcv_cntr, *rcv_cntr, false);
@@ -281,10 +282,10 @@ MPL_STATIC_INLINE_PREFIX int MPIDI_OFI_Ibcast_kary_triggered_tagged(void *buffer
281282
if (!is_root) { /* Non-root nodes send RTR to parents */
282283
s_match_bits =
283284
MPIDI_OFI_init_sendtag(comm_ptr->context_id + context_offset, comm_ptr->rank, rtr_tag);
285+
MPIDI_av_entry_t *av = MPIDIU_comm_rank_to_av(comm_ptr, parent);
284286
MPIDI_OFI_CALL_RETRY(fi_tinject
285287
(MPIDI_OFI_global.ctx[0].tx, NULL, 0,
286-
MPIDI_OFI_comm_to_phys(comm_ptr, parent, 0, 0), s_match_bits), 0,
287-
tinject);
288+
MPIDI_OFI_av_to_phys_root(av), s_match_bits), 0, tinject);
288289
}
289290

290291
if (is_root) {
@@ -296,12 +297,11 @@ MPL_STATIC_INLINE_PREFIX int MPIDI_OFI_Ibcast_kary_triggered_tagged(void *buffer
296297

297298
/* Root and intremediate nodes send data to children */
298299
for (k = 0; k < num_children; k++) {
300+
MPIDI_av_entry_t *av = MPIDIU_comm_rank_to_av(comm_ptr, first_child + k);
299301
mpi_errno = MPIDI_OFI_prepare_tagged_control_cmd(MPIDI_OFI_global.ctx[0].domain,
300302
MPIDI_OFI_global.ctx[0].tx,
301303
&((*works)[index + k]),
302-
MPIDI_OFI_comm_to_phys(comm_ptr,
303-
first_child + k, 0,
304-
0),
304+
MPIDI_OFI_av_to_phys_root(av),
305305
MPIDI_OFI_TRIGGERED_TAGGED_SEND, buffer,
306306
count * data_sz, tag, comm_ptr,
307307
first_child + k, threshold, *rcv_cntr,
@@ -424,10 +424,11 @@ MPL_STATIC_INLINE_PREFIX int MPIDI_OFI_Ibcast_knomial_triggered_rma(void *buffer
424424
/* Post recv for RTR from children; this is needed to avoid unexpected messages */
425425
for (p = (int *) utarray_front(my_tree->children); p != NULL;
426426
p = (int *) utarray_next(my_tree->children, p)) {
427+
MPIDI_av_entry_t *av = MPIDIU_comm_rank_to_av(comm_ptr, *p);
427428
mpi_errno = MPIDI_OFI_prepare_tagged_control_cmd(MPIDI_OFI_global.ctx[0].domain,
428429
MPIDI_OFI_global.ctx[0].rx,
429430
&((*works)[i + j]),
430-
MPIDI_OFI_comm_to_phys(comm_ptr, *p, 0, 0),
431+
MPIDI_OFI_av_to_phys_root(av),
431432
MPIDI_OFI_TRIGGERED_TAGGED_RECV, NULL, 0,
432433
rtr_tag, comm_ptr, *p, 0, *rcv_cntr,
433434
*rcv_cntr, false);
@@ -443,10 +444,9 @@ MPL_STATIC_INLINE_PREFIX int MPIDI_OFI_Ibcast_knomial_triggered_rma(void *buffer
443444
if (!is_root) { /* Non-root nodes send RTR to parents */
444445
s_match_bits =
445446
MPIDI_OFI_init_sendtag(comm_ptr->context_id + context_offset, comm_ptr->rank, rtr_tag);
446-
MPIDI_OFI_CALL_RETRY(fi_tinject
447-
(MPIDI_OFI_global.ctx[0].tx, NULL, 0,
448-
MPIDI_OFI_comm_to_phys(comm_ptr, parent, 0, 0), s_match_bits), 0,
449-
tinject);
447+
MPIDI_av_entry_t *av = MPIDIU_comm_rank_to_av(comm_ptr, parent);
448+
MPIDI_OFI_CALL_RETRY(fi_tinject(MPIDI_OFI_global.ctx[0].tx, NULL, 0,
449+
MPIDI_OFI_av_to_phys_root(av), s_match_bits), 0, tinject);
450450
}
451451

452452
if (is_root) {
@@ -458,9 +458,10 @@ MPL_STATIC_INLINE_PREFIX int MPIDI_OFI_Ibcast_knomial_triggered_rma(void *buffer
458458
/* Root and intermediate nodes send data to children */
459459
for (p = (int *) utarray_front(my_tree->children); p != NULL;
460460
p = (int *) utarray_next(my_tree->children, p)) {
461+
MPIDI_av_entry_t *av = MPIDIU_comm_rank_to_av(comm_ptr, *p);
461462
mpi_errno = MPIDI_OFI_prepare_rma_control_cmd(MPIDI_OFI_global.ctx[0].domain,
462463
MPIDI_OFI_global.ctx[0].tx, &((*works)[i++]),
463-
MPIDI_OFI_comm_to_phys(comm_ptr, *p, 0, 0),
464+
MPIDI_OFI_av_to_phys_root(av),
464465
buffer, fi_mr_key(*r_mr),
465466
count * data_sz, threshold, *rcv_cntr,
466467
*snd_cntr, MPIDI_OFI_TRIGGERED_RMA_WRITE,
@@ -567,12 +568,11 @@ MPL_STATIC_INLINE_PREFIX int MPIDI_OFI_Ibcast_kary_triggered_rma(void *buffer, i
567568

568569
/* Post recv for RTR from children */
569570
for (j = 0; j < num_children; j++) {
571+
MPIDI_av_entry_t *av = MPIDIU_comm_rank_to_av(comm_ptr, first_child + j);
570572
mpi_errno = MPIDI_OFI_prepare_tagged_control_cmd(MPIDI_OFI_global.ctx[0].domain,
571573
MPIDI_OFI_global.ctx[0].rx,
572574
&((*works)[i + j]),
573-
MPIDI_OFI_comm_to_phys(comm_ptr,
574-
first_child + j, 0,
575-
0),
575+
MPIDI_OFI_av_to_phys_root(av),
576576
MPIDI_OFI_TRIGGERED_TAGGED_RECV, NULL, 0,
577577
rtr_tag, comm_ptr, first_child + j, 0,
578578
*rcv_cntr, *rcv_cntr, false);
@@ -587,10 +587,10 @@ MPL_STATIC_INLINE_PREFIX int MPIDI_OFI_Ibcast_kary_triggered_rma(void *buffer, i
587587
if (!is_root) { /* Non-root nodes send RTR to parents; this is needed to avoid unexpected messages */
588588
s_match_bits =
589589
MPIDI_OFI_init_sendtag(comm_ptr->context_id + context_offset, comm_ptr->rank, rtr_tag);
590+
MPIDI_av_entry_t *av = MPIDIU_comm_rank_to_av(comm_ptr, parent);
590591
MPIDI_OFI_CALL_RETRY(fi_tinject
591592
(MPIDI_OFI_global.ctx[0].tx, NULL, 0,
592-
MPIDI_OFI_comm_to_phys(comm_ptr, parent, 0, 0), s_match_bits), 0,
593-
tinject);
593+
MPIDI_OFI_av_to_phys_root(av), s_match_bits), 0, tinject);
594594
}
595595

596596
if (is_root) {
@@ -602,9 +602,10 @@ MPL_STATIC_INLINE_PREFIX int MPIDI_OFI_Ibcast_kary_triggered_rma(void *buffer, i
602602

603603
/* Root and intremediate nodes send data to children */
604604
for (k = 0; k < num_children; k++) {
605+
MPIDI_av_entry_t *av = MPIDIU_comm_rank_to_av(comm_ptr, first_child + k);
605606
MPIDI_OFI_prepare_rma_control_cmd(MPIDI_OFI_global.ctx[0].domain,
606607
MPIDI_OFI_global.ctx[0].tx, &((*works)[i + k]),
607-
MPIDI_OFI_comm_to_phys(comm_ptr, first_child + k, 0, 0),
608+
MPIDI_OFI_av_to_phys_root(av),
608609
buffer, fi_mr_key(*r_mr),
609610
count * data_sz, threshold, *rcv_cntr, *snd_cntr,
610611
MPIDI_OFI_TRIGGERED_RMA_WRITE, false);

0 commit comments

Comments
 (0)