Skip to content

Commit 5959ce7

Browse files
shasson5shasson5
authored
UCT/IB: Split dp_ordering flag for DV/DevX (#10813)
1 parent e16f386 commit 5959ce7

File tree

10 files changed

+75
-37
lines changed

10 files changed

+75
-37
lines changed

‎src/uct/ib/mlx5/dc/dc_mlx5.c‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1616,7 +1616,8 @@ static UCS_CLASS_INIT_FUNC(uct_dc_mlx5_iface_t, uct_md_h tl_md, uct_worker_h wor
16161616
init_attr.flags |= UCT_IB_TM_SUPPORTED;
16171617
}
16181618

1619-
if (md->dp_ordering_cap.dc == UCT_IB_MLX5_DP_ORDERING_OOO_ALL) {
1619+
if ((md->dp_ordering_cap_devx.dc == UCT_IB_MLX5_DP_ORDERING_OOO_ALL) ||
1620+
md->ddp_support_dv.dc) {
16201621
init_attr.flags |= UCT_IB_DDP_SUPPORTED;
16211622
}
16221623

@@ -1628,7 +1629,8 @@ static UCS_CLASS_INIT_FUNC(uct_dc_mlx5_iface_t, uct_md_h tl_md, uct_worker_h wor
16281629
init_attr.cq_len[UCT_IB_DIR_TX] = sq_length * self->tx.ndci;
16291630

16301631
status = uct_rc_mlx5_dp_ordering_ooo_init(md, &self->super,
1631-
md->dp_ordering_cap.dc,
1632+
md->dp_ordering_cap_devx.dc,
1633+
md->ddp_support_dv.dc,
16321634
&config->rc_mlx5_common, "dc");
16331635
if (status != UCS_OK) {
16341636
return status;

‎src/uct/ib/mlx5/dc/dc_mlx5_devx.c‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -50,9 +50,9 @@ ucs_status_t uct_dc_mlx5_iface_devx_create_dct(uct_dc_mlx5_iface_t *iface)
5050
UCT_IB_MLX5DV_SET(dctc, dctc, atomic_mode,
5151
uct_ib_mlx5_get_atomic_mode(ib_iface));
5252
UCT_IB_MLX5DV_SET(dctc, dctc, dp_ordering_0,
53-
UCS_BIT_GET(iface->super.config.dp_ordering, 0));
53+
UCS_BIT_GET(iface->super.config.dp_ordering_devx, 0));
5454
UCT_IB_MLX5DV_SET(dctc, dctc, dp_ordering_1,
55-
UCS_BIT_GET(iface->super.config.dp_ordering, 1));
55+
UCS_BIT_GET(iface->super.config.dp_ordering_devx, 1));
5656
UCT_IB_MLX5DV_SET(dctc, dctc, dp_ordering_force,
5757
iface->super.config.dp_ordering_force);
5858

‎src/uct/ib/mlx5/dv/ib_mlx5dv_md.c‎

Lines changed: 22 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -38,8 +38,9 @@ uct_ib_mlx5_md_check_odp_common(uct_ib_mlx5_md_t *md, const char **reason_ptr)
3838
}
3939

4040
/* Issue 4238670 */
41-
if ((md->dp_ordering_cap.rc == UCT_IB_MLX5_DP_ORDERING_OOO_ALL) ||
42-
(md->dp_ordering_cap.dc == UCT_IB_MLX5_DP_ORDERING_OOO_ALL)) {
41+
if ((md->dp_ordering_cap_devx.rc == UCT_IB_MLX5_DP_ORDERING_OOO_ALL) ||
42+
(md->dp_ordering_cap_devx.dc == UCT_IB_MLX5_DP_ORDERING_OOO_ALL) ||
43+
md->ddp_support_dv.rc || md->ddp_support_dv.dc) {
4344
*reason_ptr = "ODP does not work with DDP";
4445
return 0;
4546
}
@@ -2022,19 +2023,19 @@ static void uct_ib_mlx5_devx_check_dp_ordering(uct_ib_mlx5_md_t *md, void *cap,
20222023
uct_ib_device_t *dev)
20232024
{
20242025
if (UCT_IB_MLX5DV_GET(cmd_hca_cap, cap, dp_ordering_ooo_all_rc)) {
2025-
md->dp_ordering_cap.rc = UCT_IB_MLX5_DP_ORDERING_OOO_ALL;
2026+
md->dp_ordering_cap_devx.rc = UCT_IB_MLX5_DP_ORDERING_OOO_ALL;
20262027
} else if (UCT_IB_MLX5DV_GET(cmd_hca_cap, cap, dp_ordering_ooo_rw_rc)) {
2027-
md->dp_ordering_cap.rc = UCT_IB_MLX5_DP_ORDERING_OOO_RW;
2028+
md->dp_ordering_cap_devx.rc = UCT_IB_MLX5_DP_ORDERING_OOO_RW;
20282029
} else {
2029-
md->dp_ordering_cap.rc = UCT_IB_MLX5_DP_ORDERING_IBTA;
2030+
md->dp_ordering_cap_devx.rc = UCT_IB_MLX5_DP_ORDERING_IBTA;
20302031
}
20312032

20322033
if (UCT_IB_MLX5DV_GET(cmd_hca_cap, cap, dp_ordering_ooo_all_dc)) {
2033-
md->dp_ordering_cap.dc = UCT_IB_MLX5_DP_ORDERING_OOO_ALL;
2034+
md->dp_ordering_cap_devx.dc = UCT_IB_MLX5_DP_ORDERING_OOO_ALL;
20342035
} else if (UCT_IB_MLX5DV_GET(cmd_hca_cap, cap, dp_ordering_ooo_rw_dc)) {
2035-
md->dp_ordering_cap.dc = UCT_IB_MLX5_DP_ORDERING_OOO_RW;
2036+
md->dp_ordering_cap_devx.dc = UCT_IB_MLX5_DP_ORDERING_OOO_RW;
20362037
} else {
2037-
md->dp_ordering_cap.dc = UCT_IB_MLX5_DP_ORDERING_IBTA;
2038+
md->dp_ordering_cap_devx.dc = UCT_IB_MLX5_DP_ORDERING_IBTA;
20382039
}
20392040

20402041
if ((cap_2 != NULL) &&
@@ -2045,7 +2046,7 @@ static void uct_ib_mlx5_devx_check_dp_ordering(uct_ib_mlx5_md_t *md, void *cap,
20452046
ucs_debug("%s: dp_ordering support: force=%d ooo_rw_rc=%d ooo_rw_dc=%d",
20462047
uct_ib_device_name(dev),
20472048
!!(md->flags & UCT_IB_MLX5_MD_FLAG_DP_ORDERING_FORCE),
2048-
md->dp_ordering_cap.rc, md->dp_ordering_cap.dc);
2049+
md->dp_ordering_cap_devx.rc, md->dp_ordering_cap_devx.dc);
20492050
}
20502051

20512052
static void uct_ib_mlx5_devx_check_mkey_by_name(uct_ib_mlx5_md_t *md,
@@ -2280,6 +2281,9 @@ static void uct_ib_mlx5dv_check_dm_ksm_reg(uct_ib_mlx5_md_t *md)
22802281
#endif
22812282
}
22822283

2284+
static ucs_status_t
2285+
uct_ib_mlx5dv_check_ddp(struct ibv_context *ctx, uct_ib_mlx5_md_t *md);
2286+
22832287
ucs_status_t uct_ib_mlx5_devx_md_open_common(const char *name, size_t size,
22842288
struct ibv_device *ibv_device,
22852289
const uct_ib_md_config_t *md_config,
@@ -2477,6 +2481,11 @@ ucs_status_t uct_ib_mlx5_devx_md_open_common(const char *name, size_t size,
24772481

24782482
uct_ib_mlx5_devx_check_dp_ordering(md, cap, cap_2, dev);
24792483

2484+
status = uct_ib_mlx5dv_check_ddp(ctx, md);
2485+
if (status != UCS_OK) {
2486+
goto err_lru_cleanup;
2487+
}
2488+
24802489
uct_ib_mlx5_devx_check_odp(md, md_config, cap);
24812490

24822491
uct_ib_mlx5dv_check_direct_nic(ctx, dev, md, md_config);
@@ -3253,15 +3262,15 @@ uct_ib_mlx5dv_check_ddp(struct ibv_context *ctx, uct_ib_mlx5_md_t *md)
32533262
}
32543263

32553264
if (ctx_dv.ooo_recv_wrs_caps.max_rc > 0) {
3256-
md->dp_ordering_cap.rc = UCT_IB_MLX5_DP_ORDERING_OOO_ALL;
3265+
md->ddp_support_dv.rc = 1;
32573266
}
32583267

32593268
if (ctx_dv.ooo_recv_wrs_caps.max_dct > 0) {
3260-
md->dp_ordering_cap.dc = UCT_IB_MLX5_DP_ORDERING_OOO_ALL;
3269+
md->ddp_support_dv.dc = 1;
32613270
}
32623271
#else
3263-
md->dp_ordering_cap.rc = UCT_IB_MLX5_DP_ORDERING_IBTA;
3264-
md->dp_ordering_cap.dc = UCT_IB_MLX5_DP_ORDERING_IBTA;
3272+
md->ddp_support_dv.rc = 0;
3273+
md->ddp_support_dv.dc = 0;
32653274
#endif
32663275
return UCS_OK;
32673276
}

‎src/uct/ib/mlx5/gdaki/gdaki.c‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -466,7 +466,8 @@ static UCS_CLASS_INIT_FUNC(uct_rc_gdaki_iface_t, uct_md_h tl_md,
466466
int cuda_id;
467467

468468
status = uct_rc_mlx5_dp_ordering_ooo_init(md, &self->super,
469-
md->dp_ordering_cap.rc,
469+
md->dp_ordering_cap_devx.rc,
470+
md->ddp_support_dv.rc,
470471
&config->mlx5, "rc_gda");
471472
if (status != UCS_OK) {
472473
return status;

‎src/uct/ib/mlx5/gga/gga_mlx5.c‎

Lines changed: 5 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -790,18 +790,19 @@ static UCS_CLASS_INIT_FUNC(uct_gga_mlx5_iface_t,
790790
uct_ib_mlx5_md_t *md = ucs_derived_of(tl_md, uct_ib_mlx5_md_t);
791791
uct_ib_iface_init_attr_t init_attr = {};
792792
ucs_status_t status;
793+
uct_ib_mlx5_dp_ordering_t dp_ordering;
793794

794795
init_attr.qp_type = IBV_QPT_RC;
795796
init_attr.cq_len[UCT_IB_DIR_TX] = config->super.tx_cq_len;
796797
init_attr.max_rd_atomic = IBV_DEV_ATTR(&md->super.dev,
797798
max_qp_rd_atom);
798799
init_attr.tx_moderation = config->super.tx_cq_moderation;
799800
init_attr.dev_name = params->mode.device.dev_name;
801+
dp_ordering = ucs_min(md->dp_ordering_cap_devx.rc,
802+
UCT_IB_MLX5_DP_ORDERING_OOO_RW);
800803

801-
status = uct_rc_mlx5_dp_ordering_ooo_init(
802-
md, &self->super,
803-
ucs_min(md->dp_ordering_cap.rc, UCT_IB_MLX5_DP_ORDERING_OOO_RW),
804-
&config->rc_mlx5_common, "gga");
804+
status = uct_rc_mlx5_dp_ordering_ooo_init(md, &self->super, dp_ordering, 0,
805+
&config->rc_mlx5_common, "gga");
805806
if (status != UCS_OK) {
806807
return status;
807808
}

‎src/uct/ib/mlx5/ib_mlx5.h‎

Lines changed: 7 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -441,11 +441,16 @@ typedef struct uct_ib_mlx5_md {
441441
uint8_t log_max_dci_stream_channels;
442442
uint32_t smkey_index;
443443
struct {
444-
/* Max dp ordering level per transport,
444+
/* Max dp ordering level per transport in DevX,
445445
as listed in uct_ib_mlx5_dp_ordering_t */
446446
uint8_t rc;
447447
uint8_t dc;
448-
} dp_ordering_cap;
448+
} dp_ordering_cap_devx;
449+
struct {
450+
/* DDP support per transport in DV API */
451+
uint8_t rc;
452+
uint8_t dc;
453+
} ddp_support_dv;
449454
} uct_ib_mlx5_md_t;
450455

451456

‎src/uct/ib/mlx5/rc/rc_mlx5_common.c‎

Lines changed: 24 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -604,7 +604,8 @@ void uct_rc_mlx5_release_desc(uct_recv_desc_t *self, void *desc)
604604
ucs_status_t
605605
uct_rc_mlx5_dp_ordering_ooo_init(uct_ib_mlx5_md_t *md,
606606
uct_rc_mlx5_iface_common_t *iface,
607-
uct_ib_mlx5_dp_ordering_t dp_ordering_cap,
607+
uct_ib_mlx5_dp_ordering_t dp_ordering_cap_devx,
608+
int ddp_supported_dv,
608609
uct_rc_mlx5_iface_common_config_t *config,
609610
const char *tl_name)
610611
{
@@ -613,11 +614,25 @@ uct_rc_mlx5_dp_ordering_ooo_init(uct_ib_mlx5_md_t *md,
613614
[UCT_IB_MLX5_DP_ORDERING_OOO_RW] = "OOO_RW",
614615
[UCT_IB_MLX5_DP_ORDERING_OOO_ALL] = "OOO_ALL"
615616
};
616-
uct_ib_mlx5_dp_ordering_t max_dp_ordering = dp_ordering_cap;
617+
uct_ib_mlx5_dp_ordering_t max_dp_ordering = dp_ordering_cap_devx;
617618
uct_ib_mlx5_dp_ordering_t min_dp_ordering = UCT_IB_MLX5_DP_ORDERING_IBTA;
618619
char ar_enable_str[16], ddp_enable_str[16];
619620
int force;
620621

622+
if (!(md->flags & UCT_IB_MLX5_MD_FLAG_DEVX)) {
623+
if ((config->ddp_enable == UCS_YES) && !ddp_supported_dv) {
624+
ucs_error("%s/%s: ddp is not supported for DV",
625+
uct_ib_device_name(&md->super.dev), tl_name);
626+
return UCS_ERR_INVALID_PARAM;
627+
}
628+
629+
iface->config.ddp_enabled_dv = (config->ddp_enable != UCS_NO) &&
630+
ddp_supported_dv;
631+
iface->config.dp_ordering_devx = UCT_IB_MLX5_DP_ORDERING_IBTA;
632+
iface->config.dp_ordering_force = 0;
633+
return UCS_OK;
634+
}
635+
621636
/*
622637
* HCA has an mlxreg admin configuration to force enable adaptive routing
623638
* (AR) or not.
@@ -657,7 +672,7 @@ uct_rc_mlx5_dp_ordering_ooo_init(uct_ib_mlx5_md_t *md,
657672
* but can't force it
658673
*/
659674
force = (min_dp_ordering > UCT_IB_MLX5_DP_ORDERING_IBTA) ||
660-
(max_dp_ordering < dp_ordering_cap);
675+
(max_dp_ordering < dp_ordering_cap_devx);
661676
if ((min_dp_ordering > max_dp_ordering) ||
662677
(force && !(md->flags & UCT_IB_MLX5_MD_FLAG_DP_ORDERING_FORCE))) {
663678
ucs_config_sprintf_ternary_auto(ar_enable_str, sizeof(ar_enable_str),
@@ -667,15 +682,16 @@ uct_rc_mlx5_dp_ordering_ooo_init(uct_ib_mlx5_md_t *md,
667682
ucs_error("%s/%s: cannot set ar_enable=%s ddp_enable=%s cap=%s "
668683
"supp_force=%d (min=%s max=%s force=%d)",
669684
uct_ib_device_name(&md->super.dev), tl_name, ar_enable_str,
670-
ddp_enable_str, dp_ordering_names[dp_ordering_cap],
685+
ddp_enable_str, dp_ordering_names[dp_ordering_cap_devx],
671686
!!(md->flags & UCT_IB_MLX5_MD_FLAG_DP_ORDERING_FORCE),
672687
dp_ordering_names[min_dp_ordering],
673688
dp_ordering_names[max_dp_ordering], force);
674689
return UCS_ERR_INVALID_PARAM;
675690
}
676691

677-
iface->config.dp_ordering = max_dp_ordering;
692+
iface->config.dp_ordering_devx = max_dp_ordering;
678693
iface->config.dp_ordering_force = force;
694+
iface->config.ddp_enabled_dv = 0;
679695
return UCS_OK;
680696
}
681697

@@ -1107,7 +1123,7 @@ void uct_rc_mlx5_common_fill_dv_qp_attr(uct_rc_mlx5_iface_common_t *iface,
11071123
}
11081124

11091125
#ifdef HAVE_OOO_RECV_WRS
1110-
if (iface->config.dp_ordering == UCT_IB_MLX5_DP_ORDERING_OOO_ALL) {
1126+
if (iface->config.ddp_enabled_dv) {
11111127
dv_attr->create_flags |= MLX5DV_QP_CREATE_OOO_DP;
11121128
dv_attr->comp_mask |= MLX5DV_QP_INIT_ATTR_MASK_QP_CREATE_FLAGS;
11131129
}
@@ -1293,9 +1309,9 @@ void uct_ib_mlx5_devx_set_qpc_dp_ordering(uct_ib_mlx5_md_t *md, void *qpc,
12931309
uct_rc_mlx5_iface_common_t *iface)
12941310
{
12951311
UCT_IB_MLX5DV_SET(qpc, qpc, dp_ordering_0,
1296-
UCS_BIT_GET(iface->config.dp_ordering, 0));
1312+
UCS_BIT_GET(iface->config.dp_ordering_devx, 0));
12971313
UCT_IB_MLX5DV_SET(qpc, qpc, dp_ordering_1,
1298-
UCS_BIT_GET(iface->config.dp_ordering, 1));
1314+
UCS_BIT_GET(iface->config.dp_ordering_devx, 1));
12991315
UCT_IB_MLX5DV_SET(qpc, qpc, dp_ordering_force,
13001316
iface->config.dp_ordering_force);
13011317
}

‎src/uct/ib/mlx5/rc/rc_mlx5_common.h‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -409,8 +409,9 @@ typedef struct uct_rc_mlx5_iface_common {
409409
uint8_t atomic_fence_flag;
410410
uct_rc_mlx5_srq_topo_t srq_topo;
411411
uint8_t log_ack_req_freq;
412-
uint8_t dp_ordering;
412+
uint8_t dp_ordering_devx;
413413
uint8_t dp_ordering_force;
414+
uint8_t ddp_enabled_dv;
414415
} config;
415416
UCS_STATS_NODE_DECLARE(stats)
416417
} uct_rc_mlx5_iface_common_t;
@@ -484,6 +485,7 @@ ucs_status_t
484485
uct_rc_mlx5_dp_ordering_ooo_init(uct_ib_mlx5_md_t *md,
485486
uct_rc_mlx5_iface_common_t *iface,
486487
uct_ib_mlx5_dp_ordering_t dp_ordering_cap,
488+
int ddp_supported_dv,
487489
uct_rc_mlx5_iface_common_config_t *config,
488490
const char *tl_name);
489491

‎src/uct/ib/mlx5/rc/rc_mlx5_iface.c‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -933,12 +933,14 @@ UCS_CLASS_INIT_FUNC(uct_rc_mlx5_iface_t,
933933
init_attr.tx_moderation = config->super.tx_cq_moderation;
934934
init_attr.dev_name = params->mode.device.dev_name;
935935

936-
if (md->dp_ordering_cap.rc == UCT_IB_MLX5_DP_ORDERING_OOO_ALL) {
936+
if ((md->dp_ordering_cap_devx.rc == UCT_IB_MLX5_DP_ORDERING_OOO_ALL) ||
937+
md->ddp_support_dv.rc) {
937938
init_attr.flags |= UCT_IB_DDP_SUPPORTED;
938939
}
939940

940941
status = uct_rc_mlx5_dp_ordering_ooo_init(md, &self->super,
941-
md->dp_ordering_cap.rc,
942+
md->dp_ordering_cap_devx.rc,
943+
md->ddp_support_dv.rc,
942944
&config->rc_mlx5_common,
943945
"rc_mlx5");
944946
if (status != UCS_OK) {

‎test/gtest/uct/ib/test_ib.cc‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -457,10 +457,10 @@ class test_uct_ib_sl : public test_uct_ib_with_specific_port {
457457

458458
uct_ib_mlx5_md_t *ib_md = ucs_derived_of(uct_md, uct_ib_mlx5_md_t);
459459
int rc_has_ddp = has_transport("rc_mlx5") &&
460-
(ib_md->dp_ordering_cap.rc ==
460+
(ib_md->dp_ordering_cap_devx.rc ==
461461
UCT_IB_MLX5_DP_ORDERING_OOO_ALL);
462462
int dc_has_ddp = has_transport("dc_mlx5") &&
463-
(ib_md->dp_ordering_cap.dc ==
463+
(ib_md->dp_ordering_cap_devx.dc ==
464464
UCT_IB_MLX5_DP_ORDERING_OOO_ALL);
465465
return rc_has_ddp || dc_has_ddp;
466466
}

0 commit comments

Comments
 (0)