Skip to content

Commit cdc9e27

Browse files
Guofeng Yuesshaulnv
authored andcommitted
Perftest: Support selecting congestion control algorithms
Support configuring congestion control algorithms with hns direct verbs. New option: --congest_type Usage example: ./ib_send_bw -d hns_0 --congest_type=DCQCN ./ib_send_bw -d hns_0 --congest_type=DCQCN 192.168.100.100 Signed-off-by: Guofeng Yue <[email protected]> Signed-off-by: Junxian Huang <[email protected]>
1 parent 8867689 commit cdc9e27

File tree

6 files changed

+92
-12
lines changed

6 files changed

+92
-12
lines changed

Makefile.am

Lines changed: 12 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -104,41 +104,41 @@ LIBMLX4=
104104
endif
105105

106106
ib_send_bw_SOURCES = src/send_bw.c src/multicast_resources.c src/multicast_resources.h
107-
ib_send_bw_LDADD = libperftest.a $(LIBUMAD) $(LIBMATH) $(LIBMLX4) $(LIBMLX5) $(LIBEFA)
107+
ib_send_bw_LDADD = libperftest.a $(LIBUMAD) $(LIBMATH) $(LIBMLX4) $(LIBMLX5) $(LIBEFA) $(LIBHNS)
108108

109109
ib_send_lat_SOURCES = src/send_lat.c src/multicast_resources.c src/multicast_resources.h
110-
ib_send_lat_LDADD = libperftest.a $(LIBUMAD) $(LIBMATH) $(LIBMLX4) $(LIBMLX5) $(LIBEFA)
110+
ib_send_lat_LDADD = libperftest.a $(LIBUMAD) $(LIBMATH) $(LIBMLX4) $(LIBMLX5) $(LIBEFA) $(LIBHNS)
111111

112112
ib_write_lat_SOURCES = src/write_lat.c
113-
ib_write_lat_LDADD = libperftest.a $(LIBMATH) $(LIBMLX4) $(LIBMLX5) $(LIBEFA)
113+
ib_write_lat_LDADD = libperftest.a $(LIBMATH) $(LIBMLX4) $(LIBMLX5) $(LIBEFA) $(LIBHNS)
114114

115115
ib_write_bw_SOURCES = src/write_bw.c
116-
ib_write_bw_LDADD = libperftest.a $(LIBMATH) $(LIBMLX4) $(LIBMLX5) $(LIBEFA)
116+
ib_write_bw_LDADD = libperftest.a $(LIBMATH) $(LIBMLX4) $(LIBMLX5) $(LIBEFA) $(LIBHNS)
117117

118118
ib_read_lat_SOURCES = src/read_lat.c
119-
ib_read_lat_LDADD = libperftest.a $(LIBMATH) $(LIBMLX4) $(LIBMLX5) $(LIBEFA)
119+
ib_read_lat_LDADD = libperftest.a $(LIBMATH) $(LIBMLX4) $(LIBMLX5) $(LIBEFA) $(LIBHNS)
120120

121121
ib_read_bw_SOURCES = src/read_bw.c
122-
ib_read_bw_LDADD = libperftest.a $(LIBMATH) $(LIBMLX4) $(LIBMLX5) $(LIBEFA)
122+
ib_read_bw_LDADD = libperftest.a $(LIBMATH) $(LIBMLX4) $(LIBMLX5) $(LIBEFA) $(LIBHNS)
123123

124124
ib_atomic_lat_SOURCES = src/atomic_lat.c
125-
ib_atomic_lat_LDADD = libperftest.a $(LIBMATH) $(LIBMLX4) $(LIBMLX5) $(LIBEFA)
125+
ib_atomic_lat_LDADD = libperftest.a $(LIBMATH) $(LIBMLX4) $(LIBMLX5) $(LIBEFA) $(LIBHNS)
126126

127127
ib_atomic_bw_SOURCES = src/atomic_bw.c
128-
ib_atomic_bw_LDADD = libperftest.a $(LIBMATH) $(LIBMLX4) $(LIBMLX5) $(LIBEFA)
128+
ib_atomic_bw_LDADD = libperftest.a $(LIBMATH) $(LIBMLX4) $(LIBMLX5) $(LIBEFA) $(LIBHNS)
129129

130130
if HAVE_RAW_ETH
131131
raw_ethernet_bw_SOURCES = src/raw_ethernet_send_bw.c
132-
raw_ethernet_bw_LDADD = libperftest.a $(LIBMATH) $(LIBMLX4) $(LIBMLX5) $(LIBEFA)
132+
raw_ethernet_bw_LDADD = libperftest.a $(LIBMATH) $(LIBMLX4) $(LIBMLX5) $(LIBEFA) $(LIBHNS)
133133

134134
raw_ethernet_lat_SOURCES = src/raw_ethernet_send_lat.c
135-
raw_ethernet_lat_LDADD = libperftest.a $(LIBMATH) $(LIBMLX4) $(LIBMLX5) $(LIBEFA)
135+
raw_ethernet_lat_LDADD = libperftest.a $(LIBMATH) $(LIBMLX4) $(LIBMLX5) $(LIBEFA) $(LIBHNS)
136136

137137
raw_ethernet_burst_lat_SOURCES = src/raw_ethernet_send_burst_lat.c
138-
raw_ethernet_burst_lat_LDADD = libperftest.a $(LIBMATH) $(LIBMLX4) $(LIBMLX5) $(LIBEFA)
138+
raw_ethernet_burst_lat_LDADD = libperftest.a $(LIBMATH) $(LIBMLX4) $(LIBMLX5) $(LIBEFA) $(LIBHNS)
139139

140140
raw_ethernet_fs_rate_SOURCES = src/raw_ethernet_fs_rate.c
141-
raw_ethernet_fs_rate_LDADD = libperftest.a $(LIBMATH) $(LIBMLX4) $(LIBMLX5) $(LIBEFA)
141+
raw_ethernet_fs_rate_LDADD = libperftest.a $(LIBMATH) $(LIBMLX4) $(LIBMLX5) $(LIBEFA) $(LIBHNS)
142142

143143
else
144144
raw_ethernet_bw_SOURCES =

configure.ac

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -382,6 +382,13 @@ if [test $HAVE_MLX5DV_LIB = yes] && [test $HAVE_MLX5DV = yes]; then
382382
AC_SUBST([LIBMLX5])
383383
fi
384384

385+
AC_CHECK_LIB([hns], [hnsdv_query_device], [HAVE_HNSDV=yes LIBHNS=-lhns], [HAVE_HNSDV=no])
386+
AM_CONDITIONAL([HAVE_HNSDV], [test "x$HAVE_HNSDV" = "xyes"])
387+
if [test $HAVE_HNSDV = yes]; then
388+
AC_DEFINE([HAVE_HNSDV], [1], [Have hns Direct Verbs support])
389+
AC_SUBST([LIBHNS])
390+
fi
391+
385392
CFLAGS="-g -Wall -D_GNU_SOURCE -O3 $CFLAGS"
386393
LDFLAGS="$LDFLAGS"
387394
LIBS=$LIBS" -lpthread"

src/perftest_parameters.c

Lines changed: 57 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -34,6 +34,9 @@ static const char *portStates[] = {"Nop","Down","Init","Armed","","Active Defer"
3434
static const char *qp_state[] = {"OFF","ON"};
3535
static const char *exchange_state[] = {"Ethernet","rdma_cm"};
3636
static const char *atomicTypesStr[] = {"CMP_AND_SWAP","FETCH_AND_ADD"};
37+
#ifdef HAVE_HNSDV
38+
static const char *congestStr[] = {"DCQCN","LDCP","HC3","DIP"};
39+
#endif
3740

3841
/******************************************************************************
3942
* parse_mac_from_str.
@@ -438,6 +441,11 @@ static void usage(const char *argv0, VerbType verb, TestType tst, int connection
438441
printf(" --cpu_util ");
439442
printf(" Show CPU Utilization in report, valid only in Duration mode \n");
440443

444+
#ifdef HAVE_HNSDV
445+
printf(" --congest_type=<DCQCN, LDCP, HC3, DIP> ");
446+
printf(" Use the hnsdv interface to set congestion control algorithm.\n");
447+
#endif
448+
441449
if (tst != FS_RATE) {
442450
printf(" --dlid ");
443451
printf(" Set a Destination LID instead of getting it from the other side.\n");
@@ -863,6 +871,7 @@ static void init_perftest_params(struct perftest_parameters *user_param)
863871
user_param->source_ip = NULL;
864872
user_param->has_source_ip = 0;
865873
user_param->use_write_with_imm = 0;
874+
user_param->congest_type = OFF;
866875
}
867876

868877
static int open_file_write(const char* file_path)
@@ -961,6 +970,25 @@ static void change_conn_type(int *cptr, VerbType verb, const char *optarg)
961970
exit(1);
962971
}
963972
}
973+
974+
#ifdef HAVE_HNSDV
975+
static void set_congest_type(int *cgtr, const char *optarg)
976+
{
977+
if (strcmp(congestStr[0], optarg) == 0) {
978+
*cgtr = HNSDV_QP_CREATE_ENABLE_DCQCN;
979+
} else if (strcmp(congestStr[1], optarg) == 0) {
980+
*cgtr = HNSDV_QP_CREATE_ENABLE_LDCP;
981+
} else if (strcmp(congestStr[2], optarg) == 0) {
982+
*cgtr = HNSDV_QP_CREATE_ENABLE_HC3;
983+
} else if (strcmp(congestStr[3], optarg) == 0) {
984+
*cgtr = HNSDV_QP_CREATE_ENABLE_DIP;
985+
} else {
986+
fprintf(stderr, " Invalid congest type. Please choose from {DCQCN,LDCP,HC3,DIP}\n");
987+
exit(1);
988+
}
989+
}
990+
#endif
991+
964992
/******************************************************************************
965993
*
966994
******************************************************************************/
@@ -1757,6 +1785,23 @@ static void force_dependecies(struct perftest_parameters *user_param)
17571785
}
17581786
}
17591787

1788+
#ifdef HAVE_HNSDV
1789+
if (user_param->congest_type) {
1790+
if (user_param->work_rdma_cm == ON)
1791+
{
1792+
printf(RESULT_LINE);
1793+
fprintf(stderr, "rdma_cm does not support setting congest type.\n");
1794+
exit(1);
1795+
}
1796+
1797+
if (user_param->connection_type == XRC || user_param->connection_type == UD) {
1798+
printf(RESULT_LINE);
1799+
fprintf(stdout, "XRC/UD does not support setting congest type.\n");
1800+
exit(1);
1801+
}
1802+
}
1803+
#endif
1804+
17601805
return;
17611806
}
17621807
/******************************************************************************
@@ -2274,6 +2319,9 @@ int parser(struct perftest_parameters *user_param,char *argv[], int argc)
22742319
static int credentials_path_flag = 0;
22752320
static int data_enc_key_app_path_flag = 0;
22762321
#endif
2322+
#ifdef HAVE_HNSDV
2323+
static int congest_type_flag = 0;
2324+
#endif
22772325

22782326
char *server_ip = NULL;
22792327
char *client_ip = NULL;
@@ -2427,6 +2475,9 @@ int parser(struct perftest_parameters *user_param,char *argv[], int argc)
24272475
#if defined HAVE_OOO_ATTR
24282476
{.name = "use_ooo", .has_arg = 0, .flag = &use_ooo_flag, .val = 1},
24292477
#endif
2478+
#ifdef HAVE_HNSDV
2479+
{ .name = "congest_type", .has_arg = 1, .flag = &congest_type_flag, .val = 1},
2480+
#endif
24302481
{.name = "bind_source_ip", .has_arg = 1, .flag = &source_ip_flag, .val = 1},
24312482
{.name = "write_with_imm", .has_arg = 0, .flag = &use_write_with_imm_flag, .val = 1 },
24322483
{0}
@@ -2721,6 +2772,12 @@ int parser(struct perftest_parameters *user_param,char *argv[], int argc)
27212772
free(duplicates_checker);
27222773
return FAILURE;
27232774
case 0: /* required for long options to work. */
2775+
#ifdef HAVE_HNSDV
2776+
if (congest_type_flag) {
2777+
set_congest_type(&user_param->congest_type, optarg);
2778+
congest_type_flag = 0;
2779+
}
2780+
#endif
27242781
if (pkey_flag) {
27252782
CHECK_VALUE(user_param->pkey_index,int,"Pkey index",not_int_ptr);
27262783
pkey_flag = 0;

src/perftest_parameters.h

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -523,6 +523,7 @@ struct perftest_parameters {
523523
int recv_post_list;
524524
int duration;
525525
int use_srq;
526+
int congest_type;
526527
int use_xrc;
527528
int use_rss;
528529
int srq_exists;

src/perftest_resources.c

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2219,6 +2219,9 @@ struct ibv_qp* ctx_qp_create(struct pingpong_context *ctx,
22192219
struct efadv_qp_init_attr efa_attr = {};
22202220
#endif
22212221
#endif
2222+
#ifdef HAVE_HNSDV
2223+
struct hnsdv_qp_init_attr hns_attr = {};
2224+
#endif
22222225

22232226
attr.send_cq = ctx->send_cq;
22242227
attr.recv_cq = (user_param->verb == SEND || user_param->verb == WRITE_IMM) ? ctx->recv_cq : ctx->send_cq;
@@ -2382,6 +2385,15 @@ struct ibv_qp* ctx_qp_create(struct pingpong_context *ctx,
23822385
#endif // HAVE_AES_XTS
23832386
else
23842387
#endif // HAVE_MLX5DV
2388+
2389+
#ifdef HAVE_HNSDV
2390+
if (user_param->congest_type) {
2391+
hns_attr.comp_mask = HNSDV_QP_INIT_ATTR_MASK_QP_CONGEST_TYPE;
2392+
hns_attr.congest_type = user_param->congest_type;
2393+
qp = hnsdv_create_qp(ctx->context, &attr_ex, &hns_attr);
2394+
}
2395+
else
2396+
#endif //HAVE_HNSDV
23852397
qp = ibv_create_qp_ex(ctx->context, &attr_ex);
23862398
}
23872399
else

src/perftest_resources.h

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -55,6 +55,9 @@
5555
#if defined(HAVE_MLX5DV)
5656
#include <infiniband/mlx5dv.h>
5757
#endif
58+
#if defined(HAVE_HNSDV)
59+
#include <infiniband/hnsdv.h>
60+
#endif
5861
#include <rdma/rdma_cma.h>
5962
#include <stdint.h>
6063
#if defined(__FreeBSD__)

0 commit comments

Comments
 (0)