Skip to content

Commit b6164bc

Browse files
sshaulnvsshaulnv
committed
Perftest: Add dynamic CQE polling mechanism
This commit introduces a dynamic CQE polling mechanism that automatically adjusts the number of completion queue entries (CQEs) polled in each iteration, based on the actual completion rate. The feature aims to optimize polling performance by matching the poll batch size to the workload characteristics. Key features: - Starts with maximum poll size (1024) and adjusts down based on usage - Adapts within first 25% of iterations or max(1000, num_qps * 3) iterations. - Uses threshold of 85% to determine if current size is appropriate - Stabilizes after 10 consecutive stable iterations - Configurable min (16), max (1024) sizes The adaptation works by: 1. If completions == current_size: increase size by step 2. If completions < 85% of current_size: decrease size by step 3. If same completion count for 10 iterations: lock at current size 4. After adaptation period: continue with last stable size Limitations and special cases: - Supported over BW tests only. - Not supported in bidirectional send/write_with_imm tests - Disabled for small iteration counts (< 3000) to avoid overhead The feature can be disabled using --disable_dynamic_polling flag. Implementation details: - Centralizes polling logic in poll_cq_adaptive() function - Uses dyn_poll_ctx to manage configuration and state - Maintains separate RX/TX adaptation for bidirectional tests - Preserves existing CQE poll behavior when disabled Example usage: $ ib_send_bw # Dynamic polling enabled by default $ ib_send_bw --disable_dynamic_polling # Use fixed polling size Signed-off-by: Shmuel Shaul <[email protected]>
1 parent b86c061 commit b6164bc

File tree

4 files changed

+186
-12
lines changed

4 files changed

+186
-12
lines changed

‎src/perftest_parameters.c‎

Lines changed: 27 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -470,6 +470,9 @@ static void usage(const char *argv0, VerbType verb, TestType tst, int connection
470470
if (tst == BW) {
471471
printf(" -y, --limit_msgrate=<value> ");
472472
printf(" Set verifier limit for Msg Rate\n");
473+
474+
printf(" --disable_dynamic_polling ");
475+
printf(" Disable dynamic CQE polling adaptation (default enabled)\n");
473476
}
474477

475478
if (connection_type != RawEth) {
@@ -1003,6 +1006,7 @@ static void init_perftest_params(struct perftest_parameters *user_param)
10031006
user_param->tph_mem_type = -1;
10041007
user_param->cpu_id = -1;
10051008
user_param->processing_hints = -1;
1009+
user_param->dynamic_cqe_poll = ON;
10061010
}
10071011

10081012
static int open_file_write(const char* file_path)
@@ -2011,7 +2015,16 @@ static void force_dependecies(struct perftest_parameters *user_param)
20112015
}
20122016
#endif
20132017

2014-
if (check_intense_polling(user_param)) {
2018+
/* Disable dynamic polling for small iteration counts, duplex send/write_with_imm, and all LAT tests */
2019+
if ((user_param->test_type == ITERATIONS && user_param->iters < 3000) ||
2020+
(user_param->tst == BW && user_param->duplex &&
2021+
(user_param->verb == SEND || user_param->use_write_with_imm)) ||
2022+
(user_param->tst == LAT)) {
2023+
printf("Disabling dynamic polling\n");
2024+
user_param->dynamic_cqe_poll = OFF;
2025+
}
2026+
2027+
if (user_param->dynamic_cqe_poll == OFF && check_intense_polling(user_param)) {
20152028
printf("Increasing CQE polling batch to %d\n", CTX_POLL_BATCH_INTENSE);
20162029
user_param->cqe_poll = CTX_POLL_BATCH_INTENSE;
20172030
}
@@ -2597,6 +2610,7 @@ int parser(struct perftest_parameters *user_param,char *argv[], int argc)
25972610
#endif
25982611
static int connectionless_flag = 0;
25992612
static int cqe_poll_flag = 0;
2613+
static int disable_dynamic_polling_flag = 0;
26002614
#ifdef HAVE_REG_MR_EX
26012615
static int tph_mem_flag = 0;
26022616
static int cpu_id_flag = 0;
@@ -2784,6 +2798,7 @@ int parser(struct perftest_parameters *user_param,char *argv[], int argc)
27842798
{ .name = "tph_mem", .has_arg = 1, .flag = &tph_mem_flag, .val = 1},
27852799
{ .name = "cpu_id", .has_arg = 1, .flag = &cpu_id_flag, .val = 1},
27862800
{ .name = "ph", .has_arg = 1, .flag = &processing_hints_flag, .val = 1},
2801+
{ .name = "disable_dynamic_polling", .has_arg = 0, .flag = &disable_dynamic_polling_flag, .val = 1},
27872802
#endif
27882803
{0}
27892804
};
@@ -3540,9 +3555,15 @@ int parser(struct perftest_parameters *user_param,char *argv[], int argc)
35403555
unsolicited_write_flag = 0;
35413556
}
35423557
#endif
3558+
if (disable_dynamic_polling_flag) {
3559+
user_param->dynamic_cqe_poll = OFF;
3560+
disable_dynamic_polling_flag = 0;
3561+
}
3562+
35433563
if (cqe_poll_flag) {
35443564
CHECK_VALUE_IN_RANGE(user_param->cqe_poll,uint16_t,1,65535,"CQE Poll",not_int_ptr);
35453565
user_param->use_cqe_poll = ON;
3566+
user_param->dynamic_cqe_poll = OFF;
35463567
cqe_poll_flag = 0;
35473568
}
35483569
#ifdef HAVE_REG_MR_EX
@@ -4017,7 +4038,11 @@ void ctx_print_test_info(struct perftest_parameters *user_param)
40174038

40184039
if (user_param->tst == BW) {
40194040
printf(" CQ Moderation : %d\n", user_param->cq_mod);
4020-
printf(" CQE Poll Batch : %hu\n", user_param->cqe_poll);
4041+
if (user_param->dynamic_cqe_poll == ON) {
4042+
printf(" CQE Poll Batch : Dynamic\n");
4043+
} else {
4044+
printf(" CQE Poll Batch : %hu\n", user_param->cqe_poll);
4045+
}
40214046
}
40224047

40234048
printf(" Mtu : %lu[B]\n",user_param->connection_type == RawEth ? user_param->curr_mtu : MTU_SIZE(user_param->curr_mtu));

‎src/perftest_parameters.h‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -678,6 +678,7 @@ struct perftest_parameters {
678678
int tph_mem_type;
679679
int cpu_id;
680680
int processing_hints;
681+
int dynamic_cqe_poll;
681682
};
682683

683684
struct report_options {

‎src/perftest_resources.c‎

Lines changed: 129 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -35,6 +35,76 @@ static enum ibv_wr_opcode opcode_atomic_array[] = {IBV_WR_ATOMIC_CMP_AND_SWP,IBV
3535
#define CPU_UTILITY "/proc/stat"
3636
#define DC_KEY 0xffeeddcc
3737

38+
/* Initialize dynamic polling context */
39+
static struct dyn_poll_ctx *init_dyn_poll_ctx(struct perftest_parameters *user_param)
40+
{
41+
struct dyn_poll_ctx *ctx;
42+
ALLOCATE(ctx, struct dyn_poll_ctx, 1);
43+
44+
/* Initialize with const config */
45+
ctx->config.min = 16;
46+
ctx->config.max = 1024;
47+
ctx->config.stabilize = 10;
48+
ctx->config.threshold = 0.85;
49+
50+
ctx->state.curr_size = user_param->dynamic_cqe_poll ? 1024 : user_param->cqe_poll;
51+
ctx->state.stable_iters = 0;
52+
ctx->state.last_ne = 0;
53+
54+
ctx->stabilization_iters = (user_param->test_type == ITERATIONS) ?
55+
MIN(user_param->iters / 4, MAX_QP_NUM) :
56+
MAX(user_param->num_of_qps * 3, 1000);
57+
58+
return ctx;
59+
}
60+
61+
static __always_inline int poll_cq_adaptive(
62+
struct ibv_cq *cq,
63+
struct ibv_wc *wc,
64+
const struct dyn_cqe_poll_config *config,
65+
struct dyn_poll_state *state,
66+
int *dynamic_enabled)
67+
{
68+
int ne = ibv_poll_cq(cq, state->curr_size, wc);
69+
70+
if (ne == state->curr_size && state->curr_size < config->max) {
71+
state->curr_size = (uint16_t)MIN((state->curr_size + ne) / 2, config->max);
72+
state->stable_iters = 0;
73+
} else if (ne < state->curr_size * config->threshold) {
74+
if (state->curr_size > config->min) {
75+
state->curr_size = (uint16_t)MAX((state->curr_size + ne) / 2, config->min);
76+
state->stable_iters = 0;
77+
}
78+
} else if (ne == state->last_ne) {
79+
if (++state->stable_iters >= config->stabilize) {
80+
*dynamic_enabled = 0;
81+
}
82+
} else {
83+
state->stable_iters = 0;
84+
}
85+
state->last_ne = ne;
86+
87+
return ne;
88+
}
89+
static __always_inline int poll_completions(
90+
struct ibv_cq *cq,
91+
struct ibv_wc *wc,
92+
struct dyn_poll_ctx *dyn_ctx,
93+
uint64_t curr_count,
94+
int *dynamic_enabled)
95+
{
96+
if (*dynamic_enabled && curr_count < dyn_ctx->stabilization_iters) {
97+
return poll_cq_adaptive(
98+
cq,
99+
wc,
100+
&dyn_ctx->config,
101+
&dyn_ctx->state,
102+
dynamic_enabled
103+
);
104+
}
105+
return ibv_poll_cq(cq, dyn_ctx->state.curr_size, wc);
106+
}
107+
38108
struct perftest_parameters* duration_param;
39109
struct check_alive_data check_alive_data;
40110

@@ -3774,12 +3844,18 @@ int run_iter_bw(struct pingpong_context *ctx,struct perftest_parameters *user_pa
37743844
int address_offset = 0;
37753845
int flows_burst_iter = 0;
37763846

3847+
struct dyn_poll_ctx *dyn_ctx = init_dyn_poll_ctx(user_param);
3848+
if (!dyn_ctx) {
3849+
fprintf(stderr, "Failed to allocate dynamic polling context\n");
3850+
return FAILURE;
3851+
}
3852+
37773853
#ifdef HAVE_IBV_WR_API
37783854
if (user_param->connection_type != RawEth)
37793855
ctx_post_send_work_request_func_pointer(ctx, user_param);
37803856
#endif
37813857

3782-
ALLOCATE(wc ,struct ibv_wc ,user_param->cqe_poll);
3858+
ALLOCATE(wc ,struct ibv_wc ,dyn_ctx->config.max);
37833859
if (user_param->test_type == DURATION) {
37843860
duration_param=user_param;
37853861
duration_param->state = START_STATE;
@@ -3932,7 +4008,14 @@ int run_iter_bw(struct pingpong_context *ctx,struct perftest_parameters *user_pa
39324008
goto cleaning;
39334009
}
39344010
}
3935-
ne = ibv_poll_cq(ctx->send_cq, user_param->cqe_poll, wc);
4011+
/* Dynamic CQE poll size adaptation */
4012+
ne = poll_completions(
4013+
ctx->send_cq,
4014+
wc,
4015+
dyn_ctx,
4016+
totccnt,
4017+
&user_param->dynamic_cqe_poll);
4018+
39364019
if (ne > 0) {
39374020
for (i = 0; i < ne; i++) {
39384021
wc_id = (int)wc[i].wr_id;
@@ -3975,7 +4058,7 @@ int run_iter_bw(struct pingpong_context *ctx,struct perftest_parameters *user_pa
39754058
user_param->tcompleted[0] = get_cycles();
39764059

39774060
cleaning:
3978-
4061+
free(dyn_ctx);
39794062
free(wc);
39804063
return return_value;
39814064
}
@@ -4026,12 +4109,18 @@ int run_iter_bw_server(struct pingpong_context *ctx, struct perftest_parameters
40264109
int recv_flows_burst = 0;
40274110
int address_flows_offset =0;
40284111

4112+
struct dyn_poll_ctx *dyn_ctx = init_dyn_poll_ctx(user_param);
4113+
if (!dyn_ctx) {
4114+
fprintf(stderr, "Failed to allocate dynamic polling context\n");
4115+
return FAILURE;
4116+
}
4117+
40294118
#ifdef HAVE_IBV_WR_API
40304119
if (user_param->connection_type != RawEth)
40314120
ctx_post_send_work_request_func_pointer(ctx, user_param);
40324121
#endif
40334122

4034-
ALLOCATE(wc ,struct ibv_wc ,user_param->cqe_poll);
4123+
ALLOCATE(wc ,struct ibv_wc ,dyn_ctx->config.max);
40354124
ALLOCATE(swc ,struct ibv_wc ,user_param->tx_depth);
40364125

40374126
ALLOCATE(rcnt_for_qp,uint64_t,user_param->num_of_qps);
@@ -4074,7 +4163,13 @@ int run_iter_bw_server(struct pingpong_context *ctx, struct perftest_parameters
40744163
if (user_param->test_type == DURATION && user_param->state == END_STATE)
40754164
break;
40764165

4077-
ne = ibv_poll_cq(ctx->recv_cq,user_param->cqe_poll,wc);
4166+
/* Dynamic CQE poll size adaptation */
4167+
ne = poll_completions(
4168+
ctx->recv_cq,
4169+
wc,
4170+
dyn_ctx,
4171+
rcnt,
4172+
&user_param->dynamic_cqe_poll);
40784173

40794174
if (ne > 0) {
40804175
if (firstRx) {
@@ -4205,7 +4300,7 @@ int run_iter_bw_server(struct pingpong_context *ctx, struct perftest_parameters
42054300
if (clean_scq_credit(tot_scredit, ctx, user_param))
42064301
return_value = FAILURE;
42074302
}
4208-
4303+
free(dyn_ctx);
42094304
check_alive_data.last_totrcnt=0;
42104305
free(wc);
42114306
free(rcnt_for_qp);
@@ -4245,7 +4340,13 @@ int run_iter_bw_infinitely(struct pingpong_context *ctx,struct perftest_paramete
42454340
ctx_post_send_work_request_func_pointer(ctx, user_param);
42464341
#endif
42474342

4248-
ALLOCATE(wc ,struct ibv_wc ,user_param->cqe_poll);
4343+
struct dyn_poll_ctx *dyn_ctx = init_dyn_poll_ctx(user_param);
4344+
if (!dyn_ctx) {
4345+
fprintf(stderr, "Failed to allocate dynamic polling context\n");
4346+
return FAILURE;
4347+
}
4348+
4349+
ALLOCATE(wc ,struct ibv_wc ,dyn_ctx->config.max);
42494350
ALLOCATE(scnt_for_qp,uint64_t,user_param->num_of_qps);
42504351
memset(scnt_for_qp,0,sizeof(uint64_t)*user_param->num_of_qps);
42514352

@@ -4307,7 +4408,12 @@ int run_iter_bw_infinitely(struct pingpong_context *ctx,struct perftest_paramete
43074408
}
43084409
}
43094410
if (totccnt < totscnt) {
4310-
ne = ibv_poll_cq(ctx->send_cq,user_param->cqe_poll,wc);
4411+
ne = poll_completions(
4412+
ctx->send_cq,
4413+
wc,
4414+
dyn_ctx,
4415+
totccnt,
4416+
&user_param->dynamic_cqe_poll);
43114417

43124418
if (ne > 0) {
43134419

@@ -4336,6 +4442,7 @@ int run_iter_bw_infinitely(struct pingpong_context *ctx,struct perftest_paramete
43364442
}
43374443
}
43384444
cleaning:
4445+
free(dyn_ctx);
43394446
free(scnt_for_qp);
43404447
free(wc);
43414448
return return_value;
@@ -4361,7 +4468,13 @@ int run_iter_bw_infinitely_server(struct pingpong_context *ctx, struct perftest_
43614468
ctx_post_send_work_request_func_pointer(ctx, user_param);
43624469
#endif
43634470

4364-
ALLOCATE(wc ,struct ibv_wc ,user_param->cqe_poll);
4471+
struct dyn_poll_ctx *dyn_ctx = init_dyn_poll_ctx(user_param);
4472+
if (!dyn_ctx) {
4473+
fprintf(stderr, "Failed to allocate dynamic polling context\n");
4474+
return FAILURE;
4475+
}
4476+
4477+
ALLOCATE(wc ,struct ibv_wc ,dyn_ctx->config.max);
43654478
ALLOCATE(swc ,struct ibv_wc ,user_param->tx_depth);
43664479

43674480
ALLOCATE(rcnt_for_qp,uint64_t,user_param->num_of_qps);
@@ -4394,7 +4507,12 @@ int run_iter_bw_infinitely_server(struct pingpong_context *ctx, struct perftest_
43944507

43954508
while (1) {
43964509

4397-
ne = ibv_poll_cq(ctx->recv_cq,user_param->cqe_poll,wc);
4510+
ne = poll_completions(
4511+
ctx->recv_cq,
4512+
wc,
4513+
dyn_ctx,
4514+
user_param->iters,
4515+
&user_param->dynamic_cqe_poll);
43984516

43994517
if (ne > 0) {
44004518

@@ -4475,6 +4593,7 @@ int run_iter_bw_infinitely_server(struct pingpong_context *ctx, struct perftest_
44754593
}
44764594

44774595
cleaning:
4596+
free(dyn_ctx);
44784597
free(wc);
44794598
free(swc);
44804599
free(rcnt_for_qp);

‎src/perftest_resources.h‎

Lines changed: 29 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -145,6 +145,13 @@
145145
_a > _b ? _a : _b; \
146146
})
147147

148+
#define MIN(a, b) \
149+
({ \
150+
typeof(a) _a = (a); \
151+
typeof(b) _b = (b); \
152+
_a < _b ? _a : _b; \
153+
})
154+
148155
/******************************************************************************
149156
* Perftest resources Structures and data types.
150157
******************************************************************************/
@@ -168,6 +175,28 @@ struct cma {
168175
int disconnects_left;
169176
};
170177

178+
/* Dynamic CQE polling configuration */
179+
struct dyn_cqe_poll_config {
180+
uint16_t min;
181+
uint16_t max;
182+
uint16_t stabilize;
183+
double threshold;
184+
};
185+
186+
/* Dynamic CQE polling state */
187+
struct dyn_poll_state {
188+
uint16_t curr_size;
189+
int stable_iters;
190+
int last_ne;
191+
};
192+
193+
/* Helper structure to hold dynamic polling context */
194+
struct dyn_poll_ctx {
195+
struct dyn_cqe_poll_config config;
196+
struct dyn_poll_state state;
197+
int stabilization_iters;
198+
};
199+
171200
struct pingpong_context {
172201
struct cma cma_master;
173202
struct rdma_event_channel *cm_channel;

0 commit comments

Comments
 (0)