diff --git a/man/perftest.1 b/man/perftest.1 index 789770cf..b5251339 100644 --- a/man/perftest.1 +++ b/man/perftest.1 @@ -471,6 +471,15 @@ many different options and modes. .B --write_with_imm Use write-with-immediate verb instead of write. Write tests only. +.TP +.B --data_validation= +Perform data validation on transferred packets. + random: Data is randomized. + serial: Data is filled with sequential numeric series. Use --data_start_value= to set starting point. + pattern: Data is filled using contents of file provided by --payload_file_path=. +.TP +.B --data_start_value +Starting value for serial data validation. Set to 0 by default. .SS RawEth only options .TP .B -B, --source_mac diff --git a/src/perftest_communication.c b/src/perftest_communication.c index 5c89dd0f..fb58d2c4 100755 --- a/src/perftest_communication.c +++ b/src/perftest_communication.c @@ -308,7 +308,7 @@ static int ethernet_write_keys(struct pingpong_dest *my_dest, } else { char msg[KEY_MSG_SIZE_GID]; sprintf(msg,KEY_PRINT_FMT_GID, my_dest->lid,my_dest->out_reads, - my_dest->qpn,my_dest->psn, my_dest->rkey, my_dest->vaddr, + my_dest->qpn,my_dest->psn, my_dest->rkey, my_dest->vaddr, my_dest->data_validation_hint, my_dest->gid.raw[0],my_dest->gid.raw[1], my_dest->gid.raw[2],my_dest->gid.raw[3], my_dest->gid.raw[4],my_dest->gid.raw[5], @@ -404,6 +404,12 @@ static int ethernet_read_keys(struct pingpong_dest *rem_dest, rem_dest->vaddr = strtoull(tmp, NULL, 16); /*VA*/ + pstr += term - pstr + 1; + term = strpbrk(pstr, ":"); + memcpy(tmp, pstr, term - pstr); + tmp[term - pstr] = 0; + rem_dest->data_validation_hint = (uint32_t)strtoul(tmp, NULL, 16); /*DATA_VALIDATION_HINT*/ + for (i = 0; i < 15; ++i) { pstr += term - pstr + 1; term = strpbrk(pstr, ":"); diff --git a/src/perftest_communication.h b/src/perftest_communication.h index 738c83cd..613ccf21 100755 --- a/src/perftest_communication.h +++ b/src/perftest_communication.h @@ -73,14 +73,14 @@ #define hton_int(x) (int) htonl((uint32_t) (x)) #define KEY_MSG_SIZE (59) /* Message size without gid. */ -#define KEY_MSG_SIZE_GID (108) /* Message size with gid (MGID as well). */ +#define KEY_MSG_SIZE_GID (129) /* Message size with gid (MGID as well). */ #define SYNC_SPEC_ID (5) /* The Format of the message we pass through sockets , without passing Gid. */ #define KEY_PRINT_FMT "%04x:%04x:%06x:%06x:%08x:%016llx:%08x" /* The Format of the message we pass through sockets (With Gid). */ -#define KEY_PRINT_FMT_GID "%04x:%04x:%06x:%06x:%08x:%016llx:%02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x:%08x:" +#define KEY_PRINT_FMT_GID "%04x:%04x:%06x:%06x:%08x:%016llx:%08x:%02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x:%08x:" /* The Basic print format for all verbs. */ #define BASIC_ADDR_FMT " %s address: LID %#04x QPN %#06x PSN %#06x" diff --git a/src/perftest_parameters.c b/src/perftest_parameters.c index 85ace6c3..5414f85b 100755 --- a/src/perftest_parameters.c +++ b/src/perftest_parameters.c @@ -36,6 +36,7 @@ static const char *portStates[] = {"Nop","Down","Init","Armed","","Active Defer" static const char *qp_state[] = {"OFF","ON"}; static const char *exchange_state[] = {"Ethernet","rdma_cm"}; static const char *atomicTypesStr[] = {"CMP_AND_SWAP","FETCH_AND_ADD"}; +static const char *dataValidationTypesStr[] = {"none","random", "serial", "pattern"}; #ifdef HAVE_HNSDV static const char *congestStr[] = {"DCQCN","LDCP","HC3","DIP"}; #endif @@ -621,6 +622,16 @@ static void usage(const char *argv0, VerbType verb, TestType tst, int connection printf(" --run_infinitely "); printf(" Run test forever, print results every seconds (SYMMETRIC)\n"); + + printf(" --data_validation= "); + printf(" Perform data validation on transferred packets\n"); + printf(" random: Data is randomized.\n"); + printf(" serial: Data is filled with sequential numeric series. Use --data_start_value= to set starting point.\n"); + printf(" pattern: Data is filled using contents of file provided by --payload_file_path=.\n"); + + printf(" --data_start_value "); + printf(" Starting value for serial data validation. Set to 0 by default.\n"); + } if (connection_type != RawEth) { @@ -1013,6 +1024,8 @@ static void init_perftest_params(struct perftest_parameters *user_param) user_param->cpu_id = -1; user_param->processing_hints = -1; user_param->dynamic_cqe_poll = ON; + user_param->data_validation = NONE; + user_param->data_start_value = 0; } static int open_file_write(const char* file_path) @@ -2082,6 +2095,44 @@ static void force_dependecies(struct perftest_parameters *user_param) } #endif + if (user_param->data_validation) { + if (user_param->post_list != user_param->tx_depth || user_param->recv_post_list != user_param->rx_depth) { + printf(RESULT_LINE); + fprintf(stderr, " Invalid data validation qps configuration. Post list size should be equal to corresponding queue depth.\n"); + exit(1); + } + + if (user_param->tst != BW || user_param->verb != WRITE_IMM) { + printf(RESULT_LINE); + fprintf(stderr, " Data validation can only be used with write with immediate BW test.\n"); + exit(1); + } + + if (user_param->duplex) { + printf(RESULT_LINE); + fprintf(stderr, "Bidirectional mode not supported in data validation.\n"); + exit(1); + } + + if (user_param->has_payload_modification && user_param->data_validation != PATTERN) { + printf(RESULT_LINE); + fprintf(stderr, "Payload modification input is not supported with random or serial data validation.\n"); + exit(1); + } + + if (user_param->mr_per_qp) { + printf(RESULT_LINE); + fprintf(stderr, "MR per QP is not supported in data validation.\n"); + exit(1); + } + + if (user_param->data_validation == PATTERN && !user_param->has_payload_modification) { + printf(RESULT_LINE); + fprintf(stderr, "Payload modification input is required for pattern data validation.\n"); + exit(1); + } + } + return; } /****************************************************************************** @@ -2634,6 +2685,8 @@ int parser(struct perftest_parameters *user_param,char *argv[], int argc) static int recv_post_list_flag = 0; static int payload_flag = 0; static int use_write_with_imm_flag = 0; + static int data_validation_flag = 0; + static int data_start_value_flag = 0; #ifdef HAVE_SRD_WITH_UNSOLICITED_WRITE_RECV static int unsolicited_write_flag = 0; #endif @@ -2860,6 +2913,8 @@ int parser(struct perftest_parameters *user_param,char *argv[], int argc) #ifdef HAVE_SIG_OFFLOAD {.name = "sig_offload", .has_arg = 0, .flag = &sig_offload_flag, .val = 1 }, #endif + {.name = "data_validation", .has_arg = 1, .flag = &data_validation_flag, .val = 1}, + {.name = "data_start_value", .has_arg = 1, .flag = &data_start_value_flag, .val = 1}, {0} }; if (!duplicates_checker) { @@ -3609,6 +3664,27 @@ int parser(struct perftest_parameters *user_param,char *argv[], int argc) user_param->verb = WRITE_IMM; use_write_with_imm_flag = 0; } + if (data_validation_flag) { + + int i, types_array_size = GET_ARRAY_SIZE(dataValidationTypesStr); + for (i = 1; i < types_array_size; i++) { + if (strcmp(dataValidationTypesStr[i],optarg) == 0) { + user_param->data_validation = i; + break; + } + } + + if (i == types_array_size) { + fprintf(stderr, " Invalid data validation type flag. Please use random, serial or pattern.\n"); + return FAILURE; + } + + data_validation_flag = 0; + } + if (data_start_value_flag) { + user_param->data_start_value = (uint32_t)strtoul(optarg, NULL, 10); + data_start_value_flag = 0; + } #ifdef HAVE_SRD_WITH_UNSOLICITED_WRITE_RECV if (unsolicited_write_flag) { user_param->use_unsolicited_write = 1; diff --git a/src/perftest_parameters.h b/src/perftest_parameters.h index 91db980d..452b22f8 100755 --- a/src/perftest_parameters.h +++ b/src/perftest_parameters.h @@ -405,6 +405,9 @@ enum rate_limiter_units {MEGA_BYTE_PS, GIGA_BIT_PS, PACKET_PS}; /*Types rate limit*/ enum rate_limiter_types {HW_RATE_LIMIT, SW_RATE_LIMIT, PP_RATE_LIMIT, DISABLE_RATE_LIMIT}; +/*Types data validation*/ +enum data_validation_types {NONE, RANDOM, SERIAL, PATTERN}; + /* Verbosity Levels for test report */ enum verbosity_level {FULL_VERBOSITY=-1, OUTPUT_BW=0, OUTPUT_MR, OUTPUT_LAT }; @@ -686,6 +689,8 @@ struct perftest_parameters { int processing_hints; int dynamic_cqe_poll; int sig_offload; + enum data_validation_types data_validation; + uint32_t data_start_value; }; struct report_options { diff --git a/src/perftest_resources.c b/src/perftest_resources.c index 94e95864..30b3019e 100755 --- a/src/perftest_resources.c +++ b/src/perftest_resources.c @@ -476,7 +476,8 @@ static inline int _new_post_send(struct pingpong_context *ctx, ibv_wr_rdma_write_imm( ctx->qpx[index], wr->wr.rdma.rkey, - wr->wr.rdma.remote_addr, 0); + wr->wr.rdma.remote_addr, + wr->imm_data); break; case IBV_WR_RDMA_READ: ibv_wr_rdma_read( @@ -1156,7 +1157,6 @@ int alloc_ctx(struct pingpong_context *ctx,struct perftest_parameters *user_para { uint64_t tarr_size; int num_of_qps_factor; - ctx->cycle_buffer = user_param->cycle_buffer; ctx->cache_line_size = user_param->cache_line_size; ALLOC(user_param->port_by_qp, uint64_t, user_param->num_of_qps); @@ -1189,6 +1189,9 @@ int alloc_ctx(struct pingpong_context *ctx,struct perftest_parameters *user_para #endif ALLOC(ctx->mr, struct ibv_mr*, user_param->num_of_qps); ALLOC(ctx->buf, void*, user_param->num_of_qps); + if (user_param->data_validation && user_param->machine == SERVER) { + ALLOC(ctx->validation_buf, void*, user_param->num_of_qps); + } if ((user_param->tst == BW || user_param->tst == LAT_BY_BW) && (user_param->machine == CLIENT || user_param->duplex)) { @@ -1231,8 +1234,17 @@ int alloc_ctx(struct pingpong_context *ctx,struct perftest_parameters *user_para user_param->num_of_qps * user_param->recv_post_list); ALLOC(ctx->rx_buffer_addr, uint64_t, user_param->num_of_qps); } - if (user_param->mac_fwd == ON ) + + if (user_param->mac_fwd == ON ) { ctx->cycle_buffer = user_param->size * user_param->rx_depth; + } else if (user_param->data_validation) { + if (user_param->machine == CLIENT) + ctx->cycle_buffer = INC(user_param->size, ctx->cache_line_size) * user_param->post_list; + else + ctx->cycle_buffer = INC(user_param->size, ctx->cache_line_size) * user_param->recv_post_list; + } else { + ctx->cycle_buffer = user_param->cycle_buffer; + } ctx->size = user_param->size; @@ -1307,6 +1319,11 @@ void dealloc_ctx(struct pingpong_context *ctx,struct perftest_parameters *user_p free(ctx->mr); if (ctx->buf != NULL) free(ctx->buf); + if (user_param->machine == SERVER && ctx->validation_buf != NULL) { + if (ctx->validation_buf[0]) + ctx->memory->free_buffer(ctx->memory, 0, ctx->validation_buf[0], ctx->buff_size); + free(ctx->validation_buf); + } if ((user_param->tst == BW || user_param->tst == LAT_BY_BW) && (user_param->machine == CLIENT || user_param->duplex)) { if (ctx->my_addr != NULL) free(ctx->my_addr); @@ -1875,28 +1892,57 @@ static int setup_mr_flags(struct perftest_parameters *user_param) return flags; } +static void generate_buffer_data(struct pingpong_context *ctx, + struct perftest_parameters *user_param, + void* buf) +{ + uint64_t i; + uint32_t *buf_ptr = (uint32_t*)buf; + uint32_t current_data; + + if (user_param->data_validation) { + if (user_param->machine == SERVER) { + current_data = ctx->data_validation_hint; + } else { + if (user_param->data_validation == SERIAL) { + current_data = user_param->data_start_value; + } else { + current_data = init_perftest_rand_state(); + } + ctx->data_validation_hint = current_data; + } + } else { + current_data = init_perftest_rand_state(); + } + + if (user_param->has_payload_modification) { + for (i = 0; i < ctx->buff_size; i++) { + ((char*)buf)[i] = user_param->payload_content[i % user_param->payload_length]; + } + } else { + for (i = 0; i < ctx->buff_size/4; i++) { + if (user_param->data_validation == SERIAL) { + buf_ptr[i] = htonl(current_data); + current_data++; + } else { + buf_ptr[i] = htonl(perftest_rand(¤t_data)); + } + } + } +} + static void initialize_buffer_content(struct pingpong_context *ctx, struct perftest_parameters *user_param, - int qp_index, bool can_init_mem) + void* buf, bool can_init_mem) { if (!can_init_mem) return; - uint32_t rng_state = init_perftest_rand_state(); - if ((user_param->verb == WRITE || user_param->verb == WRITE_IMM) && user_param->tst == LAT) { - memset(ctx->buf[qp_index], 0, ctx->buff_size); + if (((user_param->verb == WRITE || user_param->verb == WRITE_IMM) && user_param->tst == LAT) || + (user_param->data_validation && user_param->machine == SERVER)) { + memset(buf, 0, ctx->buff_size); } else { - uint64_t i; - if (user_param->has_payload_modification) { - for (i = 0; i < ctx->buff_size; i++) { - ((char*)ctx->buf[qp_index])[i] = user_param->payload_content[i % user_param->payload_length]; - } - } else { - uint32_t *buf_ptr = (uint32_t*)ctx->buf[qp_index]; - for (i = 0; i < ctx->buff_size/4; i++) { - buf_ptr[i] = perftest_rand(&rng_state); - } - } + generate_buffer_data(ctx, user_param, buf); } } @@ -2069,7 +2115,7 @@ int create_single_mr(struct pingpong_context *ctx, struct perftest_parameters *u } /* Initialize buffer content */ - initialize_buffer_content(ctx, user_param, qp_index, can_init_mem); + initialize_buffer_content(ctx, user_param, ctx->buf[qp_index], can_init_mem); return SUCCESS; } @@ -2155,6 +2201,27 @@ static int create_payload(struct perftest_parameters *user_param) return 0; } +int create_data_validation_reference_buffer(struct pingpong_context *ctx, struct perftest_parameters *user_param) { + bool can_init_mem = true; + int dmabuf_fd = 0; + uint64_t dmabuf_offset; + + if (ctx->memory->allocate_buffer(ctx->memory, user_param->cycle_buffer, ctx->buff_size, + &dmabuf_fd, &dmabuf_offset, &ctx->validation_buf[0], + &can_init_mem)) { + return FAILURE; + } + + generate_buffer_data(ctx, user_param, ctx->validation_buf[0]); + + for (int i = 1; i < user_param->num_of_qps; i++) { + ctx->validation_buf[i] = ctx->validation_buf[0] + (i * INC(user_param->size, ctx->cache_line_size) * user_param->tx_depth); + } + + return SUCCESS; +} + + /****************************************************************************** * ******************************************************************************/ @@ -2186,8 +2253,13 @@ int create_mr(struct pingpong_context *ctx, struct perftest_parameters *user_par mr_index++; } else { ctx->mr[i] = ctx->mr[0]; - // cppcheck-suppress arithOperationsOnVoidPointer - ctx->buf[i] = ctx->buf[0] + (i*BUFF_SIZE(ctx->size, ctx->cycle_buffer)); + if (user_param->machine == CLIENT || !user_param->data_validation) { + // cppcheck-suppress arithOperationsOnVoidPointer + ctx->buf[i] = ctx->buf[0] + (i*BUFF_SIZE(ctx->size, ctx->cycle_buffer)); + } else { + // cppcheck-suppress arithOperationsOnVoidPointer + ctx->buf[i] = ctx->buf[0] + (user_param->num_of_qps + i) * ctx->send_qp_buff_size; + } } } @@ -3602,7 +3674,8 @@ void ctx_set_send_reg_wqes(struct pingpong_context *ctx, ctx->sge_list[i*user_param->post_list +j].addr = ctx->sge_list[i*user_param->post_list + (j-1)].addr; - if ((user_param->tst == BW || user_param->tst == LAT_BY_BW) && user_param->size <= (ctx->cycle_buffer / 2)) + if (((user_param->tst == BW || user_param->tst == LAT_BY_BW) && user_param->size <= (ctx->cycle_buffer / 2)) || + user_param->data_validation) increase_loc_addr(&ctx->sge_list[i*user_param->post_list +j],user_param->size, j-1,ctx->my_addr[i],0,ctx->cache_line_size,ctx->cycle_buffer); } @@ -3611,6 +3684,14 @@ void ctx_set_send_reg_wqes(struct pingpong_context *ctx, ctx->wr[i*user_param->post_list + j].num_sge = MAX_SEND_SGE; ctx->wr[i*user_param->post_list + j].wr_id = build_wr_id(i * user_param->post_list + j, i); + /* In data validation the immediate field is used to send the address offset + * from the beginning of the QP buffer for the receiver to know where to start + * to validate from in the validation reference buffer. + */ + if (user_param->data_validation) { + ctx->wr[i*user_param->post_list + j].imm_data = ctx->sge_list[i*user_param->post_list +j].addr - (uintptr_t)ctx->buf[i]; + } + if (j == (user_param->post_list - 1)) { ctx->wr[i*user_param->post_list + j].next = NULL; } else { @@ -3639,7 +3720,8 @@ void ctx_set_send_reg_wqes(struct pingpong_context *ctx, ctx->wr[i*user_param->post_list + j].wr.rdma.remote_addr = ctx->wr[i*user_param->post_list + (j-1)].wr.rdma.remote_addr; - if ((user_param->tst == BW || user_param->tst == LAT_BY_BW ) && user_param->size <= (ctx->cycle_buffer / 2)) + if (((user_param->tst == BW || user_param->tst == LAT_BY_BW ) && user_param->size <= (ctx->cycle_buffer / 2)) || + user_param->data_validation) increase_rem_addr(&ctx->wr[i*user_param->post_list + j],user_param->size, j-1,ctx->rem_addr[i],WRITE,ctx->cache_line_size,ctx->cycle_buffer); } @@ -3751,7 +3833,8 @@ int ctx_set_recv_wqes(struct pingpong_context *ctx,struct perftest_parameters *u if (j > 0) { ctx->recv_sge_list[i * user_param->recv_post_list + j].addr = ctx->recv_sge_list[i * user_param->recv_post_list + j - 1].addr; - if ((user_param->tst == BW || user_param->tst == LAT_BY_BW) && user_param->size <= (ctx->cycle_buffer / 2)) { + if (((user_param->tst == BW || user_param->tst == LAT_BY_BW) && user_param->size <= (ctx->cycle_buffer / 2)) || + user_param->data_validation) { increase_loc_addr(&ctx->recv_sge_list[i * user_param->recv_post_list + j], user_param->size, j-1, @@ -4259,6 +4342,10 @@ int run_iter_bw_server(struct pingpong_context *ctx, struct perftest_parameters uintptr_t primary_recv_addr = ctx->recv_sge_list[0].addr; int recv_flows_burst = 0; int address_flows_offset =0; + uint32_t recv_offset; + uint32_t data_length; + uint64_t expected_data_addr; + uint64_t actual_data_addr; struct dyn_poll_ctx *dyn_ctx = init_dyn_poll_ctx(user_param); if (!dyn_ctx) { @@ -4336,6 +4423,20 @@ int run_iter_bw_server(struct pingpong_context *ctx, struct perftest_parameters return_value = FAILURE; goto cleaning; } + + if (user_param->data_validation) { + recv_offset = wc[i].imm_data; + data_length = wc[i].byte_len; + expected_data_addr = (uint64_t)ctx->validation_buf[qp_index] + recv_offset; + actual_data_addr = ctx->rx_buffer_addr[qp_index] + recv_offset; + + if (memcmp((void*)expected_data_addr, (void*)actual_data_addr, data_length)) { + fprintf(stderr, "Data validation comparison failed.\n"); + return_value = FAILURE; + goto cleaning; + } + } + rcnt_for_qp[qp_index]++; rcnt++; unused_recv_for_qp[qp_index]++; diff --git a/src/perftest_resources.h b/src/perftest_resources.h index e5d726d1..fe3ce54c 100755 --- a/src/perftest_resources.h +++ b/src/perftest_resources.h @@ -301,6 +301,8 @@ struct pingpong_context { #ifdef HAVE_REG_MR_EX struct ibv_dmah *dmah; #endif + uint32_t data_validation_hint; + void **validation_buf; }; struct pingpong_dest { @@ -313,6 +315,7 @@ struct pingpong_context { union ibv_gid gid; unsigned srqn; int gid_index; + uint32_t data_validation_hint; }; /****************************************************************************** @@ -985,6 +988,23 @@ int create_single_mr(struct pingpong_context *ctx, int create_mr(struct pingpong_context *ctx, struct perftest_parameters *user_param); +/* create_data_validation_reference_buffer + * + * Description : + * + * Creates a memory buffer to be used for data validation scenarios. + * Takes into consideration all user parameters and test type. + * + * Parameters : + * ctx - Resources sructure. + * user_param - the perftest parameters. + * + * Return Value : SUCCESS, FAILURE. + * + */ +int create_data_validation_reference_buffer(struct pingpong_context *ctx, + struct perftest_parameters *user_param); + /* alloc_hugapage_region * * Description : diff --git a/src/write_bw.c b/src/write_bw.c index 14b9e128..618c9bc6 100755 --- a/src/write_bw.c +++ b/src/write_bw.c @@ -222,12 +222,24 @@ int main(int argc, char *argv[]) ctx_print_pingpong_data(&rem_dest[i],&user_comm); } + if (user_param.machine == CLIENT && user_param.data_validation) { + my_dest->data_validation_hint = ctx.data_validation_hint; + } + /* An additional handshake is required after moving qp to RTR. */ if (ctx_hand_shake(&user_comm,&my_dest[0],&rem_dest[0])) { fprintf(stderr," Failed to exchange data between server and clients\n"); goto destroy_context; } + if (user_param.machine == SERVER && user_param.data_validation) { + ctx.data_validation_hint = rem_dest->data_validation_hint; + if (create_data_validation_reference_buffer(&ctx, &user_param)) { + fprintf(stderr," Failed to allocate and randomize data validation buffer at server side\n"); + goto free_mem; + } + } + if (user_param.output == FULL_VERBOSITY) { if (user_param.report_per_port) { printf(RESULT_LINE_PER_PORT);