Perftest: disable Scatter2CQE with gpudirect send_lat

mlx5 driver enables SCATETR2CQE feature by default – up to 64B payloads.
Meaning, messages that are up-to 64B, can be scattered to CQE on the
responder side.

Peer memory in general, and specifically here with GPUDirect enabled
doesn’t work with this feature, and it must be disabled.

Signed-off-by: Shmuel Shaul <[email protected]>

Author: sshaulnv

src/perftest_parameters.c

@@ -1153,6 +1153,21 @@ int check_intense_polling(struct perftest_parameters *user_param){
 	return OFF;
 }
 
+/******************************************************************************
+ *
+ ******************************************************************************/
+int disable_mlx5_scatter_to_cqe() {
+    const char *var_name = "MLX5_SCATTER_TO_CQE";
+    const char *var_value = "0";
+
+    if (setenv(var_name, var_value, 1)) {
+		fprintf(stderr,"Failed to set environment variable\n");
+		return FAILURE;
+	}
+
+	return SUCCESS;
+}
+
 /******************************************************************************
  *
  ******************************************************************************/
@@ -1897,6 +1912,13 @@ static void force_dependecies(struct perftest_parameters *user_param)
 		exit(1);
 	}
 
+	if (user_param->memory_type == MEMORY_CUDA && user_param->verb == SEND && (user_param->size <= 64 || user_param->test_method == RUN_ALL)) {
+		printf(RESULT_LINE);
+		printf("Scatter2CQE (size <= 64) is not supported with GPUDirect send tests: setting MLX5_SCATTER_TO_CQE=0\n");
+		if (disable_mlx5_scatter_to_cqe())
+			exit(1);
+	}
+
 	if (user_param->memory_type == MEMORY_CUDA && (int)user_param->size <= user_param->inline_size) {
 		printf(RESULT_LINE);
 		fprintf(stderr,"Perftest doesn't support CUDA tests with inline messages\n");