UCT/IB/MLX5: Fix DEVX for not WC-supported UAR (#11310)

* UCT/IB/MLX5: Fix DEVX for not WC-supported UAR

Fix two separate bugs related to UCX assuming UAR pages of HCA always
supporting WC (write combining) and not checking
UCT_IB_MLX5_MD_FLAG_UAR_USE_WC flag :
- Doorbell path hardcoded bf_size=256, forcing BF mode even if UAR is
  NC-mapped
- Device memory was allocated and copied data via MMIO writes even if it
  was not supported (because it requires WC)

* UCT/IB/MLX5: PR fix - code convention

Author: nbellalou

src/uct/ib/mlx5/dv/ib_mlx5_dv.c

@@ -133,6 +133,7 @@ ucs_status_t uct_ib_mlx5_devx_create_qp_common(uct_ib_iface_t *iface,
 {
     uct_ib_mlx5_md_t *md = ucs_derived_of(iface->super.md, uct_ib_mlx5_md_t);
     uct_ib_device_t *dev = &md->super.dev;
+    uint64_t bf_size     = 0;
     char in[UCT_IB_MLX5DV_ST_SZ_BYTES(create_qp_in)]           = {};
     char out[UCT_IB_MLX5DV_ST_SZ_BYTES(create_qp_out)]         = {};
     char in_2init[UCT_IB_MLX5DV_ST_SZ_BYTES(rst2init_qp_in)]   = {};
@@ -144,8 +145,12 @@ ucs_status_t uct_ib_mlx5_devx_create_qp_common(uct_ib_iface_t *iface,
 
     uct_ib_iface_fill_attr(iface, &attr->super);
 
+    if (md->flags & UCT_IB_MLX5_MD_FLAG_UAR_USE_WC) {
+        bf_size = UCT_IB_MLX5_BF_REG_SIZE;
+    }
+
     status = uct_ib_mlx5_get_mmio_mode(iface->super.worker, attr->mmio_mode, 0,
-                                       UCT_IB_MLX5_BF_REG_SIZE, &mmio_mode);
+                                       bf_size, &mmio_mode);
     if (status != UCS_OK) {
         goto err;
     }

src/uct/ib/mlx5/rc/rc_mlx5_common.c

@@ -1046,6 +1046,11 @@ uct_rc_mlx5_iface_common_dm_init(uct_rc_mlx5_iface_common_t *iface,
         goto fallback;
     }
 
+    if (!(uct_ib_mlx5_iface_md(&rc_iface->super)->flags &
+          UCT_IB_MLX5_MD_FLAG_UAR_USE_WC)) {
+        goto fallback;
+    }
+
     iface->dm.dm = uct_worker_tl_data_get(iface->super.super.super.worker,
                                           UCT_IB_MLX5_WORKER_DM_KEY,
                                           uct_mlx5_dm_data_t,