UCT/IB: disable indirect atomic registration for direct nic (#11111)

Signed-off-by: Roie Danino <rdanino@nvidia.com>

Author: roiedanino

src/uct/ib/base/ib_md.h

@@ -65,6 +65,8 @@ enum {
                                                         threads */
     UCT_IB_MEM_IMPORTED              = UCS_BIT(3), /**< The memory handle was
                                                         created by mem_attach */
+    UCT_IB_MEM_DIRECT_NIC            = UCS_BIT(6), /**< The memory handle was
+                                                        registered using Direct NIC */
 #if ENABLE_PARAMS_CHECK
     UCT_IB_MEM_ACCESS_REMOTE_RMA     = UCS_BIT(4), /**< RMA access was requested
                                                         for the memory region */

src/uct/ib/mlx5/dv/ib_mlx5dv_md.c

@@ -855,6 +855,7 @@ uct_ib_mlx5_devx_reg_mr(uct_ib_mlx5_md_t *md, uct_ib_mlx5_devx_mem_t *memh,
                                                                 length, params,
                                                                 access_flags);
     if (memh->mrs[mr_type].super.ib != NULL) {
+        memh->super.flags |= UCT_IB_MEM_DIRECT_NIC;
         goto out;
     }
 
@@ -2960,6 +2961,7 @@ uct_ib_mlx5_devx_mkey_pack(uct_md_h uct_md, uct_mem_h uct_memh,
         ((memh->super.flags & UCT_IB_MEM_ACCESS_REMOTE_ATOMIC) ||
          uct_ib_mlx5_devx_memh_has_ro(md, memh)) &&
         !(memh->super.flags & UCT_IB_MEM_IMPORTED) &&
+        !(memh->super.flags & UCT_IB_MEM_DIRECT_NIC) &&
         md->super.config.enable_indirect_atomic &&
         ucs_test_all_flags(md->flags,
                            UCT_IB_MLX5_MD_FLAG_KSM |