Merge pull request #10910 from Sergei-Lebedev/topic/config_gdaki_lat

UCT/GDA/TEST: add config parameter for gda latency thresh

Author: yosefe

src/uct/ib/base/ib_md.c

@@ -119,6 +119,11 @@ ucs_config_field_t uct_ib_md_config_table[] = {
      "Use GPU Direct RDMA for HCA to access GPU pages directly\n",
      ucs_offsetof(uct_ib_md_config_t, enable_gpudirect_rdma), UCS_CONFIG_TYPE_TERNARY},
 
+    {"GPU_IB_DISTANCE_LATENCY_THRESH", "300ns",
+     "Skip GPU device if the distance latency to the IB device is greater than this value.",
+     ucs_offsetof(uct_ib_md_config_t, ext.gpu_ib_distance_latency_thresh),
+     UCS_CONFIG_TYPE_TIME},
+
     {"PCI_BW", "",
      "Maximum effective data transfer rate of PCI bus connected to HCA\n",
      ucs_offsetof(uct_ib_md_config_t, pci_bw), UCS_CONFIG_TYPE_ARRAY(pci_bw)},

src/uct/ib/base/ib_md.h

@@ -109,6 +109,7 @@ typedef struct uct_ib_md_ext_config {
     unsigned long            reg_retry_cnt; /**< Memory registration retry count */
     unsigned                 smkey_block_size; /**< Mkey indexes in a symmetric block */
     int                      direct_nic; /**< Direct NIC with GPU functionality */
+    double                   gpu_ib_distance_latency_thresh; /**< Threshold to filter GPU<->IB distance */
 } uct_ib_md_ext_config_t;
 
 

src/uct/ib/mlx5/gdaki/gdaki.c

@@ -24,15 +24,15 @@ typedef struct {
 } uct_rc_gdaki_iface_config_t;
 
 ucs_config_field_t uct_rc_gdaki_iface_config_table[] = {
-  {UCT_IB_CONFIG_PREFIX, "", NULL,
-   ucs_offsetof(uct_rc_gdaki_iface_config_t, super),
-   UCS_CONFIG_TYPE_TABLE(uct_rc_iface_common_config_table)},
+    {UCT_IB_CONFIG_PREFIX, "", NULL,
+     ucs_offsetof(uct_rc_gdaki_iface_config_t, super),
+     UCS_CONFIG_TYPE_TABLE(uct_rc_iface_common_config_table)},
 
-  {UCT_IB_CONFIG_PREFIX, "", NULL,
-   ucs_offsetof(uct_rc_gdaki_iface_config_t, mlx5),
-   UCS_CONFIG_TYPE_TABLE(uct_rc_mlx5_common_config_table)},
+    {UCT_IB_CONFIG_PREFIX, "", NULL,
+     ucs_offsetof(uct_rc_gdaki_iface_config_t, mlx5),
+     UCS_CONFIG_TYPE_TABLE(uct_rc_mlx5_common_config_table)},
 
-  {NULL}
+    {NULL}
 };
 
 
@@ -657,7 +657,7 @@ uct_gdaki_query_tl_devices(uct_md_h tl_md,
         }
 
         /* TODO this logic should be done in UCP */
-        if (dist.latency > 300.0 / UCS_NSEC_PER_SEC) {
+        if (dist.latency > md->super.config.gpu_ib_distance_latency_thresh) {
             continue;
         }
 

test/gtest/ucp/test_ucp_device.cc

@@ -70,6 +70,7 @@ void test_ucp_device::get_test_variants(std::vector<ucp_test_variant> &variants)
 void test_ucp_device::init()
 {
     m_env.push_back(new ucs::scoped_setenv("UCX_CUDA_IPC_ENABLE_SAME_PROCESS", "y"));
+    m_env.push_back(new ucs::scoped_setenv("UCX_IB_GPU_IB_DISTANCE_LATENCY_THRESH", "1000ns"));
     ucp_test::init();
     sender().connect(&receiver(), get_ep_params());
     if (!is_loopback()) {