UCT/IB/ROCE: Use ipv6 non link-local GID in priority

Author: tvegas1

src/uct/ib/base/ib_device.c

@@ -849,6 +849,11 @@ const char *uct_ib_gid_str(const union ibv_gid *gid, char *str, size_t max_size)
     return str;
 }
 
+static int uct_ib_gid_is_ipv6_ll(const union ibv_gid *gid)
+{
+    return ((gid->raw[0] == 0xfe) && ((gid->raw[1] & 0xc0) == 0x80));
+}
+
 static int uct_ib_device_is_addr_ipv4_mcast(const struct in6_addr *raw,
                                             const uint32_t addr_last_bits)
 {
@@ -1074,11 +1079,16 @@ uct_ib_device_select_gid(uct_ib_device_t *dev, uint8_t port_num,
                          uct_ib_device_gid_info_t *gid_info)
 {
     static const size_t max_str_len                     = 200;
-    static const uct_ib_roce_version_info_t roce_prio[] = {
-        {UCT_IB_DEVICE_ROCE_V2, AF_INET},
-        {UCT_IB_DEVICE_ROCE_V2, AF_INET6},
-        {UCT_IB_DEVICE_ROCE_V1, AF_INET},
-        {UCT_IB_DEVICE_ROCE_V1, AF_INET6}
+    struct {
+        uct_ib_roce_version_info_t info;
+        int                        skip_ll; /* link-local not allowed when true */
+    } roce_prio[] = {
+        {{UCT_IB_DEVICE_ROCE_V2, AF_INET}},
+        {{UCT_IB_DEVICE_ROCE_V2, AF_INET6}, 1},
+        {{UCT_IB_DEVICE_ROCE_V2, AF_INET6}},
+        {{UCT_IB_DEVICE_ROCE_V1, AF_INET}},
+        {{UCT_IB_DEVICE_ROCE_V2, AF_INET6}, 1},
+        {{UCT_IB_DEVICE_ROCE_V1, AF_INET6}}
     };
     int gid_tbl_len         = uct_ib_device_port_attr(dev, port_num)->gid_tbl_len;
     ucs_status_t status     = UCS_OK;
@@ -1108,8 +1118,9 @@ uct_ib_device_select_gid(uct_ib_device_t *dev, uint8_t port_num,
                 goto out;
             }
 
-            if ((roce_prio[prio_idx].ver         == gid_info_tmp.roce_info.ver) &&
-                (roce_prio[prio_idx].addr_family == gid_info_tmp.roce_info.addr_family) &&
+            if ((roce_prio[prio_idx].info.ver         == gid_info_tmp.roce_info.ver) &&
+                (roce_prio[prio_idx].info.addr_family == gid_info_tmp.roce_info.addr_family) &&
+                (!roce_prio[prio_idx].skip_ll || !uct_ib_gid_is_ipv6_ll(&gid_info_tmp.gid)) &&
                 uct_ib_device_test_roce_gid_index(dev, port_num, &gid_info_tmp.gid, i) &&
                 uct_ib_device_match_roce_subnet(&gid_info_tmp, &subnets,
                                                 subnet_strs->mode)) {