Skip to content

Commit e8a2d2e

Browse files
brminichbrminich
committed
UCP: CR Comments p2
1 parent bbe8ce7 commit e8a2d2e

File tree

1 file changed

+71
-79
lines changed

1 file changed

+71
-79
lines changed

‎src/ucp/core/ucp_context.c‎

Lines changed: 71 additions & 79 deletions
Original file line numberDiff line numberDiff line change
@@ -1607,30 +1607,6 @@ static ucs_status_t ucp_check_resources(ucp_context_h context,
16071607
return ucp_check_tl_names(context);
16081608
}
16091609

1610-
static void
1611-
ucp_context_update_md_maps(ucp_context_h context, ucs_memory_type_t mem_type,
1612-
unsigned md_index)
1613-
{
1614-
const uct_md_attr_v2_t *md_attr = &context->tl_mds[md_index].attr;
1615-
1616-
if (!(md_attr->flags & UCT_MD_FLAG_REG)) {
1617-
return;
1618-
}
1619-
1620-
if (md_attr->reg_mem_types & UCS_BIT(mem_type)) {
1621-
context->reg_md_map[mem_type] |= UCS_BIT(md_index);
1622-
}
1623-
1624-
if (md_attr->cache_mem_types & UCS_BIT(mem_type)) {
1625-
context->cache_md_map[mem_type] |= UCS_BIT(md_index);
1626-
}
1627-
1628-
if ((context->config.ext.gva_enable != UCS_CONFIG_OFF) &&
1629-
(md_attr->gva_mem_types & UCS_BIT(mem_type))) {
1630-
context->gva_md_map[mem_type] |= UCS_BIT(md_index);
1631-
}
1632-
}
1633-
16341610
static ucs_status_t
16351611
ucp_add_component_resources(ucp_context_h context, ucp_rsc_index_t cmpt_index,
16361612
ucs_string_set_t avail_devices[],
@@ -1641,16 +1617,14 @@ ucp_add_component_resources(ucp_context_h context, ucp_rsc_index_t cmpt_index,
16411617
{
16421618
const ucp_tl_cmpt_t *tl_cmpt = &context->tl_cmpts[cmpt_index];
16431619
size_t avail_mds = config->max_component_mds;
1620+
uint64_t mem_type_mask = UCS_BIT(UCS_MEMORY_TYPE_HOST);
16441621
uct_component_attr_t uct_component_attr;
16451622
unsigned num_tl_resources;
16461623
ucs_status_t status;
16471624
ucp_rsc_index_t i;
1625+
const uct_md_attr_v2_t *md_attr;
16481626
unsigned md_index;
1649-
uint64_t mem_type_mask;
16501627
uint64_t mem_type_bitmap;
1651-
ucs_memory_type_t mem_type;
1652-
const uct_md_attr_v2_t *md_attr;
1653-
int md_supports_nonblock_reg;
16541628

16551629
/* List memory domain resources */
16561630
uct_component_attr.field_mask = UCT_COMPONENT_ATTR_FIELD_MD_RESOURCES |
@@ -1664,7 +1638,6 @@ ucp_add_component_resources(ucp_context_h context, ucp_rsc_index_t cmpt_index,
16641638
}
16651639

16661640
/* Open all memory domains */
1667-
mem_type_mask = UCS_BIT(UCS_MEMORY_TYPE_HOST);
16681641
for (i = 0; i < tl_cmpt->attr.md_resource_count; ++i) {
16691642
if (avail_mds == 0) {
16701643
ucs_debug("only first %zu domains kept for component %s with %u "
@@ -1712,47 +1685,6 @@ ucp_add_component_resources(ucp_context_h context, ucp_rsc_index_t cmpt_index,
17121685
mem_type_mask |= mem_type_bitmap;
17131686
}
17141687

1715-
if (md_attr->flags & UCT_MD_FLAG_REG) {
1716-
ucs_memory_type_for_each(mem_type) {
1717-
/* Record availability of non-blocking registration support */
1718-
md_supports_nonblock_reg = md_attr->reg_nonblock_mem_types &
1719-
UCS_BIT(mem_type);
1720-
if (md_supports_nonblock_reg) {
1721-
context->reg_nb_supported_mem_types |= UCS_BIT(mem_type);
1722-
}
1723-
1724-
if ((context->config.ext.reg_nb_mem_types & UCS_BIT(mem_type)) &&
1725-
!md_supports_nonblock_reg) {
1726-
if (md_attr->reg_mem_types & UCS_BIT(mem_type)) {
1727-
/* Keep map of MDs supporting blocking registration
1728-
* if non-blocking registration is requested for the
1729-
* given memory type. In some cases blocking
1730-
* registration maybe required anyway (e.g. internal
1731-
* staging buffers for rndv pipeline protocols). */
1732-
context->reg_block_md_map[mem_type] |= UCS_BIT(md_index);
1733-
}
1734-
continue;
1735-
}
1736-
1737-
ucp_context_update_md_maps(context, mem_type, md_index);
1738-
}
1739-
}
1740-
1741-
if (md_attr->flags & UCT_MD_FLAG_EXPORTED_MKEY) {
1742-
context->export_md_map |= UCS_BIT(md_index);
1743-
}
1744-
1745-
if (md_attr->flags & UCT_MD_FLAG_REG_DMABUF) {
1746-
context->dmabuf_reg_md_map |= UCS_BIT(md_index);
1747-
}
1748-
1749-
ucs_for_each_bit(mem_type, md_attr->dmabuf_mem_types) {
1750-
/* In case of multiple providers, take the first one */
1751-
if (context->dmabuf_mds[mem_type] == UCP_NULL_RESOURCE) {
1752-
context->dmabuf_mds[mem_type] = md_index;
1753-
}
1754-
}
1755-
17561688
++context->num_mds;
17571689
}
17581690

@@ -1789,24 +1721,84 @@ static ucs_status_t ucp_fill_aux_tls(ucs_string_set_t *aux_tls)
17891721
return UCS_OK;
17901722
}
17911723

1724+
static void
1725+
ucp_update_memtype_md_map(uint64_t mem_types_map, ucs_memory_type_t mem_type,
1726+
ucp_md_index_t md_index, ucp_md_map_t *md_map_p)
1727+
{
1728+
if (mem_types_map & UCS_BIT(mem_type)) {
1729+
*md_map_p |= UCS_BIT(md_index);
1730+
}
1731+
}
1732+
17921733
static void ucp_fill_resources_reg_md_map_update(ucp_context_h context)
17931734
{
17941735
UCS_STRING_BUFFER_ONSTACK(strb, 256);
1736+
ucp_md_map_t reg_block_md_map = 0;
1737+
ucp_md_map_t reg_nonblock_md_map = 0;
17951738
ucs_memory_type_t mem_type;
17961739
ucp_md_index_t md_index;
1740+
const uct_md_attr_v2_t *md_attr;
17971741

1798-
ucs_memory_type_for_each(mem_type) {
1799-
/* Fallback: If non-blocking registration was requested for this memory
1800-
* type but no MD actually supports it, treat it as if the request was
1801-
* not set (i.e., allow blocking-capable MDs as well). */
1802-
if (context->config.ext.reg_nb_mem_types & UCS_BIT(mem_type)) {
1803-
if (!(context->reg_nb_supported_mem_types & UCS_BIT(mem_type))) {
1804-
ucs_assert(context->reg_md_map[mem_type] == 0);
1742+
for (md_index = 0; md_index < context->num_mds; ++md_index) {
1743+
md_attr = &context->tl_mds[md_index].attr;
1744+
if (md_attr->flags & UCT_MD_FLAG_EXPORTED_MKEY) {
1745+
context->export_md_map |= UCS_BIT(md_index);
1746+
}
18051747

1806-
for (md_index = 0; md_index < context->num_mds; ++md_index) {
1807-
ucp_context_update_md_maps(context, mem_type, md_index);
1748+
if (md_attr->flags & UCT_MD_FLAG_REG_DMABUF) {
1749+
context->dmabuf_reg_md_map |= UCS_BIT(md_index);
1750+
}
1751+
}
1752+
1753+
ucs_memory_type_for_each(mem_type) {
1754+
for (md_index = 0; md_index < context->num_mds; ++md_index) {
1755+
md_attr = &context->tl_mds[md_index].attr;
1756+
if (md_attr->dmabuf_mem_types & UCS_BIT(mem_type)) {
1757+
/* In case of multiple providers, take the first one */
1758+
if (context->dmabuf_mds[mem_type] == UCP_NULL_RESOURCE) {
1759+
context->dmabuf_mds[mem_type] = md_index;
18081760
}
18091761
}
1762+
1763+
if (!(md_attr->flags & UCT_MD_FLAG_REG)) {
1764+
continue;
1765+
}
1766+
1767+
ucp_update_memtype_md_map(
1768+
md_attr->reg_nonblock_mem_types, mem_type,
1769+
md_index, &reg_nonblock_md_map);
1770+
ucp_update_memtype_md_map(
1771+
md_attr->reg_mem_types, mem_type, md_index,
1772+
&reg_block_md_map);
1773+
ucp_update_memtype_md_map(
1774+
md_attr->cache_mem_types, mem_type, md_index,
1775+
&context->cache_md_map[mem_type]);
1776+
1777+
if (context->config.ext.gva_enable != UCS_CONFIG_OFF) {
1778+
ucp_update_memtype_md_map(
1779+
md_attr->gva_mem_types, mem_type, md_index,
1780+
&context->gva_md_map[mem_type]);
1781+
}
1782+
}
1783+
1784+
if (context->config.ext.reg_nb_mem_types & UCS_BIT(mem_type)) {
1785+
if (reg_nonblock_md_map != 0) {
1786+
/* Keep map of MDs supporting blocking registration
1787+
* if non-blocking registration is requested for the
1788+
* given memory type. In some cases blocking
1789+
* registration maybe required anyway (e.g. internal
1790+
* staging buffers for rndv pipeline protocols). */
1791+
context->reg_block_md_map[mem_type] =
1792+
reg_block_md_map & ~reg_nonblock_md_map;
1793+
context->reg_md_map[mem_type] = reg_nonblock_md_map;
1794+
} else {
1795+
/* Fallback: non-blocking registration was requested for this
1796+
* memory type but no MD actually supports it, treat it as if
1797+
* the request was not set (i.e., allow blocking-capable MDs as
1798+
* well). */
1799+
context->reg_block_md_map[mem_type] = reg_block_md_map;
1800+
context->reg_md_map[mem_type] = reg_block_md_map;
1801+
}
18101802
}
18111803

18121804
/* If we have a dmabuf provider for a memory type, it means we can

0 commit comments

Comments
 (0)