Skip to content

v3.1.x: btl/tcp: use a hash map for kernel IP interface indexes #5331

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 3 commits into from
Jul 2, 2018
Merged

v3.1.x: btl/tcp: use a hash map for kernel IP interface indexes #5331

Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
86ecc55
btl/tcp: use a hash map for kernel IP interface indexes
jsquyres Jun 23, 2018
13b34b4
btl/tcp: fix hash map usage
jsquyres Jun 28, 2018
d61b6c1
tcp/btl: fix a cast
jsquyres Jun 29, 2018
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
37 changes: 22 additions & 15 deletions opal/mca/btl/tcp/btl_tcp_proc.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -51,10 +51,10 @@ static void mca_btl_tcp_proc_destruct(mca_btl_tcp_proc_t* proc);

struct mca_btl_tcp_proc_data_t {
mca_btl_tcp_interface_t** local_interfaces;
int local_kindex_to_index[MAX_KERNEL_INTERFACE_INDEX];
opal_hash_table_t local_kindex_to_index;
size_t num_local_interfaces, max_local_interfaces;
size_t num_peer_interfaces;
int peer_kindex_to_index[MAX_KERNEL_INTERFACE_INDEX];
opal_hash_table_t peer_kindex_to_index;
unsigned int *best_assignment;
int max_assignment_weight;
int max_assignment_cardinality;
Expand Down Expand Up @@ -280,8 +280,6 @@ static mca_btl_tcp_interface_t** mca_btl_tcp_retrieve_local_interfaces(mca_btl_t
if( NULL == proc_data->local_interfaces )
return NULL;

memset(proc_data->local_kindex_to_index, -1, sizeof(int)*MAX_KERNEL_INTERFACE_INDEX);

/* Collect up the list of included and excluded interfaces, if any */
include = opal_argv_split(mca_btl_tcp_component.tcp_if_include,',');
exclude = opal_argv_split(mca_btl_tcp_component.tcp_if_exclude,',');
Expand All @@ -291,7 +289,8 @@ static mca_btl_tcp_interface_t** mca_btl_tcp_retrieve_local_interfaces(mca_btl_t
* the local node
*/
for( idx = opal_ifbegin(); idx >= 0; idx = opal_ifnext (idx) ) {
int kindex, index;
int kindex;
uint64_t index;
bool skip = false;

opal_ifindextoaddr (idx, (struct sockaddr*) &local_addr, sizeof (local_addr));
Expand Down Expand Up @@ -340,12 +339,12 @@ static mca_btl_tcp_interface_t** mca_btl_tcp_retrieve_local_interfaces(mca_btl_t
}

kindex = opal_ifindextokindex(idx);
index = proc_data->local_kindex_to_index[kindex];
int rc = opal_hash_table_get_value_uint32(&proc_data->local_kindex_to_index, kindex, (void**) &index);

/* create entry for this kernel index previously not seen */
if(-1 == index) {
if (OPAL_SUCCESS != rc) {
index = proc_data->num_local_interfaces++;
proc_data->local_kindex_to_index[kindex] = index;
opal_hash_table_set_value_uint32(&proc_data->local_kindex_to_index, kindex, (void*)(uintptr_t) index);

if( proc_data->num_local_interfaces == proc_data->max_local_interfaces ) {
proc_data->max_local_interfaces <<= 1;
Expand All @@ -359,7 +358,7 @@ static mca_btl_tcp_interface_t** mca_btl_tcp_retrieve_local_interfaces(mca_btl_t
mca_btl_tcp_initialise_interface(proc_data->local_interfaces[index], kindex, index);
}

local_interface = proc_data->local_interfaces[proc_data->local_kindex_to_index[kindex]];
local_interface = proc_data->local_interfaces[index];
switch(local_addr.ss_family) {
case AF_INET:
/* if AF is disabled, skip it completely */
Expand Down Expand Up @@ -420,13 +419,18 @@ int mca_btl_tcp_proc_insert( mca_btl_tcp_proc_t* btl_proc,
mca_btl_tcp_interface_t** peer_interfaces;
mca_btl_tcp_proc_data_t _proc_data, *proc_data=&_proc_data;
size_t max_peer_interfaces;
memset(proc_data, 0, sizeof(mca_btl_tcp_proc_data_t));
char str_local[128], str_remote[128];

if (NULL == (proc_hostname = opal_get_proc_hostname(btl_proc->proc_opal))) {
return OPAL_ERR_UNREACH;
}

memset(proc_data, 0, sizeof(mca_btl_tcp_proc_data_t));
OBJ_CONSTRUCT(&_proc_data.local_kindex_to_index, opal_hash_table_t);
opal_hash_table_init(&_proc_data.local_kindex_to_index, 8);
OBJ_CONSTRUCT(&_proc_data.peer_kindex_to_index, opal_hash_table_t);
opal_hash_table_init(&_proc_data.peer_kindex_to_index, 8);

#ifndef WORDS_BIGENDIAN
/* if we are little endian and our peer is not so lucky, then we
need to put all information sent to him in big endian (aka
Expand All @@ -453,7 +457,6 @@ int mca_btl_tcp_proc_insert( mca_btl_tcp_proc_t* btl_proc,
peer_interfaces = (mca_btl_tcp_interface_t**)calloc( max_peer_interfaces, sizeof(mca_btl_tcp_interface_t*) );
assert(NULL != peer_interfaces);
proc_data->num_peer_interfaces = 0;
memset(proc_data->peer_kindex_to_index, -1, sizeof(int)*MAX_KERNEL_INTERFACE_INDEX);

/*
* identify all kernel interfaces and the associated addresses of
Expand All @@ -462,17 +465,17 @@ int mca_btl_tcp_proc_insert( mca_btl_tcp_proc_t* btl_proc,

for( i = 0; i < btl_proc->proc_addr_count; i++ ) {

int index;
uint64_t index;

mca_btl_tcp_addr_t* endpoint_addr = btl_proc->proc_addrs + i;

mca_btl_tcp_proc_tosocks (endpoint_addr, &endpoint_addr_ss);

index = proc_data->peer_kindex_to_index[endpoint_addr->addr_ifkindex];
rc = opal_hash_table_get_value_uint32(&proc_data->peer_kindex_to_index, endpoint_addr->addr_ifkindex, (void**) &index);

if(-1 == index) {
if (OPAL_SUCCESS != rc) {
index = proc_data->num_peer_interfaces++;
proc_data->peer_kindex_to_index[endpoint_addr->addr_ifkindex] = index;
opal_hash_table_set_value_uint32(&proc_data->peer_kindex_to_index, endpoint_addr->addr_ifkindex, (void*)(uintptr_t) index);
if( proc_data->num_peer_interfaces == max_peer_interfaces ) {
max_peer_interfaces <<= 1;
peer_interfaces = (mca_btl_tcp_interface_t**)realloc( peer_interfaces,
Expand Down Expand Up @@ -726,6 +729,10 @@ int mca_btl_tcp_proc_insert( mca_btl_tcp_proc_t* btl_proc,
free(proc_data->weights);
free(proc_data->best_addr);
free(proc_data->best_assignment);

OBJ_DESTRUCT(&_proc_data.local_kindex_to_index);
OBJ_DESTRUCT(&_proc_data.peer_kindex_to_index);

free(a);

return rc;
Expand Down