ompi/attr: update attribute system to prepare for sessions

This commit updates how the attribute system is initialized. Instead of
being set up in MPI_Init it is initialized by the subsystems (communicator,
windows, datatypes) that use it. To keep the delta from the sessions PR
minimal the ompi_mpi_instance_retain() and ompi_mpi_instance_release()
functions are stubbed out as no-ops. This will be removed once the instance
system is added.

Signed-off-by: Nathan Hjelm <[email protected]>

Author: hjelmn

ompi/attribute/attribute.c

@@ -1,3 +1,4 @@
+/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
 /*
  * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
  *                         University Research and Technology
@@ -14,6 +15,8 @@
  *                         reserved.
  * Copyright (c) 2017      Research Organization for Information Science
  *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2018-2021 Triad National Security, LLC. All rights
+ *                         reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -242,6 +245,7 @@
 #include "ompi/win/win.h"                    /* ompi_win_t generated in [COPY|DELETE]_ATTR_CALLBACKS */
 #include "ompi/mpi/fortran/base/fint_2_int.h"
 
+
 /*
  * Macros
  */
@@ -256,9 +260,9 @@
 #define attr_datatype_f d_f_to_c_index
 #define attr_win_f w_f_to_c_index
 
-#define CREATE_KEY(key) opal_bitmap_find_and_set_first_unset_bit(key_bitmap, (key))
+#define CREATE_KEY(key) opal_bitmap_find_and_set_first_unset_bit(attr_subsys->key_bitmap, (key))
 
-#define FREE_KEY(key) opal_bitmap_clear_bit(key_bitmap, (key))
+#define FREE_KEY(key) opal_bitmap_clear_bit(attr_subsys->key_bitmap, (key))
 
 
 /* Not checking for NULL_DELETE_FN here, since according to the
@@ -406,13 +410,24 @@ typedef struct attribute_value_t {
     int av_sequence;
 } attribute_value_t;
 
+/*
+ * struct to hold state of attr subsys
+ */
+
+typedef  struct attr_subsys_t {
+    opal_object_t            super;
+    opal_hash_table_t *keyval_hash;
+    opal_bitmap_t *key_bitmap;
+ } attr_subsys_t;
 
 /*
  * Local functions
  */
 static void attribute_value_construct(attribute_value_t *item);
 static void ompi_attribute_keyval_construct(ompi_attribute_keyval_t *keyval);
 static void ompi_attribute_keyval_destruct(ompi_attribute_keyval_t *keyval);
+static void attr_subsys_construct(attr_subsys_t *subsys);
+static void attr_subsys_destruct(attr_subsys_t *subsys);
 static int set_value(ompi_attribute_type_t type, void *object,
                      opal_hash_table_t **attr_hash, int key,
                      attribute_value_t *new_attr,
@@ -425,6 +440,13 @@ static MPI_Aint translate_to_aint(attribute_value_t *val);
 
 static int compare_attr_sequence(const void *attr1, const void *attr2);
 
+/*
+ * attribute_subsys_t class
+ */
+static OBJ_CLASS_INSTANCE(attr_subsys_t,
+                          opal_object_t,
+                          attr_subsys_construct,
+                          attr_subsys_destruct);
 
 /*
  * attribute_value_t class
@@ -443,24 +465,31 @@ static OBJ_CLASS_INSTANCE(ompi_attribute_keyval_t,
                           ompi_attribute_keyval_construct,
                           ompi_attribute_keyval_destruct);
 
+/*
+ * compatibility until sessions work is finished
+ */
+static inline int ompi_mpi_instance_retain(void) {
+    return OMPI_SUCCESS;
+}
+
+static inline void ompi_mpi_instance_release(void) {
+}
 
 /*
  * Static variables
  */
 
-static opal_hash_table_t *keyval_hash;
-static opal_bitmap_t *key_bitmap;
-static int attr_sequence;
+static attr_subsys_t *attr_subsys = NULL;
 static unsigned int int_pos = 12345;
 static unsigned int integer_pos = 12345;
+static int attr_sequence;
 
 /*
  * MPI attributes are *not* high performance, so just use a One Big Lock
  * approach. However, this lock is released before a user provided callback is
  * triggered and acquired right after, allowing for recursive behaviors.
  */
-static opal_mutex_t attribute_lock;
-
+static opal_mutex_t attribute_lock = OPAL_MUTEX_STATIC_INIT; 
 
 /*
  * attribute_value_t constructor function
@@ -507,33 +536,68 @@ ompi_attribute_keyval_destruct(ompi_attribute_keyval_t *keyval)
             free(keyval->bindings_extra_state);
         }
 
-        opal_hash_table_remove_value_uint32(keyval_hash, keyval->key);
+        opal_hash_table_remove_value_uint32(attr_subsys->keyval_hash, keyval->key);
         FREE_KEY(keyval->key);
     }
 }
 
 
-/*
- * This will initialize the main list to store key- attribute
- * items. This will be called one time, during MPI_INIT().
- */
-int ompi_attr_init(void)
+int ompi_attr_get_ref(void)
+{
+    int ret = OMPI_SUCCESS;
+
+    OPAL_THREAD_LOCK(&attribute_lock);
+
+    if (NULL == attr_subsys) {
+        attr_subsys = OBJ_NEW(attr_subsys_t);
+        if (NULL == attr_subsys) {
+            ret = OMPI_ERR_OUT_OF_RESOURCE;
+            goto fn_exit;
+        }
+        if ((NULL == attr_subsys->keyval_hash) || (NULL == attr_subsys->key_bitmap)) {
+            OBJ_RELEASE(attr_subsys);
+            attr_subsys = NULL;
+            ret = OMPI_ERR_OUT_OF_RESOURCE;
+            goto fn_exit;
+        }
+    } else {
+        OBJ_RETAIN(attr_subsys);
+    }
+
+fn_exit:
+    OPAL_THREAD_UNLOCK(&attribute_lock);
+
+    return ret;
+}
+
+int ompi_attr_put_ref(void)
+{
+    if (NULL != attr_subsys) {
+        OBJ_RELEASE(attr_subsys);
+    }
+    return OMPI_SUCCESS;
+}
+
+static void attr_subsys_construct(attr_subsys_t *subsys)
 {
     int ret;
     void *bogus = (void*) 1;
     int *p = (int *) &bogus;
 
-    keyval_hash = OBJ_NEW(opal_hash_table_t);
-    if (NULL == keyval_hash) {
-        return OMPI_ERR_OUT_OF_RESOURCE;
-    }
-    key_bitmap = OBJ_NEW(opal_bitmap_t);
+    subsys->keyval_hash = OBJ_NEW(opal_hash_table_t);
+
+    subsys->key_bitmap = OBJ_NEW(opal_bitmap_t);
+
     /*
      * Set the max size to OMPI_FORTRAN_HANDLE_MAX to enforce bound
      */
-    opal_bitmap_set_max_size (key_bitmap, OMPI_FORTRAN_HANDLE_MAX);
-    if (0 != opal_bitmap_init(key_bitmap, 32)) {
-        return OMPI_ERR_OUT_OF_RESOURCE;
+    opal_bitmap_set_max_size (subsys->key_bitmap, 
+                              OMPI_FORTRAN_HANDLE_MAX);
+    ret = opal_bitmap_init(subsys->key_bitmap, 32);
+    assert(OPAL_SUCCESS == ret);
+
+    for (int i = 0; i < MPI_ATTR_PREDEFINED_KEY_MAX; i++) {
+        opal_bitmap_set_bit(subsys->key_bitmap, i);
     }
 
     for (int_pos = 0; int_pos < (sizeof(void*) / sizeof(int));
@@ -550,31 +614,21 @@ int ompi_attr_init(void)
         }
     }
 
-    OBJ_CONSTRUCT(&attribute_lock, opal_mutex_t);
-
-    if (OMPI_SUCCESS != (ret = opal_hash_table_init(keyval_hash,
-                                                    ATTR_TABLE_SIZE))) {
-        return ret;
-    }
-    if (OMPI_SUCCESS != (ret = ompi_attr_create_predefined())) {
-        return ret;
-    }
+    ret = opal_hash_table_init(subsys->keyval_hash, ATTR_TABLE_SIZE);
+    assert (OPAL_SUCCESS == ret);
 
-    return OMPI_SUCCESS;
+    attr_sequence = 0;    
 }
 
 
 /*
- * Cleanup everything during MPI_Finalize().
+ * Cleanup everything when no more refs to the attr subsys
  */
-int ompi_attr_finalize(void)
+static void attr_subsys_destruct(attr_subsys_t *subsys)
 {
     ompi_attr_free_predefined();
-    OBJ_DESTRUCT(&attribute_lock);
-    OBJ_RELEASE(keyval_hash);
-    OBJ_RELEASE(key_bitmap);
-
-    return OMPI_SUCCESS;
+    OBJ_RELEASE(subsys->keyval_hash);
+    OBJ_RELEASE(subsys->key_bitmap);
 }
 
 /*****************************************************************************/
@@ -609,10 +663,15 @@ static int ompi_attr_create_keyval_impl(ompi_attribute_type_t type,
 
     /* Create a new unique key and fill the hash */
     OPAL_THREAD_LOCK(&attribute_lock);
-    ret = CREATE_KEY(key);
+    ret = MPI_SUCCESS;
+    if (!(flags & OMPI_KEYVAL_PREDEFINED)) {
+        ret = CREATE_KEY(key);
+    }
+    
     if (OMPI_SUCCESS == ret) {
         keyval->key = *key;
-        ret = opal_hash_table_set_value_uint32(keyval_hash, *key, keyval);
+        ret = opal_hash_table_set_value_uint32(attr_subsys->keyval_hash,
+                                               *key, keyval);
     }
 
     if (OMPI_SUCCESS != ret) {
@@ -635,11 +694,22 @@ int ompi_attr_create_keyval(ompi_attribute_type_t type,
                             void *bindings_extra_state)
 {
     ompi_attribute_fortran_ptr_t es_tmp;
+    int rc;
+
+    rc = ompi_mpi_instance_retain ();
+    if (OPAL_UNLIKELY(OMPI_SUCCESS != rc)) {
+        return rc;
+    }
 
     es_tmp.c_ptr = extra_state;
-    return ompi_attr_create_keyval_impl(type, copy_attr_fn, delete_attr_fn,
-                                        key, &es_tmp, flags,
-                                        bindings_extra_state);
+    rc = ompi_attr_create_keyval_impl(type, copy_attr_fn, delete_attr_fn,
+                                      key, &es_tmp, flags,
+                                      bindings_extra_state);
+    if (OPAL_UNLIKELY(OMPI_SUCCESS != rc)) {
+        ompi_mpi_instance_release ();
+    }
+
+    return rc;
 }
 
 int ompi_attr_create_keyval_fint(ompi_attribute_type_t type,
@@ -651,6 +721,12 @@ int ompi_attr_create_keyval_fint(ompi_attribute_type_t type,
                                  void *bindings_extra_state)
 {
     ompi_attribute_fortran_ptr_t es_tmp;
+    int rc;
+
+    rc = ompi_mpi_instance_retain ();
+    if (OPAL_UNLIKELY(OMPI_SUCCESS != rc)) {
+        return rc;
+    }
 
     es_tmp.f_integer = extra_state;
 #if SIZEOF_INT == OMPI_SIZEOF_FORTRAN_INTEGER
@@ -670,6 +746,12 @@ int ompi_attr_create_keyval_aint(ompi_attribute_type_t type,
                                  void *bindings_extra_state)
 {
     ompi_attribute_fortran_ptr_t es_tmp;
+    int rc;
+
+    rc = ompi_mpi_instance_retain ();
+    if (OPAL_UNLIKELY(OMPI_SUCCESS != rc)) {
+        return rc;
+    }
 
     es_tmp.f_address = extra_state;
     return ompi_attr_create_keyval_impl(type, copy_attr_fn, delete_attr_fn,
@@ -687,7 +769,7 @@ int ompi_attr_free_keyval(ompi_attribute_type_t type, int *key,
 
     /* Find the key-value pair */
     OPAL_THREAD_LOCK(&attribute_lock);
-    ret = opal_hash_table_get_value_uint32(keyval_hash, *key,
+    ret = opal_hash_table_get_value_uint32(attr_subsys->keyval_hash, *key,
                                            (void **) &keyval);
     if ((OMPI_SUCCESS != ret) || (NULL == keyval) ||
         (keyval->attr_type != type) ||
@@ -707,6 +789,9 @@ int ompi_attr_free_keyval(ompi_attribute_type_t type, int *key,
     opal_atomic_wmb();
     OPAL_THREAD_UNLOCK(&attribute_lock);
 
+    /* balance out retain in keyval_create */
+    ompi_mpi_instance_release ();
+
     return MPI_SUCCESS;
 }
 
@@ -720,7 +805,7 @@ int ompi_attr_set_c(ompi_attribute_type_t type, void *object,
                     opal_hash_table_t **attr_hash,
                     int key, void *attribute, bool predefined)
 {
-    int ret;
+    int ret = MPI_SUCCESS;
     attribute_value_t *new_attr = OBJ_NEW(attribute_value_t);
     if (NULL == new_attr) {
         return OMPI_ERR_OUT_OF_RESOURCE;
@@ -942,7 +1027,7 @@ int ompi_attr_copy_all(ompi_attribute_type_t type, void *old_object,
 
         /* Get the keyval in the main keyval hash - so that we know
            what the copy_attr_fn is */
-        err = opal_hash_table_get_value_uint32(keyval_hash, key,
+        err = opal_hash_table_get_value_uint32(attr_subsys->keyval_hash, key,
                                                (void **) &hash_value);
         if (OMPI_SUCCESS != err) {
             /* This should not happen! */
@@ -1037,7 +1122,7 @@ static int ompi_attr_delete_impl(ompi_attribute_type_t type, void *object,
     attribute_value_t *attr;
 
     /* Check if the key is valid in the master keyval hash */
-    ret = opal_hash_table_get_value_uint32(keyval_hash, key,
+    ret = opal_hash_table_get_value_uint32(attr_subsys->keyval_hash, key,
                                            (void **) &keyval);
 
     if ((OMPI_SUCCESS != ret) || (NULL == keyval) ||
@@ -1053,7 +1138,7 @@ static int ompi_attr_delete_impl(ompi_attribute_type_t type, void *object,
         goto exit;
     }
 
-    /* Check if the key is valid for the communicator/window/dtype. If
+    /* Check if the key is valid for the communicator/window/dtype/instance. If
        yes, then delete the attribute and key entry from the object's
        hash */
     ret = opal_hash_table_get_value_uint32(attr_hash, key, (void**) &attr);
@@ -1198,7 +1283,7 @@ static int set_value(ompi_attribute_type_t type, void *object,
     /* Note that this function can be invoked by ompi_attr_copy_all()
        to set attributes on the new object (in addition to the
        top-level MPI_* functions that set attributes). */
-    ret = opal_hash_table_get_value_uint32(keyval_hash, key,
+    ret = opal_hash_table_get_value_uint32(attr_subsys->keyval_hash, key,
                                            (void **) &keyval);
 
     /* If key not found */
@@ -1242,7 +1327,7 @@ static int set_value(ompi_attribute_type_t type, void *object,
         had_old = true;
     }
 
-    ret = opal_hash_table_get_value_uint32(keyval_hash, key,
+    ret = opal_hash_table_get_value_uint32(attr_subsys->keyval_hash, key,
                                            (void **) &keyval);
     if ((OMPI_SUCCESS != ret ) || (NULL == keyval)) {
         /* Keyval has disappeared underneath us -- this shouldn't
@@ -1288,7 +1373,7 @@ static int get_value(opal_hash_table_t *attr_hash, int key,
        with the key, then the call is valid and returns FALSE in the
        flag argument */
     *flag = 0;
-    ret = opal_hash_table_get_value_uint32(keyval_hash, key,
+    ret = opal_hash_table_get_value_uint32(attr_subsys->keyval_hash, key,
                                            (void**) &keyval);
     if (OMPI_ERR_NOT_FOUND == ret) {
         return MPI_KEYVAL_INVALID;