Complete the Jan AR to create an opal event async thread

ompi/runtime/ompi_mpi_finalize.c

@@ -428,7 +428,6 @@ int ompi_mpi_finalize(void)
     }
 
     /* Leave the RTE */
-
     if (OMPI_SUCCESS != (ret = ompi_rte_finalize())) {
         return ret;
     }
@@ -440,6 +439,9 @@ int ompi_mpi_finalize(void)
         return ret;
     }
 
+    /* cleanup the progress engine */
+    opal_progress_finalize();
+
     if (OPAL_SUCCESS != (ret = opal_finalize_util())) {
         return ret;
     }

ompi/runtime/ompi_mpi_init.c

@@ -17,7 +17,7 @@
  * Copyright (c) 2008-2009 Sun Microsystems, Inc.  All rights reserved.
  * Copyright (c) 2011      Sandia National Laboratories. All rights reserved.
  * Copyright (c) 2012-2013 Inria.  All rights reserved.
- * Copyright (c) 2014      Intel, Inc. All rights reserved.
+ * Copyright (c) 2014-2015 Intel, Inc. All rights reserved.
  * Copyright (c) 2014-2015 Research Organization for Information Science
  *                         and Technology (RIST). All rights reserved.
  *
@@ -483,6 +483,16 @@ int ompi_mpi_init(int argc, char **argv, int requested, int *provided)
     /* check for timing request - get stop time and report elapsed time if so */
     OPAL_TIMING_MNEXT((&tm,"time from completion of rte_init to modex"));
 
+    /*
+     * Initialize the general progress engine
+     */
+    if (OPAL_SUCCESS != (ret = opal_progress_init())) {
+        error = "opal_progress_init";
+        goto error;
+    }
+    /* we want to tick the event library whenever possible */
+    opal_progress_event_users_increment();
+
 #if OPAL_HAVE_HWLOC
     /* if hwloc is available but didn't get setup for some
      * reason, do so now

opal/Makefile.am

@@ -75,6 +75,7 @@ nobase_opal_HEADERS = $(headers)
 endif
 
 include class/Makefile.am
+include errhandler/Makefile.am
 include memoryhooks/Makefile.am
 include runtime/Makefile.am
 include threads/Makefile.am

opal/class/opal_hotel.c

@@ -1,6 +1,7 @@
 /*
  * Copyright (c) 2012      Cisco Systems, Inc. All rights reserved.
  * Copyright (c) 2012      Los Alamos National Security, LLC. All rights reserved
+ * Copyright (c) 2015      Intel, Inc. All rights reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -33,6 +34,7 @@ static void local_eviction_callback(int fd, short flags, void *arg)
 
 
 int opal_hotel_init(opal_hotel_t *h, int num_rooms,
+                    opal_event_base_t *evbase,
                     uint32_t eviction_timeout,
                     int eviction_event_priority,
                     opal_hotel_eviction_callback_fn_t evict_callback_fn)
@@ -46,6 +48,7 @@ int opal_hotel_init(opal_hotel_t *h, int num_rooms,
     }
 
     h->num_rooms = num_rooms;
+    h->evbase = evbase;
     h->eviction_timeout.tv_usec = eviction_timeout % 1000000;
     h->eviction_timeout.tv_sec = eviction_timeout / 1000000;
     h->evict_callback_fn = evict_callback_fn;
@@ -69,7 +72,7 @@ int opal_hotel_init(opal_hotel_t *h, int num_rooms,
         h->eviction_args[i].room_num = i;
 
         /* Create this room's event (but don't add it) */
-        opal_event_set(opal_event_base,
+        opal_event_set(h->evbase,
                        &(h->rooms[i].eviction_timer_event),
                        -1, 0, local_eviction_callback,
                        &(h->eviction_args[i]));

opal/class/opal_hotel.h

@@ -111,6 +111,8 @@ typedef struct opal_hotel_t {
     /* Max number of rooms in the hotel */
     int num_rooms;
 
+    /* event base for this hotel */
+    opal_event_base_t *evbase;
     struct timeval eviction_timeout;
     opal_hotel_eviction_callback_fn_t evict_callback_fn;
 
@@ -133,6 +135,7 @@ OBJ_CLASS_DECLARATION(opal_hotel_t);
  *
  * @param hotel Pointer to a hotel (IN)
  * @param num_rooms The total number of rooms in the hotel (IN)
+ * @param evbase Pointer to the event base for timer events
  * @param eviction_timeout Max length of a stay at the hotel before
  * the eviction callback is invoked (in microseconds)
  * @param eviction_event_priority Event lib priority for the eviction timeout
@@ -147,6 +150,7 @@ OBJ_CLASS_DECLARATION(opal_hotel_t);
  *  the error indicate what went wrong in the function.
  */
 OPAL_DECLSPEC int opal_hotel_init(opal_hotel_t *hotel, int num_rooms,
+                                  opal_event_base_t *evbase,
                                   uint32_t eviction_timeout,
                                   int eviction_event_priority,
                                   opal_hotel_eviction_callback_fn_t evict_callback_fn);
@@ -199,8 +203,8 @@ static inline int opal_hotel_checkin(opal_hotel_t *hotel,
  * caller *knows* that there is a room available.
  */
 static inline void opal_hotel_checkin_with_res(opal_hotel_t *hotel,
-                                     void *occupant,
-                                     int *room_num)
+                                               void *occupant,
+                                               int *room_num)
 {
     opal_hotel_room_t *room;
 

opal/errhandler/Makefile.am

@@ -0,0 +1,17 @@
+# -*- makefile -*-
+#
+# Copyright (c) 2015      Intel, Inc.  All rights reserved.
+# $COPYRIGHT$
+#
+# Additional copyrights may follow
+#
+# $HEADER$
+#
+
+# This makefile.am does not stand on its own - it is included from opal/Makefile.am
+
+headers += \
+        errhandler/opal_errhandler.h
+
+lib@OPAL_LIB_PREFIX@open_pal_la_SOURCES += \
+        errhandler/opal_errhandler.c

opal/errhandler/opal_errhandler.c

@@ -0,0 +1,34 @@
+/*
+ * Copyright (c) 2015      Intel, Inc.  All rights reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+#include "opal_config.h"
+
+#include "opal/errhandler/opal_errhandler.h"
+
+opal_errhandler_fn_t errhandler = NULL;
+void *cbdata = NULL;
+
+void opal_register_errhandler(opal_errhandler_fn_t newerr, void *cbd)
+{
+    errhandler = newerr;
+    cbdata = cbd;
+}
+
+void opal_deregister_errhandler(void)
+{
+    errhandler = NULL;
+    cbdata = NULL;
+}
+
+void opal_invoke_errhandler(int status, opal_proc_t *proc)
+{
+    if (NULL != errhandler) {
+        errhandler(status, proc, cbdata);
+    }
+}

opal/errhandler/opal_errhandler.h

@@ -0,0 +1,25 @@
+/*
+ * Copyright (c) 2015      Intel, Inc.  All rights reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+#ifndef OPAL_ERRHANDLER_H
+#define OPAL_ERRHANDLER_H
+
+#include "opal_config.h"
+
+#include "opal/util/proc.h"
+
+typedef void (*opal_errhandler_fn_t)(int status, opal_proc_t *proc, void *cbdata);
+
+OPAL_DECLSPEC void opal_register_errhandler(opal_errhandler_fn_t errhandler, void *cbdata);
+
+OPAL_DECLSPEC void opal_deregister_errhandler(void);
+
+OPAL_DECLSPEC void opal_invoke_errhandler(int status, opal_proc_t *proc);
+
+#endif

opal/mca/btl/openib/btl_openib_fd.c

@@ -3,6 +3,7 @@
  * Copyright (c) 2009 Sandia National Laboratories. All rights reserved.
  * Copyright (c) 2015      Research Organization for Information Science
  *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2015      Intel, Inc. All rights reserved.
  *
  * $COPYRIGHT$
  *
@@ -175,7 +176,7 @@ static int service_pipe_cmd_add_fd(bool use_libevent, cmd_t *cmd)
     if (use_libevent) {
         /* Make an event for this fd */
         ri->ri_event_used = true;
-        opal_event_set(opal_event_base, &ri->ri_event, ri->ri_fd,
+        opal_event_set(opal_sync_event_base, &ri->ri_event, ri->ri_fd,
                        ri->ri_flags | OPAL_EV_PERSIST, service_fd_callback,
                        ri);
         opal_event_add(&ri->ri_event, 0);
@@ -501,7 +502,7 @@ int opal_btl_openib_fd_init(void)
 
         /* Create a libevent event that is used in the main thread
            to watch its pipe */
-        opal_event_set(opal_event_base, &main_thread_event, pipe_to_main_thread[0],
+        opal_event_set(opal_sync_event_base, &main_thread_event, pipe_to_main_thread[0],
                        OPAL_EV_READ | OPAL_EV_PERSIST,
                        main_thread_event_callback, NULL);
         opal_event_add(&main_thread_event, 0);

opal/mca/btl/openib/connect/btl_openib_connect_udcm.c

@@ -7,7 +7,7 @@
  *                         reserved.
  * Copyright (c) 2014-2015 Research Organization for Information Science
  *                         and Technology (RIST). All rights reserved.
- * Copyright (c) 2014      Intel, Inc. All rights reserved.
+ * Copyright (c) 2014-2015 Intel, Inc. All rights reserved.
  * Copyright (c) 2014      Bull SAS.  All rights reserved.
  *
  * $COPYRIGHT$
@@ -2212,7 +2212,7 @@ static void udcm_sent_message_constructor (udcm_message_sent_t *message)
 {
     memset ((char *)message + sizeof (message->super), 0,
             sizeof (*message) - sizeof (message->super));
-    opal_event_evtimer_set(opal_event_base, &message->event, udcm_send_timeout, message);
+    opal_event_evtimer_set(opal_sync_event_base, &message->event, udcm_send_timeout, message);
 }
 
 static void udcm_sent_message_destructor (udcm_message_sent_t *message)

opal/mca/btl/tcp/btl_tcp_component.c

@@ -16,7 +16,7 @@
  * Copyright (c) 2012-2015 Los Alamos National Security, LLC.  All rights
  *                         reserved.
  * Copyright (c) 2013-2015 NVIDIA Corporation.  All rights reserved.
- * Copyright (c) 2014      Intel, Inc. All rights reserved.
+ * Copyright (c) 2014-2015 Intel, Inc. All rights reserved.
  * Copyright (c) 2014-2015 Research Organization for Information Science
  *                         and Technology (RIST). All rights reserved.
  * $COPYRIGHT$
@@ -818,7 +818,7 @@ static int mca_btl_tcp_component_create_listen(uint16_t af_family)
     /* register listen port */
 #if OPAL_ENABLE_IPV6
     if (AF_INET6 == af_family) {
-        opal_event_set(opal_event_base, &mca_btl_tcp_component.tcp6_recv_event,
+        opal_event_set(opal_sync_event_base, &mca_btl_tcp_component.tcp6_recv_event,
                         mca_btl_tcp_component.tcp6_listen_sd,
                         OPAL_EV_READ|OPAL_EV_PERSIST,
                         mca_btl_tcp_component_accept_handler,
@@ -827,7 +827,7 @@ static int mca_btl_tcp_component_create_listen(uint16_t af_family)
     } else
 #endif
     {
-        opal_event_set(opal_event_base, &mca_btl_tcp_component.tcp_recv_event,
+        opal_event_set(opal_sync_event_base, &mca_btl_tcp_component.tcp_recv_event,
                         mca_btl_tcp_component.tcp_listen_sd,
                         OPAL_EV_READ|OPAL_EV_PERSIST,
                         mca_btl_tcp_component_accept_handler,
@@ -1049,7 +1049,7 @@ static void mca_btl_tcp_component_accept_handler( int incoming_sd,
 
         /* wait for receipt of peers process identifier to complete this connection */
         event = OBJ_NEW(mca_btl_tcp_event_t);
-        opal_event_set(opal_event_base, &event->event, sd, OPAL_EV_READ, mca_btl_tcp_component_recv_handler, event);
+        opal_event_set(opal_sync_event_base, &event->event, sd, OPAL_EV_READ, mca_btl_tcp_component_recv_handler, event);
         opal_event_add(&event->event, 0);
     }
 }

opal/mca/btl/tcp/btl_tcp_endpoint.c

@@ -11,7 +11,7 @@
  *                         All rights reserved.
  * Copyright (c) 2007-2008 Sun Microsystems, Inc.  All rights reserved.
  * Copyright (c) 2013      Cisco Systems, Inc.  All rights reserved.
- * Copyright (c) 2014      Intel, Inc.  All rights reserved.
+ * Copyright (c) 2014-2015 Intel, Inc.  All rights reserved.
  * Copyright (c) 2015      Research Organization for Information Science
  *                         and Technology (RIST). All rights reserved.
  * $COPYRIGHT$
@@ -309,7 +309,7 @@ static inline void mca_btl_tcp_endpoint_event_init(mca_btl_base_endpoint_t* btl_
     btl_endpoint->endpoint_cache_pos = btl_endpoint->endpoint_cache;
 #endif  /* MCA_BTL_TCP_ENDPOINT_CACHE */
 
-    opal_event_set(opal_event_base, &btl_endpoint->endpoint_recv_event,
+    opal_event_set(opal_sync_event_base, &btl_endpoint->endpoint_recv_event,
                     btl_endpoint->endpoint_sd,
                     OPAL_EV_READ|OPAL_EV_PERSIST,
                     mca_btl_tcp_endpoint_recv_handler,
@@ -320,7 +320,7 @@ static inline void mca_btl_tcp_endpoint_event_init(mca_btl_base_endpoint_t* btl_
      * will be fired only once, and when the endpoint is marked as
      * CONNECTED the event should be recreated with the correct flags.
      */
-    opal_event_set(opal_event_base, &btl_endpoint->endpoint_send_event,
+    opal_event_set(opal_sync_event_base, &btl_endpoint->endpoint_send_event,
                     btl_endpoint->endpoint_sd,
                     OPAL_EV_WRITE,
                     mca_btl_tcp_endpoint_send_handler,
@@ -509,7 +509,7 @@ void mca_btl_tcp_endpoint_accept(mca_btl_base_endpoint_t* btl_endpoint,
     assert(btl_endpoint->endpoint_sd_next == -1);
     btl_endpoint->endpoint_sd_next = sd;
 
-    opal_event_evtimer_set(opal_event_base, &btl_endpoint->endpoint_accept_event,
+    opal_event_evtimer_set(opal_sync_event_base, &btl_endpoint->endpoint_accept_event,
                            mca_btl_tcp_endpoint_complete_accept, btl_endpoint);
     opal_event_add(&btl_endpoint->endpoint_accept_event, &now);
 }
@@ -570,7 +570,7 @@ static void mca_btl_tcp_endpoint_connected(mca_btl_base_endpoint_t* btl_endpoint
     MCA_BTL_TCP_ENDPOINT_DUMP(1, btl_endpoint, true, "READY [endpoint_connected]");
 
     /* Create the send event in a persistent manner. */
-    opal_event_set(opal_event_base, &btl_endpoint->endpoint_send_event,
+    opal_event_set(opal_sync_event_base, &btl_endpoint->endpoint_send_event,
                     btl_endpoint->endpoint_sd,
                     OPAL_EV_WRITE | OPAL_EV_PERSIST,
                     mca_btl_tcp_endpoint_send_handler,

opal/mca/btl/usnic/btl_usnic_component.c

@@ -15,7 +15,7 @@
  * Copyright (c) 2008-2015 Cisco Systems, Inc.  All rights reserved.
  * Copyright (c) 2012-2014 Los Alamos National Security, LLC.  All rights
  *                         reserved.
- * Copyright (c) 2014      Intel, Inc. All rights reserved.
+ * Copyright (c) 2014-2015 Intel, Inc. All rights reserved.
  * Copyright (c) 2015      Research Organization for Information Science
  *                         and Technology (RIST). All rights reserved.
  * $COPYRIGHT$
@@ -972,7 +972,7 @@ static mca_btl_base_module_t** usnic_component_init(int* num_btl_modules,
     }
 
     /* start timer to guarantee synthetic clock advances */
-    opal_event_set(opal_event_base, &usnic_clock_timer_event,
+    opal_event_set(opal_sync_event_base, &usnic_clock_timer_event,
                    -1, 0, usnic_clock_callback,
                    &usnic_clock_timeout);
     usnic_clock_timer_event_set = true;

opal/mca/btl/usnic/btl_usnic_endpoint.c

@@ -86,6 +86,7 @@ static void endpoint_construct(mca_btl_base_endpoint_t* endpoint)
     OBJ_CONSTRUCT(&endpoint->endpoint_hotel, opal_hotel_t);
     opal_hotel_init(&endpoint->endpoint_hotel,
                     WINDOW_SIZE,
+                    opal_sync_event_base,
                     mca_btl_usnic_component.retrans_timeout,
                     0,
                     opal_btl_usnic_ack_timeout);

opal/mca/btl/usnic/btl_usnic_module.c

@@ -15,7 +15,7 @@
  * Copyright (c) 2009-2015 Cisco Systems, Inc.  All rights reserved.
  * Copyright (c) 2014      Los Alamos National Security, LLC. All rights
  *                         reserved.
- * Copyright (c) 2014      Intel, Inc. All rights reserved
+ * Copyright (c) 2014-2015 Intel, Inc. All rights reserved
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -2068,7 +2068,7 @@ static void init_async_event(opal_btl_usnic_module_t *module)
     }
 
     /* Get the fd to receive events on this device */
-    opal_event_set(opal_event_base, &(module->device_async_event), fd,
+    opal_event_set(opal_sync_event_base, &(module->device_async_event), fd,
                    OPAL_EV_READ | OPAL_EV_PERSIST,
                    module_async_event_callback, module);
     opal_event_add(&(module->device_async_event), NULL);

opal/mca/btl/usnic/btl_usnic_stats.c

@@ -1,5 +1,6 @@
 /*
  * Copyright (c) 2013-2015 Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2015      Intel, Inc. All rights reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -212,7 +213,7 @@ int opal_btl_usnic_stats_init(opal_btl_usnic_module_t *module)
         module->stats.timeout.tv_sec = mca_btl_usnic_component.stats_frequency;
         module->stats.timeout.tv_usec = 0;
 
-        opal_event_set(opal_event_base, &(module->stats.timer_event),
+        opal_event_set(opal_sync_event_base, &(module->stats.timer_event),
                        -1, EV_TIMEOUT | EV_PERSIST,
                        &usnic_stats_callback, module);
         opal_event_add(&(module->stats.timer_event),