optionally passive wait when the progress loop is idle for a while

ggouaillardet · ggouaillardet · commit 1c6ae867923d · 2017-10-12T16:24:13.000+09:00
Add the new mpi_poll_when_idle and mpi_poll_threhold MCA parameters to
control if and when the progress loop should poll() when idle and
when polling should start.

The default is not to poll when idle.

Thanks Paul Kapinos for bringing this to our attention

Signed-off-by: Gilles Gouaillardet &lt;gilles@rist.or.jp&gt;
diff --git a/ompi/runtime/ompi_mpi_init.c b/ompi/runtime/ompi_mpi_init.c
@@ -18,7 +18,7 @@
  * Copyright (c) 2011      Sandia National Laboratories. All rights reserved.
  * Copyright (c) 2012-2013 Inria.  All rights reserved.
  * Copyright (c) 2014-2017 Intel, Inc.  All rights reserved.
- * Copyright (c) 2014-2016 Research Organization for Information Science
+ * Copyright (c) 2014-2017 Research Organization for Information Science
  *                         and Technology (RIST). All rights reserved.
  * Copyright (c) 2016      Mellanox Technologies Ltd. All rights reserved.
  *
@@ -284,6 +284,8 @@ opal_list_t ompi_registered_datareps = {{0}};
 
 bool ompi_enable_timing = false;
 extern bool ompi_mpi_yield_when_idle;
+extern bool ompi_mpi_poll_when_idle;
+extern uint32_t ompi_mpi_poll_threshold;
 extern int ompi_mpi_event_tick_rate;
 
 /**
@@ -971,6 +973,12 @@ int ompi_mpi_init(int argc, char **argv, int requested, int *provided)
     /* see if yield_when_idle was specified - if so, use it */
     opal_progress_set_yield_when_idle(ompi_mpi_yield_when_idle);
 
+    /* see if poll_when_idle was specified - if so, use it */
+    opal_progress_set_poll_when_idle(ompi_mpi_poll_when_idle);
+
+    /* set the threshold to start polling when idle */
+    opal_progress_set_poll_threshold(ompi_mpi_poll_threshold);
+
     /* negative value means use default - just don't do anything */
     if (ompi_mpi_event_tick_rate >= 0) {
         opal_progress_set_event_poll_rate(ompi_mpi_event_tick_rate);
diff --git a/ompi/runtime/ompi_mpi_params.c b/ompi/runtime/ompi_mpi_params.c
@@ -17,7 +17,7 @@
  * Copyright (c) 2013-2017 Intel, Inc.  All rights reserved.
  * Copyright (c) 2015      Mellanox Technologies, Inc.
  *                         All rights reserved.
- * Copyright (c) 2016      Research Organization for Information Science
+ * Copyright (c) 2016-2017 Research Organization for Information Science
  *                         and Technology (RIST). All rights reserved.
  * $COPYRIGHT$
  *
@@ -63,6 +63,8 @@ bool ompi_have_sparse_group_storage = OPAL_INT_TO_BOOL(OMPI_GROUP_SPARSE);
 bool ompi_use_sparse_group_storage = OPAL_INT_TO_BOOL(OMPI_GROUP_SPARSE);
 
 bool ompi_mpi_yield_when_idle = false;
+bool ompi_mpi_poll_when_idle = false;
+uint32_t ompi_mpi_poll_threshold = 1000;
 int ompi_mpi_event_tick_rate = -1;
 char *ompi_mpi_show_mca_params_string = NULL;
 bool ompi_mpi_have_sparse_group_storage = !!(OMPI_GROUP_SPARSE);
@@ -115,6 +117,22 @@ int ompi_mpi_register_params(void)
                                  MCA_BASE_VAR_SCOPE_READONLY,
                                  &ompi_mpi_yield_when_idle);
 
+    ompi_mpi_poll_when_idle = false;
+    (void) mca_base_var_register("ompi", "mpi", NULL, "poll_when_idle",
+                                 "Poll the processor when waiting for MPI communication (default to false)",
+                                 MCA_BASE_VAR_TYPE_BOOL, NULL, 0, 0,
+                                 OPAL_INFO_LVL_9,
+                                 MCA_BASE_VAR_SCOPE_READONLY,
+                                 &ompi_mpi_poll_when_idle);
+
+    ompi_mpi_poll_threshold = 1000;
+    (void) mca_base_var_register("ompi", "mpi", NULL, "poll_threshold",
+                                 "Poll the processor when waiting for MPI communication (default to false)",
+                                 MCA_BASE_VAR_TYPE_UNSIGNED_INT, NULL, 0, 0,
+                                 OPAL_INFO_LVL_9,
+                                 MCA_BASE_VAR_SCOPE_READONLY,
+                                 &ompi_mpi_poll_threshold);
+
     ompi_mpi_event_tick_rate = -1;
     (void) mca_base_var_register("ompi", "mpi", NULL, "event_tick_rate",
                                  "How often to progress TCP communications (0 = never, otherwise specified in microseconds)",
diff --git a/opal/runtime/opal_progress.c b/opal/runtime/opal_progress.c
@@ -12,7 +12,7 @@
  *                         All rights reserved.
  * Copyright (c) 2006-2016 Los Alamos National Security, LLC.  All rights
  *                         reserved.
- * Copyright (c) 2015-2016 Research Organization for Information Science
+ * Copyright (c) 2015-2017 Research Organization for Information Science
  *                         and Technology (RIST). All rights reserved.
  *
  * $COPYRIGHT$
@@ -27,6 +27,7 @@
 #ifdef HAVE_SCHED_H
 #include <sched.h>
 #endif
+#include <poll.h>
 
 #include "opal/runtime/opal_progress.h"
 #include "opal/mca/event/event.h"
@@ -66,6 +67,10 @@ static size_t callbacks_lp_size = 0;
 
 /* do we want to call sched_yield() if nothing happened */
 bool opal_progress_yield_when_idle = false;
+/* do we want to poll() if nothing happened for a while */
+uint32_t opal_progress_poll_threshold = 1000;
+bool opal_progress_poll_when_idle = false;
+static uint32_t yield_count = 0;
 
 #if OPAL_PROGRESS_USE_TIMERS
 static opal_timer_t event_progress_last_time = 0;
@@ -138,6 +143,8 @@ opal_progress_init(void)
                  opal_progress_event_flag));
     OPAL_OUTPUT((debug_output, "progress: initialized yield_when_idle to: %s",
                  opal_progress_yield_when_idle ? "true" : "false"));
+    OPAL_OUTPUT((debug_output, "progress: initialized poll_when_idle to: %s",
+                 opal_progress_poll_when_idle ? "true" : "false"));
     OPAL_OUTPUT((debug_output, "progress: initialized num users to: %d",
                  num_event_users));
     OPAL_OUTPUT((debug_output, "progress: initialized poll rate to: %ld",
@@ -229,6 +236,15 @@ opal_progress(void)
         }
     }
 
+    if (opal_progress_poll_when_idle) {
+        if (events <= 0) {
+            if (yield_count++ > opal_progress_poll_threshold) {
+                poll(NULL, 0, 1);
+            }
+        } else {
+            yield_count = 0;
+        }
+    }
 #if OPAL_HAVE_SCHED_YIELD
     if (opal_progress_yield_when_idle && events <= 0) {
         /* If there is nothing to do - yield the processor - otherwise
@@ -310,6 +326,33 @@ opal_progress_set_yield_when_idle(bool yieldopt)
 }
 
 
+bool
+opal_progress_set_poll_when_idle(bool pollopt)
+{
+    bool tmp = opal_progress_poll_when_idle;
+    opal_progress_poll_when_idle = (pollopt) ? 1 : 0;
+
+    OPAL_OUTPUT((debug_output, "progress: progress_set_poll_when_idle to %s",
+                                    opal_progress_poll_when_idle ? "true" : "false"));
+
+    return tmp;
+}
+
+
+uint32_t
+opal_progress_set_poll_threshold(uint32_t thresholdopt)
+{
+    uint32_t tmp = opal_progress_poll_threshold;
+
+    opal_progress_poll_threshold = thresholdopt;
+
+    OPAL_OUTPUT((debug_output, "progress: progress_set_poll_threshold to %d",
+                                    thresholdopt));
+
+    return tmp;
+}
+
+
 void
 opal_progress_set_event_poll_rate(int polltime)
 {
diff --git a/opal/runtime/opal_progress.h b/opal/runtime/opal_progress.h
@@ -11,6 +11,8 @@
  *                         All rights reserved.
  * Copyright (c) 2006-2014 Los Alamos National Security, LLC.  All rights
  *                         reserved.
+ * Copyright (c) 2017      Research Organization for Information Science
+ *                         and Technology (RIST). All rights reserved.
  *
  * $COPYRIGHT$
  *
@@ -113,19 +115,48 @@ OPAL_DECLSPEC void opal_progress_event_users_decrement(void);
 /**
  * Set whether opal_progress() should yield when idle
  *
- * Set whether opal_progress() should yield the processor (either by
- * sched_yield() or SwitchToThread()) if no events were progressed
- * during the progress loop.  The return value of the callback
- * functions is used to determine whether or not yielding is required.
+ * Set whether opal_progress() should yield the processor by
+ * sched_yield() no events were progressed during the progress loop.
+ * The return value of the callback functions is used to determine
+ * whether or not yielding is required.
  * By default, the event loop will yield when the progress function is
- * idle.
+ * idle only when oversubscription.
  *
  * @param   yieldopt  Whether to yield when idle.
  * @return         Previous value of the yield_when_idle option.
  */
 OPAL_DECLSPEC bool opal_progress_set_yield_when_idle(bool yieldopt);
 
 
+/**
+ * Set whether opal_progress() should poll when idle for a while
+ *
+ * Set whether opal_progress() should poll if no events were progressed
+ * during the progress loop for some time.  The return value of the callback
+ * functions is used to determine whether or not polling is required.
+ * By default, the event loop will not poll when the progress function is
+ * idle.
+ *
+ * @param   thresholdopt  The threshold to start polling.
+ * @return         Previous value of the poll_threshold option.
+ */
+OPAL_DECLSPEC uint32_t opal_progress_set_poll_threshold(uint32_t thresholdopt);
+
+
+/**
+ * Set the threshold for opal_progress() to start polling when idle.
+ *
+ * Set the threshold for opal_progress() to start polling when idle.
+ * The return value of the callback functions is used to determine whether or not polling is required.
+ * By default, the event loop will poll when the progress function is
+ * idle.
+ *
+ * @param   pollopt  Whether to poll when idle.
+ * @return         Previous value of the poll_when_idle option.
+ */
+OPAL_DECLSPEC bool opal_progress_set_poll_when_idle(bool pollopt);
+
+
 /**
  * Set time between calls into the event library
  *
