optionally passive wait when the progress loop is idle for a while

ggouaillardet · ggouaillardet · commit 27b83fd43b3c · 2017-10-13T13:15:39.000+09:00
Add the new mpi_poll_when_idle_threshold MCA parameter.
This is only relevant when mpi_yield_when_idle is set.
-1 value (default) means never pool when idle
0 value means always poll when idle
a positive n value means means opal_progress() will poll() after
being invoked n times in a row and no event was available.

The default is not to poll when idle.

Thanks Paul Kapinos for bringing this to our attention

Signed-off-by: Gilles Gouaillardet &lt;gilles@rist.or.jp&gt;
diff --git a/ompi/runtime/ompi_mpi_init.c b/ompi/runtime/ompi_mpi_init.c
@@ -18,7 +18,7 @@
  * Copyright (c) 2011      Sandia National Laboratories. All rights reserved.
  * Copyright (c) 2012-2013 Inria.  All rights reserved.
  * Copyright (c) 2014-2017 Intel, Inc.  All rights reserved.
- * Copyright (c) 2014-2016 Research Organization for Information Science
+ * Copyright (c) 2014-2017 Research Organization for Information Science
  *                         and Technology (RIST). All rights reserved.
  * Copyright (c) 2016      Mellanox Technologies Ltd. All rights reserved.
  *
@@ -284,6 +284,7 @@ opal_list_t ompi_registered_datareps = {{0}};
 
 bool ompi_enable_timing = false;
 extern bool ompi_mpi_yield_when_idle;
+extern uint32_t ompi_mpi_poll_when_idle_threshold;
 extern int ompi_mpi_event_tick_rate;
 
 /**
@@ -971,6 +972,9 @@ int ompi_mpi_init(int argc, char **argv, int requested, int *provided)
     /* see if yield_when_idle was specified - if so, use it */
     opal_progress_set_yield_when_idle(ompi_mpi_yield_when_idle);
 
+    /* set the threshold to start polling when idle */
+    opal_progress_set_poll_when_idle_threshold(ompi_mpi_poll_when_idle_threshold);
+
     /* negative value means use default - just don't do anything */
     if (ompi_mpi_event_tick_rate >= 0) {
         opal_progress_set_event_poll_rate(ompi_mpi_event_tick_rate);
diff --git a/ompi/runtime/ompi_mpi_params.c b/ompi/runtime/ompi_mpi_params.c
@@ -17,7 +17,7 @@
  * Copyright (c) 2013-2017 Intel, Inc.  All rights reserved.
  * Copyright (c) 2015      Mellanox Technologies, Inc.
  *                         All rights reserved.
- * Copyright (c) 2016      Research Organization for Information Science
+ * Copyright (c) 2016-2017 Research Organization for Information Science
  *                         and Technology (RIST). All rights reserved.
  * $COPYRIGHT$
  *
@@ -63,6 +63,7 @@ bool ompi_have_sparse_group_storage = OPAL_INT_TO_BOOL(OMPI_GROUP_SPARSE);
 bool ompi_use_sparse_group_storage = OPAL_INT_TO_BOOL(OMPI_GROUP_SPARSE);
 
 bool ompi_mpi_yield_when_idle = false;
+uint32_t ompi_mpi_poll_when_idle_threshold = -1;
 int ompi_mpi_event_tick_rate = -1;
 char *ompi_mpi_show_mca_params_string = NULL;
 bool ompi_mpi_have_sparse_group_storage = !!(OMPI_GROUP_SPARSE);
@@ -115,6 +116,14 @@ int ompi_mpi_register_params(void)
                                  MCA_BASE_VAR_SCOPE_READONLY,
                                  &ompi_mpi_yield_when_idle);
 
+    ompi_mpi_poll_when_idle_threshold = -1;
+    (void) mca_base_var_register("ompi", "mpi", NULL, "poll_when_idle_threshold",
+                                 "Poll the processor after waiting for MPI communication too long",
+                                 MCA_BASE_VAR_TYPE_INT, NULL, 0, 0,
+                                 OPAL_INFO_LVL_9,
+                                 MCA_BASE_VAR_SCOPE_READONLY,
+                                 &ompi_mpi_poll_when_idle_threshold);
+
     ompi_mpi_event_tick_rate = -1;
     (void) mca_base_var_register("ompi", "mpi", NULL, "event_tick_rate",
                                  "How often to progress TCP communications (0 = never, otherwise specified in microseconds)",
diff --git a/opal/runtime/opal_progress.c b/opal/runtime/opal_progress.c
@@ -12,7 +12,7 @@
  *                         All rights reserved.
  * Copyright (c) 2006-2016 Los Alamos National Security, LLC.  All rights
  *                         reserved.
- * Copyright (c) 2015-2016 Research Organization for Information Science
+ * Copyright (c) 2015-2017 Research Organization for Information Science
  *                         and Technology (RIST). All rights reserved.
  *
  * $COPYRIGHT$
@@ -27,6 +27,11 @@
 #ifdef HAVE_SCHED_H
 #include <sched.h>
 #endif
+#if HAVE_POLL_H
+#include <poll.h>
+#elif HAVE_SYS_POLL_H
+#include <sys/poll.h>
+#endif
 
 #include "opal/runtime/opal_progress.h"
 #include "opal/mca/event/event.h"
@@ -66,6 +71,9 @@ static size_t callbacks_lp_size = 0;
 
 /* do we want to call sched_yield() if nothing happened */
 bool opal_progress_yield_when_idle = false;
+/* do we want to poll() if nothing happened for a while */
+int opal_progress_poll_when_idle_threshold = -1;
+static int yield_count = 0;
 
 #if OPAL_PROGRESS_USE_TIMERS
 static opal_timer_t event_progress_last_time = 0;
@@ -229,16 +237,25 @@ opal_progress(void)
         }
     }
 
+    if (OPAL_UNLIKELY(opal_progress_yield_when_idle)) {
+        if (events <= 0) {
+            if (opal_progress_poll_when_idle_threshold < 0 || yield_count < opal_progress_poll_when_idle_threshold) {
+                yield_count++;
 #if OPAL_HAVE_SCHED_YIELD
-    if (opal_progress_yield_when_idle && events <= 0) {
-        /* If there is nothing to do - yield the processor - otherwise
-         * we could consume the processor for the entire time slice. If
-         * the processor is oversubscribed - this will result in a best-case
-         * latency equivalent to the time-slice.
-         */
-        sched_yield();
-    }
+                /* If there is nothing to do - yield the processor - otherwise
+                 * we could consume the processor for the entire time slice. If
+                 * the processor is oversubscribed - this will result in a best-case
+                 * latency equivalent to the time-slice.
+                 */
+                sched_yield();
 #endif  /* defined(HAVE_SCHED_YIELD) */
+            } else {
+                poll(NULL, 0, 1);
+            }
+        } else {
+            yield_count = 0;
+        }
+    }
 }
 
 
@@ -310,6 +327,20 @@ opal_progress_set_yield_when_idle(bool yieldopt)
 }
 
 
+int
+opal_progress_set_poll_when_idle_threshold(int thresholdopt)
+{
+    int tmp = opal_progress_poll_when_idle_threshold;
+
+    opal_progress_poll_when_idle_threshold = thresholdopt;
+
+    OPAL_OUTPUT((debug_output, "progress: progress_set_poll_threshold to %d",
+                                    thresholdopt));
+
+    return tmp;
+}
+
+
 void
 opal_progress_set_event_poll_rate(int polltime)
 {
diff --git a/opal/runtime/opal_progress.h b/opal/runtime/opal_progress.h
@@ -11,6 +11,8 @@
  *                         All rights reserved.
  * Copyright (c) 2006-2014 Los Alamos National Security, LLC.  All rights
  *                         reserved.
+ * Copyright (c) 2017      Research Organization for Information Science
+ *                         and Technology (RIST). All rights reserved.
  *
  * $COPYRIGHT$
  *
@@ -113,19 +115,35 @@ OPAL_DECLSPEC void opal_progress_event_users_decrement(void);
 /**
  * Set whether opal_progress() should yield when idle
  *
- * Set whether opal_progress() should yield the processor (either by
- * sched_yield() or SwitchToThread()) if no events were progressed
- * during the progress loop.  The return value of the callback
- * functions is used to determine whether or not yielding is required.
+ * Set whether opal_progress() should yield the processor by
+ * sched_yield() no events were progressed during the progress loop.
+ * The return value of the callback functions is used to determine
+ * whether or not yielding is required.
  * By default, the event loop will yield when the progress function is
- * idle.
+ * idle only when oversubscription.
  *
  * @param   yieldopt  Whether to yield when idle.
  * @return         Previous value of the yield_when_idle option.
  */
 OPAL_DECLSPEC bool opal_progress_set_yield_when_idle(bool yieldopt);
 
 
+/**
+ * Set whether opal_progress() should poll when idle for a while
+ *
+ * Set whether opal_progress() should poll if no events were progressed
+ * during the progress loop for some time.  The return value of the callback
+ * functions is used to determine whether or not polling is required.
+ * By default, the event loop will not poll when the progress function is
+ * idle.
+ * -1 means no polling and 0 means always poll.
+ *
+ * @param   thresholdopt  The threshold to start polling.
+ * @return         Previous value of the poll_threshold option.
+ */
+OPAL_DECLSPEC int opal_progress_set_poll_when_idle_threshold(int thresholdopt);
+
+
 /**
  * Set time between calls into the event library
  *
