opal/thread: clean up and add additional OPAL_THREAD macros

hjelmn · hjelmn · commit 170161ecdd67 · 2016-07-29T07:03:12.000-06:00
This commit expands the OPAL_THREAD macros to include 32- and 64-bit atomic swap. Additionally, macro declararations have been updated to include both OPAL_THREAD_* and OPAL_ATOMIC_*. Before this commit the former was used with add and the later with cmpset. This commit remove the OMPI_THREAD_MULTIPLE macros around the OPAL_THREAD_* function. This should not make a difference in performance as opal_uses_threads is #defined to false when OMPI_THREAD_MULTIPLE is 0. The compiler should optimize out the check on opal_using_threads. Signed-off-by: Nathan Hjelm <hjelmn@me.com> (cherry picked from commit aac6112) Signed-off-by: Nathan Hjelm <hjelmn@lanl.gov>
diff --git a/ompi/mca/osc/sm/osc_sm_active_target.c b/ompi/mca/osc/sm/osc_sm_active_target.c
@@ -1,7 +1,7 @@
 /* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
 /*
  * Copyright (c) 2012      Sandia National Laboratories.  All rights reserved.
- * Copyright (c) 2014-2015 Los Alamos National Security, LLC. All rights
+ * Copyright (c) 2014-2016 Los Alamos National Security, LLC. All rights
  *                         reserved.
  * Copyright (c) 2014      The University of Tennessee and The University
  *                         of Tennessee Research Foundation.  All rights
@@ -133,7 +133,7 @@ ompi_osc_sm_start(struct ompi_group_t *group,
 
     OBJ_RETAIN(group);
 
-    if (!OPAL_ATOMIC_CMPSET(&module->start_group, NULL, group)) {
+    if (!OPAL_ATOMIC_CMPSET_PTR(&module->start_group, NULL, group)) {
         OBJ_RELEASE(group);
         return OMPI_ERR_RMA_SYNC;
     }
@@ -185,7 +185,7 @@ ompi_osc_sm_complete(struct ompi_win_t *win)
     opal_atomic_mb();
 
     group = module->start_group;
-    if (NULL == group || !OPAL_ATOMIC_CMPSET(&module->start_group, group, NULL)) {
+    if (NULL == group || !OPAL_ATOMIC_CMPSET_PTR(&module->start_group, group, NULL)) {
         return OMPI_ERR_RMA_SYNC;
     }
 
diff --git a/ompi/mpi/c/comm_get_errhandler.c b/ompi/mpi/c/comm_get_errhandler.c
@@ -1,3 +1,4 @@
+/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
 /*
  * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
  *                         University Research and Technology
@@ -12,6 +13,8 @@
  * Copyright (c) 2007-2009 Cisco Systems, Inc.  All rights reserved.
  * Copyright (c) 2015      Research Organization for Information Science
  *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2016      Los Alamos National Security, LLC. All rights
+ *                         reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -64,7 +67,7 @@ int MPI_Comm_get_errhandler(MPI_Comm comm, MPI_Errhandler *errhandler)
      error_handler became atomic. */
   do {
       tmp = comm->error_handler;
-  } while (!OPAL_ATOMIC_CMPSET(&(comm->error_handler), tmp, tmp));
+  } while (!OPAL_ATOMIC_CMPSET_PTR(&(comm->error_handler), tmp, tmp));
 
   /* Retain the errhandler, corresponding to object refcount decrease
      in errhandler_free.c. */
diff --git a/ompi/mpi/c/comm_set_errhandler.c b/ompi/mpi/c/comm_set_errhandler.c
@@ -1,3 +1,4 @@
+/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
 /*
  * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
  *                         University Research and Technology
@@ -11,6 +12,8 @@
  *                         All rights reserved.
  * Copyright (c) 2015      Research Organization for Information Science
  *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2016      Los Alamos National Security, LLC. All rights
+ *                         reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -67,9 +70,7 @@ int MPI_Comm_set_errhandler(MPI_Comm comm, MPI_Errhandler errhandler)
     /* Ditch the old errhandler, and decrement its refcount.  On 64
        bits environments we have to make sure the reading of the
        error_handler became atomic. */
-    do {
-        tmp = comm->error_handler;
-    } while (!OPAL_ATOMIC_CMPSET(&(comm->error_handler), tmp, errhandler));
+    tmp = OPAL_ATOMIC_SWAP_PTR(&comm->error_handler, errhandler);
     OBJ_RELEASE(tmp);
 
     /* All done */
diff --git a/ompi/mpi/c/file_get_errhandler.c b/ompi/mpi/c/file_get_errhandler.c
@@ -1,3 +1,4 @@
+/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
 /*
  * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
  *                         University Research and Technology
@@ -12,6 +13,8 @@
  * Copyright (c) 2008      Cisco Systems, Inc.  All rights reserved.
  * Copyright (c) 2015      Research Organization for Information Science
  *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2016      Los Alamos National Security, LLC. All rights
+ *                         reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -63,7 +66,7 @@ int MPI_File_get_errhandler( MPI_File file, MPI_Errhandler *errhandler)
      error_handler became atomic. */
   do {
       tmp = file->error_handler;
-  } while (!OPAL_ATOMIC_CMPSET(&(file->error_handler), tmp, tmp));
+  } while (!OPAL_ATOMIC_CMPSET_PTR(&(file->error_handler), tmp, tmp));
 
   /* Retain the errhandler, corresponding to object refcount
      decrease in errhandler_free.c. */
diff --git a/ompi/mpi/c/file_set_errhandler.c b/ompi/mpi/c/file_set_errhandler.c
@@ -1,3 +1,4 @@
+/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
 /*
  * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
  *                         University Research and Technology
@@ -12,6 +13,8 @@
  * Copyright (c) 2008      Cisco Systems, Inc.  All rights reserved.
  * Copyright (c) 2015      Research Organization for Information Science
  *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2016      Los Alamos National Security, LLC. All rights
+ *                         reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -66,9 +69,7 @@ int MPI_File_set_errhandler( MPI_File file, MPI_Errhandler errhandler)
     /* Ditch the old errhandler, and decrement its refcount.  On 64
        bits environments we have to make sure the reading of the
        error_handler became atomic. */
-    do {
-        tmp = file->error_handler;
-    } while (!OPAL_ATOMIC_CMPSET(&(file->error_handler), tmp, errhandler));
+    tmp = OPAL_ATOMIC_SWAP_PTR (&file->error_handler, errhandler);
     OBJ_RELEASE(tmp);
 
     /* All done */
diff --git a/ompi/mpi/c/win_get_errhandler.c b/ompi/mpi/c/win_get_errhandler.c
@@ -1,3 +1,4 @@
+/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
 /*
  * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
  *                         University Research and Technology
@@ -12,6 +13,8 @@
  * Copyright (c) 2008-2009 Cisco Systems, Inc.  All rights reserved.
  * Copyright (c) 2015      Research Organization for Information Science
  *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2016      Los Alamos National Security, LLC. All rights
+ *                         reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -56,7 +59,7 @@ int MPI_Win_get_errhandler(MPI_Win win, MPI_Errhandler *errhandler)
        error_handler became atomic. */
     do {
         tmp = win->error_handler;
-    } while (!OPAL_ATOMIC_CMPSET(&(win->error_handler), tmp, tmp));
+    } while (!OPAL_ATOMIC_CMPSET_PTR(&(win->error_handler), tmp, tmp));
 
     /* Retain the errhandler, corresponding to object refcount
        decrease in errhandler_free.c. */
diff --git a/ompi/mpi/c/win_set_errhandler.c b/ompi/mpi/c/win_set_errhandler.c
@@ -1,3 +1,4 @@
+/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
 /*
  * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
  *                         University Research and Technology
@@ -12,6 +13,8 @@
  * Copyright (c) 2008-2009 Cisco Systems, Inc.  All rights reserved.
  * Copyright (c) 2015      Research Organization for Information Science
  *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2016      Los Alamos National Security, LLC. All rights
+ *                         reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -61,9 +64,7 @@ int MPI_Win_set_errhandler(MPI_Win win, MPI_Errhandler errhandler)
     /* Ditch the old errhandler, and decrement its refcount.  On 64
        bits environments we have to make sure the reading of the
        error_handler became atomic. */
-    do {
-        tmp = win->error_handler;
-    } while (!OPAL_ATOMIC_CMPSET(&(win->error_handler), tmp, errhandler));
+    tmp = OPAL_ATOMIC_SWAP_PTR(&win->error_handler, errhandler);
     OBJ_RELEASE(tmp);
 
     /* All done */
diff --git a/opal/threads/thread_usage.h b/opal/threads/thread_usage.h
@@ -26,6 +26,7 @@
 #define OPAL_THREAD_USAGE_H
 
 #include "opal/sys/atomic.h"
+#include "opal/prefetch.h"
 #include "opal_config.h"
 
 /**
@@ -101,113 +102,85 @@ static inline bool opal_set_using_threads(bool have)
  * Use an atomic operation for increment/decrement if opal_using_threads()
  * indicates that threads are in use by the application or library.
  */
+#define OPAL_THREAD_DEFINE_ATOMIC_ADD(type, suffix)     \
+static inline type opal_thread_add_ ## suffix (volatile type *addr, type delta) \
+{                                                                       \
+    if (OPAL_UNLIKELY(opal_using_threads())) {                          \
+        return opal_atomic_add_ ## suffix (addr, delta);                \
+    }                                                                   \
+                                                                        \
+    return (*addr += delta);                                            \
+}
 
-#if OMPI_ENABLE_THREAD_MULTIPLE
-static inline int32_t
-OPAL_THREAD_ADD32(volatile int32_t *addr, int delta)
-{
-    int32_t ret;
-
-    if (opal_using_threads()) {
-        ret = opal_atomic_add_32(addr, delta);
-    } else {
-        ret = (*addr += delta);
-    }
-
-    return ret;
+#define OPAL_THREAD_DEFINE_ATOMIC_CMPSET(type, addr_type, suffix)       \
+static inline bool opal_thread_cmpset_bool_ ## suffix (volatile addr_type *addr, type compare, type value) \
+{                                                                       \
+    if (OPAL_UNLIKELY(opal_using_threads())) {                          \
+        return opal_atomic_cmpset_ ## suffix ((volatile type *) addr, compare, value); \
+    }                                                                   \
+                                                                        \
+    if ((type) *addr == compare) {                                      \
+        ((type *) addr)[0] = value;                                     \
+        return true;                                                    \
+    }                                                                   \
+                                                                        \
+    return false;                                                       \
 }
-#else
-static inline int32_t
-OPAL_THREAD_ADD32(volatile int32_t *addr, int delta)
-{
-    int32_t ret;
-    ret = (*addr += delta);
-    return ret;
+
+#define OPAL_THREAD_DEFINE_ATOMIC_SWAP(type, addr_type, suffix)         \
+static inline type opal_thread_swap_ ## suffix (volatile addr_type *ptr, type newvalue) \
+{                                                                       \
+    if (opal_using_threads ()) {                                        \
+        return opal_atomic_swap_ ## suffix ((volatile type *) ptr, newvalue); \
+    }                                                                   \
+                                                                        \
+    type old = ((type *) ptr)[0];                                       \
+    ((type *) ptr)[0] = newvalue;                                       \
+                                                                        \
+    return old;                                                         \
 }
-#endif
 
-#if OPAL_HAVE_ATOMIC_MATH_64
-#if OMPI_ENABLE_THREAD_MULTIPLE
-static inline int64_t
-OPAL_THREAD_ADD64(volatile int64_t *addr, int delta)
-{
-    int64_t ret;
+OPAL_THREAD_DEFINE_ATOMIC_ADD(int32_t, 32)
+OPAL_THREAD_DEFINE_ATOMIC_ADD(size_t, size_t)
+OPAL_THREAD_DEFINE_ATOMIC_CMPSET(int32_t, int32_t, 32)
+OPAL_THREAD_DEFINE_ATOMIC_CMPSET(void *, intptr_t, ptr)
+OPAL_THREAD_DEFINE_ATOMIC_SWAP(int32_t, int32_t, 32)
+OPAL_THREAD_DEFINE_ATOMIC_SWAP(void *, intptr_t, ptr)
 
-    if (opal_using_threads()) {
-        ret = opal_atomic_add_64(addr, delta);
-    } else {
-        ret = (*addr += delta);
-    }
+#define OPAL_THREAD_ADD32 opal_thread_add_32
+#define OPAL_ATOMIC_ADD32 opal_thread_add_32
 
-    return ret;
-}
-#else
-static inline int64_t
-OPAL_THREAD_ADD64(volatile int64_t *addr, int delta)
-{
-    int64_t ret;
-    ret = (*addr += delta);
-    return ret;
-}
-#endif
-#endif
+#define OPAL_THREAD_ADD_SIZE_T opal_thread_add_size_t
+#define OPAL_ATOMIC_ADD_SIZE_T opal_thread_add_size_t
 
-#if OMPI_ENABLE_THREAD_MULTIPLE
-static inline size_t
-OPAL_THREAD_ADD_SIZE_T(volatile size_t *addr, int delta)
-{
-    size_t ret;
+#define OPAL_THREAD_CMPSET_32 opal_thread_cmpset_bool_32
+#define OPAL_ATOMIC_CMPSET_32 opal_thread_cmpset_bool_32
 
-    if (opal_using_threads()) {
-        ret = opal_atomic_add_size_t(addr, delta);
-    } else {
-        ret = (*addr += delta);
-    }
+#define OPAL_THREAD_CMPSET_PTR(x, y, z) opal_thread_cmpset_bool_ptr ((volatile intptr_t *) x, (void *) y, (void *) z)
+#define OPAL_ATOMIC_CMPSET_PTR OPAL_THREAD_CMPSET_PTR
 
-    return ret;
-}
-#else
-static inline size_t
-OPAL_THREAD_ADD_SIZE_T(volatile size_t *addr, int delta)
-{
-    size_t ret;
-    ret = (*addr += delta);
-    return ret;
-}
-#endif
+#define OPAL_THREAD_SWAP_32 opal_thread_swap_32
+#define OPAL_ATOMIC_SWAP_32 opal_thread_swap_32
 
-/* BWB: FIX ME: remove if possible */
-#define OPAL_CMPSET(x, y, z) ((*(x) == (y)) ? ((*(x) = (z)), 1) : 0)
+#define OPAL_THREAD_SWAP_PTR(x, y) opal_thread_swap_ptr ((volatile intptr_t *) x, (void *) y)
+#define OPAL_ATOMIC_SWAP_PTR OPAL_THREAD_SWAP_PTR
 
-#if OPAL_HAVE_ATOMIC_CMPSET_32
-#define OPAL_ATOMIC_CMPSET_32(x, y, z) \
-    (opal_using_threads() ? opal_atomic_cmpset_32(x, y, z) : OPAL_CMPSET(x, y, z))
-#endif
-#if OPAL_HAVE_ATOMIC_CMPSET_64
-#define OPAL_ATOMIC_CMPSET_64(x, y, z) \
-    (opal_using_threads() ? opal_atomic_cmpset_64(x, y, z) : OPAL_CMPSET(x, y, z))
-#endif
-#if OPAL_HAVE_ATOMIC_CMPSET_32 || OPAL_HAVE_ATOMIC_CMPSET_64
-#define OPAL_ATOMIC_CMPSET(x, y, z) \
-    (opal_using_threads() ? opal_atomic_cmpset(x, y, z) : OPAL_CMPSET(x, y, z))
-#endif
-#if OPAL_HAVE_ATOMIC_CMPSET_32 || OPAL_HAVE_ATOMIC_CMPSET_64
-#define OPAL_ATOMIC_CMPSET_PTR(x, y, z) \
-    (opal_using_threads() ? opal_atomic_cmpset_ptr(x, y, z) : OPAL_CMPSET(x, y, z))
-#endif
+/* define 64-bit macros is 64-bit atomic math is available */
+#if OPAL_HAVE_ATOMIC_MATH_64
 
-static inline void *opal_thread_swap_ptr (volatile void *ptr, void *newvalue)
-{
-    if (opal_using_threads ()) {
-        return opal_atomic_swap_ptr (ptr, newvalue);
-    }
+OPAL_THREAD_DEFINE_ATOMIC_ADD(int64_t, 64)
+OPAL_THREAD_DEFINE_ATOMIC_CMPSET(int64_t, int64_t, 64)
+OPAL_THREAD_DEFINE_ATOMIC_SWAP(int64_t, int64_t, 64)
 
-    void *old = ((void **) ptr)[0];
-    ((void **) ptr)[0] = newvalue;
+#define OPAL_THREAD_ADD64 opal_thread_add_64
+#define OPAL_ATOMIC_ADD64 opal_thread_add_64
 
-    return old;
-}
+#define OPAL_THREAD_CMPSET_64 opal_thread_cmpset_bool_64
+#define OPAL_ATOMIC_CMPSET_64 opal_thread_cmpset_bool_64
 
-#define OPAL_ATOMIC_SWAP_PTR(x, y) opal_thread_swap_ptr (x, y)
+#define OPAL_THREAD_SWAP_64 opal_thread_swap_64
+#define OPAL_ATOMIC_SWAP_64 opal_thread_swap_64
+
+#endif
 
 #endif /* !defined(OPAL_THREAD_USAGE_H) */
