Changes for the IBM XL compiler.

Author: corbett5

CMakeLists.txt

@@ -20,6 +20,7 @@ if( NOT is_submodule )
     option( ENABLE_BENCHMARKS "Builds benchmarks" ON )
     option( ENABLE_DOCS "Builds documentation" ON )
 
+    option( ENABLE_UMPIRE "Build with UMPIRE" ON )
     option( ENABLE_CHAI "Build with CHAI" ON )
     option( ENABLE_MPI "Build with MPI" ON )
     option( ENABLE_CUDA "Build with CUDA" OFF )

cmake/SetupTPL.cmake

@@ -68,7 +68,7 @@ if(ENABLE_CHAI)
     get_target_property(CHAI_LINK_LIBRARIES chai INTERFACE_LINK_LIBRARIES)
     list(REMOVE_ITEM CHAI_LINK_LIBRARIES RAJA)
     set_target_properties(chai
-                        PROPERTIES INTERFACE_LINK_LIBRARIES ${CHAI_LINK_LIBRARIES})
+                          PROPERTIES INTERFACE_LINK_LIBRARIES "${CHAI_LINK_LIBRARIES}")
 
     set(thirdPartyLibs ${thirdPartyLibs} chai)
 else()

host-configs/LLNL/[email protected]

@@ -0,0 +1,56 @@
+set(CONFIG_NAME "[email protected]" CACHE PATH "")
+
+# set(RAJA_DIR PATH/TO/RAJA CACHE PATH "")
+
+set(ENABLE_UMPIRE OFF CACHE BOOL "")
+set(ENABLE_CHAI OFF CACHE BOOL "")
+set(ENABLE_CALIPER OFF CACHE BOOL "")
+set(USE_ADDR2LINE ON CACHE BOOL "")
+
+# C options
+set(CMAKE_C_COMPILER /usr/tce/packages/xl/xl-2020.08.13-cuda-11.0.2/bin/xlc CACHE PATH "")
+set(CMAKE_CXX_FLAGS_RELEASE "-O3 -qxlcompatmacros -qlanglvl=extended0x -qalias=noansi -qsmp=omp -qhot -qnoeh -qsuppress=1500-029 -qsuppress=1500-036" CACHE STRING "")
+set(CMAKE_CXX_FLAGS_RELWITHDEBINFO "-O2 -g -qxlcompatmacros -qlanglvl=extended0x -qalias=noansi -qsmp=omp -qhot -qnoeh -qsuppress=1500-029 -qsuppress=1500-036" CACHE STRING "")
+set(CMAKE_CXX_FLAGS_DEBUG "-O0 -g -qsmp=omp:noopt " CACHE STRING "")
+set(CMAKE_EXE_LINKER_FLAGS "-Wl,-z,muldefs" CACHE STRING "")
+
+# C++ options
+set(CMAKE_CXX_COMPILER /usr/tce/packages/xl/xl-2020.08.13-cuda-11.0.2/bin/xlC CACHE PATH "")
+set(CMAKE_CXX_FLAGS_RELEASE "-O3 -qxlcompatmacros -qlanglvl=extended0x -qalias=noansi -qsmp=omp -qhot -qnoeh -qsuppress=1500-029 -qsuppress=1500-036" CACHE STRING "")
+set(CMAKE_CXX_FLAGS_RELWITHDEBINFO "-O2 -g -qxlcompatmacros -qlanglvl=extended0x -qalias=noansi -qsmp=omp -qhot -qnoeh -qsuppress=1500-029 -qsuppress=1500-036" CACHE STRING "")
+set(CMAKE_CXX_FLAGS_DEBUG "-O0 -g -qsmp=omp:noopt " CACHE STRING "")
+set(BLT_CXX_STANDARD 14 CACHE STRING "")
+
+# OpenMP options
+set(ENABLE_OPENMP ON CACHE BOOL "" FORCE)
+
+# MPI options
+set(ENABLE_MPI ON CACHE BOOL "")
+set(MPI_ROOT /usr/tce/packages/spectrum-mpi/spectrum-mpi-rolling-release-xl-2020.08.13-cuda-11.0.2/ CACHE PATH "")
+set(MPI_C_COMPILER         ${MPI_ROOT}/bin/mpicc  CACHE PATH "")
+set(MPI_CXX_COMPILER       ${MPI_ROOT}/bin/mpicxx CACHE PATH "")
+set(MPIEXEC                lrun CACHE STRING "")
+set(MPIEXEC_NUMPROC_FLAG   -n CACHE STRING "")
+set(ENABLE_WRAP_ALL_TESTS_WITH_MPIEXEC ON CACHE BOOL "")
+
+# Cuda options
+set(ENABLE_CUDA ON CACHE BOOL "")
+set(CUDA_TOOLKIT_ROOT_DIR /usr/tce/packages/cuda/cuda-11.0.2 CACHE STRING "")
+set(CMAKE_CUDA_HOST_COMPILER ${MPI_CXX_COMPILER} CACHE STRING "")
+set(CMAKE_CUDA_COMPILER ${CUDA_TOOLKIT_ROOT_DIR}/bin/nvcc CACHE STRING "")
+set(CUDA_ARCH sm_70 CACHE STRING "")
+set(CMAKE_CUDA_STANDARD 14 CACHE STRING "")
+set(CMAKE_CUDA_FLAGS "-restrict -arch ${CUDA_ARCH} --expt-extended-lambda -Werror cross-execution-space-call,reorder,deprecated-declarations" CACHE STRING "")
+set(CMAKE_CUDA_FLAGS_RELEASE "-O3 -DNDEBUG -Xcompiler -DNDEBUG -Xcompiler -O3 -Xcompiler -qxlcompatmacros -Xcompiler -qalias=noansi -Xcompiler -qsmp=omp -Xcompiler -qhot -Xcompiler -qnoeh -Xcompiler -qsuppress=1500-029 -Xcompiler -qsuppress=1500-036" CACHE STRING "")
+set(CMAKE_CUDA_FLAGS_RELWITHDEBINFO "-g -lineinfo ${CMAKE_CUDA_FLAGS_RELEASE}" CACHE STRING "")
+set(CMAKE_CUDA_FLAGS_DEBUG "-g -G -O0 -Xcompiler -O0" CACHE STRING "")
+
+# Uncomment this line to make nvcc output register usage for each kernel.
+# set(CMAKE_CUDA_FLAGS "${CMAKE_CUDA_FLAGS} --resource-usage" CACHE STRING "" FORCE)
+
+# GTEST options
+set(ENABLE_GTEST_DEATH_TESTS OFF CACHE BOOL "")
+set(gtest_disable_pthreads ON CACHE BOOL "")
+
+# Documentation options
+set(ENABLE_DOCS OFF CACHE BOOL "")

src/Array.hpp

@@ -210,7 +210,7 @@ class Array : public ArrayView< T,
   template< typename ... DIMS >
   LVARRAY_HOST_DEVICE
   std::enable_if_t< sizeof ... ( DIMS ) == NDIM && typeManipulation::all_of_t< std::is_integral< DIMS > ... >::value >
-  resize( DIMS... newDims )
+  resize( DIMS const ... newDims )
   {
     static_assert( sizeof ... ( DIMS ) == NDIM, "The number of arguments provided does not equal NDIM!" );
     INDEX_TYPE const oldSize = this->size();

src/ArraySlice.hpp

@@ -12,29 +12,27 @@
 
 #pragma once
 
-#ifndef NDEBUG
-  #ifndef __APPLE__
+#if !defined( NDEBUG ) && !defined( __APPLE__ ) && !defined( __ibmxl__ )
 /**
  * @brief Add GDB pretty printers the given script.
  * @param script_name The python script that contains the gdb hooks.
  * @note Taken from https://sourceware.org/gdb/onlinedocs/gdb/dotdebug_005fgdb_005fscripts-section.html
  */
 #define DEFINE_GDB_PY_SCRIPT( script_name ) \
   asm (".pushsection \".debug_gdb_scripts\", \"MS\",@progbits,1\n \
-              .byte 1 /* Python */\n \
-              .asciz \"" script_name "\"\n \
-              .popsection \n" )
-  #else
+                .byte 1 /* Python */\n \
+                .asciz \"" script_name "\"\n \
+                .popsection \n" );
+#else
 /**
  * @brief Add GDB pretty printers for OSX. This hasn't been done yet.
  * @param script_name The python script that contains the gdb hooks.
  */
 #define DEFINE_GDB_PY_SCRIPT( script_name )
-  #endif
+#endif
 
 /// Point GDB at the scripts/gdb-printers.py
-DEFINE_GDB_PY_SCRIPT( "scripts/gdb-printers.py" );
-#endif
+DEFINE_GDB_PY_SCRIPT( "scripts/gdb-printers.py" )
 
 // Source includes
 #include "LvArrayConfig.hpp"
@@ -171,7 +169,19 @@ class ArraySlice
    */
   LVARRAY_HOST_DEVICE inline constexpr
   INDEX_TYPE size() const noexcept
-  { return indexing::multiplyAll< NDIM >( m_dims ); }
+  {
+  #if defined( __ibmxl__ )
+    // Note: This used to be done with a recursive template but XL-release would produce incorrect results.
+    // Specifically in exampleArray it would return an "old" size even after being updated, strange.
+    INDEX_TYPE val = m_dims[ 0 ];
+    for( int i = 1; i < NDIM; ++i )
+    { val *= m_dims[ i ]; }
+
+    return val;
+  #else
+    return indexing::multiplyAll< NDIM >( m_dims );
+  #endif
+  }
 
   /**
    * @return Return the length of the given dimension.

src/ArrayView.hpp

@@ -303,7 +303,19 @@ class ArrayView
    */
   LVARRAY_HOST_DEVICE inline
   INDEX_TYPE size() const noexcept
-  { return indexing::multiplyAll< NDIM >( m_dims ); }
+  {
+  #if defined( __ibmxl__ )
+    // Note: This used to be done with a recursive template but XL-release would produce incorrect results.
+    // Specifically in exampleArray it would return an "old" size even after being updated, strange.
+    INDEX_TYPE val = m_dims[ 0 ];
+    for( int i = 1; i < NDIM; ++i )
+    { val *= m_dims[ i ]; }
+
+    return val;
+  #else
+    return indexing::multiplyAll< NDIM >( m_dims.data );
+  #endif
+  }
 
   /**
    * @return Return the length of the given dimension.

src/Macros.hpp

@@ -219,7 +219,7 @@
  * @param lhs expression to be evaluated and used as left-hand side in comparison
  * @param rhs expression to be evaluated and used as right-hand side in comparison
  */
-#define LVARRAY_ERROR_IF_EQ( lhs, rhs ) LVARRAY_ERROR_IF_NE_MSG( lhs, rhs, "" )
+#define LVARRAY_ERROR_IF_EQ( lhs, rhs ) LVARRAY_ERROR_IF_EQ_MSG( lhs, rhs, "" )
 
 /**
  * @brief Raise a hard error if two values are not equal.

src/limits.hpp

@@ -51,6 +51,35 @@ struct NumericLimits : public std::numeric_limits< T >
   static constexpr T denorm_min = std::numeric_limits< T >::denorm_min();
 };
 
+/**
+ * @struct NumericLimitsNC
+ * @brief The same as @c NumericLimits except the entries are not static or constexpr.
+ * @details This is useful for solving "undefined reference" errors that pop up often in lambdas.
+ * @tparam T the numeric type to query.
+ */
+template< typename T >
+struct NumericLimitsNC : public std::numeric_limits< T >
+{
+  /// The smallest finite value T can hold.
+  T const min = std::numeric_limits< T >::min();
+  /// The lowest finite value T can hold.
+  T const lowest = std::numeric_limits< T >::lowest();
+  /// The largest finite value T can hold.
+  T const max = std::numeric_limits< T >::max();
+  /// The difference between 1.0 and the next representable value (if T is floating point).
+  T const epsilon = std::numeric_limits< T >::epsilon();
+  /// The maximum rounding error (if T is a floating point).
+  T const round_error = std::numeric_limits< T >::round_error();
+  /// A positive infinity value (if T is a floating point).
+  T const infinity = std::numeric_limits< T >::infinity();
+  /// A quiet NaN (if T is a floating point).
+  T const quiet_NaN = std::numeric_limits< T >::quiet_NaN();
+  /// A signaling NaN (if T is a floating point).
+  T const signaling_NaN = std::numeric_limits< T >::signaling_NaN();
+  /// The smallest positive subnormal value (if T is a floating point).
+  T const denorm_min = std::numeric_limits< T >::denorm_min();
+};
+
 namespace internal
 {
 

src/sortedArrayManipulation.hpp

@@ -314,9 +314,21 @@ std::ptrdiff_t removeDuplicates( ITER first, ITER const last, Compare && comp=Co
   }
 
   std::ptrdiff_t numUnique = 1;
+
   ITER next = first;
-  ++next;
-  while( next != last )
+
+  /**
+   * For whatever reason the standard approach doesn't work with XL in release on device.
+   * It does some really strange things, for example `last - next == 0` and they can have identical
+   * values but `next != last`. If you print out the array each iteration it works as expected. I even
+   * tried substituting the example code for std::unique from cppreference and that exhibited the same problem.
+   * My guess is it's most likely a compiler bug.
+   */
+#if defined( __ibmxl__ ) && defined( __CUDA_ARCH__ )
+  while( arrayManipulation::iterDistance( ++next, last ) > 0 )
+#else
+  while( ++next != last )
+#endif
   {
     if( comp( *first, *next ) )
     {
@@ -327,8 +339,6 @@ std::ptrdiff_t removeDuplicates( ITER first, ITER const last, Compare && comp=Co
         *first = std::move( *next );
       }
     }
-
-    ++next;
   }
 
   return numUnique;

unitTests/testMath.cpp

@@ -55,9 +55,9 @@ struct TestMath : public ::testing::Test
   void sqrtAndInvSqrt()
   {
     using FloatingPoint = decltype( math::sqrt( T() ) );
-    FloatingPoint const epsilon = NumericLimits< FloatingPoint >::epsilon;
-    forall< POLICY >( 1, [epsilon] LVARRAY_HOST_DEVICE ( int )
+    forall< POLICY >( 1, [] LVARRAY_HOST_DEVICE ( int )
         {
+          FloatingPoint const epsilon = NumericLimitsNC< FloatingPoint >{}.epsilon;
 
           T a = 5 * 5;
           PORTABLE_EXPECT_EQ( math::sqrt( a ), 5.0 );
@@ -112,10 +112,11 @@ struct TestMathFloatingPointOnly : public ::testing::Test
 
   void trig()
   {
-    FloatingPoint const epsilon = NumericLimits< FloatingPoint >::epsilon;
-    forall< POLICY >( 1, [epsilon] LVARRAY_HOST_DEVICE ( int )
+    forall< POLICY >( 1, [] LVARRAY_HOST_DEVICE ( int )
         {
-          FloatingPoint coords[ 2 ][ 3 ] = { { 1, 2, 1.10714871779409050301 },
+          FloatingPoint const epsilon = NumericLimitsNC< FloatingPoint >{}.epsilon;
+
+          FloatingPoint const coords[ 2 ][ 3 ] = { { 1, 2, 1.10714871779409050301 },
             { 4, -1, -0.24497866312686415417 } };
 
           for( int i = 0; i < 2; ++i )
@@ -140,7 +141,12 @@ struct TestMathFloatingPointOnly : public ::testing::Test
 
             PORTABLE_EXPECT_NEAR( math::abs( math::asin( sinTheta ) ), math::abs( theta ), 1.1 * epsilon );
             PORTABLE_EXPECT_NEAR( math::abs( math::acos( cosTheta ) ), math::abs( theta ), 1.1 * epsilon );
+
+            #if defined( __ibmxl__ ) && !defined( __CUDA_ARCH__ )
+            PORTABLE_EXPECT_NEAR( math::atan2( y, x ), theta, 1.1 * epsilon );
+            #else
             PORTABLE_EXPECT_NEAR( math::atan2( y, x ), theta, epsilon );
+            #endif
           }
         } );
   }