Merge pull request #176 from torbjoernk/feature/for-juqueen

memmett · memmett · commit 9d83617aaaaa · 2015-04-13T07:50:00.000-06:00
getting up and running on JUQUEEN
diff --git a/3rdparty/CMakeLists.txt b/3rdparty/CMakeLists.txt
@@ -10,6 +10,11 @@ message(STATUS "----------------------------------------------------------------
 message(STATUS "Boost")
 
 set(Boost_USE_MULTITHREADED ON)
+if(${pfasst_BUILD_SHARED_LIBS})
+    set(Boost_USE_STATIC_LIBS OFF)
+else()
+    set(Boost_USE_STATIC_LIBS ON)
+endif()
 set(Boost_ADDITIONAL_VERSIONS "1.57" "1.57.0" ${Boost_ADDITIONAL_VERSIONS})
 
 if(${compiler_version_available} AND ${CMAKE_CXX_COMPILER_ID} MATCHES Clang)
diff --git a/CMakeLists.txt b/CMakeLists.txt
@@ -14,7 +14,7 @@ list(APPEND CMAKE_FIND_ROOT_PATH ${pfasst_SOURCE_DIR}/stack)
 set_directory_properties(PROPERTIES EP_PREFIX ${CMAKE_BINARY_DIR}/3rdparty)
 
 option(pfasst_DISABLE_LIBCXX    "Disable use of LLVM's libc++ when compiling with Clang." ON )
-option(pfasst_BUILD_SHARED_LIBS "Build shared libraries."                                 OFF)
+option(pfasst_BUILD_SHARED_LIBS "Build shared libraries."                                 ON )
 option(pfasst_BUILD_EXAMPLES    "Build example programs."                                 ON )
 CMAKE_DEPENDENT_OPTION(pfasst_INSTALL_EXAMPLES "Install example programs." ON
     "pfasst_BUILD_EXAMPLES" OFF)
@@ -25,7 +25,17 @@ option(pfasst_DEFAULT_RAND_SEED "Using a hardcoded random seed"
 
 if(${pfasst_WITH_MPI})
     find_package(MPI REQUIRED)
-    message(STATUS "Using MPI C++ Compiler: ${MPI_CXX_COMPILER}")
+    if(NOT "${CMAKE_CXX_COMPILER}" STREQUAL "${MPI_CXX_COMPILER}" OR 
+       NOT "${CMAKE_C_COMPILER}" STREQUAL "${MPI_C_COMPILER}")
+        message(STATUS "C++ Compiler: ${CMAKE_CXX_COMPILER}")
+        message(STATUS "MPI C++ Compiler Wrapper: ${MPI_CXX_COMPILER}")
+        message(WARNING "Please make sure to set CXX and CC to the MPI compiler wrappers!")
+        set(CMAKE_CXX_COMPILER "${MPI_CXX_COMPILER}")
+        set(CMAKE_C_COMPILER "${MPI_C_COMPILER}")
+        message(STATUS "Set default compilers to MPI compilers.")
+    endif()
+    message(STATUS "Using MPI C++ Compiler Wrapper: ${MPI_CXX_COMPILER}")
+    message(STATUS "USing MPI C Compiler Wrapper: ${MPI_C_COMPILER}")
     add_definitions(-DWITH_MPI)
 endif()
 
diff --git a/cmake/FindFFTW.cmake b/cmake/FindFFTW.cmake
@@ -10,9 +10,13 @@ if(FFTW_INCLUDE_PATH)
     set(FFTW_FIND_QUIETLY TRUE)
 endif(FFTW_INCLUDE_PATH)
 
-find_path(FFTW_INCLUDE_PATH fftw3.h)
+find_path(FFTW_INCLUDE_PATH fftw3.h
+    HINTS $ENV{FFTW3_INCLUDE}
+)
 
-find_library(FFTW_LIBRARIES NAMES fftw3)
+find_library(FFTW_LIBRARIES NAMES fftw3
+    HINTS $ENV{FFTW3_LIB}
+)
 
 # handle the QUIETLY and REQUIRED arguments and set FFTW_FOUND to TRUE if
 # all listed variables are TRUE
diff --git a/cmake/toolchain_juqueen.cmake b/cmake/toolchain_juqueen.cmake
@@ -0,0 +1,24 @@
+# Toolchain File for JUQUEEN @ Juelich Supercomputing Center
+# BG/Q system
+#
+# Remark: IBM XL/C++ compiler is not supported by PFASST++ due to lack of C++11 support
+#
+# IMPORTANT!!
+# Make sure you have loaded only the following modules:
+#  - gcc/4.8.1
+#  - fftw3/3.3.3
+#
+# As well make sure you have built boost::program_options according to the instructions from the
+# PFASST++ documentation on JUQUEEN.
+#
+
+message(STATUS "Please make sure you have loaded at least gcc/4.8.1")
+
+set(CMAKE_C_COMPILER /bgsys/local/gcc/4.8.1/bin/mpigcc)
+set(CMAKE_CXX_COMPILER /bgsys/local/gcc/4.8.1/bin/mpig++)
+set(CMAKE_Fortran_COMPILER /bgsys/local/gcc/4.8.1/bin/mpigfortran)
+
+set(MPI_ROOT /bgsys/local/gcc/4.8.1)
+set(MPI_C_COMPILER ${MPI_ROOT}/bin/mpigcc)
+set(MPI_CXX_COMPILER ${MPI_ROOT}/bin/mpig++)
+set(MPI_Fortran_COMPILER ${MPI_ROOT}/bin/mpigfortran)
diff --git a/doc/source/installing.md b/doc/source/installing.md
@@ -135,6 +135,11 @@ latest release from [GitHub][github_releases].
 
        make install
 
+### Supercomputers
+
+There are a few tweaks and special care require to get up and running on supercomputers.
+We have compiled a few working walk-throughs at \subpage page_supercomputers .
+
 ## Building with vanilla make
 
 A sample `Makefile` is included in the `advection_diffusion` example.  This may be of particular
diff --git a/doc/source/supercomputers.md b/doc/source/supercomputers.md
@@ -0,0 +1,69 @@
+# Usage on Supercomputers                                                     {#page_supercomputers}
+
+## JUQUEEN
+
+There are a few steps one has to complete before using _PFASST++_ on _JUQUEEN_.
+
+1. make sure the only modules loaded are `gcc/4.8.1` and `fftw3/3.3.3`
+
+2. prepare a custom installation of _Boost_
+
+   Because at time of writing only 1.47.0 is available compiled with _XL/C++_, which does not
+   support the C++11 features we learned to love.
+
+  1. download a recent version of _Boost_ (i.e. 1.57.0)
+
+  2. extract the archive and `cd` into it
+
+  3. run 
+
+         CXX=`which mpig++` CC=`which mpigcc` ./bootstrap.sh \
+           --with-libraries=program_options --prefix=<INSTALL_PLACE>
+
+     where `INSTALL_PLACE` is something like `$HOME/progs/juqueen`
+
+  4. create file called `user-config.jam` with the following content:
+
+         using gcc : 4.8.1 : mpig++ ;
+
+  5. now compile
+
+         CXX=`which mpig++` CC=`which mpigcc` ./b2 --prefix=<INSTALL_PLACE> \
+           --reconfigure link=static stage
+
+     and install
+
+         CXX=`which mpig++` CC=`which mpigcc` ./b2 --prefix=<INSTALL_PLACE> \
+           link=static install
+
+3. go to the sources of _PFASST++_, create a build folder and step into it
+
+4. run _CMake_
+
+       cmake -DCMAKE_TOOLCHAIN_FILE=../cmake/toolchain_juqueen.cmake \
+         -DCMAKE_CXX_COMPILER=`which mpig++` -DCMAKE_C_COMPILER=`which mpigcc` \
+         -DBOOST_ROOT=<INSTALL_PLACE> -Dpfasst_WITH_MPI=ON \
+         -Dpfasst_BUILD_TESTS=OFF -Dpfasst_BUILD_SHARED_LIBS=OFF ..
+
+5. run make
+
+       make
+
+6. write your _LLSubmit_ configuration file
+
+   for example for the Advection-Diffusion example:
+
+       #@job_name              = MPI_PFASST_test
+       #@comment               = "a little test of PFASST with MPI"
+       #@output                = mpi_pfasst_test_$(jobid)_$(stepid).out
+       #@error                 = mpi_pfasst_test_$(jobid)_$(stepid).err
+       #@environment           = COPY_ALL
+       #@job_type              = bluegene
+       #@notification          = never
+       #@wall_clock_limit      = 00:10:00
+       #@bg_size               = 1
+       #@bg_connectivity       = TORUS
+       #@queue
+       runjob --np 32 --ranks-per-node 32 : \
+         <PATH_TO_BUILD_DIR>/examples/advection_diffusion/mpi_pfasst \
+         -q --tend 0.64 --dt 0.01 --num_iter 8
diff --git a/examples/advection_diffusion/CMakeLists.txt b/examples/advection_diffusion/CMakeLists.txt
@@ -41,17 +41,4 @@ foreach(example ${all_advec_examples})
     endif()
 endforeach(example)
 
-if(${pfasst_WITH_MPI})
-    include_directories(${MPI_CXX_INCLUDE_PATH})
-    foreach(example ${advec_mpi_examples})
-        if(MPI_COMPILE_FLAGS)
-            set_target_properties(${example} PROPERTIES COMPILE_FLAGS "${MPI_COMPILE_FLAGS}")
-        endif()
-        if(MPI_LINK_FLAGS)
-            set_target_properties(${example} PROPERTIES LINK_FLAGS "${MPI_LINK_FLAGS}")
-        endif()
-        target_link_libraries(${example} ${MPI_CXX_LIBRARIES})
-    endforeach(example)
-endif()
-
 set(examples_to_install ${examples_to_install} ${all_advec_examples} PARENT_SCOPE)
diff --git a/examples/advection_diffusion/mpi_pfasst.cpp b/examples/advection_diffusion/mpi_pfasst.cpp
@@ -16,6 +16,7 @@ using namespace std;
 #include <fftw3.h>
 
 #include <pfasst.hpp>
+#include <pfasst/logging.hpp>
 #include <pfasst/controller/pfasst.hpp>
 #include <pfasst/mpi_communicator.hpp>
 #include <pfasst/encap/automagic.hpp>
@@ -40,17 +41,24 @@ namespace pfasst
        *
        * @ingroup AdvectionDiffusion
        */
-      error_map run_mpi_pfasst(double abs_residual_tol, size_t niters=4)
+      error_map run_mpi_pfasst(const double abs_res_tol, const double rel_res_tol,
+                               const size_t niters, const size_t nsteps, const double dt,
+                               const size_t ndofs_f, const size_t ndofs_c,
+                               const size_t nnodes_f, const size_t nnodes_c)
       {
-        const size_t nsteps = 4;
-        const double dt     = 0.01;
-
+        CLOG(INFO, "Advec") << "abs_res_tol: " << abs_res_tol << ", "
+                            << "rel_res_tol: " << rel_res_tol << ", "
+                            << "niter: " << niters << ", "
+                            << "nsteps: " << nsteps << ", "
+                            << "dt: " << dt << ", "
+                            << "ndofs (f-c): " << ndofs_f << "-" << ndofs_c << ", "
+                            << "nnodes (f-c): " << nnodes_f << "-" << nnodes_c;
         vector<pair<size_t, quadrature::QuadratureType>> nodes = {
-          { 3, quadrature::QuadratureType::GaussLobatto },
-          { 5, quadrature::QuadratureType::GaussLobatto }
+          { nnodes_c, quadrature::QuadratureType::GaussLobatto },
+          { nnodes_f, quadrature::QuadratureType::GaussLobatto }
         };
 
-        vector<size_t> ndofs = { 64, 128 };
+        vector<size_t> ndofs = { ndofs_c, ndofs_f };
 
         auto build_level = [ndofs](size_t level) {
           auto factory  = make_shared<MPIVectorFactory<double>>(ndofs[level]);
@@ -75,7 +83,7 @@ namespace pfasst
         pf.set_comm(&comm);
         pf.set_duration(0.0, nsteps * dt, dt, niters);
         pf.set_nsweeps({2, 1});
-        pf.get_finest<AdvectionDiffusionSweeper<>>()->set_residual_tolerances(abs_residual_tol, 0.0);
+        pf.get_finest<AdvectionDiffusionSweeper<>>()->set_residual_tolerances(abs_res_tol, rel_res_tol);
         pf.set_options();
         pf.run();
 
@@ -94,7 +102,22 @@ int main(int argc, char** argv)
   pfasst::init(argc, argv,
                pfasst::examples::advection_diffusion::AdvectionDiffusionSweeper<>::init_opts,
                pfasst::examples::advection_diffusion::AdvectionDiffusionSweeper<>::init_logs);
-  pfasst::examples::advection_diffusion::run_mpi_pfasst(0.0);
+
+  const double tend        = pfasst::config::get_value<double>("tend", 0.04);
+  const double dt          = pfasst::config::get_value<double>("dt", 0.01);
+  const size_t nnodes_f    = pfasst::config::get_value<size_t>("num_nodes", 5);
+  const size_t ndofs_f     = pfasst::config::get_value<size_t>("spatial_dofs", 128);
+  const size_t niters      = pfasst::config::get_value<size_t>("num_iter", 4);
+  const double abs_res_tol = pfasst::config::get_value<double>("abs_res_tol", 0.0);
+  const double rel_res_tol = pfasst::config::get_value<double>("rel_res_tol", 0.0);
+
+  const size_t nsteps = tend / dt;
+  const size_t nnodes_c = (nnodes_f + 1) / 2;
+  const size_t ndofs_c = ndofs_f / 2;
+
+  pfasst::examples::advection_diffusion::run_mpi_pfasst(abs_res_tol, rel_res_tol,
+                                                        niters, nsteps, dt,
+                                                        ndofs_f, ndofs_c, nnodes_f, nnodes_c);
   fftw_cleanup();
   MPI_Finalize();
 }
diff --git a/tests/examples/advection_diffusion/CMakeLists.txt b/tests/examples/advection_diffusion/CMakeLists.txt
@@ -27,25 +27,19 @@ if(${pfasst_WITH_MPI})
         if(MPI_COMPILE_FLAGS)
             if(pfasst_WITH_GCC_PROF AND ${CMAKE_CXX_COMPILER_ID} MATCHES GNU)
                 set_target_properties(${test}
-                    PROPERTIES COMPILE_FLAGS "${MPI_COMPILE_FLAGS} -ftest-coverage -fprofile-arcs"
+                    PROPERTIES COMPILE_FLAGS "-ftest-coverage -fprofile-arcs"
                                LINK_FLAGS "-fprofile-arcs"
                 )
-            else()
-                set_target_properties(${test} PROPERTIES COMPILE_FLAGS "${MPI_COMPILE_FLAGS}")
             endif()
         endif()
-        if(MPI_LINK_FLAGS)
-            set_target_properties(${test} PROPERTIES LINK_FLAGS "${MPI_LINK_FLAGS}")
-        endif()
         target_link_libraries(${test}
             ${3rdparty_DEPENDEND_LIBS}
             ${TESTS_3rdparty_DEPENDEND_LIBS}
             ${FFTW_LIBRARIES}
             ${pfasst_DEPENDEND_LIBS}
-            ${MPI_CXX_LIBRARIES}
         )
         add_test(NAME ${test}
-            COMMAND ${MPIEXEC} ${MPIEXEC_NUMPROC_FLAG} 4 ${MPIEXEC_PREFLAGS} ${CMAKE_BINARY_DIR}/tests/examples/advection_diffusion/${test} ${MPIEXEC_POSTFLAGS} --gtest_output=xml:${test}_out.xml
+            COMMAND ${MPIEXEC} ${MPIEXEC_NUMPROC_FLAG} 4 ${CMAKE_BINARY_DIR}/tests/examples/advection_diffusion/${test} --gtest_output=xml:${test}_out.xml
         )
     endforeach(test)
 else()
diff --git a/tests/examples/advection_diffusion/test_mpi_advection_diffusion.cpp b/tests/examples/advection_diffusion/test_mpi_advection_diffusion.cpp
@@ -24,7 +24,7 @@ TEST(ErrorTest, MPIPFASST)
 {
   typedef error_map::value_type vtype;
 
-  auto errors = run_mpi_pfasst(0.0);
+  auto errors = run_mpi_pfasst(0.0, 0.0, 4, 4, 0.01, 128, 64, 5, 3);
   auto get_step  = [](const vtype x) { return get<0>(get<0>(x)); };
   auto get_iter  = [](const vtype x) { return get<1>(get<0>(x)); };
   auto get_error = [](const vtype x) { return get<1>(x); };
@@ -44,7 +44,7 @@ TEST(AdaptiveErrorTest, MPIPFASST)
 {
   typedef error_map::value_type vtype;
 
-  auto errors = run_mpi_pfasst(1.e-8, 12);
+  auto errors = run_mpi_pfasst(1.e-8, 0.0, 12, 4, 0.01, 128, 64, 5, 3);
   auto get_step  = [](const vtype x) { return get<0>(get<0>(x)); };
   auto get_iter  = [](const vtype x) { return get<1>(get<0>(x)); };
   auto get_error = [](const vtype x) { return get<1>(x); };
@@ -69,6 +69,9 @@ int main(int argc, char** argv)
 {
   testing::InitGoogleTest(&argc, argv);
   MPI_Init(&argc, &argv);
+  pfasst::init(argc, argv,
+               pfasst::examples::advection_diffusion::AdvectionDiffusionSweeper<>::init_opts,
+               pfasst::examples::advection_diffusion::AdvectionDiffusionSweeper<>::init_logs);
   int result = 1, max_result;  // GTest return value 1 (failure), 0 (success)
   result = RUN_ALL_TESTS();
   MPI_Allreduce(&result, &max_result, 1, MPI_INT, MPI_MAX, MPI_COMM_WORLD);
