Merge pull request #46 from memmett/feature/mpi-pf

Add first pass at PFASST algorithm. Closes #32 as well.

Author: memmett

.travis.yml

@@ -1,22 +1,28 @@
 language:
   - cpp
 
-compiler:
-  - gcc
-  - clang
-
+# why does Travis not allow for multi-dimensional build matrices?!?
+#  -> https://github.com/travis-ci/travis-ci/issues/1519
 env:
-  - CMAKE_BUILD_TYPE="Release"
-  - CMAKE_BUILD_TYPE="Debug"
+  matrix:
+    - CMAKE_BUILD_TYPE="Release" WITH_MPI=OFF CXX=clang++
+    - CMAKE_BUILD_TYPE="Release" WITH_MPI=OFF CXX=g++
+    - CMAKE_BUILD_TYPE="Release" WITH_MPI=ON CXX=g++
+    - CMAKE_BUILD_TYPE="Debug" WITH_MPI=OFF CXX=clang++
+    - CMAKE_BUILD_TYPE="Debug" WITH_MPI=OFF CXX=g++
+    - CMAKE_BUILD_TYPE="Debug" WITH_MPI=ON CXX=g++
 
 before_install:
   - sudo add-apt-repository ppa:apokluda/boost1.53 --yes
   - sudo add-apt-repository ppa:ubuntu-toolchain-r/test --yes # libstdc++-4.8
-  - if [ "${CXX}" == "clang++" ]; then sudo add-apt-repository --yes ppa:h-rayflood/llvm; fi # clang++-3.2
+  - if [ "${CXX}" == "clang++" ]; then sudo add-apt-repository --yes ppa:h-rayflood/llvm; fi  # clang++-3.2
   - sudo apt-get update
 
+
 install:
+  # note: clang needs g++-4.8 it for libstdc++ update
   - if [ "${CXX}" == "clang++" ]; then sudo apt-get -qq install clang-3.2; fi
+  - if [ "${WITH_MPI}" == "ON" ]; then sudo apt-get -qq install mpich2 libmpich2-dev; fi
   - sudo apt-get install g++-4.8 # clang need it for libstdc++ update
   - sudo apt-get install libboost1.53-all-dev libfftw3-3 libfftw3-dev
 
@@ -28,7 +34,7 @@ before_script:
 script:
   - mkdir build
   - cd build
-  - cmake -DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE} -Dpfasst_DISABLE_LIBCXX=ON -Dpfasst_BUILD_TESTS=ON -Dpfasst_BUILD_EXAMPLES=ON ..
+  - cmake -DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE} -Dpfasst_WITH_MPI=${WITH_MPI} -Dpfasst_DISABLE_LIBCXX=ON -Dpfasst_BUILD_TESTS=ON -Dpfasst_BUILD_EXAMPLES=ON ..
   - make
   - make test
-
+  - make run_example_all

CMakeLists.txt

@@ -1,7 +1,4 @@
 cmake_minimum_required(VERSION 2.8)
-
-set(CMAKE_VERBOSE_MAKEFILE ON)
-
 project(pfasst)
 
 list(APPEND CMAKE_MODULE_PATH ${pfasst_SOURCE_DIR}/cmake)
@@ -11,15 +8,23 @@ include(ExternalProject)
 # Set default ExternalProject root directory
 set_directory_properties(PROPERTIES EP_PREFIX ${CMAKE_BINARY_DIR}/3rdparty)
 
-option(pfasst_DISABLE_LIBCXX "Disable use of LLVM's libc++ when compiling with Clang." ON)
-option(pfasst_BUILD_EXAMPLES "Build example programs." ON)
-option(pfasst_BUILD_TESTS "Build test suite for PFASST." ON)
+option(pfasst_DISABLE_LIBCXX "Disable use of LLVM's libc++ when compiling with Clang." ON )
+option(pfasst_BUILD_EXAMPLES "Build example programs."                                 ON )
+option(pfasst_BUILD_TESTS    "Build test suite for PFASST."                            ON )
+option(pfasst_WITH_MPI       "Build with MPI enabled."                                 OFF)
 
 # output directories
 set(CMAKE_RUNTIME_OUTPUT_DIRECTORY "${pfasst_SOURCE_DIR}/dist/bin")
 set(CMAKE_LIBRARY_OUTPUT_DIRECTORY "${pfasst_SOURCE_DIR}/dist/lib")
 set(CMAKE_ARCHIVE_OUTPUT_DIRECTORY "${pfasst_SOURCE_DIR}/dist/lib")
 
+if(${pfasst_WITH_MPI})
+    find_package(MPI REQUIRED)
+    # set(CMAKE_C_COMPILER ${MPI_C_COMPILER})
+    # set(CMAKE_CXX_COMPILER ${MPI_CXX_COMPILER})
+    message(STATUS "Using MPI C++ Compiler: ${MPI_CXX_COMPILER}")
+endif()
+
 # check for C++11 support
 if(${CMAKE_CXX_COMPILER_ID} MATCHES GNU)
     check_cxx_compiler_flag(-std=c++11 HAVE_STD11)
@@ -97,5 +102,7 @@ endif()
 message(STATUS "********************************************************************************")
 
 message(STATUS "C++ Compiler ID: ${CMAKE_CXX_COMPILER_ID}")
+message(STATUS "C++ Compiler Names: ${CMAKE_CXX_COMPILER_NAMES}")
+message(STATUS "C++ Compiler Version: ${CMAKE_CXX_COMPILER_VERSION}")
 message(STATUS "C++ Flags: ${CMAKE_CXX_FLAGS}")
 message(STATUS "C++ link flags: ${CMAKE_CXX_LINK_FLAGS}")

examples/CMakeLists.txt

@@ -1 +1,9 @@
+set(all_example_runs)
+
 add_subdirectory(advection_diffusion)
+
+add_custom_target(run_example_all
+    DEPENDS ${all_example_runs}
+    COMMENT "Running all examples" VERBATIM
+)
+message(STATUS "  - run_example_all")

examples/advection_diffusion/CMakeLists.txt

@@ -1,3 +1,6 @@
+set(all_example_runs ${all_example_runs})
+set(all_advec_example_runs)
+
 message(STATUS "  advection_diffusion")
 include_directories(
     ${3rdparty_INCLUDES}
@@ -11,7 +14,20 @@ set(advec_examples
     serial_mlsdc_autobuild
 )
 
-foreach(example ${advec_examples})
+set(advec_mpi_examples)
+
+if(${pfasst_WITH_MPI})
+    set(advec_mpi_examples
+        mpi_pfasst
+    )
+endif()
+
+set(all_advec_examples
+    ${advec_examples}
+    ${advec_mpi_examples}
+)
+
+foreach(example ${all_advec_examples})
     add_executable(${example} ${CMAKE_CURRENT_SOURCE_DIR}/${example}.cpp)
     if(NOT FFTW_FOUND)
       add_dependencies(${example} fftw3)
@@ -23,3 +39,47 @@ foreach(example ${advec_examples})
         PROPERTIES RUNTIME_OUTPUT_DIRECTORY ${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/examples/advection_diffusion
     )
 endforeach(example)
+
+message(STATUS "    creating make targets to run examples")
+foreach(example ${advec_examples})
+    add_custom_target(run_example_advec_${example}
+        COMMAND ${example}
+        DEPENDS ${example}
+        WORKING_DIRECTORY ${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/examples/advection_diffusion
+        COMMENT "Running Advection-Diffusion example: ${example}" VERBATIM
+    )
+    list(APPEND all_advec_example_runs run_example_advec_${example})
+    message(STATUS "      - run_example_advec_${example}")
+endforeach(example)
+
+if(${pfasst_WITH_MPI})
+    include_directories(${MPI_CXX_INCLUDE_PATH})
+    foreach(example ${advec_mpi_examples})
+	if(MPI_COMPILE_FLAGS)
+            set_target_properties(${example} PROPERTIES COMPILE_FLAGS "${MPI_COMPILE_FLAGS}")
+        endif()
+        if(MPI_LINK_FLAGS)
+            set_target_properties(${example} PROPERTIES LINK_FLAGS "${MPI_LINK_FLAGS}")
+        endif()
+        target_link_libraries(${example} ${MPI_CXX_LIBRARIES})
+    endforeach(example)
+    message(STATUS "    creating make targets to run MPI examples")
+    foreach(example ${advec_mpi_examples})
+        add_custom_target(run_example_advec_${example}
+            COMMAND ${MPIEXEC} ${MPIEXEC_NUMPROC_FLAG} 4 ${MPIEXEC_PREFLAGS} ${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/examples/advection_diffusion/${example} ${MPIEXEC_POSTFLAGS}
+            DEPENDS ${example}
+            WORKING_DIRECTORY ${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/examples/advection_diffusion
+            COMMENT "Running Advection-Diffusion example: ${example}" VERBATIM
+        )
+        list(APPEND all_advec_example_runs run_example_advec_${example})
+        message(STATUS "      - run_example_advec_${example}")
+    endforeach(example)
+endif()
+
+add_custom_target(run_example_advec_all
+  DEPENDS ${all_advec_example_runs}
+  COMMENT "Running all Advection-Diffusion examples" VERBATIM
+)
+message(STATUS "    - run_example_advec_all")
+
+set(all_example_runs ${all_advec_example_runs} PARENT_SCOPE)

examples/advection_diffusion/mpi_pfasst.cpp

@@ -0,0 +1,69 @@
+/*
+ * Advection/diffusion example using an encapsulated IMEX sweeper.
+ *
+ * This example uses MPI PFASST.
+ */
+
+#include <memory>
+#include <cassert>
+
+#include <mpi.h>
+#include <fftw3.h>
+
+#include <pfasst.hpp>
+#include <pfasst/pfasst.hpp>
+#include <pfasst/mpi_communicator.hpp>
+#include <pfasst/encap/automagic.hpp>
+#include <pfasst/encap/mpi_vector.hpp>
+
+#include "advection_diffusion_sweeper.hpp"
+#include "spectral_transfer_1d.hpp"
+
+using namespace std;
+using namespace pfasst;
+using namespace pfasst::encap;
+using namespace pfasst::mpi;
+
+int main(int argc, char** argv)
+{
+  MPI_Init(&argc, &argv);
+
+  const size_t nsteps = 4;
+  const double dt     = 0.01;
+  const size_t niters = 4;
+
+  vector<pair<size_t, string>> nodes = {
+    { 3, "gauss-lobatto" },
+    { 5, "gauss-lobatto" }
+  };
+
+  vector<size_t> ndofs = { 64, 128 };
+
+  auto build_level = [ndofs](size_t level) {
+    auto factory  = make_shared<MPIVectorFactory<double>>(ndofs[level]);
+    auto sweeper  = make_shared<AdvectionDiffusionSweeper<>>(ndofs[level]);
+    auto transfer = make_shared<SpectralTransfer1D<>>();
+
+    return AutoBuildTuple<>(sweeper, transfer, factory);
+  };
+
+  auto initial = [](shared_ptr<EncapSweeper<>> sweeper, shared_ptr<Encapsulation<>> q0) {
+    auto ad = dynamic_pointer_cast<AdvectionDiffusionSweeper<>>(sweeper);
+    assert(ad);
+    ad->exact(q0, 0.0);
+  };
+
+  MPICommunicator comm(MPI_COMM_WORLD);
+  PFASST<> pf;
+
+  auto_build(pf, nodes, build_level);
+  auto_setup(pf, initial);
+
+  pf.set_comm(&comm);
+  pf.set_duration(0.0, nsteps * dt, dt, niters);
+  pf.run();
+
+  fftw_cleanup();
+
+  MPI_Finalize();
+}

include/pfasst/controller.hpp

@@ -9,6 +9,7 @@
 #include <memory>
 #include <cassert>
 #include <iterator>
+#include <iostream>
 
 #include "interfaces.hpp"
 
@@ -31,7 +32,7 @@ namespace pfasst
 
     public:
       //! @{
-      void setup()
+      virtual void setup()
       {
         for (auto l = coarsest(); l <= finest(); ++l) {
 	  l.current()->set_controller(this);
@@ -108,24 +109,24 @@ namespace pfasst
        * implementing a `RandomAccessIterator`.
        */
       class LevelIter
-        : iterator<random_access_iterator_tag, shared_ptr<ISweeper<time>>, size_t,
+        : iterator<random_access_iterator_tag, shared_ptr<ISweeper<time>>, int,
                    ISweeper<time>*, ISweeper<time>>
       {
           Controller* ts;
 
         public:
-          typedef size_t                     difference_type;
+          typedef int                        difference_type;
           typedef shared_ptr<ISweeper<time>> value_type;
           typedef ISweeper<time>*            pointer;
           typedef ISweeper<time>             reference;
           typedef random_access_iterator_tag iterator_category;
 
-          size_t level;
+          int level;
 
           //! @{
-          LevelIter(size_t level, Controller* ts)
+          LevelIter(int level, Controller* ts)
             : ts(ts), level(level)
-          {}
+          { }
           //! @}
 
           //! @{
@@ -187,6 +188,12 @@ namespace pfasst
         return step;
       }
 
+      void set_step(size_t n)
+      {
+        t += (n - step)*dt;
+        step = n;
+      }
+
       time get_time_step()
       {
         return dt;

include/pfasst/encap/encap_sweeper.hpp

@@ -25,6 +25,29 @@ namespace pfasst
 
       public:
 
+        virtual void post(ICommunicator* comm, int tag)
+        {
+          this->get_state(0)->post(comm, tag);
+        }
+
+        virtual void send(ICommunicator* comm, int tag, bool blocking)
+        {
+          this->get_state(this->get_nodes().size() - 1)->send(comm, tag, blocking);
+        }
+
+        virtual void recv(ICommunicator* comm, int tag, bool blocking)
+        {
+          this->get_state(0)->recv(comm, tag, blocking);
+        }
+
+        virtual void broadcast(ICommunicator* comm)
+        {
+	  if (comm->rank() == comm->size() - 1) {
+	    this->get_state(0)->copy(this->get_state(this->get_nodes().size() - 1));
+	  }
+          this->get_state(0)->broadcast(comm);
+        }
+
         void set_nodes(vector<time> nodes)
         {
           this->nodes = nodes;
@@ -73,7 +96,7 @@ namespace pfasst
           return this->get_state(this->get_nodes().size() - 1);
         }
 
-      virtual void evaluate(size_t /*m*/)
+        virtual void evaluate(size_t /*m*/)
         {
           throw NotImplementedYet("sweeper");
         }

include/pfasst/encap/encapsulation.hpp

@@ -21,7 +21,7 @@ namespace pfasst
     typedef enum EncapType { solution, function } EncapType;
 
     /**
-     * basic encapsulation.
+     * Data/solution encapsulation.
      *
      * An Encapsulation provides basic functionality of the user data used by PFASST such as
      * mathematical operation @em axpy \\(y=ax+y\\) and packing/unpacking for message passing.
@@ -38,11 +38,19 @@ namespace pfasst
 
         //! @{
         // required for time-parallel communications
-        virtual void send()
+        virtual void post(ICommunicator* /*comm*/, int /*tag*/) { }
+
+        virtual void send(ICommunicator* /*comm*/, int /*tag*/, bool /*blocking*/)
+        {
+          throw NotImplementedYet("pfasst");
+        }
+
+        virtual void recv(ICommunicator* /*comm*/, int /*tag*/, bool /*blocking*/)
         {
           throw NotImplementedYet("pfasst");
         }
-        virtual void recv()
+
+        virtual void broadcast(ICommunicator* /*comm*/)
         {
           throw NotImplementedYet("pfasst");
         }