Skip to content

Commit 9238cba

Browse files
memmettmemmett
committed
Merge pull request #180 from torbjoernk/feature/boris-pfasst
Boris-PFASST with serial spatial solver
2 parents e400a8e + b5f38e4 commit 9238cba

File tree

5 files changed

+302
-22
lines changed

5 files changed

+302
-22
lines changed

‎examples/boris/CMakeLists.txt‎

Lines changed: 31 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -2,35 +2,52 @@ set(examples_to_install ${examples_to_install})
22

33
message(STATUS " boris")
44

5-
option(pfasst_BORIS_SAME_LEVELS "Use identical levels with MLSDC" ON)
5+
option(pfasst_BORIS_SAME_LEVELS "Use identical levels with MLSDC" OFF)
66

77
include_directories(
88
${3rdparty_INCLUDES}
99
${pfasst_INCLUDES}
1010
)
1111
add_subdirectory(bindings)
1212

13-
if(NOT ${pfasst_WITH_MPI})
13+
if(${pfasst_WITH_MPI})
14+
set(boris_examples
15+
boris_pfasst
16+
)
17+
else()
1418
set(boris_examples
1519
boris_sdc
1620
boris_mlsdc
1721
)
22+
endif()
1823

24+
foreach(example ${boris_examples})
25+
add_executable(${example} ${CMAKE_CURRENT_SOURCE_DIR}/${example}.cpp)
26+
target_link_libraries(${example}
27+
${3rdparty_DEPENDEND_LIBS}
28+
simple_physics_solver
29+
)
30+
if(${pfasst_NUM_DEPENDEND_TARGETS} GREATER 0)
31+
add_dependencies(${example} ${pfasst_DEPENDEND_TARGETS})
32+
endif()
33+
add_dependencies(${example} simple_physics_solver)
34+
if(pfasst_INSTALL_EXAMPLES)
35+
install(TARGETS ${example} RUNTIME DESTINATION bin)
36+
endif()
37+
endforeach(example)
38+
39+
if(${pfasst_WITH_MPI})
40+
include_directories(${MPI_CXX_INCLUDE_PATH})
1941
foreach(example ${boris_examples})
20-
add_executable(${example} ${CMAKE_CURRENT_SOURCE_DIR}/${example}.cpp)
21-
target_link_libraries(${example}
22-
${3rdparty_DEPENDEND_LIBS}
23-
simple_physics_solver
24-
)
25-
if(${pfasst_NUM_DEPENDEND_TARGETS} GREATER 0)
26-
add_dependencies(${example} ${pfasst_DEPENDEND_TARGETS})
42+
if(MPI_COMPILE_FLAGS)
43+
set_target_properties(${example} PROPERTIES COMPILE_FLAGS "${MPI_COMPILE_FLAGS}")
2744
endif()
28-
add_dependencies(${example} simple_physics_solver)
29-
if(pfasst_INSTALL_EXAMPLES)
30-
install(TARGETS ${example} RUNTIME DESTINATION bin)
45+
if(MPI_LINK_FLAGS)
46+
set_target_properties(${example} PROPERTIES LINK_FLAGS "${MPI_LINK_FLAGS}")
3147
endif()
48+
target_link_libraries(${example} ${MPI_CXX_LIBRARIES})
3249
endforeach(example)
33-
50+
else()
3451
if(pfasst_BORIS_SAME_LEVELS)
3552
get_target_property(boris_mlsdc_definitions boris_mlsdc COMPILE_DEFINITIONS)
3653
if(${boris_mlsdc_definitions})
@@ -42,6 +59,7 @@ if(NOT ${pfasst_WITH_MPI})
4259
endif()
4360
endif()
4461

62+
4563
add_feature_info(Dummy-Boris-MLSDC
4664
pfasst_BORIS_SAME_LEVELS
4765
"use identical levels for Boris-MLSDC"

‎examples/boris/boris_pfasst.cpp‎

Lines changed: 120 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,120 @@
1+
#include <memory>
2+
3+
#include <pfasst.hpp>
4+
#include <pfasst/config.hpp>
5+
#include <pfasst/logging.hpp>
6+
#include <pfasst/controller/pfasst.hpp>
7+
#include <pfasst/mpi_communicator.hpp>
8+
using namespace pfasst;
9+
10+
#include "particle.hpp"
11+
#include "particle_cloud.hpp"
12+
#include "bindings/wrapper_interface.hpp"
13+
#include "bindings/wrapper_simple_physics_solver.hpp"
14+
#include "boris_sweeper.hpp"
15+
#include "injective_transfer.hpp"
16+
17+
18+
namespace pfasst
19+
{
20+
namespace examples
21+
{
22+
namespace boris
23+
{
24+
template<typename scalar>
25+
error_map<scalar> run_boris_pfasst(const size_t nsteps, const scalar dt, const size_t nnodes,
26+
const size_t nparticles, const size_t niters,
27+
const double abs_res_tol, const double rel_res_tol)
28+
{
29+
PFASST<> controller;
30+
mpi::MPICommunicator comm(MPI_COMM_WORLD);
31+
controller.set_comm(&comm);
32+
33+
const double mass = 1.0;
34+
const double charge = 1.0;
35+
36+
shared_ptr<bindings::WrapperInterface<double, double>> impl_solver = \
37+
make_shared<bindings::WrapperSimplePhysicsSolver<double, double>>();
38+
bindings::setup(dynamic_pointer_cast<bindings::WrapperSimplePhysicsSolver<double, double>>(impl_solver));
39+
40+
// fine level
41+
auto quad1 = quadrature::quadrature_factory<double>(nnodes,
42+
quadrature::QuadratureType::GaussLobatto);
43+
auto factory1 = make_shared<ParticleCloudFactory<double>>(nparticles, 3, mass, charge);
44+
string data_file1 = "s" + to_string(nsteps) + "_i" + to_string(niters)
45+
+ "_dt" + to_string(dt) + "_m" + to_string(nnodes)
46+
+ "_p" + to_string(nparticles) + "_level1.csv";
47+
auto sweeper1 = make_shared<BorisSweeper<double, double>>(impl_solver, data_file1);
48+
auto transfer1 = make_shared<InjectiveTransfer<double, double>>();
49+
sweeper1->set_quadrature(quad1);
50+
sweeper1->set_factory(factory1);
51+
sweeper1->set_residual_tolerances(abs_res_tol, rel_res_tol);
52+
controller.add_level(sweeper1, transfer1);
53+
54+
// coarse level
55+
auto quad2 = quadrature::quadrature_factory<double>(nnodes,
56+
quadrature::QuadratureType::GaussLobatto);
57+
auto factory2 = make_shared<ParticleCloudFactory<double>>(nparticles, 3, mass, charge);
58+
string data_file2 = "s" + to_string(nsteps) + "_i" + to_string(niters)
59+
+ "_dt" + to_string(dt) + "_m" + to_string(nnodes)
60+
+ "_p" + to_string(nparticles) + "_level2.csv";
61+
auto sweeper2 = make_shared<BorisSweeper<double, double>>(impl_solver, data_file2);
62+
auto transfer2 = make_shared<InjectiveTransfer<double, double>>();
63+
sweeper2->set_quadrature(quad2);
64+
sweeper2->set_factory(factory2);
65+
controller.add_level(sweeper2, transfer2);
66+
67+
controller.set_duration(0.0, nsteps*dt, dt, niters);
68+
controller.set_options();
69+
controller.setup();
70+
71+
shared_ptr<Particle<double>> center = make_shared<Particle<double>>();
72+
center->pos()[0] = 10;
73+
center->vel()[0] = 100;
74+
center->vel()[2] = 100;
75+
76+
auto fine_sweeper = controller.get_finest<BorisSweeper<double, double>>();
77+
shared_ptr<ParticleCloud<double>> q0 = \
78+
dynamic_pointer_cast<ParticleCloud<double>>(fine_sweeper->get_start_state());
79+
q0->distribute_around_center(center);
80+
CLOG(INFO, "Boris") << "Initial Particle (fine) : "
81+
<< *(dynamic_pointer_cast<ParticleCloud<double>>(fine_sweeper->get_start_state()));
82+
fine_sweeper->set_initial_energy();
83+
84+
controller.run();
85+
86+
return fine_sweeper->get_errors();
87+
}
88+
} // ::pfasst::examples::boris
89+
} // ::pfasst::examples
90+
} // ::pfasst
91+
92+
#ifndef PFASST_UNIT_TESTING
93+
int main(int argc, char** argv)
94+
{
95+
MPI_Init(&argc, &argv);
96+
pfasst::init(argc, argv, pfasst::examples::boris::init_opts<double>);
97+
pfasst::log::add_custom_logger("Boris");
98+
pfasst::log::add_custom_logger("BorisTransfer");
99+
pfasst::log::add_custom_logger("Solver");
100+
101+
const size_t nsteps = pfasst::config::get_value<size_t>("num_steps", 1);
102+
const double dt = pfasst::config::get_value<double>("delta_step", 0.015625);
103+
const size_t nnodes = pfasst::config::get_value<size_t>("num_nodes", 5);
104+
const size_t nparticles = pfasst::config::get_value<size_t>("num_particles", 1);
105+
const size_t niters = pfasst::config::get_value<size_t>("num_iter", 2);
106+
const double abs_res_tol = pfasst::config::get_value<double>("abs_res_tol", 0.0);
107+
const double rel_res_tol = pfasst::config::get_value<double>("rel_res_tol", 0.0);
108+
109+
LOG(INFO) << "nsteps=" << nsteps << ", "
110+
<< "dt=" << dt << ", "
111+
<< "nnodes=" << nnodes << ", "
112+
<< "nparticles=" << nparticles << ", "
113+
<< "niter=" << niters << ", "
114+
<< "abs res=" << abs_res_tol << ", "
115+
<< "rel res=" << rel_res_tol;
116+
117+
pfasst::examples::boris::run_boris_pfasst<double>(nsteps, dt, nnodes, nparticles, niters, abs_res_tol, rel_res_tol);
118+
MPI_Finalize();
119+
}
120+
#endif

‎examples/boris/boris_sweeper_impl.hpp‎

Lines changed: 10 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -830,29 +830,31 @@ namespace pfasst
830830
template<typename scalar, typename time>
831831
void BorisSweeper<scalar, time>::post(ICommunicator* comm, int tag)
832832
{
833-
UNUSED(comm); UNUSED(tag);
834-
// TODO: implement BorisSweeper::post
833+
this->start_particles->post(comm, tag);
835834
}
836835

837836
template<typename scalar, typename time>
838837
void BorisSweeper<scalar, time>::send(ICommunicator* comm, int tag, bool blocking)
839838
{
840-
UNUSED(comm); UNUSED(tag); UNUSED(blocking);
841-
// TODO: implement BorisSweeper::send
839+
this->end_particles->send(comm, tag, blocking);
842840
}
843841

844842
template<typename scalar, typename time>
845843
void BorisSweeper<scalar, time>::recv(ICommunicator* comm, int tag, bool blocking)
846844
{
847-
UNUSED(comm); UNUSED(tag); UNUSED(blocking);
848-
// TODO: implement BorisSweeper::recv
845+
this->start_particles->recv(comm, tag, blocking);
846+
if (this->quadrature->left_is_node()) {
847+
this->particles[0]->copy(this->start_particles);
848+
}
849849
}
850850

851851
template<typename scalar, typename time>
852852
void BorisSweeper<scalar, time>::broadcast(ICommunicator* comm)
853853
{
854-
UNUSED(comm);
855-
// TODO: implement BorisSweeper::broadcast
854+
if (comm->rank() == comm->size() - 1) {
855+
this->start_particles->copy(this->end_particles);
856+
}
857+
this->start_particles->broadcast(comm);
856858
}
857859

858860
template<typename scalar, typename time>

‎examples/boris/particle_cloud.hpp‎

Lines changed: 24 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -9,8 +9,11 @@
99
#include <vector>
1010
using namespace std;
1111

12-
#include <pfasst/encap/encapsulation.hpp>
1312
#include <pfasst/logging.hpp>
13+
#include <pfasst/encap/encapsulation.hpp>
14+
#ifdef WITH_MPI
15+
#include <pfasst/mpi_communicator.hpp>
16+
#endif
1417

1518
#include "particle.hpp"
1619

@@ -47,6 +50,17 @@ namespace pfasst
4750
precision _default_charge;
4851
precision _default_mass;
4952

53+
#ifdef WITH_MPI
54+
//! @{
55+
vector<MPI_Request> recv_request;
56+
vector<MPI_Request> send_request;
57+
//! @}
58+
59+
//! @{
60+
inline mpi::MPICommunicator& as_mpi(ICommunicator* comm);
61+
//! @}
62+
#endif
63+
5064
public:
5165
explicit ParticleCloud(const size_t num_particles = 0,
5266
const size_t dim = 3,
@@ -83,6 +97,15 @@ namespace pfasst
8397
// TODO: unify behaviour with particle_util::norm0 (e.g. norm_max vs. norm0 (==sqrt(^2))
8498
virtual precision norm0() const;
8599

100+
#ifdef WITH_MPI
101+
//! @{
102+
virtual void post(ICommunicator* comm, int tag) override;
103+
virtual void recv(ICommunicator* comm, int tag, bool blocking) override;
104+
virtual void send(ICommunicator* comm, int tag, bool blocking) override;
105+
virtual void broadcast(ICommunicator* comm) override;
106+
//! @}
107+
#endif
108+
86109
virtual void log(el::base::type::ostream_t& os) const;
87110
};
88111

0 commit comments

Comments
 (0)