From fda6b53fef4166695596c6727de72042f47e2934 Mon Sep 17 00:00:00 2001 From: Victor Mateevitsi Date: Thu, 12 Dec 2024 20:15:59 +0000 Subject: [PATCH 01/14] Added Ascent cmake option --- CMakeLists.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/CMakeLists.txt b/CMakeLists.txt index 15a0dfe13..38d2a9e32 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -207,6 +207,7 @@ if (ENABLE_FFT) endif () option (ENABLE_CATALYST "Build example with Catalyst enabled" OFF) +option (ENABLE_ASCENT "Build example with Ascent enabled" OFF) option (ENABLE_SOLVERS "Enable IPPL solvers" ON) add_subdirectory (src) From c07c6e4921280545bef37f8e481d677bfa95ee3f Mon Sep 17 00:00:00 2001 From: goebbert1 Date: Tue, 17 Dec 2024 10:10:01 +0100 Subject: [PATCH 02/14] ensure only `--pvscript`-args are passed to the catalyst-adapter --- alpine/PenningTrap.cpp | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/alpine/PenningTrap.cpp b/alpine/PenningTrap.cpp index 3f351a10b..b899d2968 100644 --- a/alpine/PenningTrap.cpp +++ b/alpine/PenningTrap.cpp @@ -48,7 +48,14 @@ int main(int argc, char* argv[]) { { #ifdef ENABLE_CATALYST - CatalystAdaptor::Initialize(argc, argv); + for (int i = 1; i < argc; ++i) { + if (std::string(argv[i]) == "--pvscript" && i + 1 < argc) { + // reduce the argument list + char* reducedArgv[] = { argv[0], argv[i + 1] }; + CatalystAdaptor::Initialize(2, reducedArgv); + break; + } + } #endif Inform msg(TestName); From 0d9f5cc635fe8182e8d5e7e1b4feb489502fede2 Mon Sep 17 00:00:00 2001 From: goebbert1 Date: Tue, 17 Dec 2024 10:12:03 +0100 Subject: [PATCH 03/14] pass script as filename+args in catalyst-adapter --- src/Stream/InSitu/CatalystAdaptor.h | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/src/Stream/InSitu/CatalystAdaptor.h b/src/Stream/InSitu/CatalystAdaptor.h index 514df25c2..2244bb211 100644 --- a/src/Stream/InSitu/CatalystAdaptor.h +++ b/src/Stream/InSitu/CatalystAdaptor.h @@ -56,8 +56,12 @@ namespace CatalystAdaptor { void Initialize(int argc, char* argv[]) { conduit_cpp::Node node; - for (int cc = 1; cc < argc; ++cc) { - node["catalyst/scripts/script" + std::to_string(cc - 1)].set_string(argv[cc]); + std::cout << "pvscript path: " << argv[1] << std::endl; + node["catalyst/scripts/script/filename"].set_string(argv[1]); + for (int cc = 2; cc < argc; ++cc) { + std::cout << "pvscript args: " << argv[cc] << std::endl; + conduit_cpp::Node list_entry = node["catalyst/scripts/script/args"].append(); + list_entry.set(argv[cc]); } try { node["catalyst_load/implementation"] = getenv("CATALYST_IMPLEMENTATION_NAME"); From fd9a616157f6a40ba39e549a6974481fd8f53b6e Mon Sep 17 00:00:00 2001 From: goebbert1 Date: Mon, 6 Jan 2025 00:01:52 +0100 Subject: [PATCH 04/14] initial commit --- alpine/CatalystAdaptor.h | 327 ++++++++++++++++++++++++++++ alpine/PenningTrapManager.h | 20 +- src/CMakeLists.txt | 2 +- src/Stream/InSitu/CatalystAdaptor.h | 194 +++++++++++++++++ test/stream/CMakeLists.txt | 12 +- 5 files changed, 539 insertions(+), 16 deletions(-) create mode 100644 alpine/CatalystAdaptor.h diff --git a/alpine/CatalystAdaptor.h b/alpine/CatalystAdaptor.h new file mode 100644 index 000000000..78a5d024c --- /dev/null +++ b/alpine/CatalystAdaptor.h @@ -0,0 +1,327 @@ +#ifndef CatalystAdaptor_h +#define CatalystAdaptor_h + +#include "Ippl.h" + +#include +#include +#include +#include +#include +#include +#include + +#include "Utility/IpplException.h" + + +namespace CatalystAdaptor { + + template + using FieldVariant = std::variant*, VField_t*>; + + template + using FieldPair = std::pair>; + + template + using ParticlePair = std::pair > >; + + + using View_vector = + Kokkos::View***, Kokkos::LayoutLeft, Kokkos::HostSpace>; + inline void setData(conduit_cpp::Node& node, const View_vector& view) { + node["electrostatic/association"].set_string("element"); + node["electrostatic/topology"].set_string("mesh"); + node["electrostatic/volume_dependent"].set_string("false"); + + auto length = std::size(view); + + // offset is zero as we start without the ghost cells + // stride is 1 as we have every index of the array + node["electrostatic/values/x"].set_external(&view.data()[0][0], length, 0, 1); + node["electrostatic/values/y"].set_external(&view.data()[0][1], length, 0, 1); + node["electrostatic/values/z"].set_external(&view.data()[0][2], length, 0, 1); + } + + + using View_scalar = Kokkos::View; + inline void setData(conduit_cpp::Node& node, const View_scalar& view) { + node["density/association"].set_string("element"); + node["density/topology"].set_string("mesh"); + node["density/volume_dependent"].set_string("false"); + + node["density/values"].set_external(view.data(), view.size()); + } + + + void Initialize(int argc, char* argv[]) { + conduit_cpp::Node node; + std::cout << "pvscript path: " << argv[1] << std::endl; + node["catalyst/scripts/script/filename"].set_string(argv[1]); + for (int cc = 2; cc < argc; ++cc) { + std::cout << "pvscript args: " << argv[cc] << std::endl; + conduit_cpp::Node list_entry = node["catalyst/scripts/script/args"].append(); + list_entry.set(argv[cc]); + } + try { + node["catalyst_load/implementation"] = getenv("CATALYST_IMPLEMENTATION_NAME"); + node["catalyst_load/search_paths/paraview"] = getenv("CATALYST_IMPLEMENTATION_PATHS"); + } catch (...) { + throw IpplException("CatalystAdaptor::Initialize", + "no environmental variable for CATALYST_IMPLEMENTATION_NAME or " + "CATALYST_IMPLEMENTATION_PATHS found"); + } + // TODO: catch catalyst error also with IpplException + catalyst_status err = catalyst_initialize(conduit_cpp::c_node(&node)); + if (err != catalyst_status_ok) { + std::cerr << "Failed to initialize Catalyst: " << err << std::endl; + } + } + + + void Finalize() { + conduit_cpp::Node node; + catalyst_status err = catalyst_finalize(conduit_cpp::c_node(&node)); + if (err != catalyst_status_ok) { + std::cerr << "Failed to finalize Catalyst: " << err << std::endl; + } + } + + + void Execute_Particle( + const auto& particleContainer, + const auto& R_host, const auto& P_host, const auto& q_host, const auto& ID_host, + const std::string& particlesName, + conduit_cpp::Node& node) { + + // channel for particles + auto channel = node["catalyst/channels/ippl_" + particlesName]; + channel["type"].set_string("mesh"); + + // in data channel now we adhere to conduits mesh blueprint definition + auto mesh = channel["data"]; + mesh["coordsets/coords/type"].set_string("explicit"); + + //mesh["coordsets/coords/values/x"].set_external(&layout_view.data()[0][0], particleContainer->getLocalNum(), 0, sizeof(double)*3); + //mesh["coordsets/coords/values/y"].set_external(&layout_view.data()[0][1], particleContainer->getLocalNum(), 0, sizeof(double)*3); + //mesh["coordsets/coords/values/z"].set_external(&layout_view.data()[0][2], particleContainer->getLocalNum(), 0, sizeof(double)*3); + mesh["coordsets/coords/values/x"].set_external(&R_host.data()[0][0], particleContainer->getLocalNum(), 0, sizeof(double)*3); + mesh["coordsets/coords/values/y"].set_external(&R_host.data()[0][1], particleContainer->getLocalNum(), 0, sizeof(double)*3); + mesh["coordsets/coords/values/z"].set_external(&R_host.data()[0][2], particleContainer->getLocalNum(), 0, sizeof(double)*3); + + mesh["topologies/mesh/type"].set_string("unstructured"); + mesh["topologies/mesh/coordset"].set_string("coords"); + mesh["topologies/mesh/elements/shape"].set_string("point"); + //mesh["topologies/mesh/elements/connectivity"].set_external(particleContainer->ID.getView().data(),particleContainer->getLocalNum()); + mesh["topologies/mesh/elements/connectivity"].set_external(ID_host.data(),particleContainer->getLocalNum()); + + //auto charge_view = particleContainer->getQ().getView(); + + // add values for scalar charge field + auto fields = mesh["fields"]; + fields["charge/association"].set_string("vertex"); + fields["charge/topology"].set_string("mesh"); + fields["charge/volume_dependent"].set_string("false"); + + //fields["charge/values"].set_external(particleContainer->q.getView().data(), particleContainer->getLocalNum()); + fields["charge/values"].set_external(q_host.data(), particleContainer->getLocalNum()); + + // add values for vector velocity field + //auto velocity_view = particleContainer->P.getView(); + fields["velocity/association"].set_string("vertex"); + fields["velocity/topology"].set_string("mesh"); + fields["velocity/volume_dependent"].set_string("false"); + + //fields["velocity/values/x"].set_external(&velocity_view.data()[0][0], particleContainer->getLocalNum(),0 ,sizeof(double)*3); + //fields["velocity/values/y"].set_external(&velocity_view.data()[0][1], particleContainer->getLocalNum(),0 ,sizeof(double)*3); + //fields["velocity/values/z"].set_external(&velocity_view.data()[0][2], particleContainer->getLocalNum(),0 ,sizeof(double)*3); + fields["velocity/values/x"].set_external(&P_host.data()[0][0], particleContainer->getLocalNum(),0 ,sizeof(double)*3); + fields["velocity/values/y"].set_external(&P_host.data()[0][1], particleContainer->getLocalNum(),0 ,sizeof(double)*3); + fields["velocity/values/z"].set_external(&P_host.data()[0][2], particleContainer->getLocalNum(),0 ,sizeof(double)*3); + + fields["position/association"].set_string("vertex"); + fields["position/topology"].set_string("mesh"); + fields["position/volume_dependent"].set_string("false"); + + //fields["position/values/x"].set_external(&layout_view.data()[0][0], particleContainer->getLocalNum(), 0, sizeof(double)*3); + //fields["position/values/y"].set_external(&layout_view.data()[0][1], particleContainer->getLocalNum(), 0, sizeof(double)*3); + //fields["position/values/z"].set_external(&layout_view.data()[0][2], particleContainer->getLocalNum(), 0, sizeof(double)*3); + fields["position/values/x"].set_external(&R_host.data()[0][0], particleContainer->getLocalNum(), 0, sizeof(double)*3); + fields["position/values/y"].set_external(&R_host.data()[0][1], particleContainer->getLocalNum(), 0, sizeof(double)*3); + fields["position/values/z"].set_external(&R_host.data()[0][2], particleContainer->getLocalNum(), 0, sizeof(double)*3); + } + + + template // == ippl::Field, Cell>* + void Execute_Field(Field* field, const std::string& fieldName, + Kokkos::View& host_view_layout_left, + conduit_cpp::Node& node) { + static_assert(Field::dim == 3, "CatalystAdaptor only supports 3D"); + + // A) define mesh + + // add catalyst channel named ippl_"field", as fields is reserved + auto channel = node["catalyst/channels/ippl_" + fieldName]; + channel["type"].set_string("mesh"); + + // in data channel now we adhere to conduits mesh blueprint definition + auto mesh = channel["data"]; + mesh["coordsets/coords/type"].set_string("uniform"); + + // number of points in specific dimension + std::string field_node_dim{"coordsets/coords/dims/i"}; + std::string field_node_origin{"coordsets/coords/origin/x"}; + std::string field_node_spacing{"coordsets/coords/spacing/dx"}; + + for (unsigned int iDim = 0; iDim < field->get_mesh().getGridsize().dim; ++iDim) { + // add dimension + mesh[field_node_dim].set(field->getLayout().getLocalNDIndex()[iDim].length() + 1); + + // add origin + mesh[field_node_origin].set( + field->get_mesh().getOrigin()[iDim] + field->getLayout().getLocalNDIndex()[iDim].first() + * field->get_mesh().getMeshSpacing(iDim)); + + // add spacing + mesh[field_node_spacing].set(field->get_mesh().getMeshSpacing(iDim)); + + // increment last char in string + ++field_node_dim.back(); + ++field_node_origin.back(); + ++field_node_spacing.back(); + } + + // add topology + mesh["topologies/mesh/type"].set_string("uniform"); + mesh["topologies/mesh/coordset"].set_string("coords"); + std::string field_node_origin_topo{"topologies/mesh/origin/x"}; + for (unsigned int iDim = 0; iDim < field->get_mesh().getGridsize().dim; ++iDim) { + // shift origin + mesh[field_node_origin_topo].set(field->get_mesh().getOrigin()[iDim] + + field->getLayout().getLocalNDIndex()[iDim].first() + * field->get_mesh().getMeshSpacing(iDim)); + + // increment last char in string ('x' becomes 'y' becomes 'z') + ++field_node_origin_topo.back(); + } + + // B) Set the field values + + // Initialize the existing Kokkos::View + host_view_layout_left = Kokkos::View( + "host_view_layout_left", + field->getLayout().getLocalNDIndex()[0].length(), + field->getLayout().getLocalNDIndex()[1].length(), + field->getLayout().getLocalNDIndex()[2].length()); + + // Creates a host-accessible mirror view and copies the data from the device view to the host. + auto host_view = + Kokkos::create_mirror_view_and_copy(Kokkos::HostSpace(), field->getView()); + + // Copy data from field to the memory+style which will be passed to Catalyst + auto nGhost = field->getNghost(); + for (size_t i = 0; i < field->getLayout().getLocalNDIndex()[0].length(); ++i) { + for (size_t j = 0; j < field->getLayout().getLocalNDIndex()[1].length(); ++j) { + for (size_t k = 0; k < field->getLayout().getLocalNDIndex()[2].length(); ++k) { + host_view_layout_left(i, j, k) = host_view(i + nGhost, j + nGhost, k + nGhost); + } + } + } + + // Add values and subscribe to data + auto fields = mesh["fields"]; + setData(fields, host_view_layout_left); + } + + + template + void Execute(int cycle, double time, int rank, + // const auto& /* std::shared_ptr >& */ particle, + const std::vector>& particles, + const std::vector>& fields) { + + // catalyst blueprint definition + // https://docs.paraview.org/en/latest/Catalyst/blueprints.html + // + // conduit blueprint definition (v.8.3) + // https://llnl-conduit.readthedocs.io/en/latest/blueprint_mesh.html + conduit_cpp::Node node; + + // add time/cycle information + auto state = node["catalyst/state"]; + state["cycle"].set(cycle); + state["time"].set(time); + state["domain_id"].set(rank); + + // Handle particles + + std::map>::HostMirror> R_host_map; + std::map>::HostMirror> P_host_map; + std::map::HostMirror> q_host_map; + std::map::HostMirror> ID_host_map; + + // Loop over all particle container + for (const auto& particlesPair : particles) + { + const std::string& particlesName = particlesPair.first; + const auto particleContainer = particlesPair.second; + + assert((particleContainer->ID.getView().data() != nullptr) && "ID view should not be nullptr, might be missing the right execution space"); + + R_host_map[particlesName] = particleContainer->R.getHostMirror(); + P_host_map[particlesName] = particleContainer->P.getHostMirror(); + q_host_map[particlesName] = particleContainer->q.getHostMirror(); + ID_host_map[particlesName] = particleContainer->ID.getHostMirror(); + + Kokkos::deep_copy(R_host_map[particlesName], particleContainer->R.getView()); + Kokkos::deep_copy(P_host_map[particlesName], particleContainer->P.getView()); + Kokkos::deep_copy(q_host_map[particlesName], particleContainer->q.getView()); + Kokkos::deep_copy(ID_host_map[particlesName], particleContainer->ID.getView()); + + Execute_Particle( + particleContainer, + R_host_map[particlesName], P_host_map[particlesName], q_host_map[particlesName], ID_host_map[particlesName], + particlesName, + node); + } + + + // Handle fields + + // Map of all Kokkos::Views. This keeps a reference on all Kokkos::Views + // which ensures that Kokkos does not free the memory before the end of this function. + std::map::view_type::data_type, Kokkos::LayoutLeft, Kokkos::HostSpace> > scalar_host_views; + std::map::view_type::data_type, Kokkos::LayoutLeft, Kokkos::HostSpace> > vector_host_views; + + // Loop over all fields + for (const auto& fieldPair : fields) + { + const std::string& fieldName = fieldPair.first; + const auto& fieldVariant = fieldPair.second; + + // If field is a _scalar_ field + if (std::holds_alternative*>(fieldVariant)) { + Field_t* field = std::get*>(fieldVariant); + // == ippl::Field, Cell>* + + Execute_Field(field, fieldName, scalar_host_views[fieldName], node); + } + // If field is a _vector_ field + else if (std::holds_alternative*>(fieldVariant)) { + VField_t* field = std::get*>(fieldVariant); + // == ippl::Field, 3, ippl::UniformCartesian, Cell>* + + Execute_Field(field, fieldName, vector_host_views[fieldName], node); + } + } + + // Pass Conduit node to Catalyst + catalyst_status err = catalyst_execute(conduit_cpp::c_node(&node)); + if (err != catalyst_status_ok) { + std::cerr << "Failed to execute Catalyst: " << err << std::endl; + } + + } + +} // namespace CatalystAdaptor + +#endif diff --git a/alpine/PenningTrapManager.h b/alpine/PenningTrapManager.h index 08fe37f99..6ac50f110 100644 --- a/alpine/PenningTrapManager.h +++ b/alpine/PenningTrapManager.h @@ -15,8 +15,7 @@ #include "Random/Randn.h" #ifdef ENABLE_CATALYST -#include -#include "Stream/InSitu/CatalystAdaptor.h" +#include "CatalystAdaptor.h" #endif using view_type = typename ippl::detail::ViewType, 1>::view_type; @@ -284,14 +283,17 @@ class PenningTrapManager : public AlpineManager { // scatter the charge onto the underlying grid this->par2grid(); + #ifdef ENABLE_CATALYST - std::optional node = std::nullopt; - //CatalystAdaptor::Execute_Particle(it, this->time_m, ippl::Comm->rank(), pc, node); - auto *rho = &this->fcontainer_m->getRho(); - CatalystAdaptor::Execute_Field(it, this->time_m, ippl::Comm->rank(), *rho, node); - //auto *E = &this->fcontainer_m->getE(); - //CatalystAdaptor::Execute_Field(it, this->time_m, ippl::Comm->rank(), *E, node); - //CatalystAdaptor::Execute_Field_Particle(it, this->time_m, ippl::Comm->rank(), *E, pc); + std::vector> particles = { + {"particle", std::shared_ptr >(pc)}, + }; + std::vector> fields = { + {"E", CatalystAdaptor::FieldVariant(&this->fcontainer_m->getE())}, + {"roh", CatalystAdaptor::FieldVariant(&this->fcontainer_m->getRho())}, + {"phi", CatalystAdaptor::FieldVariant(&this->fcontainer_m->getPhi())}, + }; + CatalystAdaptor::Execute(it, this->time_m, ippl::Comm->rank(), particles, fields); #endif // Field solve diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index b5fd3eb26..b5b65a456 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -82,7 +82,7 @@ add_subdirectory (Expression) add_subdirectory (Types) add_subdirectory (Partition) add_subdirectory (Stream) -add_subdirectory (Stream/InSitu) +# add_subdirectory (Stream/InSitu) add_subdirectory (Random) if (ENABLE_SOLVERS) diff --git a/src/Stream/InSitu/CatalystAdaptor.h b/src/Stream/InSitu/CatalystAdaptor.h index 2244bb211..13b4f31e7 100644 --- a/src/Stream/InSitu/CatalystAdaptor.h +++ b/src/Stream/InSitu/CatalystAdaptor.h @@ -10,12 +10,49 @@ #include #include #include +#include +#include #include "Utility/IpplException.h" +// ---- +// instead of including alpine/datatypes.h +template +using Mesh_t = ippl::UniformCartesian; + +template +using Centering_t = typename Mesh_t::DefaultCentering; + +template +using Field = ippl::Field, Centering_t, ViewArgs...>; + +template +using Vector_t = ippl::Vector; + +template +using Field_t = Field; + +template +using VField_t = Field, Dim, ViewArgs...>; + +// instead of including alpine/AlpineManager.h +//template +//using ParticleContainer_t = ParticleContainer; +// ---- + namespace CatalystAdaptor { + template + using FieldVariant = std::variant*, VField_t*>; + + template + using FieldPair = std::pair>; + + //template + //using ParticlePair = std::pair > >; + + using View_vector = Kokkos::View***, Kokkos::LayoutLeft, Kokkos::HostSpace>; inline void setData(conduit_cpp::Node& node, const View_vector& view) { @@ -116,6 +153,7 @@ namespace CatalystAdaptor { auto nGhost = field.getNghost(); + // Creates a host-accessible mirror view and copies the data from the device view to the host. typename Field::view_type::host_mirror_type host_view = Kokkos::create_mirror_view_and_copy(Kokkos::HostSpace(), field.getView()); @@ -308,6 +346,162 @@ namespace CatalystAdaptor { std::cerr << "Failed to finalize Catalyst: " << err << std::endl; } } + + +// === NEW ================================= + + + template // == ippl::Field, Cell>* + void Execute_Field_new(Field* field, const std::string& fieldName, + Kokkos::View& host_view_layout_left, + conduit_cpp::Node& node) { + static_assert(Field::dim == 3, "CatalystAdaptor only supports 3D"); + + // A) define mesh + + // add catalyst channel named ippl_"field", as fields is reserved + auto channel = node["catalyst/channels/ippl_" + fieldName]; + channel["type"].set_string("mesh"); + + // in data channel now we adhere to conduits mesh blueprint definition + auto mesh = channel["data"]; + mesh["coordsets/coords/type"].set_string("uniform"); + + // number of points in specific dimension + std::string field_node_dim{"coordsets/coords/dims/i"}; + std::string field_node_origin{"coordsets/coords/origin/x"}; + std::string field_node_spacing{"coordsets/coords/spacing/dx"}; + + for (unsigned int iDim = 0; iDim < field->get_mesh().getGridsize().dim; ++iDim) { + // add dimension + mesh[field_node_dim].set(field->getLayout().getLocalNDIndex()[iDim].length() + 1); + + // add origin + mesh[field_node_origin].set( + field->get_mesh().getOrigin()[iDim] + field->getLayout().getLocalNDIndex()[iDim].first() + * field->get_mesh().getMeshSpacing(iDim)); + + // add spacing + mesh[field_node_spacing].set(field->get_mesh().getMeshSpacing(iDim)); + + // increment last char in string + ++field_node_dim.back(); + ++field_node_origin.back(); + ++field_node_spacing.back(); + } + + // add topology + mesh["topologies/mesh/type"].set_string("uniform"); + mesh["topologies/mesh/coordset"].set_string("coords"); + std::string field_node_origin_topo{"topologies/mesh/origin/x"}; + for (unsigned int iDim = 0; iDim < field->get_mesh().getGridsize().dim; ++iDim) { + // shift origin + mesh[field_node_origin_topo].set(field->get_mesh().getOrigin()[iDim] + + field->getLayout().getLocalNDIndex()[iDim].first() + * field->get_mesh().getMeshSpacing(iDim)); + + // increment last char in string ('x' becomes 'y' becomes 'z') + ++field_node_origin_topo.back(); + } + + // B) Set the field values + + // Initialize the existing Kokkos::View + host_view_layout_left = Kokkos::View( + "host_view_layout_left", + field->getLayout().getLocalNDIndex()[0].length(), + field->getLayout().getLocalNDIndex()[1].length(), + field->getLayout().getLocalNDIndex()[2].length()); + + // Creates a host-accessible mirror view and copies the data from the device view to the host. + auto host_view = + Kokkos::create_mirror_view_and_copy(Kokkos::HostSpace(), field->getView()); + + // Copy data from field to the memory+style which will be passed to Catalyst + auto nGhost = field->getNghost(); + for (size_t i = 0; i < field->getLayout().getLocalNDIndex()[0].length(); ++i) { + for (size_t j = 0; j < field->getLayout().getLocalNDIndex()[1].length(); ++j) { + for (size_t k = 0; k < field->getLayout().getLocalNDIndex()[2].length(); ++k) { + host_view_layout_left(i, j, k) = host_view(i + nGhost, j + nGhost, k + nGhost); + } + } + } + + // Add values and subscribe to data + auto fields = mesh["fields"]; + setData(fields, host_view_layout_left); + } + + + template + void Execute(int cycle, double time, int rank, + const auto& /* std::shared_ptr >& */ particle, + const std::vector>& fields) { + + // catalyst blueprint definition + // https://docs.paraview.org/en/latest/Catalyst/blueprints.html + // + // conduit blueprint definition (v.8.3) + // https://llnl-conduit.readthedocs.io/en/latest/blueprint_mesh.html + conduit_cpp::Node node; + + // add time/cycle information + auto state = node["catalyst/state"]; + state["cycle"].set(cycle); + state["time"].set(time); + state["domain_id"].set(rank); + + // Handle particles + + // Handle fields + + // Map of all Kokkos::Views. This keeps a reference on all Kokkos::Views + // which ensures that Kokkos does not free the memory before the end of this function. + std::map::view_type::data_type, Kokkos::LayoutLeft, Kokkos::HostSpace> > scalar_host_views; + std::map::view_type::data_type, Kokkos::LayoutLeft, Kokkos::HostSpace> > vector_host_views; + + // Loop over all fields + for (const auto& fieldPair : fields) + { + const std::string& fieldName = fieldPair.first; + const auto& fieldVariant = fieldPair.second; + + // If field is a _scalar_ field + if (std::holds_alternative*>(fieldVariant)) { + Field_t* field = std::get*>(fieldVariant); + // == ippl::Field, Cell>* + + Kokkos::View::view_type::data_type, Kokkos::LayoutLeft, Kokkos::HostSpace> host_view_layout_left; + Execute_Field_new(field, fieldName, host_view_layout_left, node); + scalar_host_views[fieldName] = host_view_layout_left; + } + // If field is a _vector_ field + else if (std::holds_alternative*>(fieldVariant)) { + VField_t* field = std::get*>(fieldVariant); + // == ippl::Field, 3, ippl::UniformCartesian, Cell>* + + Kokkos::View::view_type::data_type, Kokkos::LayoutLeft, Kokkos::HostSpace> host_view_layout_left; + Execute_Field_new(field, fieldName, host_view_layout_left, node); + vector_host_views[fieldName] = host_view_layout_left; + } + } + + // Pass Conduit node to Catalyst + catalyst_status err = catalyst_execute(conduit_cpp::c_node(&node)); + if (err != catalyst_status_ok) { + std::cerr << "Failed to execute Catalyst: " << err << std::endl; + } + + } + +// void Execute(int cycle, double time, int rank, +// const auto& /* std::shared_ptr >& */ particle, const std::string& particle_name, bool particle_onoff, +// const auto& /* ippl::Field, 3, ippl::UniformCartesian, Cell>*& */ E, const std::string& E_name, bool E_onoff, +// const auto& /* ippl::Field, Cell>*& */ rho, const std::string& rho_name, bool rho_onoff, +// const auto& /* ippl::Field, Cell>*& */ phi, const std::string& phi_name, bool phi_onoff) { +// // todo +// } + } // namespace CatalystAdaptor #endif diff --git a/test/stream/CMakeLists.txt b/test/stream/CMakeLists.txt index 7248e5f5f..daa8fe893 100644 --- a/test/stream/CMakeLists.txt +++ b/test/stream/CMakeLists.txt @@ -13,12 +13,12 @@ link_directories ( set (IPPL_LIBS ippl) set (COMPILE_FLAGS ${OPAL_CXX_FLAGS}) -add_executable (TestCatalystAdaptor TestCatalystAdaptor.cpp) -target_link_libraries ( - TestCatalystAdaptor - ${IPPL_LIBS} - ${MPI_CXX_LIBRARIES} -) +#add_executable (TestCatalystAdaptor TestCatalystAdaptor.cpp) +#target_link_libraries ( +# TestCatalystAdaptor +# ${IPPL_LIBS} +# ${MPI_CXX_LIBRARIES} +#) # vi: set et ts=4 sw=4 sts=4: From 4ca0a044d7f08abd01e14d8eb3ada3d15e768924 Mon Sep 17 00:00:00 2001 From: goebbert1 Date: Mon, 6 Jan 2025 00:04:57 +0100 Subject: [PATCH 05/14] remove old files --- src/Stream/InSitu/CMakeLists.txt | 3 +- src/Stream/InSitu/CatalystAdaptor.h | 507 ---------------------------- test/CMakeLists.txt | 1 - 3 files changed, 1 insertion(+), 510 deletions(-) delete mode 100644 src/Stream/InSitu/CatalystAdaptor.h diff --git a/src/Stream/InSitu/CMakeLists.txt b/src/Stream/InSitu/CMakeLists.txt index 6b3ddbcfc..dfbb7234b 100644 --- a/src/Stream/InSitu/CMakeLists.txt +++ b/src/Stream/InSitu/CMakeLists.txt @@ -2,7 +2,6 @@ set (_SRCS ) set (_HDRS - CatalystAdaptor.h ) include_DIRECTORIES ( @@ -21,4 +20,4 @@ install (FILES ${_HDRS} DESTINATION include/Stream/InSitu) # cmake-tab-width: 4 # indent-tabs-mode: nil # require-final-newline: nil -# End: \ No newline at end of file +# End: diff --git a/src/Stream/InSitu/CatalystAdaptor.h b/src/Stream/InSitu/CatalystAdaptor.h deleted file mode 100644 index 13b4f31e7..000000000 --- a/src/Stream/InSitu/CatalystAdaptor.h +++ /dev/null @@ -1,507 +0,0 @@ -// SPDX-FileCopyrightText: Copyright (c) Kitware Inc. -// SPDX-License-Identifier: BSD-3-Clause -#ifndef CatalystAdaptor_h -#define CatalystAdaptor_h - -#include "Ippl.h" - -#include -#include -#include -#include -#include -#include -#include - -#include "Utility/IpplException.h" - -// ---- -// instead of including alpine/datatypes.h -template -using Mesh_t = ippl::UniformCartesian; - -template -using Centering_t = typename Mesh_t::DefaultCentering; - -template -using Field = ippl::Field, Centering_t, ViewArgs...>; - -template -using Vector_t = ippl::Vector; - -template -using Field_t = Field; - -template -using VField_t = Field, Dim, ViewArgs...>; - -// instead of including alpine/AlpineManager.h -//template -//using ParticleContainer_t = ParticleContainer; -// ---- - - -namespace CatalystAdaptor { - - template - using FieldVariant = std::variant*, VField_t*>; - - template - using FieldPair = std::pair>; - - //template - //using ParticlePair = std::pair > >; - - - using View_vector = - Kokkos::View***, Kokkos::LayoutLeft, Kokkos::HostSpace>; - inline void setData(conduit_cpp::Node& node, const View_vector& view) { - node["electrostatic/association"].set_string("element"); - node["electrostatic/topology"].set_string("mesh"); - node["electrostatic/volume_dependent"].set_string("false"); - - auto length = std::size(view); - - // offset is zero as we start without the ghost cells - // stride is 1 as we have every index of the array - node["electrostatic/values/x"].set_external(&view.data()[0][0], length, 0, 1); - node["electrostatic/values/y"].set_external(&view.data()[0][1], length, 0, 1); - node["electrostatic/values/z"].set_external(&view.data()[0][2], length, 0, 1); - } - - using View_scalar = Kokkos::View; - inline void setData(conduit_cpp::Node& node, const View_scalar& view) { - node["density/association"].set_string("element"); - node["density/topology"].set_string("mesh"); - node["density/volume_dependent"].set_string("false"); - - node["density/values"].set_external(view.data(), view.size()); - } - - inline void callCatalystExecute(const conduit_cpp::Node& node) { - - // TODO: we should add here this IPPL-INFO stuff - //if ( static auto called {false}; !std::exchange(called, true) ) { - // catalyst_conduit_node_print(conduit_cpp::c_node(&node)); - //} - - catalyst_status err = catalyst_execute(conduit_cpp::c_node(&node)); - if (err != catalyst_status_ok) { - std::cerr << "Failed to execute Catalyst: " << err << std::endl; - } - } - - void Initialize(int argc, char* argv[]) { - conduit_cpp::Node node; - std::cout << "pvscript path: " << argv[1] << std::endl; - node["catalyst/scripts/script/filename"].set_string(argv[1]); - for (int cc = 2; cc < argc; ++cc) { - std::cout << "pvscript args: " << argv[cc] << std::endl; - conduit_cpp::Node list_entry = node["catalyst/scripts/script/args"].append(); - list_entry.set(argv[cc]); - } - try { - node["catalyst_load/implementation"] = getenv("CATALYST_IMPLEMENTATION_NAME"); - node["catalyst_load/search_paths/paraview"] = getenv("CATALYST_IMPLEMENTATION_PATHS"); - } catch (...) { - throw IpplException("CatalystAdaptor::Initialize", - "no environmental variable for CATALYST_IMPLEMENTATION_NAME or " - "CATALYST_IMPLEMENTATION_PATHS found"); - } - // TODO: catch catalyst error also with IpplException - catalyst_status err = catalyst_initialize(conduit_cpp::c_node(&node)); - if (err != catalyst_status_ok) { - std::cerr << "Failed to initialize Catalyst: " << err << std::endl; - } - } - - - void Initialize_Adios(int argc, char* argv[]) - { - conduit_cpp::Node node; - for (int cc = 1; cc < argc; ++cc) - { - if (strstr(argv[cc], "xml")) - { - node["adios/config_filepath"].set_string(argv[cc]); - } - else - { - node["catalyst/scripts/script" +std::to_string(cc - 1)].set_string(argv[cc]); - } - } - node["catalyst_load/implementation"] = getenv("CATALYST_IMPLEMENTATION_NAME"); - catalyst_status err = catalyst_initialize(conduit_cpp::c_node(&node)); - if (err != catalyst_status_ok) - { - std::cerr << "Failed to initialize Catalyst: " << err << std::endl; - } - } - - - template - std::optional Execute_Field(int cycle, double time, int rank, Field& field, std::optional& node_in) { - static_assert(Field::dim == 3, "CatalystAdaptor only supports 3D"); - // catalyst blueprint definition - // https://docs.paraview.org/en/latest/Catalyst/blueprints.html - // - // conduit blueprint definition (v.8.3) - // https://llnl-conduit.readthedocs.io/en/latest/blueprint_mesh.html - conduit_cpp::Node node; - if (node_in) - node = node_in.value(); - - auto nGhost = field.getNghost(); - - // Creates a host-accessible mirror view and copies the data from the device view to the host. - typename Field::view_type::host_mirror_type host_view = - Kokkos::create_mirror_view_and_copy(Kokkos::HostSpace(), field.getView()); - - Kokkos::View - host_view_layout_left("host_view_layout_left", - field.getLayout().getLocalNDIndex()[0].length(), - field.getLayout().getLocalNDIndex()[1].length(), - field.getLayout().getLocalNDIndex()[2].length()); - - for (size_t i = 0; i < field.getLayout().getLocalNDIndex()[0].length(); ++i) { - for (size_t j = 0; j < field.getLayout().getLocalNDIndex()[1].length(); ++j) { - for (size_t k = 0; k < field.getLayout().getLocalNDIndex()[2].length(); ++k) { - host_view_layout_left(i, j, k) = host_view(i + nGhost, j + nGhost, k + nGhost); - } - } - } - - - // add time/cycle information - auto state = node["catalyst/state"]; - state["cycle"].set(cycle); - state["time"].set(time); - state["domain_id"].set(rank); - - // add catalyst channel named ippl_field, as fields is reserved - auto channel = node["catalyst/channels/ippl_field"]; - channel["type"].set_string("mesh"); - - // in data channel now we adhere to conduits mesh blueprint definition - auto mesh = channel["data"]; - mesh["coordsets/coords/type"].set_string("uniform"); - - // number of points in specific dimension - std::string field_node_dim{"coordsets/coords/dims/i"}; - std::string field_node_origin{"coordsets/coords/origin/x"}; - std::string field_node_spacing{"coordsets/coords/spacing/dx"}; - - for (unsigned int iDim = 0; iDim < field.get_mesh().getGridsize().dim; ++iDim) { - // add dimension - mesh[field_node_dim].set(field.getLayout().getLocalNDIndex()[iDim].length() + 1); - - // add origin - mesh[field_node_origin].set( - field.get_mesh().getOrigin()[iDim] + field.getLayout().getLocalNDIndex()[iDim].first() - * field.get_mesh().getMeshSpacing(iDim)); - - // add spacing - mesh[field_node_spacing].set(field.get_mesh().getMeshSpacing(iDim)); - - // increment last char in string - ++field_node_dim.back(); - ++field_node_origin.back(); - ++field_node_spacing.back(); - } - - // add topology - mesh["topologies/mesh/type"].set_string("uniform"); - mesh["topologies/mesh/coordset"].set_string("coords"); - std::string field_node_origin_topo{"topologies/mesh/origin/x"}; - for (unsigned int iDim = 0; iDim < field.get_mesh().getGridsize().dim; ++iDim) { - // shift origin - mesh[field_node_origin_topo].set(field.get_mesh().getOrigin()[iDim] - + field.getLayout().getLocalNDIndex()[iDim].first() - * field.get_mesh().getMeshSpacing(iDim)); - - // increment last char in string - ++field_node_origin_topo.back(); - } - - // add values and subscribe to data - auto fields = mesh["fields"]; - setData(fields, host_view_layout_left); - - // as we have a local copy of the field, the catalyst_execute needs to be called - // within this scope otherwise the memory location might be already overwritten - if (node_in == std::nullopt) - { - callCatalystExecute(node); - return {}; - } - else - return node; - - } - - template - std::optional Execute_Particle(int cycle, double time, int rank, ParticleContainer& particleContainer, std::optional& node_in) { - assert((particleContainer->ID.getView().data() != nullptr) && "ID view should not be nullptr, might be missing the right execution space"); - - //auto layout_view = particleContainer->R.getView(); - typename ippl::ParticleAttrib>::HostMirror R_host = particleContainer->R.getHostMirror(); - typename ippl::ParticleAttrib>::HostMirror P_host = particleContainer->P.getHostMirror(); - typename ippl::ParticleAttrib::HostMirror q_host = particleContainer->q.getHostMirror(); - typename ippl::ParticleAttrib::HostMirror ID_host = particleContainer->ID.getHostMirror(); - Kokkos::deep_copy(R_host, particleContainer->R.getView()); - Kokkos::deep_copy(P_host, particleContainer->P.getView()); - Kokkos::deep_copy(q_host, particleContainer->q.getView()); - Kokkos::deep_copy(ID_host, particleContainer->ID.getView()); - - // if node is passed in, append data to it - conduit_cpp::Node node; - if (node_in) - node = node_in.value(); - - // add time/cycle information - auto state = node["catalyst/state"]; - state["cycle"].set(cycle); - state["time"].set(time); - state["domain_id"].set(rank); - - // channel for particles - auto channel = node["catalyst/channels/ippl_particle"]; - channel["type"].set_string("mesh"); - - // in data channel now we adhere to conduits mesh blueprint definition - auto mesh = channel["data"]; - mesh["coordsets/coords/type"].set_string("explicit"); - - //mesh["coordsets/coords/values/x"].set_external(&layout_view.data()[0][0], particleContainer->getLocalNum(), 0, sizeof(double)*3); - //mesh["coordsets/coords/values/y"].set_external(&layout_view.data()[0][1], particleContainer->getLocalNum(), 0, sizeof(double)*3); - //mesh["coordsets/coords/values/z"].set_external(&layout_view.data()[0][2], particleContainer->getLocalNum(), 0, sizeof(double)*3); - mesh["coordsets/coords/values/x"].set_external(&R_host.data()[0][0], particleContainer->getLocalNum(), 0, sizeof(double)*3); - mesh["coordsets/coords/values/y"].set_external(&R_host.data()[0][1], particleContainer->getLocalNum(), 0, sizeof(double)*3); - mesh["coordsets/coords/values/z"].set_external(&R_host.data()[0][2], particleContainer->getLocalNum(), 0, sizeof(double)*3); - - mesh["topologies/mesh/type"].set_string("unstructured"); - mesh["topologies/mesh/coordset"].set_string("coords"); - mesh["topologies/mesh/elements/shape"].set_string("point"); - //mesh["topologies/mesh/elements/connectivity"].set_external(particleContainer->ID.getView().data(),particleContainer->getLocalNum()); - mesh["topologies/mesh/elements/connectivity"].set_external(ID_host.data(),particleContainer->getLocalNum()); - - //auto charge_view = particleContainer->getQ().getView(); - - // add values for scalar charge field - auto fields = mesh["fields"]; - fields["charge/association"].set_string("vertex"); - fields["charge/topology"].set_string("mesh"); - fields["charge/volume_dependent"].set_string("false"); - - //fields["charge/values"].set_external(particleContainer->q.getView().data(), particleContainer->getLocalNum()); - fields["charge/values"].set_external(q_host.data(), particleContainer->getLocalNum()); - - // add values for vector velocity field - //auto velocity_view = particleContainer->P.getView(); - fields["velocity/association"].set_string("vertex"); - fields["velocity/topology"].set_string("mesh"); - fields["velocity/volume_dependent"].set_string("false"); - - //fields["velocity/values/x"].set_external(&velocity_view.data()[0][0], particleContainer->getLocalNum(),0 ,sizeof(double)*3); - //fields["velocity/values/y"].set_external(&velocity_view.data()[0][1], particleContainer->getLocalNum(),0 ,sizeof(double)*3); - //fields["velocity/values/z"].set_external(&velocity_view.data()[0][2], particleContainer->getLocalNum(),0 ,sizeof(double)*3); - fields["velocity/values/x"].set_external(&P_host.data()[0][0], particleContainer->getLocalNum(),0 ,sizeof(double)*3); - fields["velocity/values/y"].set_external(&P_host.data()[0][1], particleContainer->getLocalNum(),0 ,sizeof(double)*3); - fields["velocity/values/z"].set_external(&P_host.data()[0][2], particleContainer->getLocalNum(),0 ,sizeof(double)*3); - - fields["position/association"].set_string("vertex"); - fields["position/topology"].set_string("mesh"); - fields["position/volume_dependent"].set_string("false"); - - //fields["position/values/x"].set_external(&layout_view.data()[0][0], particleContainer->getLocalNum(), 0, sizeof(double)*3); - //fields["position/values/y"].set_external(&layout_view.data()[0][1], particleContainer->getLocalNum(), 0, sizeof(double)*3); - //fields["position/values/z"].set_external(&layout_view.data()[0][2], particleContainer->getLocalNum(), 0, sizeof(double)*3); - fields["position/values/x"].set_external(&R_host.data()[0][0], particleContainer->getLocalNum(), 0, sizeof(double)*3); - fields["position/values/y"].set_external(&R_host.data()[0][1], particleContainer->getLocalNum(), 0, sizeof(double)*3); - fields["position/values/z"].set_external(&R_host.data()[0][2], particleContainer->getLocalNum(), 0, sizeof(double)*3); - - // this node we can return as the pointer to velocity and charge is globally valid - if (node_in == std::nullopt) - { - callCatalystExecute(node); - return {}; - } - else - return node; - } - - - template - void Execute_Field_Particle(int cycle, double time, int rank, Field& field, ParticleContainer& particle) { - - auto node = std::make_optional(); - node = CatalystAdaptor::Execute_Particle(cycle, time, rank, particle, node); - CatalystAdaptor::Execute_Field(cycle, time, rank, field, node); - } - - void Finalize() { - conduit_cpp::Node node; - catalyst_status err = catalyst_finalize(conduit_cpp::c_node(&node)); - if (err != catalyst_status_ok) { - std::cerr << "Failed to finalize Catalyst: " << err << std::endl; - } - } - - -// === NEW ================================= - - - template // == ippl::Field, Cell>* - void Execute_Field_new(Field* field, const std::string& fieldName, - Kokkos::View& host_view_layout_left, - conduit_cpp::Node& node) { - static_assert(Field::dim == 3, "CatalystAdaptor only supports 3D"); - - // A) define mesh - - // add catalyst channel named ippl_"field", as fields is reserved - auto channel = node["catalyst/channels/ippl_" + fieldName]; - channel["type"].set_string("mesh"); - - // in data channel now we adhere to conduits mesh blueprint definition - auto mesh = channel["data"]; - mesh["coordsets/coords/type"].set_string("uniform"); - - // number of points in specific dimension - std::string field_node_dim{"coordsets/coords/dims/i"}; - std::string field_node_origin{"coordsets/coords/origin/x"}; - std::string field_node_spacing{"coordsets/coords/spacing/dx"}; - - for (unsigned int iDim = 0; iDim < field->get_mesh().getGridsize().dim; ++iDim) { - // add dimension - mesh[field_node_dim].set(field->getLayout().getLocalNDIndex()[iDim].length() + 1); - - // add origin - mesh[field_node_origin].set( - field->get_mesh().getOrigin()[iDim] + field->getLayout().getLocalNDIndex()[iDim].first() - * field->get_mesh().getMeshSpacing(iDim)); - - // add spacing - mesh[field_node_spacing].set(field->get_mesh().getMeshSpacing(iDim)); - - // increment last char in string - ++field_node_dim.back(); - ++field_node_origin.back(); - ++field_node_spacing.back(); - } - - // add topology - mesh["topologies/mesh/type"].set_string("uniform"); - mesh["topologies/mesh/coordset"].set_string("coords"); - std::string field_node_origin_topo{"topologies/mesh/origin/x"}; - for (unsigned int iDim = 0; iDim < field->get_mesh().getGridsize().dim; ++iDim) { - // shift origin - mesh[field_node_origin_topo].set(field->get_mesh().getOrigin()[iDim] - + field->getLayout().getLocalNDIndex()[iDim].first() - * field->get_mesh().getMeshSpacing(iDim)); - - // increment last char in string ('x' becomes 'y' becomes 'z') - ++field_node_origin_topo.back(); - } - - // B) Set the field values - - // Initialize the existing Kokkos::View - host_view_layout_left = Kokkos::View( - "host_view_layout_left", - field->getLayout().getLocalNDIndex()[0].length(), - field->getLayout().getLocalNDIndex()[1].length(), - field->getLayout().getLocalNDIndex()[2].length()); - - // Creates a host-accessible mirror view and copies the data from the device view to the host. - auto host_view = - Kokkos::create_mirror_view_and_copy(Kokkos::HostSpace(), field->getView()); - - // Copy data from field to the memory+style which will be passed to Catalyst - auto nGhost = field->getNghost(); - for (size_t i = 0; i < field->getLayout().getLocalNDIndex()[0].length(); ++i) { - for (size_t j = 0; j < field->getLayout().getLocalNDIndex()[1].length(); ++j) { - for (size_t k = 0; k < field->getLayout().getLocalNDIndex()[2].length(); ++k) { - host_view_layout_left(i, j, k) = host_view(i + nGhost, j + nGhost, k + nGhost); - } - } - } - - // Add values and subscribe to data - auto fields = mesh["fields"]; - setData(fields, host_view_layout_left); - } - - - template - void Execute(int cycle, double time, int rank, - const auto& /* std::shared_ptr >& */ particle, - const std::vector>& fields) { - - // catalyst blueprint definition - // https://docs.paraview.org/en/latest/Catalyst/blueprints.html - // - // conduit blueprint definition (v.8.3) - // https://llnl-conduit.readthedocs.io/en/latest/blueprint_mesh.html - conduit_cpp::Node node; - - // add time/cycle information - auto state = node["catalyst/state"]; - state["cycle"].set(cycle); - state["time"].set(time); - state["domain_id"].set(rank); - - // Handle particles - - // Handle fields - - // Map of all Kokkos::Views. This keeps a reference on all Kokkos::Views - // which ensures that Kokkos does not free the memory before the end of this function. - std::map::view_type::data_type, Kokkos::LayoutLeft, Kokkos::HostSpace> > scalar_host_views; - std::map::view_type::data_type, Kokkos::LayoutLeft, Kokkos::HostSpace> > vector_host_views; - - // Loop over all fields - for (const auto& fieldPair : fields) - { - const std::string& fieldName = fieldPair.first; - const auto& fieldVariant = fieldPair.second; - - // If field is a _scalar_ field - if (std::holds_alternative*>(fieldVariant)) { - Field_t* field = std::get*>(fieldVariant); - // == ippl::Field, Cell>* - - Kokkos::View::view_type::data_type, Kokkos::LayoutLeft, Kokkos::HostSpace> host_view_layout_left; - Execute_Field_new(field, fieldName, host_view_layout_left, node); - scalar_host_views[fieldName] = host_view_layout_left; - } - // If field is a _vector_ field - else if (std::holds_alternative*>(fieldVariant)) { - VField_t* field = std::get*>(fieldVariant); - // == ippl::Field, 3, ippl::UniformCartesian, Cell>* - - Kokkos::View::view_type::data_type, Kokkos::LayoutLeft, Kokkos::HostSpace> host_view_layout_left; - Execute_Field_new(field, fieldName, host_view_layout_left, node); - vector_host_views[fieldName] = host_view_layout_left; - } - } - - // Pass Conduit node to Catalyst - catalyst_status err = catalyst_execute(conduit_cpp::c_node(&node)); - if (err != catalyst_status_ok) { - std::cerr << "Failed to execute Catalyst: " << err << std::endl; - } - - } - -// void Execute(int cycle, double time, int rank, -// const auto& /* std::shared_ptr >& */ particle, const std::string& particle_name, bool particle_onoff, -// const auto& /* ippl::Field, 3, ippl::UniformCartesian, Cell>*& */ E, const std::string& E_name, bool E_onoff, -// const auto& /* ippl::Field, Cell>*& */ rho, const std::string& rho_name, bool rho_onoff, -// const auto& /* ippl::Field, Cell>*& */ phi, const std::string& phi_name, bool phi_onoff) { -// // todo -// } - -} // namespace CatalystAdaptor - -#endif diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index 1f784cc0c..9cd986c1e 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -29,7 +29,6 @@ add_subdirectory (random) add_subdirectory (serialization) if (ENABLE_CATALYST) - add_subdirectory (stream) endif () # vi: set et ts=4 sw=4 sts=4: From 5d8129fe0a86e87014fbcfcc8e8787c91fbdf19a Mon Sep 17 00:00:00 2001 From: goebbert1 Date: Mon, 6 Jan 2025 11:36:13 +0100 Subject: [PATCH 06/14] fix path --- alpine/PenningTrap.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/alpine/PenningTrap.cpp b/alpine/PenningTrap.cpp index b899d2968..66c88aac7 100644 --- a/alpine/PenningTrap.cpp +++ b/alpine/PenningTrap.cpp @@ -40,7 +40,7 @@ const char* TestName = "PenningTrap"; #include "PenningTrapManager.h" #ifdef ENABLE_CATALYST -#include "Stream/InSitu/CatalystAdaptor.h" +#include "CatalystAdaptor.h" #endif int main(int argc, char* argv[]) { From 5b3012942849e939f56cf1c260ef2334e1565533 Mon Sep 17 00:00:00 2001 From: kim17 Date: Mon, 6 Jan 2025 15:04:50 +0100 Subject: [PATCH 07/14] Steering --- alpine/CatalystAdaptor.h | 46 ++++++++++++- alpine/PenningTrap.cpp | 15 +++-- alpine/PenningTrapManager.h | 9 +-- src/Stream/InSitu/proxy.xml | 54 +++++++++++++++ src/Stream/InSitu/steering.py | 119 ++++++++++++++++++++++++++++++++++ 5 files changed, 233 insertions(+), 10 deletions(-) create mode 100644 src/Stream/InSitu/proxy.xml create mode 100644 src/Stream/InSitu/steering.py diff --git a/alpine/CatalystAdaptor.h b/alpine/CatalystAdaptor.h index 78a5d024c..d65399ae3 100644 --- a/alpine/CatalystAdaptor.h +++ b/alpine/CatalystAdaptor.h @@ -56,12 +56,16 @@ namespace CatalystAdaptor { void Initialize(int argc, char* argv[]) { conduit_cpp::Node node; std::cout << "pvscript path: " << argv[1] << std::endl; - node["catalyst/scripts/script/filename"].set_string(argv[1]); + std::cout << "pvproxy path: " << argv[2] << std::endl; + node["catalyst/scripts/script/filename"].set(argv[1]); + node["catalyst/proxies/proxy"].set(argv[2]); + /* for (int cc = 2; cc < argc; ++cc) { std::cout << "pvscript args: " << argv[cc] << std::endl; conduit_cpp::Node list_entry = node["catalyst/scripts/script/args"].append(); list_entry.set(argv[cc]); } + */ try { node["catalyst_load/implementation"] = getenv("CATALYST_IMPLEMENTATION_NAME"); node["catalyst_load/search_paths/paraview"] = getenv("CATALYST_IMPLEMENTATION_PATHS"); @@ -232,12 +236,47 @@ namespace CatalystAdaptor { setData(fields, host_view_layout_left); } + void AddSteerableChannel(conduit_cpp::Node& node, double scaleFactor) { + auto steerable = node["catalyst/channels/steerable"]; + steerable["type"].set("mesh"); + + auto steerable_data = steerable["data"]; + steerable_data["coordsets/coords/type"].set_string("explicit"); + steerable_data["coordsets/coords/values/x"].set_float64_vector({ 1 }); + steerable_data["coordsets/coords/values/y"].set_float64_vector({ 2 }); + steerable_data["coordsets/coords/values/z"].set_float64_vector({ 3 }); + steerable_data["topologies/mesh/type"].set("unstructured"); + steerable_data["topologies/mesh/coordset"].set("coords"); + steerable_data["topologies/mesh/elements/shape"].set("point"); + steerable_data["topologies/mesh/elements/connectivity"].set_int32_vector({ 0 }); + steerable_data["fields/steerable/association"].set("vertex"); + steerable_data["fields/steerable/topology"].set("mesh"); + steerable_data["fields/steerable/volume_dependent"].set("false"); + steerable_data["fields/steerable/values"].set_float64_vector({scaleFactor}); + } + + void Results( double& scaleFactor) { + + conduit_cpp::Node results; + catalyst_status err = catalyst_results(conduit_cpp::c_node(&results)); + + if (err != catalyst_status_ok) + { + std::cerr << "Failed to execute Catalyst-results: " << err << std::endl; + } + else + { + std::cout << "Result Node dump:" << std::endl; + const std::string value_path = "catalyst/steerable/fields/scalefactor/values"; + scaleFactor = results[value_path].to_double(); + } + } template void Execute(int cycle, double time, int rank, // const auto& /* std::shared_ptr >& */ particle, const std::vector>& particles, - const std::vector>& fields) { + const std::vector>& fields, double& scaleFactor) { // catalyst blueprint definition // https://docs.paraview.org/en/latest/Catalyst/blueprints.html @@ -314,12 +353,15 @@ namespace CatalystAdaptor { } } + AddSteerableChannel(node, scaleFactor); + // Pass Conduit node to Catalyst catalyst_status err = catalyst_execute(conduit_cpp::c_node(&node)); if (err != catalyst_status_ok) { std::cerr << "Failed to execute Catalyst: " << err << std::endl; } + Results(scaleFactor); } } // namespace CatalystAdaptor diff --git a/alpine/PenningTrap.cpp b/alpine/PenningTrap.cpp index 66c88aac7..6ca15c754 100644 --- a/alpine/PenningTrap.cpp +++ b/alpine/PenningTrap.cpp @@ -48,14 +48,20 @@ int main(int argc, char* argv[]) { { #ifdef ENABLE_CATALYST + char* script = nullptr; + char* proxy = nullptr; for (int i = 1; i < argc; ++i) { if (std::string(argv[i]) == "--pvscript" && i + 1 < argc) { - // reduce the argument list - char* reducedArgv[] = { argv[0], argv[i + 1] }; - CatalystAdaptor::Initialize(2, reducedArgv); - break; + script = argv[i+1]; + i++; + } + if (std::string(argv[i]) == "--pvproxy" && i+1 < argc) { + proxy = argv[i+1]; + i++; } } + char* reducedArgv[] = { argv[0], script, proxy}; + CatalystAdaptor::Initialize(2, reducedArgv); #endif Inform msg(TestName); @@ -106,3 +112,4 @@ int main(int argc, char* argv[]) { return 0; } + diff --git a/alpine/PenningTrapManager.h b/alpine/PenningTrapManager.h index 6ac50f110..40ee89cc3 100644 --- a/alpine/PenningTrapManager.h +++ b/alpine/PenningTrapManager.h @@ -27,10 +27,11 @@ class PenningTrapManager : public AlpineManager { using FieldContainer_t = FieldContainer; using FieldSolver_t= FieldSolver; using LoadBalancer_t= LoadBalancer; + double scaleFactor; PenningTrapManager(size_type totalP_, int nt_, Vector_t &nr_, double lbt_, std::string& solver_, std::string& stepMethod_) - : AlpineManager(totalP_, nt_, nr_, lbt_, solver_, stepMethod_){} + : AlpineManager(totalP_, nt_, nr_, lbt_, solver_, stepMethod_),scaleFactor(30){} ~PenningTrapManager(){} @@ -228,7 +229,7 @@ class PenningTrapManager : public AlpineManager { double alpha = this->alpha_m; double Bext = this->Bext_m; double DrInv = this->DrInv_m; - double V0 = 30 * this->length_m[2]; + double V0 = scaleFactor * this->length_m[2]; Vector_t length = this->length_m; Vector_t origin = this->origin_m; double dt = this->dt_m; @@ -291,9 +292,9 @@ class PenningTrapManager : public AlpineManager { std::vector> fields = { {"E", CatalystAdaptor::FieldVariant(&this->fcontainer_m->getE())}, {"roh", CatalystAdaptor::FieldVariant(&this->fcontainer_m->getRho())}, - {"phi", CatalystAdaptor::FieldVariant(&this->fcontainer_m->getPhi())}, + //{"phi", CatalystAdaptor::FieldVariant(&this->fcontainer_m->getPhi())}, }; - CatalystAdaptor::Execute(it, this->time_m, ippl::Comm->rank(), particles, fields); + CatalystAdaptor::Execute(it, this->time_m, ippl::Comm->rank(), particles, fields, scaleFactor); #endif // Field solve diff --git a/src/Stream/InSitu/proxy.xml b/src/Stream/InSitu/proxy.xml new file mode 100644 index 000000000..040c0c128 --- /dev/null +++ b/src/Stream/InSitu/proxy.xml @@ -0,0 +1,54 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/src/Stream/InSitu/steering.py b/src/Stream/InSitu/steering.py new file mode 100644 index 000000000..d33df3e4a --- /dev/null +++ b/src/Stream/InSitu/steering.py @@ -0,0 +1,119 @@ +# script-version: 2.0 +# for more details check https://www.paraview.org/paraview-docs/latest/cxx/CatalystPythonScriptsV2.html +from paraview import print_info +from paraview.simple import * +from paraview import catalyst +import time + +#### disable automatic camera reset on 'Show' +paraview.simple._DisableFirstRenderCameraReset() + +# print start marker +print_info("begin '%s'", __name__) + +# ---------------------------------------------------------------- +# setup the data processing pipelines +# ---------------------------------------------------------------- + +# registrationName must match the channel name used in the 'CatalystAdaptor'. +ippl_field = PVTrivialProducer(registrationName='ippl_E') +ippl_particle = PVTrivialProducer(registrationName='ippl_particle') + + +from paraview.simple import LoadPlugin, CreateSteerableParameters + +# SteerableParameters 생성 +try: + steerable_parameters = CreateSteerableParameters("SteerableParameters") + print("SteerableParameters loaded successfully.") +except Exception as e: + print(f"Error loading SteerableParameters: {e}") + +# ---------------------------------------------------------------- +# setup animation scene, tracks and keyframes +# note: the Get..() functions create a new object, if needed +# ---------------------------------------------------------------- + +# get the time-keeper +timeKeeper1 = GetTimeKeeper() + +# initialize the timekeeper + +# get time animation track +timeAnimationCue1 = GetTimeTrack() + +# initialize the animation track + +# get animation scene +animationScene1 = GetAnimationScene() + +# initialize the animation scene +animationScene1.Cues = timeAnimationCue1 +animationScene1.AnimationTime = 0.0 + +# initialize the animation scene + +# ---------------------------------------------------------------- +# setup extractors +# ---------------------------------------------------------------- + +# create extractor (U = unstructured) +# vTI1 = CreateExtractor('VTI', ippl_field, registrationName='VTI1') +# vTI1.Trigger = 'Time Step' +# vTI1.Writer.FileName = 'ippl_field_{timestep:06d}.vti' + +# create extractor (PD=point data) +#vTPD2 = CreateExtractor('VTPD', ippl_particle, registrationName='VTPD2') +#vTPD2.Trigger = 'Time Step' +#vTPD2.Writer.FileName = 'ippl_particle_{timestep:06d}.vtpd' + +# ---------------------------------------------------------------- +# restore active source +#SetActiveSource(ippl_particle) +SetActiveSource(ippl_field) +# ---------------------------------------------------------------- + +# ------------------------------------------------------------------------------ +# Catalyst options +options = catalyst.Options() +options.GlobalTrigger = 'Time Step' +options.EnableCatalystLive = 1 +options.CatalystLiveTrigger = 'Time Step' +#options.ExtractsOutputDirectory = 'data_vtpd' + +# ------------------------------------------------------------------------------ +def catalyst_initialize(): + print_info("in '%s::catalyst_initialize'", __name__) + + +# ------------------------------------------------------------------------------ +def catalyst_execute(info): + print_info("in '%s::catalyst_execute'", __name__) + + global ippl_field + ippl_field.UpdatePipeline() + global ippl_particle + ippl_particle.UpdatePipeline() + + global steerable_parameters + steerable_parameters.scaleFactor[0] = 31 + info.cycle + + print("-----------------------------------") + print("executing (cycle={}, time={})".format(info.cycle, info.time)) + #print("field bounds :", ippl_field.GetDataInformation().GetBounds()) + #print("particle bounds:", ippl_particle.GetDataInformation().GetBounds()) + + # In a real simulation sleep is not needed. We use it here to slow down the + # "simulation" and make sure ParaView client can catch up with the produced + # results instead of having all of them flashing at once. + if options.EnableCatalystLive: + time.sleep(5) + + +# ------------------------------------------------------------------------------ +def catalyst_finalize(): + print_info("in '%s::catalyst_finalize'", __name__) + + +# print end marker +print_info("end '%s'", __name__) \ No newline at end of file From 225d33b8417c38a9bcfc1e7f55b4c686c0d14de4 Mon Sep 17 00:00:00 2001 From: Victor Mateevitsi Date: Thu, 23 Jan 2025 20:53:56 +0000 Subject: [PATCH 08/14] WIP: Ascent instrumentation --- CMakeLists.txt | 3 +- alpine/PenningTrap.cpp | 4 +- alpine/PenningTrapManager.h | 12 +- src/CMakeLists.txt | 9 ++ src/Stream/InSitu/AscentAdaptor.h | 242 ++++++++++++++++++++++++++++++ src/Stream/InSitu/CMakeLists.txt | 1 + 6 files changed, 266 insertions(+), 5 deletions(-) create mode 100644 src/Stream/InSitu/AscentAdaptor.h diff --git a/CMakeLists.txt b/CMakeLists.txt index 38d2a9e32..6dd79ce11 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -23,7 +23,6 @@ macro(remove_flag_from_target _target _flag) endif() endmacro() - if (NOT CMAKE_CONFIGURATION_TYPES AND NOT CMAKE_BUILD_TYPE) set (CMAKE_BUILD_TYPE RelWithDebInfo CACHE STRING @@ -207,7 +206,7 @@ if (ENABLE_FFT) endif () option (ENABLE_CATALYST "Build example with Catalyst enabled" OFF) -option (ENABLE_ASCENT "Build example with Ascent enabled" OFF) +option (ENABLE_ASCENT "Build example with Ascent enabled" ON) option (ENABLE_SOLVERS "Enable IPPL solvers" ON) add_subdirectory (src) diff --git a/alpine/PenningTrap.cpp b/alpine/PenningTrap.cpp index 3f351a10b..6dbb8f9f6 100644 --- a/alpine/PenningTrap.cpp +++ b/alpine/PenningTrap.cpp @@ -39,8 +39,8 @@ const char* TestName = "PenningTrap"; #include "Manager/PicManager.h" #include "PenningTrapManager.h" -#ifdef ENABLE_CATALYST -#include "Stream/InSitu/CatalystAdaptor.h" +#ifdef ENABLE_ASCENT +#include "Stream/InSitu/AscentAdaptor.h" #endif int main(int argc, char* argv[]) { diff --git a/alpine/PenningTrapManager.h b/alpine/PenningTrapManager.h index 08fe37f99..d0fbb364c 100644 --- a/alpine/PenningTrapManager.h +++ b/alpine/PenningTrapManager.h @@ -19,6 +19,10 @@ #include "Stream/InSitu/CatalystAdaptor.h" #endif +#ifdef ENABLE_ASCENT +#include "Stream/InSitu/AscentAdaptor.h" +#endif + using view_type = typename ippl::detail::ViewType, 1>::view_type; template @@ -285,7 +289,7 @@ class PenningTrapManager : public AlpineManager { // scatter the charge onto the underlying grid this->par2grid(); #ifdef ENABLE_CATALYST - std::optional node = std::nullopt; + std::optional node = std::nullopt; //CatalystAdaptor::Execute_Particle(it, this->time_m, ippl::Comm->rank(), pc, node); auto *rho = &this->fcontainer_m->getRho(); CatalystAdaptor::Execute_Field(it, this->time_m, ippl::Comm->rank(), *rho, node); @@ -294,6 +298,12 @@ class PenningTrapManager : public AlpineManager { //CatalystAdaptor::Execute_Field_Particle(it, this->time_m, ippl::Comm->rank(), *E, pc); #endif +#ifdef ENABLE_ASCENT + std::optional node = std::nullopt; + auto *rho = &this->fcontainer_m->getRho(); + AscentAdaptor::Execute_Field(it, this->time_m, ippl::Comm->rank(), *rho, node); +#endif + // Field solve IpplTimings::startTimer(SolveTimer); this->fsolver_m->runSolver(); diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index b5fd3eb26..2a898c0fd 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -98,10 +98,17 @@ if (ENABLE_CATALYST) message (STATUS "Found catalyst_DIR: ${catalyst_DIR}") endif() +if (ENABLE_ASCENT) + find_package(Ascent REQUIRED) + message (STATUS "Enable Ascent") + message (STATUS "Found ascent_DIR: ${Ascent_DIR}") +endif() + if (ENABLE_AMR) add_subdirectory(AmrParticle) endif () include_directories ( + $<$:${ASCENT_INSTALL_PREFIX}/include/ascent> BEFORE ${CMAKE_CURRENT_SOURCE_DIR} ) @@ -119,6 +126,7 @@ message("Timer fences for this build: ${TimerFences}") target_compile_definitions(ippl PUBLIC $<$:-DENABLE_FFT> $<$:-DENABLE_CATALYST> + $<$:-DENABLE_ASCENT> IPPL_ENABLE_TIMER_FENCES=${TimerFences} ) @@ -126,6 +134,7 @@ target_link_libraries(ippl PUBLIC Kokkos::kokkos $<$:Heffte> $<$:catalyst::catalyst> + $<$:ascent::ascent_mpi> ) install (TARGETS ippl DESTINATION lib) diff --git a/src/Stream/InSitu/AscentAdaptor.h b/src/Stream/InSitu/AscentAdaptor.h new file mode 100644 index 000000000..48c92155f --- /dev/null +++ b/src/Stream/InSitu/AscentAdaptor.h @@ -0,0 +1,242 @@ +// SPDX-FileCopyrightText: Copyright (c) Kitware Inc. +// SPDX-License-Identifier: BSD-3-Clause +#ifndef AsecntAdaptor_h +#define AsecntAdaptor_h + +#include "Ippl.h" + +#include +#include +#include +#include +#include + +#include "Utility/IpplException.h" + + +namespace AscentAdaptor { + + ascent::Ascent mAscent; + + using View_vector = + Kokkos::View***, Kokkos::LayoutLeft, Kokkos::HostSpace>; + inline void setData(conduit::Node& node, const View_vector& view) { + node["electrostatic/association"].set_string("element"); + node["electrostatic/topology"].set_string("mesh"); + node["electrostatic/volume_dependent"].set_string("false"); + + auto length = std::size(view); + + // offset is zero as we start without the ghost cells + // stride is 1 as we have every index of the array + node["electrostatic/values/x"].set_external(&view.data()[0][0], length, 0, 1); + node["electrostatic/values/y"].set_external(&view.data()[0][1], length, 0, 1); + node["electrostatic/values/z"].set_external(&view.data()[0][2], length, 0, 1); + } + + using View_scalar = Kokkos::View; + inline void setData(conduit::Node& node, const View_scalar& view) { + node["density/association"].set_string("element"); + node["density/topology"].set_string("mesh"); + node["density/volume_dependent"].set_string("false"); + + node["density/values"].set_external(view.data(), view.size()); + } + + void Initialize(int argc, char* argv[]) { + MPI_Comm ascent_comm; + + // Split communicator based on the task ID + MPI_Comm_dup(MPI_COMM_WORLD, &ascent_comm); + + conduit::Node ascent_opts; + ascent_opts["mpi_comm"] = MPI_Comm_c2f(ascent_comm); + mAscent.open(ascent_opts); + } + + template + std::optional Execute_Field(int cycle, double time, int rank, Field& field, std::optional& node_in) { + static_assert(Field::dim == 3, "AscentAdaptor only supports 3D"); + // conduit blueprint definition (v.8.3) + // https://llnl-conduit.readthedocs.io/en/latest/blueprint_mesh.html + conduit::Node mesh; + if (node_in) + mesh = node_in.value(); + + auto nGhost = field.getNghost(); + + typename Field::view_type::host_mirror_type host_view = + Kokkos::create_mirror_view_and_copy(Kokkos::HostSpace(), field.getView()); + + Kokkos::View + host_view_layout_left("host_view_layout_left", + field.getLayout().getLocalNDIndex()[0].length(), + field.getLayout().getLocalNDIndex()[1].length(), + field.getLayout().getLocalNDIndex()[2].length()); + + for (size_t i = 0; i < field.getLayout().getLocalNDIndex()[0].length(); ++i) { + for (size_t j = 0; j < field.getLayout().getLocalNDIndex()[1].length(); ++j) { + for (size_t k = 0; k < field.getLayout().getLocalNDIndex()[2].length(); ++k) { + host_view_layout_left(i, j, k) = host_view(i + nGhost, j + nGhost, k + nGhost); + } + } + } + + + mesh["state/cycle"] = cycle; + mesh["state/time"] = time; + + mesh["coordsets/coords/type"].set_string("uniform"); + + // number of points in specific dimension + std::string field_node_dim{"coordsets/coords/dims/i"}; + std::string field_node_origin{"coordsets/coords/origin/x"}; + std::string field_node_spacing{"coordsets/coords/spacing/dx"}; + + for (unsigned int iDim = 0; iDim < field.get_mesh().getGridsize().dim; ++iDim) { + // add dimension + mesh[field_node_dim].set(field.getLayout().getLocalNDIndex()[iDim].length() + 1); + + // add origin + mesh[field_node_origin].set( + field.get_mesh().getOrigin()[iDim] + field.getLayout().getLocalNDIndex()[iDim].first() + * field.get_mesh().getMeshSpacing(iDim)); + + // add spacing + mesh[field_node_spacing].set(field.get_mesh().getMeshSpacing(iDim)); + + // increment last char in string + ++field_node_dim.back(); + ++field_node_origin.back(); + ++field_node_spacing.back(); + } + + // add topology + mesh["topologies/mesh/type"].set_string("uniform"); + mesh["topologies/mesh/coordset"].set_string("coords"); + std::string field_node_origin_topo{"topologies/mesh/origin/x"}; + for (unsigned int iDim = 0; iDim < field.get_mesh().getGridsize().dim; ++iDim) { + // shift origin + mesh[field_node_origin_topo].set(field.get_mesh().getOrigin()[iDim] + + field.getLayout().getLocalNDIndex()[iDim].first() + * field.get_mesh().getMeshSpacing(iDim)); + + // increment last char in string + ++field_node_origin_topo.back(); + } + + // add values and subscribe to data + auto fields = mesh["fields"]; + setData(fields, host_view_layout_left); + + // as we have a local copy of the field, the catalyst_execute needs to be called + // within this scope otherwise the memory location might be already overwritten + if (node_in == std::nullopt) + { + //callCatalystExecute(node); + return {}; + } + else + return mesh; + + } + + template + std::optional Execute_Particle(int cycle, double time, int rank, ParticleContainer& particleContainer, std::optional& node_in) { + assert((particleContainer->ID.getView().data() != nullptr) && "ID view should not be nullptr, might be missing the right execution space"); + + //auto layout_view = particleContainer->R.getView(); + typename ippl::ParticleAttrib>::HostMirror R_host = particleContainer->R.getHostMirror(); + typename ippl::ParticleAttrib>::HostMirror P_host = particleContainer->P.getHostMirror(); + typename ippl::ParticleAttrib::HostMirror q_host = particleContainer->q.getHostMirror(); + typename ippl::ParticleAttrib::HostMirror ID_host = particleContainer->ID.getHostMirror(); + Kokkos::deep_copy(R_host, particleContainer->R.getView()); + Kokkos::deep_copy(P_host, particleContainer->P.getView()); + Kokkos::deep_copy(q_host, particleContainer->q.getView()); + Kokkos::deep_copy(ID_host, particleContainer->ID.getView()); + + // if node is passed in, append data to it + conduit::Node mesh; + if (node_in) + mesh = node_in.value(); + + + mesh["state/cycle"] = cycle; + mesh["state/time"] = time; + + mesh["coordsets/coords/type"].set_string("explicit"); + + //mesh["coordsets/coords/values/x"].set_external(&layout_view.data()[0][0], particleContainer->getLocalNum(), 0, sizeof(double)*3); + //mesh["coordsets/coords/values/y"].set_external(&layout_view.data()[0][1], particleContainer->getLocalNum(), 0, sizeof(double)*3); + //mesh["coordsets/coords/values/z"].set_external(&layout_view.data()[0][2], particleContainer->getLocalNum(), 0, sizeof(double)*3); + mesh["coordsets/coords/values/x"].set_external(&R_host.data()[0][0], particleContainer->getLocalNum(), 0, sizeof(double)*3); + mesh["coordsets/coords/values/y"].set_external(&R_host.data()[0][1], particleContainer->getLocalNum(), 0, sizeof(double)*3); + mesh["coordsets/coords/values/z"].set_external(&R_host.data()[0][2], particleContainer->getLocalNum(), 0, sizeof(double)*3); + + mesh["topologies/mesh/type"].set_string("unstructured"); + mesh["topologies/mesh/coordset"].set_string("coords"); + mesh["topologies/mesh/elements/shape"].set_string("point"); + //mesh["topologies/mesh/elements/connectivity"].set_external(particleContainer->ID.getView().data(),particleContainer->getLocalNum()); + mesh["topologies/mesh/elements/connectivity"].set_external(ID_host.data(),particleContainer->getLocalNum()); + + //auto charge_view = particleContainer->getQ().getView(); + + // add values for scalar charge field + auto fields = mesh["fields"]; + fields["charge/association"].set_string("vertex"); + fields["charge/topology"].set_string("mesh"); + fields["charge/volume_dependent"].set_string("false"); + + //fields["charge/values"].set_external(particleContainer->q.getView().data(), particleContainer->getLocalNum()); + fields["charge/values"].set_external(q_host.data(), particleContainer->getLocalNum()); + + // add values for vector velocity field + //auto velocity_view = particleContainer->P.getView(); + fields["velocity/association"].set_string("vertex"); + fields["velocity/topology"].set_string("mesh"); + fields["velocity/volume_dependent"].set_string("false"); + + //fields["velocity/values/x"].set_external(&velocity_view.data()[0][0], particleContainer->getLocalNum(),0 ,sizeof(double)*3); + //fields["velocity/values/y"].set_external(&velocity_view.data()[0][1], particleContainer->getLocalNum(),0 ,sizeof(double)*3); + //fields["velocity/values/z"].set_external(&velocity_view.data()[0][2], particleContainer->getLocalNum(),0 ,sizeof(double)*3); + fields["velocity/values/x"].set_external(&P_host.data()[0][0], particleContainer->getLocalNum(),0 ,sizeof(double)*3); + fields["velocity/values/y"].set_external(&P_host.data()[0][1], particleContainer->getLocalNum(),0 ,sizeof(double)*3); + fields["velocity/values/z"].set_external(&P_host.data()[0][2], particleContainer->getLocalNum(),0 ,sizeof(double)*3); + + fields["position/association"].set_string("vertex"); + fields["position/topology"].set_string("mesh"); + fields["position/volume_dependent"].set_string("false"); + + //fields["position/values/x"].set_external(&layout_view.data()[0][0], particleContainer->getLocalNum(), 0, sizeof(double)*3); + //fields["position/values/y"].set_external(&layout_view.data()[0][1], particleContainer->getLocalNum(), 0, sizeof(double)*3); + //fields["position/values/z"].set_external(&layout_view.data()[0][2], particleContainer->getLocalNum(), 0, sizeof(double)*3); + fields["position/values/x"].set_external(&R_host.data()[0][0], particleContainer->getLocalNum(), 0, sizeof(double)*3); + fields["position/values/y"].set_external(&R_host.data()[0][1], particleContainer->getLocalNum(), 0, sizeof(double)*3); + fields["position/values/z"].set_external(&R_host.data()[0][2], particleContainer->getLocalNum(), 0, sizeof(double)*3); + + // this node we can return as the pointer to velocity and charge is globally valid + if (node_in == std::nullopt) + { + //callCatalystExecute(node); + return {}; + } + else + return mesh; + } + + + template + void Execute_Field_Particle(int cycle, double time, int rank, Field& field, ParticleContainer& particle) { + + auto node = std::make_optional(); + node = AscentAdaptor::Execute_Particle(cycle, time, rank, particle, node); + AscentAdaptor::Execute_Field(cycle, time, rank, field, node); + } + + void Finalize() { + conduit::Node node; + + } +} // namespace CatalystAdaptor + +#endif diff --git a/src/Stream/InSitu/CMakeLists.txt b/src/Stream/InSitu/CMakeLists.txt index 6b3ddbcfc..9d95b0bcb 100644 --- a/src/Stream/InSitu/CMakeLists.txt +++ b/src/Stream/InSitu/CMakeLists.txt @@ -2,6 +2,7 @@ set (_SRCS ) set (_HDRS + AscentAdaptor.h CatalystAdaptor.h ) From 0bb7c587aec9f1945435189102960da4882adfec Mon Sep 17 00:00:00 2001 From: Victor Mateevitsi Date: Mon, 27 Jan 2025 21:22:03 +0000 Subject: [PATCH 09/14] Added ascent instrumentation --- alpine/PenningTrap.cpp | 8 +++ alpine/PenningTrapManager.h | 4 +- src/Stream/InSitu/AscentAdaptor.h | 103 +++++++++++++++--------------- 3 files changed, 62 insertions(+), 53 deletions(-) diff --git a/alpine/PenningTrap.cpp b/alpine/PenningTrap.cpp index 6dbb8f9f6..32a571db9 100644 --- a/alpine/PenningTrap.cpp +++ b/alpine/PenningTrap.cpp @@ -51,6 +51,10 @@ int main(int argc, char* argv[]) { CatalystAdaptor::Initialize(argc, argv); #endif +#ifdef ENABLE_ASCENT + AscentAdaptor::Initialize(argc, argv); +#endif + Inform msg(TestName); Inform msg2all(TestName, INFORM_ALL_NODES); @@ -90,6 +94,10 @@ int main(int argc, char* argv[]) { CatalystAdaptor::Finalize(); #endif +#ifdef ENABLE_ASCENT + AscentAdaptor::Finalize(); +#endif + IpplTimings::stopTimer(mainTimer); IpplTimings::print(); diff --git a/alpine/PenningTrapManager.h b/alpine/PenningTrapManager.h index d0fbb364c..5e255a3c3 100644 --- a/alpine/PenningTrapManager.h +++ b/alpine/PenningTrapManager.h @@ -299,9 +299,9 @@ class PenningTrapManager : public AlpineManager { #endif #ifdef ENABLE_ASCENT - std::optional node = std::nullopt; auto *rho = &this->fcontainer_m->getRho(); - AscentAdaptor::Execute_Field(it, this->time_m, ippl::Comm->rank(), *rho, node); + //AscentAdaptor::Execute_Field(it, this->time_m, ippl::Comm->rank(), *rho); + AscentAdaptor::Execute_Particle(it, this->time_m, ippl::Comm->rank(), pc); #endif // Field solve diff --git a/src/Stream/InSitu/AscentAdaptor.h b/src/Stream/InSitu/AscentAdaptor.h index 48c92155f..9a381320c 100644 --- a/src/Stream/InSitu/AscentAdaptor.h +++ b/src/Stream/InSitu/AscentAdaptor.h @@ -55,13 +55,11 @@ namespace AscentAdaptor { } template - std::optional Execute_Field(int cycle, double time, int rank, Field& field, std::optional& node_in) { + std::optional Execute_Field(int cycle, double time, int rank, Field& field) { static_assert(Field::dim == 3, "AscentAdaptor only supports 3D"); // conduit blueprint definition (v.8.3) // https://llnl-conduit.readthedocs.io/en/latest/blueprint_mesh.html conduit::Node mesh; - if (node_in) - mesh = node_in.value(); auto nGhost = field.getNghost(); @@ -126,23 +124,26 @@ namespace AscentAdaptor { } // add values and subscribe to data - auto fields = mesh["fields"]; + auto &fields = mesh["fields"]; setData(fields, host_view_layout_left); - // as we have a local copy of the field, the catalyst_execute needs to be called - // within this scope otherwise the memory location might be already overwritten - if (node_in == std::nullopt) + conduit::Node verify_info; + if(!conduit::blueprint::mesh::verify(mesh, verify_info)) { - //callCatalystExecute(node); - return {}; + std::cerr << "Mesh verification failed!" << std::endl; + verify_info.print(); + exit(EXIT_FAILURE); } - else - return mesh; + conduit::Node actions; + mAscent.publish(mesh); + mAscent.execute(actions); + + return mesh; } template - std::optional Execute_Particle(int cycle, double time, int rank, ParticleContainer& particleContainer, std::optional& node_in) { + std::optional Execute_Particle(int cycle, double time, int rank, ParticleContainer& particleContainer) { assert((particleContainer->ID.getView().data() != nullptr) && "ID view should not be nullptr, might be missing the right execution space"); //auto layout_view = particleContainer->R.getView(); @@ -154,87 +155,87 @@ namespace AscentAdaptor { Kokkos::deep_copy(P_host, particleContainer->P.getView()); Kokkos::deep_copy(q_host, particleContainer->q.getView()); Kokkos::deep_copy(ID_host, particleContainer->ID.getView()); - - // if node is passed in, append data to it + conduit::Node mesh; - if (node_in) - mesh = node_in.value(); - mesh["state/cycle"] = cycle; mesh["state/time"] = time; - mesh["coordsets/coords/type"].set_string("explicit"); + mesh["coordsets/particle_coords/type"].set_string("explicit"); //mesh["coordsets/coords/values/x"].set_external(&layout_view.data()[0][0], particleContainer->getLocalNum(), 0, sizeof(double)*3); //mesh["coordsets/coords/values/y"].set_external(&layout_view.data()[0][1], particleContainer->getLocalNum(), 0, sizeof(double)*3); //mesh["coordsets/coords/values/z"].set_external(&layout_view.data()[0][2], particleContainer->getLocalNum(), 0, sizeof(double)*3); - mesh["coordsets/coords/values/x"].set_external(&R_host.data()[0][0], particleContainer->getLocalNum(), 0, sizeof(double)*3); - mesh["coordsets/coords/values/y"].set_external(&R_host.data()[0][1], particleContainer->getLocalNum(), 0, sizeof(double)*3); - mesh["coordsets/coords/values/z"].set_external(&R_host.data()[0][2], particleContainer->getLocalNum(), 0, sizeof(double)*3); + mesh["coordsets/particle_coords/values/x"].set_external(&R_host.data()[0][0], particleContainer->getLocalNum(), 0, sizeof(double)*3); + mesh["coordsets/particle_coords/values/y"].set_external(&R_host.data()[0][1], particleContainer->getLocalNum(), 0, sizeof(double)*3); + mesh["coordsets/particle_coords/values/z"].set_external(&R_host.data()[0][2], particleContainer->getLocalNum(), 0, sizeof(double)*3); - mesh["topologies/mesh/type"].set_string("unstructured"); - mesh["topologies/mesh/coordset"].set_string("coords"); - mesh["topologies/mesh/elements/shape"].set_string("point"); + mesh["topologies/particle_topo/type"].set_string("unstructured"); + mesh["topologies/particle_topo/coordset"].set_string("particle_coords"); + mesh["topologies/particle_topo/elements/shape"].set_string("point"); //mesh["topologies/mesh/elements/connectivity"].set_external(particleContainer->ID.getView().data(),particleContainer->getLocalNum()); - mesh["topologies/mesh/elements/connectivity"].set_external(ID_host.data(),particleContainer->getLocalNum()); + mesh["topologies/particle_topo/elements/connectivity"].set_external(ID_host.data(),particleContainer->getLocalNum()); //auto charge_view = particleContainer->getQ().getView(); // add values for scalar charge field - auto fields = mesh["fields"]; - fields["charge/association"].set_string("vertex"); - fields["charge/topology"].set_string("mesh"); - fields["charge/volume_dependent"].set_string("false"); + auto &fields = mesh["fields"]; + fields["particle_charge/association"].set_string("vertex"); + fields["particle_charge/topology"].set_string("particle_topo"); + fields["particle_charge/volume_dependent"].set_string("false"); //fields["charge/values"].set_external(particleContainer->q.getView().data(), particleContainer->getLocalNum()); - fields["charge/values"].set_external(q_host.data(), particleContainer->getLocalNum()); + fields["particle_charge/values"].set_external(q_host.data(), particleContainer->getLocalNum()); // add values for vector velocity field //auto velocity_view = particleContainer->P.getView(); - fields["velocity/association"].set_string("vertex"); - fields["velocity/topology"].set_string("mesh"); - fields["velocity/volume_dependent"].set_string("false"); + fields["particle_velocity/association"].set_string("vertex"); + fields["particle_velocity/topology"].set_string("particle_topo"); + fields["particle_velocity/volume_dependent"].set_string("false"); //fields["velocity/values/x"].set_external(&velocity_view.data()[0][0], particleContainer->getLocalNum(),0 ,sizeof(double)*3); //fields["velocity/values/y"].set_external(&velocity_view.data()[0][1], particleContainer->getLocalNum(),0 ,sizeof(double)*3); //fields["velocity/values/z"].set_external(&velocity_view.data()[0][2], particleContainer->getLocalNum(),0 ,sizeof(double)*3); - fields["velocity/values/x"].set_external(&P_host.data()[0][0], particleContainer->getLocalNum(),0 ,sizeof(double)*3); - fields["velocity/values/y"].set_external(&P_host.data()[0][1], particleContainer->getLocalNum(),0 ,sizeof(double)*3); - fields["velocity/values/z"].set_external(&P_host.data()[0][2], particleContainer->getLocalNum(),0 ,sizeof(double)*3); + fields["particle_velocity/values/x"].set_external(&P_host.data()[0][0], particleContainer->getLocalNum(),0 ,sizeof(double)*3); + fields["particle_velocity/values/y"].set_external(&P_host.data()[0][1], particleContainer->getLocalNum(),0 ,sizeof(double)*3); + fields["particle_velocity/values/z"].set_external(&P_host.data()[0][2], particleContainer->getLocalNum(),0 ,sizeof(double)*3); - fields["position/association"].set_string("vertex"); - fields["position/topology"].set_string("mesh"); - fields["position/volume_dependent"].set_string("false"); + fields["particle_position/association"].set_string("vertex"); + fields["particle_position/topology"].set_string("particle_topo"); + fields["particle_position/volume_dependent"].set_string("false"); //fields["position/values/x"].set_external(&layout_view.data()[0][0], particleContainer->getLocalNum(), 0, sizeof(double)*3); //fields["position/values/y"].set_external(&layout_view.data()[0][1], particleContainer->getLocalNum(), 0, sizeof(double)*3); //fields["position/values/z"].set_external(&layout_view.data()[0][2], particleContainer->getLocalNum(), 0, sizeof(double)*3); - fields["position/values/x"].set_external(&R_host.data()[0][0], particleContainer->getLocalNum(), 0, sizeof(double)*3); - fields["position/values/y"].set_external(&R_host.data()[0][1], particleContainer->getLocalNum(), 0, sizeof(double)*3); - fields["position/values/z"].set_external(&R_host.data()[0][2], particleContainer->getLocalNum(), 0, sizeof(double)*3); + fields["particle_position/values/x"].set_external(&R_host.data()[0][0], particleContainer->getLocalNum(), 0, sizeof(double)*3); + fields["particle_position/values/y"].set_external(&R_host.data()[0][1], particleContainer->getLocalNum(), 0, sizeof(double)*3); + fields["particle_position/values/z"].set_external(&R_host.data()[0][2], particleContainer->getLocalNum(), 0, sizeof(double)*3); - // this node we can return as the pointer to velocity and charge is globally valid - if (node_in == std::nullopt) + conduit::Node verify_info; + if(!conduit::blueprint::mesh::verify(mesh, verify_info)) { - //callCatalystExecute(node); - return {}; + std::cerr << "Mesh verification failed!" << std::endl; + verify_info.print(); + exit(EXIT_FAILURE); } - else - return mesh; + conduit::Node actions; + mAscent.publish(mesh); + mAscent.execute(actions); + + return mesh; } template void Execute_Field_Particle(int cycle, double time, int rank, Field& field, ParticleContainer& particle) { - auto node = std::make_optional(); - node = AscentAdaptor::Execute_Particle(cycle, time, rank, particle, node); - AscentAdaptor::Execute_Field(cycle, time, rank, field, node); + AscentAdaptor::Execute_Particle(cycle, time, rank, particle); + AscentAdaptor::Execute_Field(cycle, time, rank, field); } void Finalize() { conduit::Node node; + mAscent.close(); } } // namespace CatalystAdaptor From acc4877af8cb2b33b8acb7f458f67e55eea43734 Mon Sep 17 00:00:00 2001 From: Victor Mateevitsi Date: Mon, 27 Jan 2025 22:58:39 +0000 Subject: [PATCH 10/14] WIP --- alpine/AscentAdaptor.h | 304 ++++++++++++++++++++++++++++++ alpine/PenningTrap.cpp | 10 +- alpine/PenningTrapManager.h | 14 +- src/Stream/InSitu/AscentAdaptor.h | 243 ------------------------ 4 files changed, 323 insertions(+), 248 deletions(-) create mode 100644 alpine/AscentAdaptor.h delete mode 100644 src/Stream/InSitu/AscentAdaptor.h diff --git a/alpine/AscentAdaptor.h b/alpine/AscentAdaptor.h new file mode 100644 index 000000000..a280fed2d --- /dev/null +++ b/alpine/AscentAdaptor.h @@ -0,0 +1,304 @@ +#ifndef AsecntAdaptor_h +#define AsecntAdaptor_h + +#include "Ippl.h" + +#include +#include +#include +#include +#include + +#include "Utility/IpplException.h" + + +namespace AscentAdaptor { + + ascent::Ascent mAscent; + int mFrequency = 1; + + template + using FieldVariant = std::variant*, VField_t*>; + + template + using FieldPair = std::pair>; + + template + using ParticlePair = std::pair > >; + + using View_vector = + Kokkos::View***, Kokkos::LayoutLeft, Kokkos::HostSpace>; + inline void setData(conduit::Node& node, const View_vector& view) { + node["electrostatic/association"].set_string("element"); + node["electrostatic/topology"].set_string("mesh"); + node["electrostatic/volume_dependent"].set_string("false"); + + auto length = std::size(view); + + // offset is zero as we start without the ghost cells + // stride is 1 as we have every index of the array + node["electrostatic/values/x"].set_external(&view.data()[0][0], length, 0, 1); + node["electrostatic/values/y"].set_external(&view.data()[0][1], length, 0, 1); + node["electrostatic/values/z"].set_external(&view.data()[0][2], length, 0, 1); + } + + using View_scalar = Kokkos::View; + inline void setData(conduit::Node& node, const View_scalar& view) { + node["density/association"].set_string("element"); + node["density/topology"].set_string("mesh"); + node["density/volume_dependent"].set_string("false"); + + node["density/values"].set_external(view.data(), view.size()); + } + + void Initialize(int frequency) { + MPI_Comm ascent_comm; + mFrequency = frequency; + + // Split communicator based on the task ID + MPI_Comm_dup(MPI_COMM_WORLD, &ascent_comm); + + conduit::Node ascent_opts; + ascent_opts["mpi_comm"] = MPI_Comm_c2f(ascent_comm); + mAscent.open(ascent_opts); + } + + void Finalize() { + conduit::Node node; + mAscent.close(); + + } + + + void Execute_Particle( + const auto& particleContainer, + const auto& R_host, const auto& P_host, const auto& q_host, const auto& ID_host, + conduit::Node& node) { + + + node["coordsets/particle_coords/type"].set_string("explicit"); + + //mesh["coordsets/coords/values/x"].set_external(&layout_view.data()[0][0], particleContainer->getLocalNum(), 0, sizeof(double)*3); + //mesh["coordsets/coords/values/y"].set_external(&layout_view.data()[0][1], particleContainer->getLocalNum(), 0, sizeof(double)*3); + //mesh["coordsets/coords/values/z"].set_external(&layout_view.data()[0][2], particleContainer->getLocalNum(), 0, sizeof(double)*3); + node["coordsets/particle_coords/values/x"].set_external(&R_host.data()[0][0], particleContainer->getLocalNum(), 0, sizeof(double)*3); + node["coordsets/particle_coords/values/y"].set_external(&R_host.data()[0][1], particleContainer->getLocalNum(), 0, sizeof(double)*3); + node["coordsets/particle_coords/values/z"].set_external(&R_host.data()[0][2], particleContainer->getLocalNum(), 0, sizeof(double)*3); + + node["topologies/particle_topo/type"].set_string("unstructured"); + node["topologies/particle_topo/coordset"].set_string("particle_coords"); + node["topologies/particle_topo/elements/shape"].set_string("point"); + //mesh["topologies/mesh/elements/connectivity"].set_external(particleContainer->ID.getView().data(),particleContainer->getLocalNum()); + node["topologies/particle_topo/elements/connectivity"].set_external(ID_host.data(),particleContainer->getLocalNum()); + + //auto charge_view = particleContainer->getQ().getView(); + + // add values for scalar charge field + auto &fields = node["fields"]; + fields["particle_charge/association"].set_string("vertex"); + fields["particle_charge/topology"].set_string("particle_topo"); + fields["particle_charge/volume_dependent"].set_string("false"); + + //fields["charge/values"].set_external(particleContainer->q.getView().data(), particleContainer->getLocalNum()); + fields["particle_charge/values"].set_external(q_host.data(), particleContainer->getLocalNum()); + + // add values for vector velocity field + //auto velocity_view = particleContainer->P.getView(); + fields["particle_velocity/association"].set_string("vertex"); + fields["particle_velocity/topology"].set_string("particle_topo"); + fields["particle_velocity/volume_dependent"].set_string("false"); + + //fields["velocity/values/x"].set_external(&velocity_view.data()[0][0], particleContainer->getLocalNum(),0 ,sizeof(double)*3); + //fields["velocity/values/y"].set_external(&velocity_view.data()[0][1], particleContainer->getLocalNum(),0 ,sizeof(double)*3); + //fields["velocity/values/z"].set_external(&velocity_view.data()[0][2], particleContainer->getLocalNum(),0 ,sizeof(double)*3); + fields["particle_velocity/values/x"].set_external(&P_host.data()[0][0], particleContainer->getLocalNum(),0 ,sizeof(double)*3); + fields["particle_velocity/values/y"].set_external(&P_host.data()[0][1], particleContainer->getLocalNum(),0 ,sizeof(double)*3); + fields["particle_velocity/values/z"].set_external(&P_host.data()[0][2], particleContainer->getLocalNum(),0 ,sizeof(double)*3); + + fields["particle_position/association"].set_string("vertex"); + fields["particle_position/topology"].set_string("particle_topo"); + fields["particle_position/volume_dependent"].set_string("false"); + + //fields["position/values/x"].set_external(&layout_view.data()[0][0], particleContainer->getLocalNum(), 0, sizeof(double)*3); + //fields["position/values/y"].set_external(&layout_view.data()[0][1], particleContainer->getLocalNum(), 0, sizeof(double)*3); + //fields["position/values/z"].set_external(&layout_view.data()[0][2], particleContainer->getLocalNum(), 0, sizeof(double)*3); + fields["particle_position/values/x"].set_external(&R_host.data()[0][0], particleContainer->getLocalNum(), 0, sizeof(double)*3); + fields["particle_position/values/y"].set_external(&R_host.data()[0][1], particleContainer->getLocalNum(), 0, sizeof(double)*3); + fields["particle_position/values/z"].set_external(&R_host.data()[0][2], particleContainer->getLocalNum(), 0, sizeof(double)*3); + + conduit::Node verify_info; + if(!conduit::blueprint::mesh::verify(node, verify_info)) + { + std::cerr << "Mesh verification failed!" << std::endl; + verify_info.print(); + exit(EXIT_FAILURE); + } + } + + + template + void Execute_Field(Field* field, const std::string& fieldName, + Kokkos::View& host_view_layout_left, + conduit::Node& node) { + static_assert(Field::dim == 3, "AscentAdaptor only supports 3D"); + + + node["coordsets/coords/type"].set_string("uniform"); + + // number of points in specific dimension + std::string field_node_dim{"coordsets/coords/dims/i"}; + std::string field_node_origin{"coordsets/coords/origin/x"}; + std::string field_node_spacing{"coordsets/coords/spacing/dx"}; + + for (unsigned int iDim = 0; iDim < field->get_mesh().getGridsize().dim; ++iDim) { + // add dimension + node[field_node_dim].set(field->getLayout().getLocalNDIndex()[iDim].length() + 1); + + // add origin + node[field_node_origin].set( + field->get_mesh().getOrigin()[iDim] + field->getLayout().getLocalNDIndex()[iDim].first() + * field->get_mesh().getMeshSpacing(iDim)); + + // add spacing + node[field_node_spacing].set(field->get_mesh().getMeshSpacing(iDim)); + + // increment last char in string + ++field_node_dim.back(); + ++field_node_origin.back(); + ++field_node_spacing.back(); + } + + // add topology + node["topologies/mesh/type"].set_string("uniform"); + node["topologies/mesh/coordset"].set_string("coords"); + std::string field_node_origin_topo{"topologies/mesh/origin/x"}; + for (unsigned int iDim = 0; iDim < field->get_mesh().getGridsize().dim; ++iDim) { + // shift origin + node[field_node_origin_topo].set(field->get_mesh().getOrigin()[iDim] + + field->getLayout().getLocalNDIndex()[iDim].first() + * field->get_mesh().getMeshSpacing(iDim)); + + // increment last char in string + ++field_node_origin_topo.back(); + } + + // B) Set the field values + + // Initialize the existing Kokkos::View + host_view_layout_left = Kokkos::View( + "host_view_layout_left", + field->getLayout().getLocalNDIndex()[0].length(), + field->getLayout().getLocalNDIndex()[1].length(), + field->getLayout().getLocalNDIndex()[2].length()); + + // Creates a host-accessible mirror view and copies the data from the device view to the host. + auto host_view = + Kokkos::create_mirror_view_and_copy(Kokkos::HostSpace(), field->getView()); + + // Copy data from field to the memory+style which will be passed to Catalyst + auto nGhost = field->getNghost(); + for (size_t i = 0; i < field->getLayout().getLocalNDIndex()[0].length(); ++i) { + for (size_t j = 0; j < field->getLayout().getLocalNDIndex()[1].length(); ++j) { + for (size_t k = 0; k < field->getLayout().getLocalNDIndex()[2].length(); ++k) { + host_view_layout_left(i, j, k) = host_view(i + nGhost, j + nGhost, k + nGhost); + } + } + } + + auto &fields = node["fields"]; + setData(fields, host_view_layout_left); + + conduit::Node verify_info; + if(!conduit::blueprint::mesh::verify(node, verify_info)) + { + std::cerr << "Mesh verification failed!" << std::endl; + verify_info.print(); + exit(EXIT_FAILURE); + } + } + + + + template + void Execute(int cycle, double time, int rank, + // const auto& /* std::shared_ptr >& */ particle, + const std::vector>& particles, + const std::vector>& fields, double& scaleFactor) { + conduit::Node node; + + // add time/cycle information + auto state = node["state"]; + state["cycle"].set(cycle); + state["time"].set(time); + + // Handle particles + + std::map>::HostMirror> R_host_map; + std::map>::HostMirror> P_host_map; + std::map::HostMirror> q_host_map; + std::map::HostMirror> ID_host_map; + + // Loop over all particle container + for (const auto& particlesPair : particles) + { + const std::string& particlesName = particlesPair.first; + const auto particleContainer = particlesPair.second; + + assert((particleContainer->ID.getView().data() != nullptr) && "ID view should not be nullptr, might be missing the right execution space"); + + R_host_map[particlesName] = particleContainer->R.getHostMirror(); + P_host_map[particlesName] = particleContainer->P.getHostMirror(); + q_host_map[particlesName] = particleContainer->q.getHostMirror(); + ID_host_map[particlesName] = particleContainer->ID.getHostMirror(); + + Kokkos::deep_copy(R_host_map[particlesName], particleContainer->R.getView()); + Kokkos::deep_copy(P_host_map[particlesName], particleContainer->P.getView()); + Kokkos::deep_copy(q_host_map[particlesName], particleContainer->q.getView()); + Kokkos::deep_copy(ID_host_map[particlesName], particleContainer->ID.getView()); + + Execute_Particle( + particleContainer, + R_host_map[particlesName], P_host_map[particlesName], q_host_map[particlesName], ID_host_map[particlesName], + particlesName, + node); + } + + + // Handle fields + + // Map of all Kokkos::Views. This keeps a reference on all Kokkos::Views + // which ensures that Kokkos does not free the memory before the end of this function. + std::map::view_type::data_type, Kokkos::LayoutLeft, Kokkos::HostSpace> > scalar_host_views; + std::map::view_type::data_type, Kokkos::LayoutLeft, Kokkos::HostSpace> > vector_host_views; + + // Loop over all fields + for (const auto& fieldPair : fields) + { + const std::string& fieldName = fieldPair.first; + const auto& fieldVariant = fieldPair.second; + + // If field is a _scalar_ field + if (std::holds_alternative*>(fieldVariant)) { + Field_t* field = std::get*>(fieldVariant); + // == ippl::Field, Cell>* + + Execute_Field(field, fieldName, scalar_host_views[fieldName], node); + } + // If field is a _vector_ field + else if (std::holds_alternative*>(fieldVariant)) { + VField_t* field = std::get*>(fieldVariant); + // == ippl::Field, 3, ippl::UniformCartesian, Cell>* + + Execute_Field(field, fieldName, vector_host_views[fieldName], node); + } + } + + conduit::Node actions; + mAscent.publish(node); + mAscent.execute(actions); + + + } +} // namespace CatalystAdaptor + +#endif diff --git a/alpine/PenningTrap.cpp b/alpine/PenningTrap.cpp index 89b53f486..5c272af96 100644 --- a/alpine/PenningTrap.cpp +++ b/alpine/PenningTrap.cpp @@ -70,7 +70,15 @@ int main(int argc, char* argv[]) { #endif #ifdef ENABLE_ASCENT - AscentAdaptor::Initialize(argc, argv); + int frequency = 1; + for (int i = 1; i < argc; ++i) { + if (std::string(argv[i]) == "--frequency" && i + 1 < argc) { + frequency = atoi(argv[i+1]); + i++; + } + } + + AscentAdaptor::Initialize(frequency); #endif Inform msg(TestName); diff --git a/alpine/PenningTrapManager.h b/alpine/PenningTrapManager.h index 33149d368..fc3daf77f 100644 --- a/alpine/PenningTrapManager.h +++ b/alpine/PenningTrapManager.h @@ -19,7 +19,7 @@ #endif #ifdef ENABLE_ASCENT -#include "Stream/InSitu/AscentAdaptor.h" +#include "AscentAdaptor.h" #endif using view_type = typename ippl::detail::ViewType, 1>::view_type; @@ -302,9 +302,15 @@ class PenningTrapManager : public AlpineManager { #endif #ifdef ENABLE_ASCENT - auto *rho = &this->fcontainer_m->getRho(); - //AscentAdaptor::Execute_Field(it, this->time_m, ippl::Comm->rank(), *rho); - AscentAdaptor::Execute_Particle(it, this->time_m, ippl::Comm->rank(), pc); + std::vector> particles = { + {"particle", std::shared_ptr >(pc)}, + }; + std::vector> fields = { + {"E", AscentAdaptor::FieldVariant(&this->fcontainer_m->getE())}, + {"roh", AscentAdaptor::FieldVariant(&this->fcontainer_m->getRho())}, + //{"phi", CatalystAdaptor::FieldVariant(&this->fcontainer_m->getPhi())}, + }; + AscentAdaptor::Execute(it, this->time_m, ippl::Comm->rank(), particles, fields, scaleFactor); #endif // Field solve diff --git a/src/Stream/InSitu/AscentAdaptor.h b/src/Stream/InSitu/AscentAdaptor.h deleted file mode 100644 index 9a381320c..000000000 --- a/src/Stream/InSitu/AscentAdaptor.h +++ /dev/null @@ -1,243 +0,0 @@ -// SPDX-FileCopyrightText: Copyright (c) Kitware Inc. -// SPDX-License-Identifier: BSD-3-Clause -#ifndef AsecntAdaptor_h -#define AsecntAdaptor_h - -#include "Ippl.h" - -#include -#include -#include -#include -#include - -#include "Utility/IpplException.h" - - -namespace AscentAdaptor { - - ascent::Ascent mAscent; - - using View_vector = - Kokkos::View***, Kokkos::LayoutLeft, Kokkos::HostSpace>; - inline void setData(conduit::Node& node, const View_vector& view) { - node["electrostatic/association"].set_string("element"); - node["electrostatic/topology"].set_string("mesh"); - node["electrostatic/volume_dependent"].set_string("false"); - - auto length = std::size(view); - - // offset is zero as we start without the ghost cells - // stride is 1 as we have every index of the array - node["electrostatic/values/x"].set_external(&view.data()[0][0], length, 0, 1); - node["electrostatic/values/y"].set_external(&view.data()[0][1], length, 0, 1); - node["electrostatic/values/z"].set_external(&view.data()[0][2], length, 0, 1); - } - - using View_scalar = Kokkos::View; - inline void setData(conduit::Node& node, const View_scalar& view) { - node["density/association"].set_string("element"); - node["density/topology"].set_string("mesh"); - node["density/volume_dependent"].set_string("false"); - - node["density/values"].set_external(view.data(), view.size()); - } - - void Initialize(int argc, char* argv[]) { - MPI_Comm ascent_comm; - - // Split communicator based on the task ID - MPI_Comm_dup(MPI_COMM_WORLD, &ascent_comm); - - conduit::Node ascent_opts; - ascent_opts["mpi_comm"] = MPI_Comm_c2f(ascent_comm); - mAscent.open(ascent_opts); - } - - template - std::optional Execute_Field(int cycle, double time, int rank, Field& field) { - static_assert(Field::dim == 3, "AscentAdaptor only supports 3D"); - // conduit blueprint definition (v.8.3) - // https://llnl-conduit.readthedocs.io/en/latest/blueprint_mesh.html - conduit::Node mesh; - - auto nGhost = field.getNghost(); - - typename Field::view_type::host_mirror_type host_view = - Kokkos::create_mirror_view_and_copy(Kokkos::HostSpace(), field.getView()); - - Kokkos::View - host_view_layout_left("host_view_layout_left", - field.getLayout().getLocalNDIndex()[0].length(), - field.getLayout().getLocalNDIndex()[1].length(), - field.getLayout().getLocalNDIndex()[2].length()); - - for (size_t i = 0; i < field.getLayout().getLocalNDIndex()[0].length(); ++i) { - for (size_t j = 0; j < field.getLayout().getLocalNDIndex()[1].length(); ++j) { - for (size_t k = 0; k < field.getLayout().getLocalNDIndex()[2].length(); ++k) { - host_view_layout_left(i, j, k) = host_view(i + nGhost, j + nGhost, k + nGhost); - } - } - } - - - mesh["state/cycle"] = cycle; - mesh["state/time"] = time; - - mesh["coordsets/coords/type"].set_string("uniform"); - - // number of points in specific dimension - std::string field_node_dim{"coordsets/coords/dims/i"}; - std::string field_node_origin{"coordsets/coords/origin/x"}; - std::string field_node_spacing{"coordsets/coords/spacing/dx"}; - - for (unsigned int iDim = 0; iDim < field.get_mesh().getGridsize().dim; ++iDim) { - // add dimension - mesh[field_node_dim].set(field.getLayout().getLocalNDIndex()[iDim].length() + 1); - - // add origin - mesh[field_node_origin].set( - field.get_mesh().getOrigin()[iDim] + field.getLayout().getLocalNDIndex()[iDim].first() - * field.get_mesh().getMeshSpacing(iDim)); - - // add spacing - mesh[field_node_spacing].set(field.get_mesh().getMeshSpacing(iDim)); - - // increment last char in string - ++field_node_dim.back(); - ++field_node_origin.back(); - ++field_node_spacing.back(); - } - - // add topology - mesh["topologies/mesh/type"].set_string("uniform"); - mesh["topologies/mesh/coordset"].set_string("coords"); - std::string field_node_origin_topo{"topologies/mesh/origin/x"}; - for (unsigned int iDim = 0; iDim < field.get_mesh().getGridsize().dim; ++iDim) { - // shift origin - mesh[field_node_origin_topo].set(field.get_mesh().getOrigin()[iDim] - + field.getLayout().getLocalNDIndex()[iDim].first() - * field.get_mesh().getMeshSpacing(iDim)); - - // increment last char in string - ++field_node_origin_topo.back(); - } - - // add values and subscribe to data - auto &fields = mesh["fields"]; - setData(fields, host_view_layout_left); - - conduit::Node verify_info; - if(!conduit::blueprint::mesh::verify(mesh, verify_info)) - { - std::cerr << "Mesh verification failed!" << std::endl; - verify_info.print(); - exit(EXIT_FAILURE); - } - conduit::Node actions; - mAscent.publish(mesh); - mAscent.execute(actions); - - return mesh; - - } - - template - std::optional Execute_Particle(int cycle, double time, int rank, ParticleContainer& particleContainer) { - assert((particleContainer->ID.getView().data() != nullptr) && "ID view should not be nullptr, might be missing the right execution space"); - - //auto layout_view = particleContainer->R.getView(); - typename ippl::ParticleAttrib>::HostMirror R_host = particleContainer->R.getHostMirror(); - typename ippl::ParticleAttrib>::HostMirror P_host = particleContainer->P.getHostMirror(); - typename ippl::ParticleAttrib::HostMirror q_host = particleContainer->q.getHostMirror(); - typename ippl::ParticleAttrib::HostMirror ID_host = particleContainer->ID.getHostMirror(); - Kokkos::deep_copy(R_host, particleContainer->R.getView()); - Kokkos::deep_copy(P_host, particleContainer->P.getView()); - Kokkos::deep_copy(q_host, particleContainer->q.getView()); - Kokkos::deep_copy(ID_host, particleContainer->ID.getView()); - - conduit::Node mesh; - - mesh["state/cycle"] = cycle; - mesh["state/time"] = time; - - mesh["coordsets/particle_coords/type"].set_string("explicit"); - - //mesh["coordsets/coords/values/x"].set_external(&layout_view.data()[0][0], particleContainer->getLocalNum(), 0, sizeof(double)*3); - //mesh["coordsets/coords/values/y"].set_external(&layout_view.data()[0][1], particleContainer->getLocalNum(), 0, sizeof(double)*3); - //mesh["coordsets/coords/values/z"].set_external(&layout_view.data()[0][2], particleContainer->getLocalNum(), 0, sizeof(double)*3); - mesh["coordsets/particle_coords/values/x"].set_external(&R_host.data()[0][0], particleContainer->getLocalNum(), 0, sizeof(double)*3); - mesh["coordsets/particle_coords/values/y"].set_external(&R_host.data()[0][1], particleContainer->getLocalNum(), 0, sizeof(double)*3); - mesh["coordsets/particle_coords/values/z"].set_external(&R_host.data()[0][2], particleContainer->getLocalNum(), 0, sizeof(double)*3); - - mesh["topologies/particle_topo/type"].set_string("unstructured"); - mesh["topologies/particle_topo/coordset"].set_string("particle_coords"); - mesh["topologies/particle_topo/elements/shape"].set_string("point"); - //mesh["topologies/mesh/elements/connectivity"].set_external(particleContainer->ID.getView().data(),particleContainer->getLocalNum()); - mesh["topologies/particle_topo/elements/connectivity"].set_external(ID_host.data(),particleContainer->getLocalNum()); - - //auto charge_view = particleContainer->getQ().getView(); - - // add values for scalar charge field - auto &fields = mesh["fields"]; - fields["particle_charge/association"].set_string("vertex"); - fields["particle_charge/topology"].set_string("particle_topo"); - fields["particle_charge/volume_dependent"].set_string("false"); - - //fields["charge/values"].set_external(particleContainer->q.getView().data(), particleContainer->getLocalNum()); - fields["particle_charge/values"].set_external(q_host.data(), particleContainer->getLocalNum()); - - // add values for vector velocity field - //auto velocity_view = particleContainer->P.getView(); - fields["particle_velocity/association"].set_string("vertex"); - fields["particle_velocity/topology"].set_string("particle_topo"); - fields["particle_velocity/volume_dependent"].set_string("false"); - - //fields["velocity/values/x"].set_external(&velocity_view.data()[0][0], particleContainer->getLocalNum(),0 ,sizeof(double)*3); - //fields["velocity/values/y"].set_external(&velocity_view.data()[0][1], particleContainer->getLocalNum(),0 ,sizeof(double)*3); - //fields["velocity/values/z"].set_external(&velocity_view.data()[0][2], particleContainer->getLocalNum(),0 ,sizeof(double)*3); - fields["particle_velocity/values/x"].set_external(&P_host.data()[0][0], particleContainer->getLocalNum(),0 ,sizeof(double)*3); - fields["particle_velocity/values/y"].set_external(&P_host.data()[0][1], particleContainer->getLocalNum(),0 ,sizeof(double)*3); - fields["particle_velocity/values/z"].set_external(&P_host.data()[0][2], particleContainer->getLocalNum(),0 ,sizeof(double)*3); - - fields["particle_position/association"].set_string("vertex"); - fields["particle_position/topology"].set_string("particle_topo"); - fields["particle_position/volume_dependent"].set_string("false"); - - //fields["position/values/x"].set_external(&layout_view.data()[0][0], particleContainer->getLocalNum(), 0, sizeof(double)*3); - //fields["position/values/y"].set_external(&layout_view.data()[0][1], particleContainer->getLocalNum(), 0, sizeof(double)*3); - //fields["position/values/z"].set_external(&layout_view.data()[0][2], particleContainer->getLocalNum(), 0, sizeof(double)*3); - fields["particle_position/values/x"].set_external(&R_host.data()[0][0], particleContainer->getLocalNum(), 0, sizeof(double)*3); - fields["particle_position/values/y"].set_external(&R_host.data()[0][1], particleContainer->getLocalNum(), 0, sizeof(double)*3); - fields["particle_position/values/z"].set_external(&R_host.data()[0][2], particleContainer->getLocalNum(), 0, sizeof(double)*3); - - conduit::Node verify_info; - if(!conduit::blueprint::mesh::verify(mesh, verify_info)) - { - std::cerr << "Mesh verification failed!" << std::endl; - verify_info.print(); - exit(EXIT_FAILURE); - } - conduit::Node actions; - mAscent.publish(mesh); - mAscent.execute(actions); - - return mesh; - } - - - template - void Execute_Field_Particle(int cycle, double time, int rank, Field& field, ParticleContainer& particle) { - - AscentAdaptor::Execute_Particle(cycle, time, rank, particle); - AscentAdaptor::Execute_Field(cycle, time, rank, field); - } - - void Finalize() { - conduit::Node node; - mAscent.close(); - - } -} // namespace CatalystAdaptor - -#endif From 7d5d3948eca97baac6df755e05f503285dd6c8b3 Mon Sep 17 00:00:00 2001 From: Victor Mateevitsi Date: Tue, 28 Jan 2025 16:35:52 +0000 Subject: [PATCH 11/14] Updated code to latest Jens Henrik's version --- alpine/AscentAdaptor.h | 75 +++++++++++++++++++++--------------------- 1 file changed, 38 insertions(+), 37 deletions(-) diff --git a/alpine/AscentAdaptor.h b/alpine/AscentAdaptor.h index a280fed2d..05eecc82e 100644 --- a/alpine/AscentAdaptor.h +++ b/alpine/AscentAdaptor.h @@ -28,9 +28,9 @@ namespace AscentAdaptor { using View_vector = Kokkos::View***, Kokkos::LayoutLeft, Kokkos::HostSpace>; - inline void setData(conduit::Node& node, const View_vector& view) { + inline void setData(conduit::Node& node, const View_vector& view, const std::string& fieldName) { node["electrostatic/association"].set_string("element"); - node["electrostatic/topology"].set_string("mesh"); + node["electrostatic/topology"].set_string(fieldName + "_mesh"); node["electrostatic/volume_dependent"].set_string("false"); auto length = std::size(view); @@ -43,9 +43,9 @@ namespace AscentAdaptor { } using View_scalar = Kokkos::View; - inline void setData(conduit::Node& node, const View_scalar& view) { + inline void setData(conduit::Node& node, const View_scalar& view, const std::string& fieldName) { node["density/association"].set_string("element"); - node["density/topology"].set_string("mesh"); + node["density/topology"].set_string(fieldName + "_mesh"); node["density/volume_dependent"].set_string("false"); node["density/values"].set_external(view.data(), view.size()); @@ -73,58 +73,60 @@ namespace AscentAdaptor { void Execute_Particle( const auto& particleContainer, const auto& R_host, const auto& P_host, const auto& q_host, const auto& ID_host, + const std::string& particlesName, conduit::Node& node) { + std::cout << "Adding particles: " << particlesName << std::endl; - node["coordsets/particle_coords/type"].set_string("explicit"); + node["coordsets/" + particlesName + "_coords/type"].set_string("explicit"); //mesh["coordsets/coords/values/x"].set_external(&layout_view.data()[0][0], particleContainer->getLocalNum(), 0, sizeof(double)*3); //mesh["coordsets/coords/values/y"].set_external(&layout_view.data()[0][1], particleContainer->getLocalNum(), 0, sizeof(double)*3); //mesh["coordsets/coords/values/z"].set_external(&layout_view.data()[0][2], particleContainer->getLocalNum(), 0, sizeof(double)*3); - node["coordsets/particle_coords/values/x"].set_external(&R_host.data()[0][0], particleContainer->getLocalNum(), 0, sizeof(double)*3); - node["coordsets/particle_coords/values/y"].set_external(&R_host.data()[0][1], particleContainer->getLocalNum(), 0, sizeof(double)*3); - node["coordsets/particle_coords/values/z"].set_external(&R_host.data()[0][2], particleContainer->getLocalNum(), 0, sizeof(double)*3); + node["coordsets/" + particlesName + "_coords/values/x"].set_external(&R_host.data()[0][0], particleContainer->getLocalNum(), 0, sizeof(double)*3); + node["coordsets/" + particlesName + "_coords/values/y"].set_external(&R_host.data()[0][1], particleContainer->getLocalNum(), 0, sizeof(double)*3); + node["coordsets/" + particlesName + "_coords/values/z"].set_external(&R_host.data()[0][2], particleContainer->getLocalNum(), 0, sizeof(double)*3); - node["topologies/particle_topo/type"].set_string("unstructured"); - node["topologies/particle_topo/coordset"].set_string("particle_coords"); - node["topologies/particle_topo/elements/shape"].set_string("point"); + node["topologies/" + particlesName + "_topo/type"].set_string("unstructured"); + node["topologies/" + particlesName + "_topo/coordset"].set_string(particlesName + "_coords"); + node["topologies/" + particlesName + "_topo/elements/shape"].set_string("point"); //mesh["topologies/mesh/elements/connectivity"].set_external(particleContainer->ID.getView().data(),particleContainer->getLocalNum()); - node["topologies/particle_topo/elements/connectivity"].set_external(ID_host.data(),particleContainer->getLocalNum()); + node["topologies/" + particlesName + "_topo/elements/connectivity"].set_external(ID_host.data(),particleContainer->getLocalNum()); //auto charge_view = particleContainer->getQ().getView(); // add values for scalar charge field auto &fields = node["fields"]; - fields["particle_charge/association"].set_string("vertex"); - fields["particle_charge/topology"].set_string("particle_topo"); - fields["particle_charge/volume_dependent"].set_string("false"); + fields[particlesName + "_charge/association"].set_string("vertex"); + fields[particlesName + "_charge/topology"].set_string(particlesName + "_topo"); + fields[particlesName + "_charge/volume_dependent"].set_string("false"); //fields["charge/values"].set_external(particleContainer->q.getView().data(), particleContainer->getLocalNum()); - fields["particle_charge/values"].set_external(q_host.data(), particleContainer->getLocalNum()); + fields[particlesName + "_charge/values"].set_external(q_host.data(), particleContainer->getLocalNum()); // add values for vector velocity field //auto velocity_view = particleContainer->P.getView(); - fields["particle_velocity/association"].set_string("vertex"); - fields["particle_velocity/topology"].set_string("particle_topo"); - fields["particle_velocity/volume_dependent"].set_string("false"); + fields[particlesName + "_velocity/association"].set_string("vertex"); + fields[particlesName + "_velocity/topology"].set_string(particlesName + "_topo"); + fields[particlesName + "_velocity/volume_dependent"].set_string("false"); //fields["velocity/values/x"].set_external(&velocity_view.data()[0][0], particleContainer->getLocalNum(),0 ,sizeof(double)*3); //fields["velocity/values/y"].set_external(&velocity_view.data()[0][1], particleContainer->getLocalNum(),0 ,sizeof(double)*3); //fields["velocity/values/z"].set_external(&velocity_view.data()[0][2], particleContainer->getLocalNum(),0 ,sizeof(double)*3); - fields["particle_velocity/values/x"].set_external(&P_host.data()[0][0], particleContainer->getLocalNum(),0 ,sizeof(double)*3); - fields["particle_velocity/values/y"].set_external(&P_host.data()[0][1], particleContainer->getLocalNum(),0 ,sizeof(double)*3); - fields["particle_velocity/values/z"].set_external(&P_host.data()[0][2], particleContainer->getLocalNum(),0 ,sizeof(double)*3); + fields[particlesName + "_velocity/values/x"].set_external(&P_host.data()[0][0], particleContainer->getLocalNum(),0 ,sizeof(double)*3); + fields[particlesName + "_velocity/values/y"].set_external(&P_host.data()[0][1], particleContainer->getLocalNum(),0 ,sizeof(double)*3); + fields[particlesName + "_velocity/values/z"].set_external(&P_host.data()[0][2], particleContainer->getLocalNum(),0 ,sizeof(double)*3); - fields["particle_position/association"].set_string("vertex"); - fields["particle_position/topology"].set_string("particle_topo"); - fields["particle_position/volume_dependent"].set_string("false"); + fields[particlesName + "_position/association"].set_string("vertex"); + fields[particlesName + "_position/topology"].set_string(particlesName + "_topo"); + fields[particlesName + "_position/volume_dependent"].set_string("false"); //fields["position/values/x"].set_external(&layout_view.data()[0][0], particleContainer->getLocalNum(), 0, sizeof(double)*3); //fields["position/values/y"].set_external(&layout_view.data()[0][1], particleContainer->getLocalNum(), 0, sizeof(double)*3); //fields["position/values/z"].set_external(&layout_view.data()[0][2], particleContainer->getLocalNum(), 0, sizeof(double)*3); - fields["particle_position/values/x"].set_external(&R_host.data()[0][0], particleContainer->getLocalNum(), 0, sizeof(double)*3); - fields["particle_position/values/y"].set_external(&R_host.data()[0][1], particleContainer->getLocalNum(), 0, sizeof(double)*3); - fields["particle_position/values/z"].set_external(&R_host.data()[0][2], particleContainer->getLocalNum(), 0, sizeof(double)*3); + fields[particlesName + "_position/values/x"].set_external(&R_host.data()[0][0], particleContainer->getLocalNum(), 0, sizeof(double)*3); + fields[particlesName + "_position/values/y"].set_external(&R_host.data()[0][1], particleContainer->getLocalNum(), 0, sizeof(double)*3); + fields[particlesName + "_position/values/z"].set_external(&R_host.data()[0][2], particleContainer->getLocalNum(), 0, sizeof(double)*3); conduit::Node verify_info; if(!conduit::blueprint::mesh::verify(node, verify_info)) @@ -141,14 +143,13 @@ namespace AscentAdaptor { Kokkos::View& host_view_layout_left, conduit::Node& node) { static_assert(Field::dim == 3, "AscentAdaptor only supports 3D"); - - node["coordsets/coords/type"].set_string("uniform"); + node["coordsets/" + fieldName + "_coords/type"].set_string("uniform"); // number of points in specific dimension - std::string field_node_dim{"coordsets/coords/dims/i"}; - std::string field_node_origin{"coordsets/coords/origin/x"}; - std::string field_node_spacing{"coordsets/coords/spacing/dx"}; + std::string field_node_dim{"coordsets/" + fieldName + "_coords/dims/i"}; + std::string field_node_origin{"coordsets/" + fieldName + "_coords/origin/x"}; + std::string field_node_spacing{"coordsets/" + fieldName + "_coords/spacing/dx"}; for (unsigned int iDim = 0; iDim < field->get_mesh().getGridsize().dim; ++iDim) { // add dimension @@ -169,9 +170,9 @@ namespace AscentAdaptor { } // add topology - node["topologies/mesh/type"].set_string("uniform"); - node["topologies/mesh/coordset"].set_string("coords"); - std::string field_node_origin_topo{"topologies/mesh/origin/x"}; + node["topologies/" + fieldName + "_mesh/type"].set_string("uniform"); + node["topologies/" + fieldName + "_mesh/coordset"].set_string(fieldName + "_coords"); + std::string field_node_origin_topo{"topologies/" + fieldName + "_mesh/origin/x"}; for (unsigned int iDim = 0; iDim < field->get_mesh().getGridsize().dim; ++iDim) { // shift origin node[field_node_origin_topo].set(field->get_mesh().getOrigin()[iDim] @@ -206,7 +207,7 @@ namespace AscentAdaptor { } auto &fields = node["fields"]; - setData(fields, host_view_layout_left); + setData(fields, host_view_layout_left, fieldName); conduit::Node verify_info; if(!conduit::blueprint::mesh::verify(node, verify_info)) From ee4b7a214c4484ff33226c99bb7785ea41f406b3 Mon Sep 17 00:00:00 2001 From: Victor Mateevitsi Date: Wed, 29 Jan 2025 22:58:05 +0000 Subject: [PATCH 12/14] Finished ascent instrumentation --- .gitignore | 2 + CMakeLists.txt | 1 + alpine/AscentAdaptor.h | 98 ++++++++++++++++++++++++++++++++++--- alpine/CMakeLists.txt | 6 +++ alpine/PenningTrap.cpp | 1 + alpine/PenningTrapManager.h | 4 +- alpine/ascent_actions.yaml | 25 ++++++++++ run.polaris.sh | 9 ++++ sourceme.sh | 10 ++++ 9 files changed, 149 insertions(+), 7 deletions(-) create mode 100644 alpine/ascent_actions.yaml create mode 100644 run.polaris.sh create mode 100644 sourceme.sh diff --git a/.gitignore b/.gitignore index e64104dbf..626491737 100644 --- a/.gitignore +++ b/.gitignore @@ -27,3 +27,5 @@ compile_commands.json __pycache__ *.~ + +.vscode diff --git a/CMakeLists.txt b/CMakeLists.txt index 6dd79ce11..7c6c27511 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -196,6 +196,7 @@ if (ENABLE_FFT) endif() FetchContent_MakeAvailable(Heffte) # Required to suppress warnings related to heffte + MESSAGE(STATUS "HEFFTE: "${Heffte_SOURCE_DIR}"") target_include_directories(Heffte SYSTEM PUBLIC "${Heffte_SOURCE_DIR}/include") set(Heffte_DIR "${Heffte_SOURCE_DIR}") remove_flag_from_target(Heffte "-Werror") diff --git a/alpine/AscentAdaptor.h b/alpine/AscentAdaptor.h index 05eecc82e..fb1a18791 100644 --- a/alpine/AscentAdaptor.h +++ b/alpine/AscentAdaptor.h @@ -73,11 +73,10 @@ namespace AscentAdaptor { void Execute_Particle( const auto& particleContainer, const auto& R_host, const auto& P_host, const auto& q_host, const auto& ID_host, + const auto& magnitude_host, const std::string& particlesName, conduit::Node& node) { - std::cout << "Adding particles: " << particlesName << std::endl; - node["coordsets/" + particlesName + "_coords/type"].set_string("explicit"); //mesh["coordsets/coords/values/x"].set_external(&layout_view.data()[0][0], particleContainer->getLocalNum(), 0, sizeof(double)*3); @@ -128,6 +127,14 @@ namespace AscentAdaptor { fields[particlesName + "_position/values/y"].set_external(&R_host.data()[0][1], particleContainer->getLocalNum(), 0, sizeof(double)*3); fields[particlesName + "_position/values/z"].set_external(&R_host.data()[0][2], particleContainer->getLocalNum(), 0, sizeof(double)*3); + fields[particlesName + "_magnitude/association"].set_string("vertex"); + fields[particlesName + "_magnitude/topology"].set_string(particlesName + "_topo"); + fields[particlesName + "_magnitude/volume_dependent"].set_string("false"); + + fields[particlesName + "_magnitude/values/x"].set_external(&magnitude_host[0], particleContainer->getLocalNum(), 0, sizeof(double)*3); + fields[particlesName + "_magnitude/values/y"].set_external(&magnitude_host[1], particleContainer->getLocalNum(), 0, sizeof(double)*3); + fields[particlesName + "_magnitude/values/z"].set_external(&magnitude_host[2], particleContainer->getLocalNum(), 0, sizeof(double)*3); + conduit::Node verify_info; if(!conduit::blueprint::mesh::verify(node, verify_info)) { @@ -218,15 +225,90 @@ namespace AscentAdaptor { } } - + std::array compute_center(const auto& particleContainer, MPI_Comm comm) { + using memory_space = typename decltype(particleContainer->R)::memory_space; + + auto R_view = particleContainer->R.getView(); // R is a Kokkos::View + auto num_particles = R_view.extent(0); + + Kokkos::View local_sum("local_sum"); + Kokkos::View global_sum("global_sum"); + + // Compute local sum using parallel reduction + Kokkos::parallel_reduce( + "ComputeLocalSum", num_particles, + KOKKOS_LAMBDA(const int i, double& sum_x, double& sum_y, double& sum_z) { + sum_x += R_view(i)[0]; + sum_y += R_view(i)[1]; + sum_z += R_view(i)[2]; + }, + local_sum(0), local_sum(1), local_sum(2)); + + // Copy local sum to host + Kokkos::deep_copy(global_sum, local_sum); + + // Get local particle count + int local_count = num_particles; + int global_count; + + // Perform MPI Allreduce to sum positions across all ranks + MPI_Allreduce(MPI_IN_PLACE, global_sum.data(), 3, MPI_DOUBLE, MPI_SUM, comm); + MPI_Allreduce(&local_count, &global_count, 1, MPI_INT, MPI_SUM, comm); + + // Compute global center + if (global_count > 0) { + global_sum(0) /= global_count; + global_sum(1) /= global_count; + global_sum(2) /= global_count; + } + + // Compute global center + std::array center = {0.0, 0.0, 0.0}; + if (global_count > 0) { + center[0] = global_sum(0) / global_count; + center[1] = global_sum(1) / global_count; + center[2] = global_sum(2) / global_count; + } + + return center; + } + + // Function to compute the magnitude of each point from the center + Kokkos::View compute_magnitude_from_center( + const auto& particleContainer, const std::array& center) { + using memory_space = typename decltype(particleContainer->R)::memory_space; + + auto R_view = particleContainer->R.getView(); + auto num_particles = R_view.extent(0); + + // Create a view to hold the magnitudes + Kokkos::View magnitude("magnitude", num_particles); + + // Compute the magnitude (distance) from the center for each particle + Kokkos::parallel_for( + "ComputeMagnitude", num_particles, KOKKOS_LAMBDA(const int i) { + double dx = R_view(i)[0] - center[0]; + double dy = R_view(i)[1] - center[1]; + double dz = R_view(i)[2] - center[2]; + magnitude(i) = sqrt(dx * dx + dy * dy + dz * dz); + }); + + // Copy magnitudes to host + Kokkos::View host_magnitude("host_magnitude", num_particles); + Kokkos::deep_copy(host_magnitude, magnitude); + + return host_magnitude; + } template - void Execute(int cycle, double time, int rank, + void Execute(int cycle, double time, // const auto& /* std::shared_ptr >& */ particle, const std::vector>& particles, - const std::vector>& fields, double& scaleFactor) { + const std::vector>& fields) { conduit::Node node; + if(mFrequency % (cycle+1) != 0) return; + // add time/cycle information auto state = node["state"]; state["cycle"].set(cycle); @@ -236,6 +318,7 @@ namespace AscentAdaptor { std::map>::HostMirror> R_host_map; std::map>::HostMirror> P_host_map; + std::map>::HostMirror> center_host_map; std::map::HostMirror> q_host_map; std::map::HostMirror> ID_host_map; @@ -256,15 +339,18 @@ namespace AscentAdaptor { Kokkos::deep_copy(P_host_map[particlesName], particleContainer->P.getView()); Kokkos::deep_copy(q_host_map[particlesName], particleContainer->q.getView()); Kokkos::deep_copy(ID_host_map[particlesName], particleContainer->ID.getView()); + + std::array center = compute_center(particleContainer, MPI_COMM_WORLD); + auto host_magnitudes = compute_magnitude_from_center(particleContainer, center); Execute_Particle( particleContainer, R_host_map[particlesName], P_host_map[particlesName], q_host_map[particlesName], ID_host_map[particlesName], + host_magnitudes, particlesName, node); } - // Handle fields // Map of all Kokkos::Views. This keeps a reference on all Kokkos::Views diff --git a/alpine/CMakeLists.txt b/alpine/CMakeLists.txt index e44b73c5b..a89fde259 100644 --- a/alpine/CMakeLists.txt +++ b/alpine/CMakeLists.txt @@ -24,6 +24,12 @@ target_link_libraries (LandauDamping ${IPPL_LIBS}) add_executable (BumponTailInstability BumponTailInstability.cpp) target_link_libraries (BumponTailInstability ${IPPL_LIBS}) +configure_file( + "${CMAKE_CURRENT_SOURCE_DIR}/ascent_actions.yaml" + "${CMAKE_CURRENT_BINARY_DIR}/ascent_actions.yaml" + COPYONLY +) + # vi: set et ts=4 sw=4 sts=4: # Local Variables: diff --git a/alpine/PenningTrap.cpp b/alpine/PenningTrap.cpp index 5c272af96..34b732768 100644 --- a/alpine/PenningTrap.cpp +++ b/alpine/PenningTrap.cpp @@ -74,6 +74,7 @@ int main(int argc, char* argv[]) { for (int i = 1; i < argc; ++i) { if (std::string(argv[i]) == "--frequency" && i + 1 < argc) { frequency = atoi(argv[i+1]); + std::cout << "Frequency: " << frequency << std::endl; i++; } } diff --git a/alpine/PenningTrapManager.h b/alpine/PenningTrapManager.h index fc3daf77f..8459009e0 100644 --- a/alpine/PenningTrapManager.h +++ b/alpine/PenningTrapManager.h @@ -302,6 +302,7 @@ class PenningTrapManager : public AlpineManager { #endif #ifdef ENABLE_ASCENT + std::vector> particles = { {"particle", std::shared_ptr >(pc)}, }; @@ -310,7 +311,8 @@ class PenningTrapManager : public AlpineManager { {"roh", AscentAdaptor::FieldVariant(&this->fcontainer_m->getRho())}, //{"phi", CatalystAdaptor::FieldVariant(&this->fcontainer_m->getPhi())}, }; - AscentAdaptor::Execute(it, this->time_m, ippl::Comm->rank(), particles, fields, scaleFactor); + AscentAdaptor::Execute(it, this->time_m , particles, fields); + #endif // Field solve diff --git a/alpine/ascent_actions.yaml b/alpine/ascent_actions.yaml new file mode 100644 index 000000000..72f908ad2 --- /dev/null +++ b/alpine/ascent_actions.yaml @@ -0,0 +1,25 @@ +- + action: "add_pipelines" + pipelines: + pl1: + f1: + type: "threshold" + params: + field: "particle_magnitude" + min_value: 0 + max_value: 1 +- + action: "add_scenes" + scenes: + s1: + plots: + p1: + type: pseudocolor + field: particle_charge + points: + radius: 0.1 + pipeline: "pl1" + renders: + r1: + image_prefix: "render_%06d" + screen_annotations: "false" diff --git a/run.polaris.sh b/run.polaris.sh new file mode 100644 index 000000000..c5afafe9c --- /dev/null +++ b/run.polaris.sh @@ -0,0 +1,9 @@ +#!/bin/bash + +NNODES=`wc -l < $PBS_NODEFILE` +NRANKS_PER_NODE=4 +NDEPTH=8 +NTHREADS=8 + +NTOTRANKS=$(( NNODES * NRANKS_PER_NODE )) +mpirun --np ${NTOTRANKS} -ppn ${NRANKS} -d ${NDEPTH} -env OMP_NUM_THREADS=${NTHREADS} depth ./PenningTrap 32 32 32 655360 400 FFT 0.01 LeapFrog --overallocate 1.0 --info 5 --frequency 2 diff --git a/sourceme.sh b/sourceme.sh new file mode 100644 index 000000000..7cb54d8c6 --- /dev/null +++ b/sourceme.sh @@ -0,0 +1,10 @@ +module reset +module use /soft/modulefiles +module switch PrgEnv-nvhpc PrgEnv-gnu +module load spack-pe-base cmake +#module load kokkos/4.2.01 +module load cudatoolkit-standalone +module load visualization/ascent + +export CXX=CC +export CC=cc From 596e042ab5e363ff4244cc5d8543102a65e7e45d Mon Sep 17 00:00:00 2001 From: Victor Mateevitsi Date: Thu, 30 Jan 2025 19:25:18 +0000 Subject: [PATCH 13/14] Added build scripts --- build-cuda-sophia.sh | 27 +++++++++++++++++++++++++++ build-cuda.sh | 26 ++++++++++++++++++++++++++ build-openmp.sh | 23 +++++++++++++++++++++++ src/Types/Variant.h | 2 +- 4 files changed, 77 insertions(+), 1 deletion(-) create mode 100755 build-cuda-sophia.sh create mode 100755 build-cuda.sh create mode 100755 build-openmp.sh diff --git a/build-cuda-sophia.sh b/build-cuda-sophia.sh new file mode 100755 index 000000000..a939f8e46 --- /dev/null +++ b/build-cuda-sophia.sh @@ -0,0 +1,27 @@ +#!/bin/bash + +export CXX=mpiCC +export CC=mpicc + +cmake -B build-cuda-sophia -S . \ + -DCMAKE_INSTALL_PREFIX=install-cuda \ + -DCMAKE_BUILD_TYPE=Release \ + -DCMAKE_CXX_COMPILER=mpiCC \ + -DCMAKE_C_COMPILER=mpicc \ + -DCMAKE_CUDA_HOST_COMPILER=mpiCC \ + -DCUDA_TOOLKIT_ROOT_DIR=/usr/local/cuda \ + -DKokkos_ENABLE_CUDA=ON \ + -DKokkos_ARCH_AMPERE80=ON \ + -DKokkos_ENABLE_OPENMP=ON \ + -DCMAKE_CXX_STANDARD=20 \ + -DCMAKE_CXX_STANDARD_REQUIRED=ON \ + -DENABLE_FFT=ON \ + -DENABLE_SOLVERS=ON \ + -DENABLE_ALPINE=True \ + -DENABLE_TESTS=ON \ + -DUSE_ALTERNATIVE_VARIANT=ON \ + -DIPPL_PLATFORMS=cuda \ + -DAscent_DIR=`pwd`/../../ascent/scripts/build_ascent/install/ascent-checkout/lib/cmake/ascent + + +cmake --build build-cuda-sophia --target install -j8 diff --git a/build-cuda.sh b/build-cuda.sh new file mode 100755 index 000000000..2b34af4f3 --- /dev/null +++ b/build-cuda.sh @@ -0,0 +1,26 @@ +#!/bin/bash + +export CXX=CC +export CC=cc + +cmake -B build-cuda -S . \ + -DCMAKE_INSTALL_PREFIX=install-cuda \ + -DCMAKE_BUILD_TYPE=Release \ + -DCMAKE_CXX_COMPILER=CC \ + -DCMAKE_C_COMPILER=cc \ + -DCMAKE_CUDA_HOST_COMPILER=cc \ + -DKokkos_ENABLE_CUDA=ON \ + -DKokkos_ARCH_AMPERE80=ON \ + -DKokkos_ENABLE_OPENMP=ON \ + -DKokkos_ENABLE_IMPL_CUDA_MALLOC_ASYNC=OFF \ + -DCMAKE_CXX_STANDARD=20 \ + -DENABLE_FFT=ON \ + -DENABLE_SOLVERS=ON \ + -DENABLE_ALPINE=True \ + -DENABLE_TESTS=ON \ + -DUSE_ALTERNATIVE_VARIANT=ON \ + -DIPPL_PLATFORMS=cuda \ + -DAscent_DIR=`pwd`/../ascent/scripts/build_ascent/install/ascent-checkout/lib/cmake/ascent + + +cmake --build build-cuda --target install -j8 diff --git a/build-openmp.sh b/build-openmp.sh new file mode 100755 index 000000000..5ec93e10d --- /dev/null +++ b/build-openmp.sh @@ -0,0 +1,23 @@ +#!/bin/bash + +export CXX=CC +export CC=cc + +cmake -B build-openmp -S . \ + -DCMAKE_INSTALL_PREFIX=install-ippl \ + -DCMAKE_BUILD_TYPE=Release \ + -DCMAKE_CXX_COMPILER=CC \ + -DCMAKE_C_COMPILER=cc \ + -DKokkos_ENABLE_CUDA=OFF \ + -DKokkos_ENABLE_OPENMP=ON \ + -DCMAKE_CXX_STANDARD=20 \ + -DENABLE_FFT=ON \ + -DENABLE_SOLVERS=ON \ + -DENABLE_ALPINE=True \ + -DENABLE_TESTS=ON \ + -DUSE_ALTERNATIVE_VARIANT=ON \ + -DIPPL_PLATFORMS=openmp \ + -DAscent_DIR=`pwd`/../ascent/scripts/build_ascent/install/ascent-checkout/lib/cmake/ascent + + +cmake --build build-openmp -j8 diff --git a/src/Types/Variant.h b/src/Types/Variant.h index e45935a13..0d469fb4e 100644 --- a/src/Types/Variant.h +++ b/src/Types/Variant.h @@ -41,7 +41,7 @@ #include #include #include -#include // in_place_index_t +#include // in_place_index_t #include #include #if __cplusplus >= 202002L From ac79285efc9a1e902a5086569f0dd43ea4736190 Mon Sep 17 00:00:00 2001 From: Victor Mateevitsi Date: Thu, 13 Feb 2025 22:41:02 +0000 Subject: [PATCH 14/14] Fixed thresholding. --- alpine/AscentAdaptor.h | 76 ++++++++++++++++-------------- alpine/ascent_actions.yaml | 13 ++++- build-cuda.sh | 26 ---------- build-openmp.sh | 23 --------- run.polaris.sh | 12 +++-- sourceme.sh => sourceme.polaris.sh | 1 - sourceme.sophia.sh | 18 +++++++ 7 files changed, 77 insertions(+), 92 deletions(-) delete mode 100755 build-cuda.sh delete mode 100755 build-openmp.sh rename sourceme.sh => sourceme.polaris.sh (88%) create mode 100644 sourceme.sophia.sh diff --git a/alpine/AscentAdaptor.h b/alpine/AscentAdaptor.h index fb1a18791..04fb15502 100644 --- a/alpine/AscentAdaptor.h +++ b/alpine/AscentAdaptor.h @@ -72,35 +72,46 @@ namespace AscentAdaptor { void Execute_Particle( const auto& particleContainer, - const auto& R_host, const auto& P_host, const auto& q_host, const auto& ID_host, + const auto& R_host, const auto& P_host, const auto& q_host, const auto& magnitude_host, const std::string& particlesName, - conduit::Node& node) { + conduit::Node& node, + std::array& center ) { node["coordsets/" + particlesName + "_coords/type"].set_string("explicit"); - - //mesh["coordsets/coords/values/x"].set_external(&layout_view.data()[0][0], particleContainer->getLocalNum(), 0, sizeof(double)*3); - //mesh["coordsets/coords/values/y"].set_external(&layout_view.data()[0][1], particleContainer->getLocalNum(), 0, sizeof(double)*3); - //mesh["coordsets/coords/values/z"].set_external(&layout_view.data()[0][2], particleContainer->getLocalNum(), 0, sizeof(double)*3); node["coordsets/" + particlesName + "_coords/values/x"].set_external(&R_host.data()[0][0], particleContainer->getLocalNum(), 0, sizeof(double)*3); node["coordsets/" + particlesName + "_coords/values/y"].set_external(&R_host.data()[0][1], particleContainer->getLocalNum(), 0, sizeof(double)*3); node["coordsets/" + particlesName + "_coords/values/z"].set_external(&R_host.data()[0][2], particleContainer->getLocalNum(), 0, sizeof(double)*3); - node["topologies/" + particlesName + "_topo/type"].set_string("unstructured"); + node["topologies/" + particlesName + "_topo/type"].set_string("points"); node["topologies/" + particlesName + "_topo/coordset"].set_string(particlesName + "_coords"); - node["topologies/" + particlesName + "_topo/elements/shape"].set_string("point"); - //mesh["topologies/mesh/elements/connectivity"].set_external(particleContainer->ID.getView().data(),particleContainer->getLocalNum()); - node["topologies/" + particlesName + "_topo/elements/connectivity"].set_external(ID_host.data(),particleContainer->getLocalNum()); - //auto charge_view = particleContainer->getQ().getView(); + node["topologies/" + particlesName + "_topo/type"].set_string("points"); + node["topologies/" + particlesName + "_topo/coordset"].set_string(particlesName + "_coords"); - // add values for scalar charge field + /* Particle_center */ + node["coordsets/" + particlesName + "_center_coords/type"].set_string("explicit"); + + node["coordsets/" + particlesName + "_center_coords/values/x"].set(¢er[0], 1); + node["coordsets/" + particlesName + "_center_coords/values/y"].set(¢er[1], 1); + node["coordsets/" + particlesName + "_center_coords/values/z"].set(¢er[2], 1); + node["topologies/" + particlesName + "_center_topo/type"].set_string("points"); + node["topologies/" + particlesName + "_center_topo/coordset"].set_string(particlesName + "_center_coords"); + + std::vector dummy_field = { 1.0f }; + // add a dummy field auto &fields = node["fields"]; + fields[particlesName + "_center/association"].set_string("vertex"); + fields[particlesName + "_center/topology"].set_string(particlesName + "_center_topo"); + fields[particlesName + "_center/volume_dependent"].set_string("false"); + fields[particlesName + "_center/values"].set(dummy_field); + /* end particle center */ + + // add values for scalar charge field fields[particlesName + "_charge/association"].set_string("vertex"); fields[particlesName + "_charge/topology"].set_string(particlesName + "_topo"); fields[particlesName + "_charge/volume_dependent"].set_string("false"); - //fields["charge/values"].set_external(particleContainer->q.getView().data(), particleContainer->getLocalNum()); fields[particlesName + "_charge/values"].set_external(q_host.data(), particleContainer->getLocalNum()); // add values for vector velocity field @@ -109,9 +120,6 @@ namespace AscentAdaptor { fields[particlesName + "_velocity/topology"].set_string(particlesName + "_topo"); fields[particlesName + "_velocity/volume_dependent"].set_string("false"); - //fields["velocity/values/x"].set_external(&velocity_view.data()[0][0], particleContainer->getLocalNum(),0 ,sizeof(double)*3); - //fields["velocity/values/y"].set_external(&velocity_view.data()[0][1], particleContainer->getLocalNum(),0 ,sizeof(double)*3); - //fields["velocity/values/z"].set_external(&velocity_view.data()[0][2], particleContainer->getLocalNum(),0 ,sizeof(double)*3); fields[particlesName + "_velocity/values/x"].set_external(&P_host.data()[0][0], particleContainer->getLocalNum(),0 ,sizeof(double)*3); fields[particlesName + "_velocity/values/y"].set_external(&P_host.data()[0][1], particleContainer->getLocalNum(),0 ,sizeof(double)*3); fields[particlesName + "_velocity/values/z"].set_external(&P_host.data()[0][2], particleContainer->getLocalNum(),0 ,sizeof(double)*3); @@ -120,20 +128,17 @@ namespace AscentAdaptor { fields[particlesName + "_position/topology"].set_string(particlesName + "_topo"); fields[particlesName + "_position/volume_dependent"].set_string("false"); - //fields["position/values/x"].set_external(&layout_view.data()[0][0], particleContainer->getLocalNum(), 0, sizeof(double)*3); - //fields["position/values/y"].set_external(&layout_view.data()[0][1], particleContainer->getLocalNum(), 0, sizeof(double)*3); - //fields["position/values/z"].set_external(&layout_view.data()[0][2], particleContainer->getLocalNum(), 0, sizeof(double)*3); fields[particlesName + "_position/values/x"].set_external(&R_host.data()[0][0], particleContainer->getLocalNum(), 0, sizeof(double)*3); fields[particlesName + "_position/values/y"].set_external(&R_host.data()[0][1], particleContainer->getLocalNum(), 0, sizeof(double)*3); fields[particlesName + "_position/values/z"].set_external(&R_host.data()[0][2], particleContainer->getLocalNum(), 0, sizeof(double)*3); + fields[particlesName + "_magnitude/association"].set_string("vertex"); fields[particlesName + "_magnitude/topology"].set_string(particlesName + "_topo"); fields[particlesName + "_magnitude/volume_dependent"].set_string("false"); - fields[particlesName + "_magnitude/values/x"].set_external(&magnitude_host[0], particleContainer->getLocalNum(), 0, sizeof(double)*3); - fields[particlesName + "_magnitude/values/y"].set_external(&magnitude_host[1], particleContainer->getLocalNum(), 0, sizeof(double)*3); - fields[particlesName + "_magnitude/values/z"].set_external(&magnitude_host[2], particleContainer->getLocalNum(), 0, sizeof(double)*3); + + fields[particlesName + "_magnitude/values"].set(magnitude_host.data(), magnitude_host.extent(0)); conduit::Node verify_info; if(!conduit::blueprint::mesh::verify(node, verify_info)) @@ -247,6 +252,7 @@ namespace AscentAdaptor { // Copy local sum to host Kokkos::deep_copy(global_sum, local_sum); + // Get local particle count int local_count = num_particles; int global_count; @@ -265,12 +271,13 @@ namespace AscentAdaptor { // Compute global center std::array center = {0.0, 0.0, 0.0}; if (global_count > 0) { - center[0] = global_sum(0) / global_count; - center[1] = global_sum(1) / global_count; - center[2] = global_sum(2) / global_count; + center[0] = global_sum(0); + center[1] = global_sum(1); + center[2] = global_sum(2); } - return center; + + return {global_sum(0), global_sum(1), global_sum(2)}; } // Function to compute the magnitude of each point from the center @@ -292,7 +299,7 @@ namespace AscentAdaptor { double dz = R_view(i)[2] - center[2]; magnitude(i) = sqrt(dx * dx + dy * dy + dz * dz); }); - + // Copy magnitudes to host Kokkos::View host_magnitude("host_magnitude", num_particles); Kokkos::deep_copy(host_magnitude, magnitude); @@ -307,7 +314,7 @@ namespace AscentAdaptor { const std::vector>& fields) { conduit::Node node; - if(mFrequency % (cycle+1) != 0) return; + if((cycle+1) % mFrequency != 0) return; // add time/cycle information auto state = node["state"]; @@ -333,22 +340,22 @@ namespace AscentAdaptor { R_host_map[particlesName] = particleContainer->R.getHostMirror(); P_host_map[particlesName] = particleContainer->P.getHostMirror(); q_host_map[particlesName] = particleContainer->q.getHostMirror(); - ID_host_map[particlesName] = particleContainer->ID.getHostMirror(); + Kokkos::deep_copy(R_host_map[particlesName], particleContainer->R.getView()); Kokkos::deep_copy(P_host_map[particlesName], particleContainer->P.getView()); Kokkos::deep_copy(q_host_map[particlesName], particleContainer->q.getView()); - Kokkos::deep_copy(ID_host_map[particlesName], particleContainer->ID.getView()); std::array center = compute_center(particleContainer, MPI_COMM_WORLD); - auto host_magnitudes = compute_magnitude_from_center(particleContainer, center); + auto magnitude_host = compute_magnitude_from_center(particleContainer, center); Execute_Particle( particleContainer, - R_host_map[particlesName], P_host_map[particlesName], q_host_map[particlesName], ID_host_map[particlesName], - host_magnitudes, + R_host_map[particlesName], P_host_map[particlesName], q_host_map[particlesName], + magnitude_host, particlesName, - node); + node, + center); } // Handle fields @@ -374,7 +381,6 @@ namespace AscentAdaptor { // If field is a _vector_ field else if (std::holds_alternative*>(fieldVariant)) { VField_t* field = std::get*>(fieldVariant); - // == ippl::Field, 3, ippl::UniformCartesian, Cell>* Execute_Field(field, fieldName, vector_host_views[fieldName], node); } diff --git a/alpine/ascent_actions.yaml b/alpine/ascent_actions.yaml index 72f908ad2..9cc3d6648 100644 --- a/alpine/ascent_actions.yaml +++ b/alpine/ascent_actions.yaml @@ -6,8 +6,8 @@ type: "threshold" params: field: "particle_magnitude" - min_value: 0 - max_value: 1 + min_value: 9.2 + max_value: 25.0 - action: "add_scenes" scenes: @@ -19,7 +19,16 @@ points: radius: 0.1 pipeline: "pl1" + p2: + type: pseudocolor + field: particle_center + points: + radius: 0.5 renders: r1: image_prefix: "render_%06d" screen_annotations: "false" + camera: + look_at: [10, 0, 10] + up: [1, 0, 0] + position: [10, 40, 10] diff --git a/build-cuda.sh b/build-cuda.sh deleted file mode 100755 index 2b34af4f3..000000000 --- a/build-cuda.sh +++ /dev/null @@ -1,26 +0,0 @@ -#!/bin/bash - -export CXX=CC -export CC=cc - -cmake -B build-cuda -S . \ - -DCMAKE_INSTALL_PREFIX=install-cuda \ - -DCMAKE_BUILD_TYPE=Release \ - -DCMAKE_CXX_COMPILER=CC \ - -DCMAKE_C_COMPILER=cc \ - -DCMAKE_CUDA_HOST_COMPILER=cc \ - -DKokkos_ENABLE_CUDA=ON \ - -DKokkos_ARCH_AMPERE80=ON \ - -DKokkos_ENABLE_OPENMP=ON \ - -DKokkos_ENABLE_IMPL_CUDA_MALLOC_ASYNC=OFF \ - -DCMAKE_CXX_STANDARD=20 \ - -DENABLE_FFT=ON \ - -DENABLE_SOLVERS=ON \ - -DENABLE_ALPINE=True \ - -DENABLE_TESTS=ON \ - -DUSE_ALTERNATIVE_VARIANT=ON \ - -DIPPL_PLATFORMS=cuda \ - -DAscent_DIR=`pwd`/../ascent/scripts/build_ascent/install/ascent-checkout/lib/cmake/ascent - - -cmake --build build-cuda --target install -j8 diff --git a/build-openmp.sh b/build-openmp.sh deleted file mode 100755 index 5ec93e10d..000000000 --- a/build-openmp.sh +++ /dev/null @@ -1,23 +0,0 @@ -#!/bin/bash - -export CXX=CC -export CC=cc - -cmake -B build-openmp -S . \ - -DCMAKE_INSTALL_PREFIX=install-ippl \ - -DCMAKE_BUILD_TYPE=Release \ - -DCMAKE_CXX_COMPILER=CC \ - -DCMAKE_C_COMPILER=cc \ - -DKokkos_ENABLE_CUDA=OFF \ - -DKokkos_ENABLE_OPENMP=ON \ - -DCMAKE_CXX_STANDARD=20 \ - -DENABLE_FFT=ON \ - -DENABLE_SOLVERS=ON \ - -DENABLE_ALPINE=True \ - -DENABLE_TESTS=ON \ - -DUSE_ALTERNATIVE_VARIANT=ON \ - -DIPPL_PLATFORMS=openmp \ - -DAscent_DIR=`pwd`/../ascent/scripts/build_ascent/install/ascent-checkout/lib/cmake/ascent - - -cmake --build build-openmp -j8 diff --git a/run.polaris.sh b/run.polaris.sh index c5afafe9c..5bb6812ce 100644 --- a/run.polaris.sh +++ b/run.polaris.sh @@ -1,9 +1,11 @@ #!/bin/bash NNODES=`wc -l < $PBS_NODEFILE` -NRANKS_PER_NODE=4 -NDEPTH=8 -NTHREADS=8 +NRANKS=1 +NDEPTH=32 +NTHREADS=32 -NTOTRANKS=$(( NNODES * NRANKS_PER_NODE )) -mpirun --np ${NTOTRANKS} -ppn ${NRANKS} -d ${NDEPTH} -env OMP_NUM_THREADS=${NTHREADS} depth ./PenningTrap 32 32 32 655360 400 FFT 0.01 LeapFrog --overallocate 1.0 --info 5 --frequency 2 +NTOTRANKS=$(( NNODES * NRANKS )) +echo "NUM_OF_NODES= ${NNODES} TOTAL_NUM_RANKS= ${NTOTRANKS} RANKS_PER_NODE= ${NRANKS} THREADS_PER_RANK= ${NTHREADS}" + +mpirun --np ${NTOTRANKS} -ppn ${NRANKS} -d ${NDEPTH} --cpu-bind depth -env OMP_NUM_THREADS=${NTHREADS} ./PenningTrap 32 32 32 655360 400 FFT 0.01 LeapFrog --overallocate 1.0 --info 5 --frequency 2 \ No newline at end of file diff --git a/sourceme.sh b/sourceme.polaris.sh similarity index 88% rename from sourceme.sh rename to sourceme.polaris.sh index 7cb54d8c6..b3e0c3072 100644 --- a/sourceme.sh +++ b/sourceme.polaris.sh @@ -2,7 +2,6 @@ module reset module use /soft/modulefiles module switch PrgEnv-nvhpc PrgEnv-gnu module load spack-pe-base cmake -#module load kokkos/4.2.01 module load cudatoolkit-standalone module load visualization/ascent diff --git a/sourceme.sophia.sh b/sourceme.sophia.sh new file mode 100644 index 000000000..a06da1cfb --- /dev/null +++ b/sourceme.sophia.sh @@ -0,0 +1,18 @@ +export HTTP_PROXY="http://proxy.alcf.anl.gov:3128" +export HTTPS_PROXY="http://proxy.alcf.anl.gov:3128" +export http_proxy="http://proxy.alcf.anl.gov:3128" +export https_proxy="http://proxy.alcf.anl.gov:3128" +export ftp_proxy="http://proxy.alcf.anl.gov:3128" +export no_proxy="admin,polaris-adminvm-01,localhost,*.cm.polaris.alcf.anl.gov,polaris-*,*.polaris.alcf.anl.gov,*.alcf.anl.gov" + + +module reset +module use /soft/modulefiles +module switch PrgEnv-nvhpc PrgEnv-gnu +module load spack-pe-base cmake +#module load kokkos/4.2.01 +module load cudatoolkit-standalone +module load visualization/ascent + +export CXX=CC +export CC=cc