diff --git a/.gitignore b/.gitignore index e64104dbf..626491737 100644 --- a/.gitignore +++ b/.gitignore @@ -27,3 +27,5 @@ compile_commands.json __pycache__ *.~ + +.vscode diff --git a/CMakeLists.txt b/CMakeLists.txt index 15a0dfe13..7c6c27511 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -23,7 +23,6 @@ macro(remove_flag_from_target _target _flag) endif() endmacro() - if (NOT CMAKE_CONFIGURATION_TYPES AND NOT CMAKE_BUILD_TYPE) set (CMAKE_BUILD_TYPE RelWithDebInfo CACHE STRING @@ -197,6 +196,7 @@ if (ENABLE_FFT) endif() FetchContent_MakeAvailable(Heffte) # Required to suppress warnings related to heffte + MESSAGE(STATUS "HEFFTE: "${Heffte_SOURCE_DIR}"") target_include_directories(Heffte SYSTEM PUBLIC "${Heffte_SOURCE_DIR}/include") set(Heffte_DIR "${Heffte_SOURCE_DIR}") remove_flag_from_target(Heffte "-Werror") @@ -207,6 +207,7 @@ if (ENABLE_FFT) endif () option (ENABLE_CATALYST "Build example with Catalyst enabled" OFF) +option (ENABLE_ASCENT "Build example with Ascent enabled" ON) option (ENABLE_SOLVERS "Enable IPPL solvers" ON) add_subdirectory (src) diff --git a/alpine/AscentAdaptor.h b/alpine/AscentAdaptor.h new file mode 100644 index 000000000..04fb15502 --- /dev/null +++ b/alpine/AscentAdaptor.h @@ -0,0 +1,397 @@ +#ifndef AsecntAdaptor_h +#define AsecntAdaptor_h + +#include "Ippl.h" + +#include +#include +#include +#include +#include + +#include "Utility/IpplException.h" + + +namespace AscentAdaptor { + + ascent::Ascent mAscent; + int mFrequency = 1; + + template + using FieldVariant = std::variant*, VField_t*>; + + template + using FieldPair = std::pair>; + + template + using ParticlePair = std::pair > >; + + using View_vector = + Kokkos::View***, Kokkos::LayoutLeft, Kokkos::HostSpace>; + inline void setData(conduit::Node& node, const View_vector& view, const std::string& fieldName) { + node["electrostatic/association"].set_string("element"); + node["electrostatic/topology"].set_string(fieldName + "_mesh"); + node["electrostatic/volume_dependent"].set_string("false"); + + auto length = std::size(view); + + // offset is zero as we start without the ghost cells + // stride is 1 as we have every index of the array + node["electrostatic/values/x"].set_external(&view.data()[0][0], length, 0, 1); + node["electrostatic/values/y"].set_external(&view.data()[0][1], length, 0, 1); + node["electrostatic/values/z"].set_external(&view.data()[0][2], length, 0, 1); + } + + using View_scalar = Kokkos::View; + inline void setData(conduit::Node& node, const View_scalar& view, const std::string& fieldName) { + node["density/association"].set_string("element"); + node["density/topology"].set_string(fieldName + "_mesh"); + node["density/volume_dependent"].set_string("false"); + + node["density/values"].set_external(view.data(), view.size()); + } + + void Initialize(int frequency) { + MPI_Comm ascent_comm; + mFrequency = frequency; + + // Split communicator based on the task ID + MPI_Comm_dup(MPI_COMM_WORLD, &ascent_comm); + + conduit::Node ascent_opts; + ascent_opts["mpi_comm"] = MPI_Comm_c2f(ascent_comm); + mAscent.open(ascent_opts); + } + + void Finalize() { + conduit::Node node; + mAscent.close(); + + } + + + void Execute_Particle( + const auto& particleContainer, + const auto& R_host, const auto& P_host, const auto& q_host, + const auto& magnitude_host, + const std::string& particlesName, + conduit::Node& node, + std::array& center ) { + + node["coordsets/" + particlesName + "_coords/type"].set_string("explicit"); + node["coordsets/" + particlesName + "_coords/values/x"].set_external(&R_host.data()[0][0], particleContainer->getLocalNum(), 0, sizeof(double)*3); + node["coordsets/" + particlesName + "_coords/values/y"].set_external(&R_host.data()[0][1], particleContainer->getLocalNum(), 0, sizeof(double)*3); + node["coordsets/" + particlesName + "_coords/values/z"].set_external(&R_host.data()[0][2], particleContainer->getLocalNum(), 0, sizeof(double)*3); + + node["topologies/" + particlesName + "_topo/type"].set_string("points"); + node["topologies/" + particlesName + "_topo/coordset"].set_string(particlesName + "_coords"); + + node["topologies/" + particlesName + "_topo/type"].set_string("points"); + node["topologies/" + particlesName + "_topo/coordset"].set_string(particlesName + "_coords"); + + /* Particle_center */ + node["coordsets/" + particlesName + "_center_coords/type"].set_string("explicit"); + + node["coordsets/" + particlesName + "_center_coords/values/x"].set(¢er[0], 1); + node["coordsets/" + particlesName + "_center_coords/values/y"].set(¢er[1], 1); + node["coordsets/" + particlesName + "_center_coords/values/z"].set(¢er[2], 1); + node["topologies/" + particlesName + "_center_topo/type"].set_string("points"); + node["topologies/" + particlesName + "_center_topo/coordset"].set_string(particlesName + "_center_coords"); + + std::vector dummy_field = { 1.0f }; + // add a dummy field + auto &fields = node["fields"]; + fields[particlesName + "_center/association"].set_string("vertex"); + fields[particlesName + "_center/topology"].set_string(particlesName + "_center_topo"); + fields[particlesName + "_center/volume_dependent"].set_string("false"); + fields[particlesName + "_center/values"].set(dummy_field); + /* end particle center */ + + // add values for scalar charge field + fields[particlesName + "_charge/association"].set_string("vertex"); + fields[particlesName + "_charge/topology"].set_string(particlesName + "_topo"); + fields[particlesName + "_charge/volume_dependent"].set_string("false"); + + fields[particlesName + "_charge/values"].set_external(q_host.data(), particleContainer->getLocalNum()); + + // add values for vector velocity field + //auto velocity_view = particleContainer->P.getView(); + fields[particlesName + "_velocity/association"].set_string("vertex"); + fields[particlesName + "_velocity/topology"].set_string(particlesName + "_topo"); + fields[particlesName + "_velocity/volume_dependent"].set_string("false"); + + fields[particlesName + "_velocity/values/x"].set_external(&P_host.data()[0][0], particleContainer->getLocalNum(),0 ,sizeof(double)*3); + fields[particlesName + "_velocity/values/y"].set_external(&P_host.data()[0][1], particleContainer->getLocalNum(),0 ,sizeof(double)*3); + fields[particlesName + "_velocity/values/z"].set_external(&P_host.data()[0][2], particleContainer->getLocalNum(),0 ,sizeof(double)*3); + + fields[particlesName + "_position/association"].set_string("vertex"); + fields[particlesName + "_position/topology"].set_string(particlesName + "_topo"); + fields[particlesName + "_position/volume_dependent"].set_string("false"); + + fields[particlesName + "_position/values/x"].set_external(&R_host.data()[0][0], particleContainer->getLocalNum(), 0, sizeof(double)*3); + fields[particlesName + "_position/values/y"].set_external(&R_host.data()[0][1], particleContainer->getLocalNum(), 0, sizeof(double)*3); + fields[particlesName + "_position/values/z"].set_external(&R_host.data()[0][2], particleContainer->getLocalNum(), 0, sizeof(double)*3); + + + fields[particlesName + "_magnitude/association"].set_string("vertex"); + fields[particlesName + "_magnitude/topology"].set_string(particlesName + "_topo"); + fields[particlesName + "_magnitude/volume_dependent"].set_string("false"); + + + fields[particlesName + "_magnitude/values"].set(magnitude_host.data(), magnitude_host.extent(0)); + + conduit::Node verify_info; + if(!conduit::blueprint::mesh::verify(node, verify_info)) + { + std::cerr << "Mesh verification failed!" << std::endl; + verify_info.print(); + exit(EXIT_FAILURE); + } + } + + + template + void Execute_Field(Field* field, const std::string& fieldName, + Kokkos::View& host_view_layout_left, + conduit::Node& node) { + static_assert(Field::dim == 3, "AscentAdaptor only supports 3D"); + + node["coordsets/" + fieldName + "_coords/type"].set_string("uniform"); + + // number of points in specific dimension + std::string field_node_dim{"coordsets/" + fieldName + "_coords/dims/i"}; + std::string field_node_origin{"coordsets/" + fieldName + "_coords/origin/x"}; + std::string field_node_spacing{"coordsets/" + fieldName + "_coords/spacing/dx"}; + + for (unsigned int iDim = 0; iDim < field->get_mesh().getGridsize().dim; ++iDim) { + // add dimension + node[field_node_dim].set(field->getLayout().getLocalNDIndex()[iDim].length() + 1); + + // add origin + node[field_node_origin].set( + field->get_mesh().getOrigin()[iDim] + field->getLayout().getLocalNDIndex()[iDim].first() + * field->get_mesh().getMeshSpacing(iDim)); + + // add spacing + node[field_node_spacing].set(field->get_mesh().getMeshSpacing(iDim)); + + // increment last char in string + ++field_node_dim.back(); + ++field_node_origin.back(); + ++field_node_spacing.back(); + } + + // add topology + node["topologies/" + fieldName + "_mesh/type"].set_string("uniform"); + node["topologies/" + fieldName + "_mesh/coordset"].set_string(fieldName + "_coords"); + std::string field_node_origin_topo{"topologies/" + fieldName + "_mesh/origin/x"}; + for (unsigned int iDim = 0; iDim < field->get_mesh().getGridsize().dim; ++iDim) { + // shift origin + node[field_node_origin_topo].set(field->get_mesh().getOrigin()[iDim] + + field->getLayout().getLocalNDIndex()[iDim].first() + * field->get_mesh().getMeshSpacing(iDim)); + + // increment last char in string + ++field_node_origin_topo.back(); + } + + // B) Set the field values + + // Initialize the existing Kokkos::View + host_view_layout_left = Kokkos::View( + "host_view_layout_left", + field->getLayout().getLocalNDIndex()[0].length(), + field->getLayout().getLocalNDIndex()[1].length(), + field->getLayout().getLocalNDIndex()[2].length()); + + // Creates a host-accessible mirror view and copies the data from the device view to the host. + auto host_view = + Kokkos::create_mirror_view_and_copy(Kokkos::HostSpace(), field->getView()); + + // Copy data from field to the memory+style which will be passed to Catalyst + auto nGhost = field->getNghost(); + for (size_t i = 0; i < field->getLayout().getLocalNDIndex()[0].length(); ++i) { + for (size_t j = 0; j < field->getLayout().getLocalNDIndex()[1].length(); ++j) { + for (size_t k = 0; k < field->getLayout().getLocalNDIndex()[2].length(); ++k) { + host_view_layout_left(i, j, k) = host_view(i + nGhost, j + nGhost, k + nGhost); + } + } + } + + auto &fields = node["fields"]; + setData(fields, host_view_layout_left, fieldName); + + conduit::Node verify_info; + if(!conduit::blueprint::mesh::verify(node, verify_info)) + { + std::cerr << "Mesh verification failed!" << std::endl; + verify_info.print(); + exit(EXIT_FAILURE); + } + } + + std::array compute_center(const auto& particleContainer, MPI_Comm comm) { + using memory_space = typename decltype(particleContainer->R)::memory_space; + + auto R_view = particleContainer->R.getView(); // R is a Kokkos::View + auto num_particles = R_view.extent(0); + + Kokkos::View local_sum("local_sum"); + Kokkos::View global_sum("global_sum"); + + // Compute local sum using parallel reduction + Kokkos::parallel_reduce( + "ComputeLocalSum", num_particles, + KOKKOS_LAMBDA(const int i, double& sum_x, double& sum_y, double& sum_z) { + sum_x += R_view(i)[0]; + sum_y += R_view(i)[1]; + sum_z += R_view(i)[2]; + }, + local_sum(0), local_sum(1), local_sum(2)); + + // Copy local sum to host + Kokkos::deep_copy(global_sum, local_sum); + + + // Get local particle count + int local_count = num_particles; + int global_count; + + // Perform MPI Allreduce to sum positions across all ranks + MPI_Allreduce(MPI_IN_PLACE, global_sum.data(), 3, MPI_DOUBLE, MPI_SUM, comm); + MPI_Allreduce(&local_count, &global_count, 1, MPI_INT, MPI_SUM, comm); + + // Compute global center + if (global_count > 0) { + global_sum(0) /= global_count; + global_sum(1) /= global_count; + global_sum(2) /= global_count; + } + + // Compute global center + std::array center = {0.0, 0.0, 0.0}; + if (global_count > 0) { + center[0] = global_sum(0); + center[1] = global_sum(1); + center[2] = global_sum(2); + } + + + return {global_sum(0), global_sum(1), global_sum(2)}; + } + + // Function to compute the magnitude of each point from the center + Kokkos::View compute_magnitude_from_center( + const auto& particleContainer, const std::array& center) { + using memory_space = typename decltype(particleContainer->R)::memory_space; + + auto R_view = particleContainer->R.getView(); + auto num_particles = R_view.extent(0); + + // Create a view to hold the magnitudes + Kokkos::View magnitude("magnitude", num_particles); + + // Compute the magnitude (distance) from the center for each particle + Kokkos::parallel_for( + "ComputeMagnitude", num_particles, KOKKOS_LAMBDA(const int i) { + double dx = R_view(i)[0] - center[0]; + double dy = R_view(i)[1] - center[1]; + double dz = R_view(i)[2] - center[2]; + magnitude(i) = sqrt(dx * dx + dy * dy + dz * dz); + }); + + // Copy magnitudes to host + Kokkos::View host_magnitude("host_magnitude", num_particles); + Kokkos::deep_copy(host_magnitude, magnitude); + + return host_magnitude; + } + + template + void Execute(int cycle, double time, + // const auto& /* std::shared_ptr >& */ particle, + const std::vector>& particles, + const std::vector>& fields) { + conduit::Node node; + + if((cycle+1) % mFrequency != 0) return; + + // add time/cycle information + auto state = node["state"]; + state["cycle"].set(cycle); + state["time"].set(time); + + // Handle particles + + std::map>::HostMirror> R_host_map; + std::map>::HostMirror> P_host_map; + std::map>::HostMirror> center_host_map; + std::map::HostMirror> q_host_map; + std::map::HostMirror> ID_host_map; + + // Loop over all particle container + for (const auto& particlesPair : particles) + { + const std::string& particlesName = particlesPair.first; + const auto particleContainer = particlesPair.second; + + assert((particleContainer->ID.getView().data() != nullptr) && "ID view should not be nullptr, might be missing the right execution space"); + + R_host_map[particlesName] = particleContainer->R.getHostMirror(); + P_host_map[particlesName] = particleContainer->P.getHostMirror(); + q_host_map[particlesName] = particleContainer->q.getHostMirror(); + + + Kokkos::deep_copy(R_host_map[particlesName], particleContainer->R.getView()); + Kokkos::deep_copy(P_host_map[particlesName], particleContainer->P.getView()); + Kokkos::deep_copy(q_host_map[particlesName], particleContainer->q.getView()); + + std::array center = compute_center(particleContainer, MPI_COMM_WORLD); + auto magnitude_host = compute_magnitude_from_center(particleContainer, center); + + Execute_Particle( + particleContainer, + R_host_map[particlesName], P_host_map[particlesName], q_host_map[particlesName], + magnitude_host, + particlesName, + node, + center); + } + + // Handle fields + + // Map of all Kokkos::Views. This keeps a reference on all Kokkos::Views + // which ensures that Kokkos does not free the memory before the end of this function. + std::map::view_type::data_type, Kokkos::LayoutLeft, Kokkos::HostSpace> > scalar_host_views; + std::map::view_type::data_type, Kokkos::LayoutLeft, Kokkos::HostSpace> > vector_host_views; + + // Loop over all fields + for (const auto& fieldPair : fields) + { + const std::string& fieldName = fieldPair.first; + const auto& fieldVariant = fieldPair.second; + + // If field is a _scalar_ field + if (std::holds_alternative*>(fieldVariant)) { + Field_t* field = std::get*>(fieldVariant); + // == ippl::Field, Cell>* + + Execute_Field(field, fieldName, scalar_host_views[fieldName], node); + } + // If field is a _vector_ field + else if (std::holds_alternative*>(fieldVariant)) { + VField_t* field = std::get*>(fieldVariant); + + Execute_Field(field, fieldName, vector_host_views[fieldName], node); + } + } + + conduit::Node actions; + mAscent.publish(node); + mAscent.execute(actions); + + + } +} // namespace CatalystAdaptor + +#endif diff --git a/alpine/CMakeLists.txt b/alpine/CMakeLists.txt index e44b73c5b..a89fde259 100644 --- a/alpine/CMakeLists.txt +++ b/alpine/CMakeLists.txt @@ -24,6 +24,12 @@ target_link_libraries (LandauDamping ${IPPL_LIBS}) add_executable (BumponTailInstability BumponTailInstability.cpp) target_link_libraries (BumponTailInstability ${IPPL_LIBS}) +configure_file( + "${CMAKE_CURRENT_SOURCE_DIR}/ascent_actions.yaml" + "${CMAKE_CURRENT_BINARY_DIR}/ascent_actions.yaml" + COPYONLY +) + # vi: set et ts=4 sw=4 sts=4: # Local Variables: diff --git a/src/Stream/InSitu/CatalystAdaptor.h b/alpine/CatalystAdaptor.h similarity index 50% rename from src/Stream/InSitu/CatalystAdaptor.h rename to alpine/CatalystAdaptor.h index 514df25c2..d65399ae3 100644 --- a/src/Stream/InSitu/CatalystAdaptor.h +++ b/alpine/CatalystAdaptor.h @@ -1,5 +1,3 @@ -// SPDX-FileCopyrightText: Copyright (c) Kitware Inc. -// SPDX-License-Identifier: BSD-3-Clause #ifndef CatalystAdaptor_h #define CatalystAdaptor_h @@ -10,12 +8,24 @@ #include #include #include +#include +#include #include "Utility/IpplException.h" namespace CatalystAdaptor { + template + using FieldVariant = std::variant*, VField_t*>; + + template + using FieldPair = std::pair>; + + template + using ParticlePair = std::pair > >; + + using View_vector = Kokkos::View***, Kokkos::LayoutLeft, Kokkos::HostSpace>; inline void setData(conduit_cpp::Node& node, const View_vector& view) { @@ -32,6 +42,7 @@ namespace CatalystAdaptor { node["electrostatic/values/z"].set_external(&view.data()[0][2], length, 0, 1); } + using View_scalar = Kokkos::View; inline void setData(conduit_cpp::Node& node, const View_scalar& view) { node["density/association"].set_string("element"); @@ -41,24 +52,20 @@ namespace CatalystAdaptor { node["density/values"].set_external(view.data(), view.size()); } - inline void callCatalystExecute(const conduit_cpp::Node& node) { - - // TODO: we should add here this IPPL-INFO stuff - //if ( static auto called {false}; !std::exchange(called, true) ) { - // catalyst_conduit_node_print(conduit_cpp::c_node(&node)); - //} - - catalyst_status err = catalyst_execute(conduit_cpp::c_node(&node)); - if (err != catalyst_status_ok) { - std::cerr << "Failed to execute Catalyst: " << err << std::endl; - } - } void Initialize(int argc, char* argv[]) { conduit_cpp::Node node; - for (int cc = 1; cc < argc; ++cc) { - node["catalyst/scripts/script" + std::to_string(cc - 1)].set_string(argv[cc]); + std::cout << "pvscript path: " << argv[1] << std::endl; + std::cout << "pvproxy path: " << argv[2] << std::endl; + node["catalyst/scripts/script/filename"].set(argv[1]); + node["catalyst/proxies/proxy"].set(argv[2]); + /* + for (int cc = 2; cc < argc; ++cc) { + std::cout << "pvscript args: " << argv[cc] << std::endl; + conduit_cpp::Node list_entry = node["catalyst/scripts/script/args"].append(); + list_entry.set(argv[cc]); } + */ try { node["catalyst_load/implementation"] = getenv("CATALYST_IMPLEMENTATION_NAME"); node["catalyst_load/search_paths/paraview"] = getenv("CATALYST_IMPLEMENTATION_PATHS"); @@ -75,69 +82,89 @@ namespace CatalystAdaptor { } - void Initialize_Adios(int argc, char* argv[]) - { + void Finalize() { conduit_cpp::Node node; - for (int cc = 1; cc < argc; ++cc) - { - if (strstr(argv[cc], "xml")) - { - node["adios/config_filepath"].set_string(argv[cc]); - } - else - { - node["catalyst/scripts/script" +std::to_string(cc - 1)].set_string(argv[cc]); - } - } - node["catalyst_load/implementation"] = getenv("CATALYST_IMPLEMENTATION_NAME"); - catalyst_status err = catalyst_initialize(conduit_cpp::c_node(&node)); - if (err != catalyst_status_ok) - { - std::cerr << "Failed to initialize Catalyst: " << err << std::endl; + catalyst_status err = catalyst_finalize(conduit_cpp::c_node(&node)); + if (err != catalyst_status_ok) { + std::cerr << "Failed to finalize Catalyst: " << err << std::endl; } } - template - std::optional Execute_Field(int cycle, double time, int rank, Field& field, std::optional& node_in) { - static_assert(Field::dim == 3, "CatalystAdaptor only supports 3D"); - // catalyst blueprint definition - // https://docs.paraview.org/en/latest/Catalyst/blueprints.html - // - // conduit blueprint definition (v.8.3) - // https://llnl-conduit.readthedocs.io/en/latest/blueprint_mesh.html - conduit_cpp::Node node; - if (node_in) - node = node_in.value(); + void Execute_Particle( + const auto& particleContainer, + const auto& R_host, const auto& P_host, const auto& q_host, const auto& ID_host, + const std::string& particlesName, + conduit_cpp::Node& node) { - auto nGhost = field.getNghost(); + // channel for particles + auto channel = node["catalyst/channels/ippl_" + particlesName]; + channel["type"].set_string("mesh"); - typename Field::view_type::host_mirror_type host_view = - Kokkos::create_mirror_view_and_copy(Kokkos::HostSpace(), field.getView()); + // in data channel now we adhere to conduits mesh blueprint definition + auto mesh = channel["data"]; + mesh["coordsets/coords/type"].set_string("explicit"); - Kokkos::View - host_view_layout_left("host_view_layout_left", - field.getLayout().getLocalNDIndex()[0].length(), - field.getLayout().getLocalNDIndex()[1].length(), - field.getLayout().getLocalNDIndex()[2].length()); + //mesh["coordsets/coords/values/x"].set_external(&layout_view.data()[0][0], particleContainer->getLocalNum(), 0, sizeof(double)*3); + //mesh["coordsets/coords/values/y"].set_external(&layout_view.data()[0][1], particleContainer->getLocalNum(), 0, sizeof(double)*3); + //mesh["coordsets/coords/values/z"].set_external(&layout_view.data()[0][2], particleContainer->getLocalNum(), 0, sizeof(double)*3); + mesh["coordsets/coords/values/x"].set_external(&R_host.data()[0][0], particleContainer->getLocalNum(), 0, sizeof(double)*3); + mesh["coordsets/coords/values/y"].set_external(&R_host.data()[0][1], particleContainer->getLocalNum(), 0, sizeof(double)*3); + mesh["coordsets/coords/values/z"].set_external(&R_host.data()[0][2], particleContainer->getLocalNum(), 0, sizeof(double)*3); - for (size_t i = 0; i < field.getLayout().getLocalNDIndex()[0].length(); ++i) { - for (size_t j = 0; j < field.getLayout().getLocalNDIndex()[1].length(); ++j) { - for (size_t k = 0; k < field.getLayout().getLocalNDIndex()[2].length(); ++k) { - host_view_layout_left(i, j, k) = host_view(i + nGhost, j + nGhost, k + nGhost); - } - } - } + mesh["topologies/mesh/type"].set_string("unstructured"); + mesh["topologies/mesh/coordset"].set_string("coords"); + mesh["topologies/mesh/elements/shape"].set_string("point"); + //mesh["topologies/mesh/elements/connectivity"].set_external(particleContainer->ID.getView().data(),particleContainer->getLocalNum()); + mesh["topologies/mesh/elements/connectivity"].set_external(ID_host.data(),particleContainer->getLocalNum()); + //auto charge_view = particleContainer->getQ().getView(); + + // add values for scalar charge field + auto fields = mesh["fields"]; + fields["charge/association"].set_string("vertex"); + fields["charge/topology"].set_string("mesh"); + fields["charge/volume_dependent"].set_string("false"); + + //fields["charge/values"].set_external(particleContainer->q.getView().data(), particleContainer->getLocalNum()); + fields["charge/values"].set_external(q_host.data(), particleContainer->getLocalNum()); + + // add values for vector velocity field + //auto velocity_view = particleContainer->P.getView(); + fields["velocity/association"].set_string("vertex"); + fields["velocity/topology"].set_string("mesh"); + fields["velocity/volume_dependent"].set_string("false"); + + //fields["velocity/values/x"].set_external(&velocity_view.data()[0][0], particleContainer->getLocalNum(),0 ,sizeof(double)*3); + //fields["velocity/values/y"].set_external(&velocity_view.data()[0][1], particleContainer->getLocalNum(),0 ,sizeof(double)*3); + //fields["velocity/values/z"].set_external(&velocity_view.data()[0][2], particleContainer->getLocalNum(),0 ,sizeof(double)*3); + fields["velocity/values/x"].set_external(&P_host.data()[0][0], particleContainer->getLocalNum(),0 ,sizeof(double)*3); + fields["velocity/values/y"].set_external(&P_host.data()[0][1], particleContainer->getLocalNum(),0 ,sizeof(double)*3); + fields["velocity/values/z"].set_external(&P_host.data()[0][2], particleContainer->getLocalNum(),0 ,sizeof(double)*3); + + fields["position/association"].set_string("vertex"); + fields["position/topology"].set_string("mesh"); + fields["position/volume_dependent"].set_string("false"); + + //fields["position/values/x"].set_external(&layout_view.data()[0][0], particleContainer->getLocalNum(), 0, sizeof(double)*3); + //fields["position/values/y"].set_external(&layout_view.data()[0][1], particleContainer->getLocalNum(), 0, sizeof(double)*3); + //fields["position/values/z"].set_external(&layout_view.data()[0][2], particleContainer->getLocalNum(), 0, sizeof(double)*3); + fields["position/values/x"].set_external(&R_host.data()[0][0], particleContainer->getLocalNum(), 0, sizeof(double)*3); + fields["position/values/y"].set_external(&R_host.data()[0][1], particleContainer->getLocalNum(), 0, sizeof(double)*3); + fields["position/values/z"].set_external(&R_host.data()[0][2], particleContainer->getLocalNum(), 0, sizeof(double)*3); + } - // add time/cycle information - auto state = node["catalyst/state"]; - state["cycle"].set(cycle); - state["time"].set(time); - state["domain_id"].set(rank); - // add catalyst channel named ippl_field, as fields is reserved - auto channel = node["catalyst/channels/ippl_field"]; + template // == ippl::Field, Cell>* + void Execute_Field(Field* field, const std::string& fieldName, + Kokkos::View& host_view_layout_left, + conduit_cpp::Node& node) { + static_assert(Field::dim == 3, "CatalystAdaptor only supports 3D"); + + // A) define mesh + + // add catalyst channel named ippl_"field", as fields is reserved + auto channel = node["catalyst/channels/ippl_" + fieldName]; channel["type"].set_string("mesh"); // in data channel now we adhere to conduits mesh blueprint definition @@ -149,17 +176,17 @@ namespace CatalystAdaptor { std::string field_node_origin{"coordsets/coords/origin/x"}; std::string field_node_spacing{"coordsets/coords/spacing/dx"}; - for (unsigned int iDim = 0; iDim < field.get_mesh().getGridsize().dim; ++iDim) { + for (unsigned int iDim = 0; iDim < field->get_mesh().getGridsize().dim; ++iDim) { // add dimension - mesh[field_node_dim].set(field.getLayout().getLocalNDIndex()[iDim].length() + 1); + mesh[field_node_dim].set(field->getLayout().getLocalNDIndex()[iDim].length() + 1); // add origin mesh[field_node_origin].set( - field.get_mesh().getOrigin()[iDim] + field.getLayout().getLocalNDIndex()[iDim].first() - * field.get_mesh().getMeshSpacing(iDim)); + field->get_mesh().getOrigin()[iDim] + field->getLayout().getLocalNDIndex()[iDim].first() + * field->get_mesh().getMeshSpacing(iDim)); // add spacing - mesh[field_node_spacing].set(field.get_mesh().getMeshSpacing(iDim)); + mesh[field_node_spacing].set(field->get_mesh().getMeshSpacing(iDim)); // increment last char in string ++field_node_dim.back(); @@ -171,50 +198,92 @@ namespace CatalystAdaptor { mesh["topologies/mesh/type"].set_string("uniform"); mesh["topologies/mesh/coordset"].set_string("coords"); std::string field_node_origin_topo{"topologies/mesh/origin/x"}; - for (unsigned int iDim = 0; iDim < field.get_mesh().getGridsize().dim; ++iDim) { + for (unsigned int iDim = 0; iDim < field->get_mesh().getGridsize().dim; ++iDim) { // shift origin - mesh[field_node_origin_topo].set(field.get_mesh().getOrigin()[iDim] - + field.getLayout().getLocalNDIndex()[iDim].first() - * field.get_mesh().getMeshSpacing(iDim)); + mesh[field_node_origin_topo].set(field->get_mesh().getOrigin()[iDim] + + field->getLayout().getLocalNDIndex()[iDim].first() + * field->get_mesh().getMeshSpacing(iDim)); - // increment last char in string + // increment last char in string ('x' becomes 'y' becomes 'z') ++field_node_origin_topo.back(); } - // add values and subscribe to data + // B) Set the field values + + // Initialize the existing Kokkos::View + host_view_layout_left = Kokkos::View( + "host_view_layout_left", + field->getLayout().getLocalNDIndex()[0].length(), + field->getLayout().getLocalNDIndex()[1].length(), + field->getLayout().getLocalNDIndex()[2].length()); + + // Creates a host-accessible mirror view and copies the data from the device view to the host. + auto host_view = + Kokkos::create_mirror_view_and_copy(Kokkos::HostSpace(), field->getView()); + + // Copy data from field to the memory+style which will be passed to Catalyst + auto nGhost = field->getNghost(); + for (size_t i = 0; i < field->getLayout().getLocalNDIndex()[0].length(); ++i) { + for (size_t j = 0; j < field->getLayout().getLocalNDIndex()[1].length(); ++j) { + for (size_t k = 0; k < field->getLayout().getLocalNDIndex()[2].length(); ++k) { + host_view_layout_left(i, j, k) = host_view(i + nGhost, j + nGhost, k + nGhost); + } + } + } + + // Add values and subscribe to data auto fields = mesh["fields"]; setData(fields, host_view_layout_left); + } + + void AddSteerableChannel(conduit_cpp::Node& node, double scaleFactor) { + auto steerable = node["catalyst/channels/steerable"]; + steerable["type"].set("mesh"); + + auto steerable_data = steerable["data"]; + steerable_data["coordsets/coords/type"].set_string("explicit"); + steerable_data["coordsets/coords/values/x"].set_float64_vector({ 1 }); + steerable_data["coordsets/coords/values/y"].set_float64_vector({ 2 }); + steerable_data["coordsets/coords/values/z"].set_float64_vector({ 3 }); + steerable_data["topologies/mesh/type"].set("unstructured"); + steerable_data["topologies/mesh/coordset"].set("coords"); + steerable_data["topologies/mesh/elements/shape"].set("point"); + steerable_data["topologies/mesh/elements/connectivity"].set_int32_vector({ 0 }); + steerable_data["fields/steerable/association"].set("vertex"); + steerable_data["fields/steerable/topology"].set("mesh"); + steerable_data["fields/steerable/volume_dependent"].set("false"); + steerable_data["fields/steerable/values"].set_float64_vector({scaleFactor}); + } - // as we have a local copy of the field, the catalyst_execute needs to be called - // within this scope otherwise the memory location might be already overwritten - if (node_in == std::nullopt) + void Results( double& scaleFactor) { + + conduit_cpp::Node results; + catalyst_status err = catalyst_results(conduit_cpp::c_node(&results)); + + if (err != catalyst_status_ok) { - callCatalystExecute(node); - return {}; + std::cerr << "Failed to execute Catalyst-results: " << err << std::endl; } else - return node; - + { + std::cout << "Result Node dump:" << std::endl; + const std::string value_path = "catalyst/steerable/fields/scalefactor/values"; + scaleFactor = results[value_path].to_double(); + } } - template - std::optional Execute_Particle(int cycle, double time, int rank, ParticleContainer& particleContainer, std::optional& node_in) { - assert((particleContainer->ID.getView().data() != nullptr) && "ID view should not be nullptr, might be missing the right execution space"); - - //auto layout_view = particleContainer->R.getView(); - typename ippl::ParticleAttrib>::HostMirror R_host = particleContainer->R.getHostMirror(); - typename ippl::ParticleAttrib>::HostMirror P_host = particleContainer->P.getHostMirror(); - typename ippl::ParticleAttrib::HostMirror q_host = particleContainer->q.getHostMirror(); - typename ippl::ParticleAttrib::HostMirror ID_host = particleContainer->ID.getHostMirror(); - Kokkos::deep_copy(R_host, particleContainer->R.getView()); - Kokkos::deep_copy(P_host, particleContainer->P.getView()); - Kokkos::deep_copy(q_host, particleContainer->q.getView()); - Kokkos::deep_copy(ID_host, particleContainer->ID.getView()); - - // if node is passed in, append data to it + template + void Execute(int cycle, double time, int rank, + // const auto& /* std::shared_ptr >& */ particle, + const std::vector>& particles, + const std::vector>& fields, double& scaleFactor) { + + // catalyst blueprint definition + // https://docs.paraview.org/en/latest/Catalyst/blueprints.html + // + // conduit blueprint definition (v.8.3) + // https://llnl-conduit.readthedocs.io/en/latest/blueprint_mesh.html conduit_cpp::Node node; - if (node_in) - node = node_in.value(); // add time/cycle information auto state = node["catalyst/state"]; @@ -222,88 +291,79 @@ namespace CatalystAdaptor { state["time"].set(time); state["domain_id"].set(rank); - // channel for particles - auto channel = node["catalyst/channels/ippl_particle"]; - channel["type"].set_string("mesh"); - - // in data channel now we adhere to conduits mesh blueprint definition - auto mesh = channel["data"]; - mesh["coordsets/coords/type"].set_string("explicit"); - - //mesh["coordsets/coords/values/x"].set_external(&layout_view.data()[0][0], particleContainer->getLocalNum(), 0, sizeof(double)*3); - //mesh["coordsets/coords/values/y"].set_external(&layout_view.data()[0][1], particleContainer->getLocalNum(), 0, sizeof(double)*3); - //mesh["coordsets/coords/values/z"].set_external(&layout_view.data()[0][2], particleContainer->getLocalNum(), 0, sizeof(double)*3); - mesh["coordsets/coords/values/x"].set_external(&R_host.data()[0][0], particleContainer->getLocalNum(), 0, sizeof(double)*3); - mesh["coordsets/coords/values/y"].set_external(&R_host.data()[0][1], particleContainer->getLocalNum(), 0, sizeof(double)*3); - mesh["coordsets/coords/values/z"].set_external(&R_host.data()[0][2], particleContainer->getLocalNum(), 0, sizeof(double)*3); + // Handle particles - mesh["topologies/mesh/type"].set_string("unstructured"); - mesh["topologies/mesh/coordset"].set_string("coords"); - mesh["topologies/mesh/elements/shape"].set_string("point"); - //mesh["topologies/mesh/elements/connectivity"].set_external(particleContainer->ID.getView().data(),particleContainer->getLocalNum()); - mesh["topologies/mesh/elements/connectivity"].set_external(ID_host.data(),particleContainer->getLocalNum()); + std::map>::HostMirror> R_host_map; + std::map>::HostMirror> P_host_map; + std::map::HostMirror> q_host_map; + std::map::HostMirror> ID_host_map; - //auto charge_view = particleContainer->getQ().getView(); + // Loop over all particle container + for (const auto& particlesPair : particles) + { + const std::string& particlesName = particlesPair.first; + const auto particleContainer = particlesPair.second; + + assert((particleContainer->ID.getView().data() != nullptr) && "ID view should not be nullptr, might be missing the right execution space"); + + R_host_map[particlesName] = particleContainer->R.getHostMirror(); + P_host_map[particlesName] = particleContainer->P.getHostMirror(); + q_host_map[particlesName] = particleContainer->q.getHostMirror(); + ID_host_map[particlesName] = particleContainer->ID.getHostMirror(); + + Kokkos::deep_copy(R_host_map[particlesName], particleContainer->R.getView()); + Kokkos::deep_copy(P_host_map[particlesName], particleContainer->P.getView()); + Kokkos::deep_copy(q_host_map[particlesName], particleContainer->q.getView()); + Kokkos::deep_copy(ID_host_map[particlesName], particleContainer->ID.getView()); + + Execute_Particle( + particleContainer, + R_host_map[particlesName], P_host_map[particlesName], q_host_map[particlesName], ID_host_map[particlesName], + particlesName, + node); + } - // add values for scalar charge field - auto fields = mesh["fields"]; - fields["charge/association"].set_string("vertex"); - fields["charge/topology"].set_string("mesh"); - fields["charge/volume_dependent"].set_string("false"); - //fields["charge/values"].set_external(particleContainer->q.getView().data(), particleContainer->getLocalNum()); - fields["charge/values"].set_external(q_host.data(), particleContainer->getLocalNum()); + // Handle fields - // add values for vector velocity field - //auto velocity_view = particleContainer->P.getView(); - fields["velocity/association"].set_string("vertex"); - fields["velocity/topology"].set_string("mesh"); - fields["velocity/volume_dependent"].set_string("false"); + // Map of all Kokkos::Views. This keeps a reference on all Kokkos::Views + // which ensures that Kokkos does not free the memory before the end of this function. + std::map::view_type::data_type, Kokkos::LayoutLeft, Kokkos::HostSpace> > scalar_host_views; + std::map::view_type::data_type, Kokkos::LayoutLeft, Kokkos::HostSpace> > vector_host_views; - //fields["velocity/values/x"].set_external(&velocity_view.data()[0][0], particleContainer->getLocalNum(),0 ,sizeof(double)*3); - //fields["velocity/values/y"].set_external(&velocity_view.data()[0][1], particleContainer->getLocalNum(),0 ,sizeof(double)*3); - //fields["velocity/values/z"].set_external(&velocity_view.data()[0][2], particleContainer->getLocalNum(),0 ,sizeof(double)*3); - fields["velocity/values/x"].set_external(&P_host.data()[0][0], particleContainer->getLocalNum(),0 ,sizeof(double)*3); - fields["velocity/values/y"].set_external(&P_host.data()[0][1], particleContainer->getLocalNum(),0 ,sizeof(double)*3); - fields["velocity/values/z"].set_external(&P_host.data()[0][2], particleContainer->getLocalNum(),0 ,sizeof(double)*3); + // Loop over all fields + for (const auto& fieldPair : fields) + { + const std::string& fieldName = fieldPair.first; + const auto& fieldVariant = fieldPair.second; - fields["position/association"].set_string("vertex"); - fields["position/topology"].set_string("mesh"); - fields["position/volume_dependent"].set_string("false"); + // If field is a _scalar_ field + if (std::holds_alternative*>(fieldVariant)) { + Field_t* field = std::get*>(fieldVariant); + // == ippl::Field, Cell>* - //fields["position/values/x"].set_external(&layout_view.data()[0][0], particleContainer->getLocalNum(), 0, sizeof(double)*3); - //fields["position/values/y"].set_external(&layout_view.data()[0][1], particleContainer->getLocalNum(), 0, sizeof(double)*3); - //fields["position/values/z"].set_external(&layout_view.data()[0][2], particleContainer->getLocalNum(), 0, sizeof(double)*3); - fields["position/values/x"].set_external(&R_host.data()[0][0], particleContainer->getLocalNum(), 0, sizeof(double)*3); - fields["position/values/y"].set_external(&R_host.data()[0][1], particleContainer->getLocalNum(), 0, sizeof(double)*3); - fields["position/values/z"].set_external(&R_host.data()[0][2], particleContainer->getLocalNum(), 0, sizeof(double)*3); + Execute_Field(field, fieldName, scalar_host_views[fieldName], node); + } + // If field is a _vector_ field + else if (std::holds_alternative*>(fieldVariant)) { + VField_t* field = std::get*>(fieldVariant); + // == ippl::Field, 3, ippl::UniformCartesian, Cell>* - // this node we can return as the pointer to velocity and charge is globally valid - if (node_in == std::nullopt) - { - callCatalystExecute(node); - return {}; + Execute_Field(field, fieldName, vector_host_views[fieldName], node); + } } - else - return node; - } + AddSteerableChannel(node, scaleFactor); - template - void Execute_Field_Particle(int cycle, double time, int rank, Field& field, ParticleContainer& particle) { - - auto node = std::make_optional(); - node = CatalystAdaptor::Execute_Particle(cycle, time, rank, particle, node); - CatalystAdaptor::Execute_Field(cycle, time, rank, field, node); - } - - void Finalize() { - conduit_cpp::Node node; - catalyst_status err = catalyst_finalize(conduit_cpp::c_node(&node)); + // Pass Conduit node to Catalyst + catalyst_status err = catalyst_execute(conduit_cpp::c_node(&node)); if (err != catalyst_status_ok) { - std::cerr << "Failed to finalize Catalyst: " << err << std::endl; + std::cerr << "Failed to execute Catalyst: " << err << std::endl; } + + Results(scaleFactor); } + } // namespace CatalystAdaptor #endif diff --git a/alpine/PenningTrap.cpp b/alpine/PenningTrap.cpp index 3f351a10b..34b732768 100644 --- a/alpine/PenningTrap.cpp +++ b/alpine/PenningTrap.cpp @@ -39,8 +39,13 @@ const char* TestName = "PenningTrap"; #include "Manager/PicManager.h" #include "PenningTrapManager.h" + +#ifdef ENABLE_ASCENT +#include "AscentAdaptor.h" +#endif + #ifdef ENABLE_CATALYST -#include "Stream/InSitu/CatalystAdaptor.h" +#include "CatalystAdaptor.h" #endif int main(int argc, char* argv[]) { @@ -48,7 +53,33 @@ int main(int argc, char* argv[]) { { #ifdef ENABLE_CATALYST - CatalystAdaptor::Initialize(argc, argv); + char* script = nullptr; + char* proxy = nullptr; + for (int i = 1; i < argc; ++i) { + if (std::string(argv[i]) == "--pvscript" && i + 1 < argc) { + script = argv[i+1]; + i++; + } + if (std::string(argv[i]) == "--pvproxy" && i+1 < argc) { + proxy = argv[i+1]; + i++; + } + } + char* reducedArgv[] = { argv[0], script, proxy}; + CatalystAdaptor::Initialize(2, reducedArgv); +#endif + +#ifdef ENABLE_ASCENT + int frequency = 1; + for (int i = 1; i < argc; ++i) { + if (std::string(argv[i]) == "--frequency" && i + 1 < argc) { + frequency = atoi(argv[i+1]); + std::cout << "Frequency: " << frequency << std::endl; + i++; + } + } + + AscentAdaptor::Initialize(frequency); #endif Inform msg(TestName); @@ -90,6 +121,10 @@ int main(int argc, char* argv[]) { CatalystAdaptor::Finalize(); #endif +#ifdef ENABLE_ASCENT + AscentAdaptor::Finalize(); +#endif + IpplTimings::stopTimer(mainTimer); IpplTimings::print(); @@ -99,3 +134,4 @@ int main(int argc, char* argv[]) { return 0; } + diff --git a/alpine/PenningTrapManager.h b/alpine/PenningTrapManager.h index 08fe37f99..8459009e0 100644 --- a/alpine/PenningTrapManager.h +++ b/alpine/PenningTrapManager.h @@ -15,8 +15,11 @@ #include "Random/Randn.h" #ifdef ENABLE_CATALYST -#include -#include "Stream/InSitu/CatalystAdaptor.h" +#include "CatalystAdaptor.h" +#endif + +#ifdef ENABLE_ASCENT +#include "AscentAdaptor.h" #endif using view_type = typename ippl::detail::ViewType, 1>::view_type; @@ -28,10 +31,11 @@ class PenningTrapManager : public AlpineManager { using FieldContainer_t = FieldContainer; using FieldSolver_t= FieldSolver; using LoadBalancer_t= LoadBalancer; + double scaleFactor; PenningTrapManager(size_type totalP_, int nt_, Vector_t &nr_, double lbt_, std::string& solver_, std::string& stepMethod_) - : AlpineManager(totalP_, nt_, nr_, lbt_, solver_, stepMethod_){} + : AlpineManager(totalP_, nt_, nr_, lbt_, solver_, stepMethod_),scaleFactor(30){} ~PenningTrapManager(){} @@ -229,7 +233,7 @@ class PenningTrapManager : public AlpineManager { double alpha = this->alpha_m; double Bext = this->Bext_m; double DrInv = this->DrInv_m; - double V0 = 30 * this->length_m[2]; + double V0 = scaleFactor * this->length_m[2]; Vector_t length = this->length_m; Vector_t origin = this->origin_m; double dt = this->dt_m; @@ -284,14 +288,31 @@ class PenningTrapManager : public AlpineManager { // scatter the charge onto the underlying grid this->par2grid(); + #ifdef ENABLE_CATALYST - std::optional node = std::nullopt; - //CatalystAdaptor::Execute_Particle(it, this->time_m, ippl::Comm->rank(), pc, node); - auto *rho = &this->fcontainer_m->getRho(); - CatalystAdaptor::Execute_Field(it, this->time_m, ippl::Comm->rank(), *rho, node); - //auto *E = &this->fcontainer_m->getE(); - //CatalystAdaptor::Execute_Field(it, this->time_m, ippl::Comm->rank(), *E, node); - //CatalystAdaptor::Execute_Field_Particle(it, this->time_m, ippl::Comm->rank(), *E, pc); + std::vector> particles = { + {"particle", std::shared_ptr >(pc)}, + }; + std::vector> fields = { + {"E", CatalystAdaptor::FieldVariant(&this->fcontainer_m->getE())}, + {"roh", CatalystAdaptor::FieldVariant(&this->fcontainer_m->getRho())}, + //{"phi", CatalystAdaptor::FieldVariant(&this->fcontainer_m->getPhi())}, + }; + CatalystAdaptor::Execute(it, this->time_m, ippl::Comm->rank(), particles, fields, scaleFactor); +#endif + +#ifdef ENABLE_ASCENT + + std::vector> particles = { + {"particle", std::shared_ptr >(pc)}, + }; + std::vector> fields = { + {"E", AscentAdaptor::FieldVariant(&this->fcontainer_m->getE())}, + {"roh", AscentAdaptor::FieldVariant(&this->fcontainer_m->getRho())}, + //{"phi", CatalystAdaptor::FieldVariant(&this->fcontainer_m->getPhi())}, + }; + AscentAdaptor::Execute(it, this->time_m , particles, fields); + #endif // Field solve diff --git a/alpine/ascent_actions.yaml b/alpine/ascent_actions.yaml new file mode 100644 index 000000000..9cc3d6648 --- /dev/null +++ b/alpine/ascent_actions.yaml @@ -0,0 +1,34 @@ +- + action: "add_pipelines" + pipelines: + pl1: + f1: + type: "threshold" + params: + field: "particle_magnitude" + min_value: 9.2 + max_value: 25.0 +- + action: "add_scenes" + scenes: + s1: + plots: + p1: + type: pseudocolor + field: particle_charge + points: + radius: 0.1 + pipeline: "pl1" + p2: + type: pseudocolor + field: particle_center + points: + radius: 0.5 + renders: + r1: + image_prefix: "render_%06d" + screen_annotations: "false" + camera: + look_at: [10, 0, 10] + up: [1, 0, 0] + position: [10, 40, 10] diff --git a/build-cuda-sophia.sh b/build-cuda-sophia.sh new file mode 100755 index 000000000..a939f8e46 --- /dev/null +++ b/build-cuda-sophia.sh @@ -0,0 +1,27 @@ +#!/bin/bash + +export CXX=mpiCC +export CC=mpicc + +cmake -B build-cuda-sophia -S . \ + -DCMAKE_INSTALL_PREFIX=install-cuda \ + -DCMAKE_BUILD_TYPE=Release \ + -DCMAKE_CXX_COMPILER=mpiCC \ + -DCMAKE_C_COMPILER=mpicc \ + -DCMAKE_CUDA_HOST_COMPILER=mpiCC \ + -DCUDA_TOOLKIT_ROOT_DIR=/usr/local/cuda \ + -DKokkos_ENABLE_CUDA=ON \ + -DKokkos_ARCH_AMPERE80=ON \ + -DKokkos_ENABLE_OPENMP=ON \ + -DCMAKE_CXX_STANDARD=20 \ + -DCMAKE_CXX_STANDARD_REQUIRED=ON \ + -DENABLE_FFT=ON \ + -DENABLE_SOLVERS=ON \ + -DENABLE_ALPINE=True \ + -DENABLE_TESTS=ON \ + -DUSE_ALTERNATIVE_VARIANT=ON \ + -DIPPL_PLATFORMS=cuda \ + -DAscent_DIR=`pwd`/../../ascent/scripts/build_ascent/install/ascent-checkout/lib/cmake/ascent + + +cmake --build build-cuda-sophia --target install -j8 diff --git a/run.polaris.sh b/run.polaris.sh new file mode 100644 index 000000000..5bb6812ce --- /dev/null +++ b/run.polaris.sh @@ -0,0 +1,11 @@ +#!/bin/bash + +NNODES=`wc -l < $PBS_NODEFILE` +NRANKS=1 +NDEPTH=32 +NTHREADS=32 + +NTOTRANKS=$(( NNODES * NRANKS )) +echo "NUM_OF_NODES= ${NNODES} TOTAL_NUM_RANKS= ${NTOTRANKS} RANKS_PER_NODE= ${NRANKS} THREADS_PER_RANK= ${NTHREADS}" + +mpirun --np ${NTOTRANKS} -ppn ${NRANKS} -d ${NDEPTH} --cpu-bind depth -env OMP_NUM_THREADS=${NTHREADS} ./PenningTrap 32 32 32 655360 400 FFT 0.01 LeapFrog --overallocate 1.0 --info 5 --frequency 2 \ No newline at end of file diff --git a/sourceme.polaris.sh b/sourceme.polaris.sh new file mode 100644 index 000000000..b3e0c3072 --- /dev/null +++ b/sourceme.polaris.sh @@ -0,0 +1,9 @@ +module reset +module use /soft/modulefiles +module switch PrgEnv-nvhpc PrgEnv-gnu +module load spack-pe-base cmake +module load cudatoolkit-standalone +module load visualization/ascent + +export CXX=CC +export CC=cc diff --git a/sourceme.sophia.sh b/sourceme.sophia.sh new file mode 100644 index 000000000..a06da1cfb --- /dev/null +++ b/sourceme.sophia.sh @@ -0,0 +1,18 @@ +export HTTP_PROXY="http://proxy.alcf.anl.gov:3128" +export HTTPS_PROXY="http://proxy.alcf.anl.gov:3128" +export http_proxy="http://proxy.alcf.anl.gov:3128" +export https_proxy="http://proxy.alcf.anl.gov:3128" +export ftp_proxy="http://proxy.alcf.anl.gov:3128" +export no_proxy="admin,polaris-adminvm-01,localhost,*.cm.polaris.alcf.anl.gov,polaris-*,*.polaris.alcf.anl.gov,*.alcf.anl.gov" + + +module reset +module use /soft/modulefiles +module switch PrgEnv-nvhpc PrgEnv-gnu +module load spack-pe-base cmake +#module load kokkos/4.2.01 +module load cudatoolkit-standalone +module load visualization/ascent + +export CXX=CC +export CC=cc diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index b5fd3eb26..0771560da 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -82,7 +82,7 @@ add_subdirectory (Expression) add_subdirectory (Types) add_subdirectory (Partition) add_subdirectory (Stream) -add_subdirectory (Stream/InSitu) +# add_subdirectory (Stream/InSitu) add_subdirectory (Random) if (ENABLE_SOLVERS) @@ -98,10 +98,17 @@ if (ENABLE_CATALYST) message (STATUS "Found catalyst_DIR: ${catalyst_DIR}") endif() +if (ENABLE_ASCENT) + find_package(Ascent REQUIRED) + message (STATUS "Enable Ascent") + message (STATUS "Found ascent_DIR: ${Ascent_DIR}") +endif() + if (ENABLE_AMR) add_subdirectory(AmrParticle) endif () include_directories ( + $<$:${ASCENT_INSTALL_PREFIX}/include/ascent> BEFORE ${CMAKE_CURRENT_SOURCE_DIR} ) @@ -119,6 +126,7 @@ message("Timer fences for this build: ${TimerFences}") target_compile_definitions(ippl PUBLIC $<$:-DENABLE_FFT> $<$:-DENABLE_CATALYST> + $<$:-DENABLE_ASCENT> IPPL_ENABLE_TIMER_FENCES=${TimerFences} ) @@ -126,6 +134,7 @@ target_link_libraries(ippl PUBLIC Kokkos::kokkos $<$:Heffte> $<$:catalyst::catalyst> + $<$:ascent::ascent_mpi> ) install (TARGETS ippl DESTINATION lib) diff --git a/src/Stream/InSitu/CMakeLists.txt b/src/Stream/InSitu/CMakeLists.txt index 6b3ddbcfc..dfbb7234b 100644 --- a/src/Stream/InSitu/CMakeLists.txt +++ b/src/Stream/InSitu/CMakeLists.txt @@ -2,7 +2,6 @@ set (_SRCS ) set (_HDRS - CatalystAdaptor.h ) include_DIRECTORIES ( @@ -21,4 +20,4 @@ install (FILES ${_HDRS} DESTINATION include/Stream/InSitu) # cmake-tab-width: 4 # indent-tabs-mode: nil # require-final-newline: nil -# End: \ No newline at end of file +# End: diff --git a/src/Stream/InSitu/proxy.xml b/src/Stream/InSitu/proxy.xml new file mode 100644 index 000000000..040c0c128 --- /dev/null +++ b/src/Stream/InSitu/proxy.xml @@ -0,0 +1,54 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/src/Stream/InSitu/steering.py b/src/Stream/InSitu/steering.py new file mode 100644 index 000000000..d33df3e4a --- /dev/null +++ b/src/Stream/InSitu/steering.py @@ -0,0 +1,119 @@ +# script-version: 2.0 +# for more details check https://www.paraview.org/paraview-docs/latest/cxx/CatalystPythonScriptsV2.html +from paraview import print_info +from paraview.simple import * +from paraview import catalyst +import time + +#### disable automatic camera reset on 'Show' +paraview.simple._DisableFirstRenderCameraReset() + +# print start marker +print_info("begin '%s'", __name__) + +# ---------------------------------------------------------------- +# setup the data processing pipelines +# ---------------------------------------------------------------- + +# registrationName must match the channel name used in the 'CatalystAdaptor'. +ippl_field = PVTrivialProducer(registrationName='ippl_E') +ippl_particle = PVTrivialProducer(registrationName='ippl_particle') + + +from paraview.simple import LoadPlugin, CreateSteerableParameters + +# SteerableParameters 생성 +try: + steerable_parameters = CreateSteerableParameters("SteerableParameters") + print("SteerableParameters loaded successfully.") +except Exception as e: + print(f"Error loading SteerableParameters: {e}") + +# ---------------------------------------------------------------- +# setup animation scene, tracks and keyframes +# note: the Get..() functions create a new object, if needed +# ---------------------------------------------------------------- + +# get the time-keeper +timeKeeper1 = GetTimeKeeper() + +# initialize the timekeeper + +# get time animation track +timeAnimationCue1 = GetTimeTrack() + +# initialize the animation track + +# get animation scene +animationScene1 = GetAnimationScene() + +# initialize the animation scene +animationScene1.Cues = timeAnimationCue1 +animationScene1.AnimationTime = 0.0 + +# initialize the animation scene + +# ---------------------------------------------------------------- +# setup extractors +# ---------------------------------------------------------------- + +# create extractor (U = unstructured) +# vTI1 = CreateExtractor('VTI', ippl_field, registrationName='VTI1') +# vTI1.Trigger = 'Time Step' +# vTI1.Writer.FileName = 'ippl_field_{timestep:06d}.vti' + +# create extractor (PD=point data) +#vTPD2 = CreateExtractor('VTPD', ippl_particle, registrationName='VTPD2') +#vTPD2.Trigger = 'Time Step' +#vTPD2.Writer.FileName = 'ippl_particle_{timestep:06d}.vtpd' + +# ---------------------------------------------------------------- +# restore active source +#SetActiveSource(ippl_particle) +SetActiveSource(ippl_field) +# ---------------------------------------------------------------- + +# ------------------------------------------------------------------------------ +# Catalyst options +options = catalyst.Options() +options.GlobalTrigger = 'Time Step' +options.EnableCatalystLive = 1 +options.CatalystLiveTrigger = 'Time Step' +#options.ExtractsOutputDirectory = 'data_vtpd' + +# ------------------------------------------------------------------------------ +def catalyst_initialize(): + print_info("in '%s::catalyst_initialize'", __name__) + + +# ------------------------------------------------------------------------------ +def catalyst_execute(info): + print_info("in '%s::catalyst_execute'", __name__) + + global ippl_field + ippl_field.UpdatePipeline() + global ippl_particle + ippl_particle.UpdatePipeline() + + global steerable_parameters + steerable_parameters.scaleFactor[0] = 31 + info.cycle + + print("-----------------------------------") + print("executing (cycle={}, time={})".format(info.cycle, info.time)) + #print("field bounds :", ippl_field.GetDataInformation().GetBounds()) + #print("particle bounds:", ippl_particle.GetDataInformation().GetBounds()) + + # In a real simulation sleep is not needed. We use it here to slow down the + # "simulation" and make sure ParaView client can catch up with the produced + # results instead of having all of them flashing at once. + if options.EnableCatalystLive: + time.sleep(5) + + +# ------------------------------------------------------------------------------ +def catalyst_finalize(): + print_info("in '%s::catalyst_finalize'", __name__) + + +# print end marker +print_info("end '%s'", __name__) \ No newline at end of file diff --git a/src/Types/Variant.h b/src/Types/Variant.h index e45935a13..0d469fb4e 100644 --- a/src/Types/Variant.h +++ b/src/Types/Variant.h @@ -41,7 +41,7 @@ #include #include #include -#include // in_place_index_t +#include // in_place_index_t #include #include #if __cplusplus >= 202002L diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index 1f784cc0c..9cd986c1e 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -29,7 +29,6 @@ add_subdirectory (random) add_subdirectory (serialization) if (ENABLE_CATALYST) - add_subdirectory (stream) endif () # vi: set et ts=4 sw=4 sts=4: diff --git a/test/stream/CMakeLists.txt b/test/stream/CMakeLists.txt index 7248e5f5f..daa8fe893 100644 --- a/test/stream/CMakeLists.txt +++ b/test/stream/CMakeLists.txt @@ -13,12 +13,12 @@ link_directories ( set (IPPL_LIBS ippl) set (COMPILE_FLAGS ${OPAL_CXX_FLAGS}) -add_executable (TestCatalystAdaptor TestCatalystAdaptor.cpp) -target_link_libraries ( - TestCatalystAdaptor - ${IPPL_LIBS} - ${MPI_CXX_LIBRARIES} -) +#add_executable (TestCatalystAdaptor TestCatalystAdaptor.cpp) +#target_link_libraries ( +# TestCatalystAdaptor +# ${IPPL_LIBS} +# ${MPI_CXX_LIBRARIES} +#) # vi: set et ts=4 sw=4 sts=4: