Enable Ascent In Situ Visualization in PenningTrap Example

.gitignore

@@ -27,3 +27,5 @@ compile_commands.json
 __pycache__
 
 *.~
+
+.vscode

CMakeLists.txt

@@ -23,7 +23,6 @@ macro(remove_flag_from_target _target _flag)
     endif()
 endmacro()
 
-
 if (NOT CMAKE_CONFIGURATION_TYPES AND NOT CMAKE_BUILD_TYPE)
 
     set (CMAKE_BUILD_TYPE RelWithDebInfo CACHE STRING
@@ -197,6 +196,7 @@ if (ENABLE_FFT)
         endif()
         FetchContent_MakeAvailable(Heffte)
         # Required to suppress warnings related to heffte
+        MESSAGE(STATUS "HEFFTE: "${Heffte_SOURCE_DIR}"")
         target_include_directories(Heffte SYSTEM PUBLIC "${Heffte_SOURCE_DIR}/include")
         set(Heffte_DIR "${Heffte_SOURCE_DIR}")
         remove_flag_from_target(Heffte "-Werror")
@@ -207,6 +207,7 @@ if (ENABLE_FFT)
 endif ()
 
 option (ENABLE_CATALYST "Build example with Catalyst enabled" OFF)
+option (ENABLE_ASCENT "Build example with Ascent enabled" ON)
 option (ENABLE_SOLVERS "Enable IPPL solvers" ON)
 
 add_subdirectory (src)

alpine/CMakeLists.txt

@@ -24,6 +24,12 @@ target_link_libraries (LandauDamping ${IPPL_LIBS})
 add_executable (BumponTailInstability BumponTailInstability.cpp)
 target_link_libraries (BumponTailInstability ${IPPL_LIBS})
 
+configure_file(
+    "${CMAKE_CURRENT_SOURCE_DIR}/ascent_actions.yaml"
+    "${CMAKE_CURRENT_BINARY_DIR}/ascent_actions.yaml"
+    COPYONLY
+)
+
 # vi: set et ts=4 sw=4 sts=4:
 
 # Local Variables:

alpine/PenningTrap.cpp

@@ -39,16 +39,47 @@ const char* TestName   = "PenningTrap";
 #include "Manager/PicManager.h"
 #include "PenningTrapManager.h"
 
+
+#ifdef ENABLE_ASCENT
+#include "AscentAdaptor.h"
+#endif
+
 #ifdef ENABLE_CATALYST
-#include "Stream/InSitu/CatalystAdaptor.h"
+#include "CatalystAdaptor.h"
 #endif
 
 int main(int argc, char* argv[]) {
     ippl::initialize(argc, argv);
     {
 
 #ifdef ENABLE_CATALYST
-        CatalystAdaptor::Initialize(argc, argv);
+    char* script = nullptr;
+    char* proxy = nullptr;
+    for (int i = 1; i < argc; ++i) {
+        if (std::string(argv[i]) == "--pvscript" && i + 1 < argc) {
+            script = argv[i+1]; 
+            i++;
+        }   
+        if (std::string(argv[i]) == "--pvproxy" && i+1 < argc) {
+            proxy = argv[i+1];
+            i++;
+        }
+    }
+    char* reducedArgv[] = { argv[0], script, proxy};
+    CatalystAdaptor::Initialize(2, reducedArgv);
+#endif
+
+#ifdef ENABLE_ASCENT
+        int frequency = 1;
+        for (int i = 1; i < argc; ++i) {
+            if (std::string(argv[i]) == "--frequency" && i + 1 < argc) {
+                frequency = atoi(argv[i+1]); 
+                std::cout << "Frequency: " << frequency << std::endl;
+                i++;
+            } 
+        }
+
+        AscentAdaptor::Initialize(frequency);
 #endif
 
         Inform msg(TestName);
@@ -90,6 +121,10 @@ int main(int argc, char* argv[]) {
         CatalystAdaptor::Finalize();
 #endif
 
+#ifdef ENABLE_ASCENT
+        AscentAdaptor::Finalize();
+#endif
+
 
         IpplTimings::stopTimer(mainTimer);
         IpplTimings::print();
@@ -99,3 +134,4 @@ int main(int argc, char* argv[]) {
 
     return 0;
 }
+

alpine/PenningTrapManager.h

@@ -15,8 +15,11 @@
 #include "Random/Randn.h"
 
 #ifdef ENABLE_CATALYST
-#include <optional>
-#include "Stream/InSitu/CatalystAdaptor.h"
+#include "CatalystAdaptor.h"
+#endif
+
+#ifdef ENABLE_ASCENT
+#include "AscentAdaptor.h"
 #endif
 
 using view_type = typename ippl::detail::ViewType<ippl::Vector<double, Dim>, 1>::view_type;
@@ -28,10 +31,11 @@ class PenningTrapManager : public AlpineManager<T, Dim> {
     using FieldContainer_t = FieldContainer<T, Dim>;
     using FieldSolver_t= FieldSolver<T, Dim>;
     using LoadBalancer_t= LoadBalancer<T, Dim>;
+    double scaleFactor;
 
     PenningTrapManager(size_type totalP_, int nt_, Vector_t<int, Dim> &nr_,
                        double lbt_, std::string& solver_, std::string& stepMethod_)
-        : AlpineManager<T, Dim>(totalP_, nt_, nr_, lbt_, solver_, stepMethod_){}
+        : AlpineManager<T, Dim>(totalP_, nt_, nr_, lbt_, solver_, stepMethod_),scaleFactor(30){}
 
     ~PenningTrapManager(){}
 
@@ -229,7 +233,7 @@ class PenningTrapManager : public AlpineManager<T, Dim> {
         double alpha = this->alpha_m;
         double Bext = this->Bext_m;
         double DrInv = this->DrInv_m;
-        double V0  = 30 * this->length_m[2];
+        double V0  = scaleFactor * this->length_m[2];
         Vector_t<double, Dim> length = this->length_m;
         Vector_t<double, Dim> origin = this->origin_m;
         double dt = this->dt_m;
@@ -284,14 +288,31 @@ class PenningTrapManager : public AlpineManager<T, Dim> {
 
         // scatter the charge onto the underlying grid
         this->par2grid();
+
 #ifdef ENABLE_CATALYST
-        std::optional<conduit_cpp::Node> node = std::nullopt;
-        //CatalystAdaptor::Execute_Particle(it, this->time_m, ippl::Comm->rank(),  pc, node);
-        auto *rho               = &this->fcontainer_m->getRho();
-        CatalystAdaptor::Execute_Field(it, this->time_m, ippl::Comm->rank(),  *rho, node);
-        //auto *E               = &this->fcontainer_m->getE();
-        //CatalystAdaptor::Execute_Field(it, this->time_m, ippl::Comm->rank(),  *E, node);
-        //CatalystAdaptor::Execute_Field_Particle(it, this->time_m, ippl::Comm->rank(),  *E, pc);
+        std::vector<CatalystAdaptor::ParticlePair<T, Dim>> particles = {
+            {"particle", std::shared_ptr<ParticleContainer<T, Dim> >(pc)},
+        };
+        std::vector<CatalystAdaptor::FieldPair<T, Dim>> fields = {
+            {"E",   CatalystAdaptor::FieldVariant<T, Dim>(&this->fcontainer_m->getE())},
+            {"roh", CatalystAdaptor::FieldVariant<T, Dim>(&this->fcontainer_m->getRho())},
+            //{"phi", CatalystAdaptor::FieldVariant<T, Dim>(&this->fcontainer_m->getPhi())},
+        };
+        CatalystAdaptor::Execute(it, this->time_m, ippl::Comm->rank(), particles, fields, scaleFactor);
+#endif
+
+#ifdef ENABLE_ASCENT
+
+        std::vector<AscentAdaptor::ParticlePair<T, Dim>> particles = {
+            {"particle", std::shared_ptr<ParticleContainer<T, Dim> >(pc)},
+        };
+        std::vector<AscentAdaptor::FieldPair<T, Dim>> fields = {
+            {"E",   AscentAdaptor::FieldVariant<T, Dim>(&this->fcontainer_m->getE())},
+            {"roh", AscentAdaptor::FieldVariant<T, Dim>(&this->fcontainer_m->getRho())},
+            //{"phi", CatalystAdaptor::FieldVariant<T, Dim>(&this->fcontainer_m->getPhi())},
+        };
+        AscentAdaptor::Execute(it, this->time_m ,  particles, fields);
+
 #endif
 
         // Field solve

alpine/ascent_actions.yaml

@@ -0,0 +1,34 @@
+-
+  action: "add_pipelines"
+  pipelines:
+    pl1:
+      f1:
+        type: "threshold"
+        params:
+          field: "particle_magnitude"
+          min_value: 9.2
+          max_value: 25.0
+- 
+  action: "add_scenes"
+  scenes: 
+    s1: 
+      plots: 
+        p1:
+          type: pseudocolor
+          field: particle_charge
+          points:
+            radius: 0.1
+          pipeline: "pl1"
+        p2:
+          type: pseudocolor
+          field: particle_center
+          points:
+            radius: 0.5
+      renders: 
+        r1: 
+          image_prefix: "render_%06d"
+          screen_annotations: "false"
+          camera:
+              look_at: [10, 0, 10]
+              up: [1, 0, 0]
+              position: [10, 40, 10]

build-cuda-sophia.sh

@@ -0,0 +1,27 @@
+#!/bin/bash
+
+export CXX=mpiCC
+export CC=mpicc
+
+cmake -B build-cuda-sophia -S . \
+  -DCMAKE_INSTALL_PREFIX=install-cuda \
+  -DCMAKE_BUILD_TYPE=Release \
+  -DCMAKE_CXX_COMPILER=mpiCC \
+  -DCMAKE_C_COMPILER=mpicc \
+  -DCMAKE_CUDA_HOST_COMPILER=mpiCC \
+  -DCUDA_TOOLKIT_ROOT_DIR=/usr/local/cuda \
+  -DKokkos_ENABLE_CUDA=ON \
+  -DKokkos_ARCH_AMPERE80=ON \
+  -DKokkos_ENABLE_OPENMP=ON \
+  -DCMAKE_CXX_STANDARD=20 \
+  -DCMAKE_CXX_STANDARD_REQUIRED=ON \
+  -DENABLE_FFT=ON \
+  -DENABLE_SOLVERS=ON \
+  -DENABLE_ALPINE=True \
+  -DENABLE_TESTS=ON \
+  -DUSE_ALTERNATIVE_VARIANT=ON \
+  -DIPPL_PLATFORMS=cuda \
+  -DAscent_DIR=`pwd`/../../ascent/scripts/build_ascent/install/ascent-checkout/lib/cmake/ascent 
+
+
+cmake --build build-cuda-sophia --target install -j8

run.polaris.sh

@@ -0,0 +1,11 @@
+#!/bin/bash
+
+NNODES=`wc -l < $PBS_NODEFILE` 
+NRANKS=1
+NDEPTH=32 
+NTHREADS=32
+
+NTOTRANKS=$(( NNODES * NRANKS ))
+echo "NUM_OF_NODES= ${NNODES} TOTAL_NUM_RANKS= ${NTOTRANKS} RANKS_PER_NODE= ${NRANKS} THREADS_PER_RANK= ${NTHREADS}"
+
+mpirun --np ${NTOTRANKS} -ppn ${NRANKS} -d ${NDEPTH} --cpu-bind depth -env OMP_NUM_THREADS=${NTHREADS} ./PenningTrap 32 32 32 655360 400 FFT 0.01 LeapFrog --overallocate 1.0 --info 5 --frequency 2

sourceme.polaris.sh

@@ -0,0 +1,9 @@
+module reset
+module use /soft/modulefiles
+module switch PrgEnv-nvhpc PrgEnv-gnu
+module load spack-pe-base cmake
+module load cudatoolkit-standalone
+module load visualization/ascent
+
+export CXX=CC
+export CC=cc

sourceme.sophia.sh

@@ -0,0 +1,18 @@
+export HTTP_PROXY="http://proxy.alcf.anl.gov:3128"
+export HTTPS_PROXY="http://proxy.alcf.anl.gov:3128"
+export http_proxy="http://proxy.alcf.anl.gov:3128"
+export https_proxy="http://proxy.alcf.anl.gov:3128"
+export ftp_proxy="http://proxy.alcf.anl.gov:3128"
+export no_proxy="admin,polaris-adminvm-01,localhost,*.cm.polaris.alcf.anl.gov,polaris-*,*.polaris.alcf.anl.gov,*.alcf.anl.gov"
+
+
+module reset
+module use /soft/modulefiles
+module switch PrgEnv-nvhpc PrgEnv-gnu
+module load spack-pe-base cmake
+#module load kokkos/4.2.01
+module load cudatoolkit-standalone
+module load visualization/ascent
+
+export CXX=CC
+export CC=cc

src/CMakeLists.txt

@@ -82,7 +82,7 @@ add_subdirectory (Expression)
 add_subdirectory (Types)
 add_subdirectory (Partition)
 add_subdirectory (Stream)
-add_subdirectory (Stream/InSitu)
+# add_subdirectory (Stream/InSitu)
 add_subdirectory (Random)
 
 if (ENABLE_SOLVERS)
@@ -98,10 +98,17 @@ if (ENABLE_CATALYST)
     message (STATUS "Found catalyst_DIR: ${catalyst_DIR}")
 endif()
 
+if (ENABLE_ASCENT)
+    find_package(Ascent REQUIRED)
+    message (STATUS "Enable Ascent")
+    message (STATUS "Found ascent_DIR: ${Ascent_DIR}")
+endif()
+
 if (ENABLE_AMR)
     add_subdirectory(AmrParticle)
 endif ()
 include_directories (
+    $<$<BOOL:${ENABLE_ASCENT}>:${ASCENT_INSTALL_PREFIX}/include/ascent>
     BEFORE ${CMAKE_CURRENT_SOURCE_DIR}
     )
 
@@ -119,13 +126,15 @@ message("Timer fences for this build: ${TimerFences}")
 target_compile_definitions(ippl PUBLIC
   $<$<BOOL:${ENABLE_FFT}>:-DENABLE_FFT>
   $<$<BOOL:${ENABLE_CATALYST}>:-DENABLE_CATALYST>
+  $<$<BOOL:${ENABLE_ASCENT}>:-DENABLE_ASCENT>
   IPPL_ENABLE_TIMER_FENCES=${TimerFences}
 )
 
 target_link_libraries(ippl PUBLIC
   Kokkos::kokkos
   $<$<BOOL:${ENABLE_FFT}>:Heffte>
   $<$<BOOL:${ENABLE_CATALYST}>:catalyst::catalyst>
+  $<$<BOOL:${ENABLE_ASCENT}>:ascent::ascent_mpi>
   )
 
 install (TARGETS ippl DESTINATION lib)

src/Stream/InSitu/CMakeLists.txt

@@ -2,7 +2,6 @@ set (_SRCS
 )
 
 set (_HDRS
-        CatalystAdaptor.h
 )
 
 include_DIRECTORIES (
@@ -21,4 +20,4 @@ install (FILES ${_HDRS} DESTINATION include/Stream/InSitu)
 # cmake-tab-width: 4
 # indent-tabs-mode: nil
 # require-final-newline: nil
-# End:
+# End: