Add rocTx instrumentation

README.md

@@ -10,9 +10,8 @@ Redistribution and use in source and binary forms, with or without modification,
 
 THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
-=================================
-A hipified version of Quicksilver
-=================================
+# A hipified version of Quicksilver
+
 NOTE:
 =====
 This will only work with ROCM 1.7 and later. 
@@ -42,23 +41,27 @@ I have made several changes to get this to work with hip:
 
 Compiling and running:
 ======================
-To compile you must set several environment variables in the Makefile in the src/ directory. For example, in order to compile using HIP without MPI, and up to 15 sub-iterations for a particle per kernel launch,  you could set the environment variables as follows (this is how they are set by default, you just need to point to where HIP is installed):
+To compile you must set several environment variables in the Makefile in the `src/` directory. For example, in order to compile using HIP without MPI, and up to 15 sub-iterations for a particle per kernel launch,  you could set the environment variables as follows (this is how they are set by default, you just need to point to where HIP is installed):
 
-CXX = $(HIP)/bin/hipcc<br/>
-CXXFLAGS = -I$(HIP)/include/<br/>
-CPPFLAGS = -DHAVE_HIP=1 -DMaxIt=15<br/>
+```
+CXX = $(HIP)/bin/hipcc
+CXXFLAGS = -I$(HIP)/include/
+CPPFLAGS = -DHAVE_HIP=1 -DMaxIt=15
 LDFLAGS = -L$(HIP)/lib -L$(HIP)/lib
+```
 
 To compile using HIP and MPI and up to 15 sub-iterations for a particle per kernel launch, you could set the following environment variables:
 
-CXX = $(HIP)/bin/hipcc<br/>
-CXXFLAGS1 = -I$(HIP)/include/<br/>
-CXXFLAGS2 = $(CXXFLAGS1) -I$(MPIPATH)/include<br/>
-CXXFLAGS = $(CXXFLAGS2) -pthread<br/>
-CPPFLAGS = -DHAVE_HIP=1 -DHAVE_MPI -DMaxIt=15<br/>
+```
+CXX = $(HIP)/bin/hipcc
+CXXFLAGS1 = -I$(HIP)/include/
+CXXFLAGS2 = $(CXXFLAGS1) -I$(MPIPATH)/include
+CXXFLAGS = $(CXXFLAGS2) -pthread
+CPPFLAGS = -DHAVE_HIP=1 -DHAVE_MPI -DMaxIt=15
 LDFLAGS = -L$(HIP)/lib -L$(MPIPATH)/lib -lmpicxx -lmpi
+```
 
-To run look in src/READ.ME.HOW.TO.RUN. You can also look at the examples in the .sh files found in the directories 'Examples/CORAL2_Benchmark/Problem1' or 'Examples/CORAL2_Benchmark/Problem2'.
+To run look in `src/READ.ME.HOW.TO.RUN`. You can also look at the examples in the `.sh` files found in the directories `Examples/CORAL2_Benchmark/Problem1` or `Examples/CORAL2_Benchmark/Problem2`.
 
 A Note on ROCm Versions:
 ========================

src/Makefile

@@ -78,6 +78,9 @@
 # -DUSE_NVTX        Define this for some extra NVProf profiling information.
 #                   It will create regions that can be visualized in NVVP.
 #
+# -DUSE_ROCTX       Define this for some extra ROCtx profiling information.
+#                   Needs -lroctx64 in LDFLAGS.
+#
 # -DUSE_OPENMP_NO_GPU 
 #                   Define this for runs with OPENMP_TARGET but when you do not
 #                   want to target GPU devices only the CPU

src/NVTX_Range.hh

@@ -17,6 +17,8 @@
 
 #ifdef USE_NVTX
 #include "nvToolsExt.h"
+#elif defined(USE_ROCTX)
+#include <roctracer/roctx.h>
 #endif
 
 
@@ -28,32 +30,21 @@ class NVTX_Range
    {
       #ifdef USE_NVTX
       char *result = strdup(rangeName.c_str());
-      _rangeId = nvtxRangeStartA(result);
-      _isOpen = true;
+      nvtxRangePushA(result);
+      free(result);
+      #elif defined(USE_ROCTX)
+      roctxRangePushA(rangeName.c_str());
       #endif
    }
 
    ~NVTX_Range()
    {
       #ifdef USE_NVTX
-      if (_isOpen)
-	 nvtxRangeEnd(_rangeId);
+      nvtxRangePop();
+      #elif defined(USE_ROCTX)
+      roctxRangePop();
       #endif
    }
-
-  void endRange()
-  {
-      #ifdef USE_NVTX
-      nvtxRangeEnd(_rangeId);
-      _isOpen = false;
-      #endif
-  }
-
- private:
-   #ifdef USE_NVTX
-   nvtxRangeId_t _rangeId;
-   bool _isOpen;
-   #endif
 };
 
 #endif

src/main.cc

@@ -392,16 +392,18 @@ void cycleTracking(MonteCarlo *monteCarlo, uint64_cu* tallies, uint64_cu * talli
 
             MC_FASTTIMER_START(MC_Fast_Timer::cycleTracking_MPI);
 
-            NVTX_Range collapseRange("cycleTracking_Collapse_ProcessingandProcessed");
-            my_particle_vault.collapseProcessing();
-            my_particle_vault.collapseProcessed();
-            collapseRange.endRange();
+            {
+                NVTX_Range collapseRange("cycleTracking_Collapse_ProcessingandProcessed");
+                my_particle_vault.collapseProcessing();
+                my_particle_vault.collapseProcessed();
+            }
 
 
             //Test for done - blocking on all MPI ranks
-            NVTX_Range doneRange("cycleTracking_Test_Done_New");
-            done = monteCarlo->particle_buffer->Test_Done_New( new_test_done_method );
-            doneRange.endRange();
+            {
+                NVTX_Range doneRange("cycleTracking_Test_Done_New");
+                done = monteCarlo->particle_buffer->Test_Done_New( new_test_done_method );
+            }
 
             MC_FASTTIMER_STOP(MC_Fast_Timer::cycleTracking_MPI);