Merge branch 'kinit' into 'main'

Fixes in the LibHPG RAII class to initialize/finalize HPG/Kokoks once in a process.

See merge request ardg/libra!141

Author: preshanth

.gitlab-ci.yml

@@ -151,8 +151,9 @@ test-job:
     # --- C++ GTest section ---
     cd ${base_path}/apps/src/tests/
     # Run C++ gtest binaries if they exist
-    test_binaries=("LibRATests" "test_roadrunner_robustMinus2" "test_roadrunner_robustPlus2" "test_roadrunner_snrpsf" "test_roadrunner_weight")
-
+    #test_binaries=("LibRATests" "test_roadrunner_robustMinus2" "test_roadrunner_robustPlus2" "test_roadrunner_snrpsf" "test_roadrunner_weight")
+    test_binaries=("LibRATests")
+    
     for test_binary in "${test_binaries[@]}"; do
         if [ -x "${base_path}/install/bin/tests/${test_binary}" ]; then
             echo "Running ${test_binary}..."

apps/src/RoadRunner/roadrunner.cc

@@ -339,13 +339,24 @@ auto Roadrunner(//bool& restartUI, int& argc, char** argv,
 		bool& conjBeams, float& pbLimit, vector<float>& posigdev,
 		bool& doSPWDataIter) -> RRReturnType
 {
-  LogFilter filter(LogMessage::NORMAL);
-  LogSink::globalSink().filter(filter);
+  // LogFilter filter(LogMessage::NORMAL);
+  // LogSink::globalSink().filter(filter);
   LogIO log_l(LogOrigin("roadrunner","Roadrunner_func"));
   RRReturnType rrr;
 
   try
     {
+      // set the default of rmode to be "norm"
+      if (rmode =="")
+        rmode = "norm";
+
+      // put the guard here so in the UI() users don't need to correct `rmode` if they don't know how to
+      if (weighting == "briggs" && rmode !="norm")
+	{
+	  log_l << "'rmode' parameter set to 'norm' for Briggs weighting to function correctly"
+		<<  LogIO::WARN << LogIO::POST;
+	  rmode = "norm";
+	}
       // Set safe defaults...
       casa::refim::FTMachine::Type dataCol_l=casa::refim::FTMachine::CORRECTED;
       if (imagingMode=="predict")       dataCol_l=casa::refim::FTMachine::MODEL;
@@ -368,7 +379,7 @@ auto Roadrunner(//bool& restartUI, int& argc, char** argv,
       // A RAII class instance that manages the HPG initialize/finalize scope.
       // And hpg::finalize() is called when this instance goes out of
       // scope.
-      LibHPG libhpg(ftmName=="awphpg");
+      LibHPG libhpg(ftmName=="awphpg",&std::cerr);
       //  std::atexit(tpl_finalize);
 
       bool const doSow = sowImageExt != "";
@@ -443,19 +454,6 @@ auto Roadrunner(//bool& restartUI, int& argc, char** argv,
 						  stokes, refFreqStr, mode);
       PagedImage<Float> skyImage(cgrid.shape(),cgrid.coordinates(), imageName);
 
-      // set the default of rmode to be "norm"
-      if (rmode =="")
-        rmode = "norm";
-      // put the guard here so in the UI() users don't need to correct `rmode` if they don't know how to
-      if (rmode != "norm")
-      {
-        log_l << "'rmode' parameter is not set to 'norm'. Please verify this is intentional." << LogIO::WARN;
-        log_l << "For Briggs weighting, `rmode` is automatically set to `norm` to ensure the robust parameter functions correctly." << LogIO::WARN;
-      }
-      if (weighting == "briggs" && rmode !="norm")
-        rmode = "norm";
-        
-
       //      cgrid.table().markForDelete();
 
       // Setup the weighting scheme in the supplied VI2

apps/src/RoadRunner/roadrunner.h

@@ -115,7 +115,8 @@ class LibHPG
    * @brief Constructor for the LibHPG class.
    * @param usingHPG A boolean indicating whether to use HPG. Default is true.
    */
-  LibHPG(const bool usingHPG=true): init_hpg(usingHPG)
+  LibHPG(const bool usingHPG=true, std::ostream* os=nullptr):
+    init_hpg(usingHPG),hpg_initialized(false),os_p(os)
   {
     initialize();
   }
@@ -125,43 +126,65 @@ class LibHPG
    */
   bool initialize()
   {
-    bool ret = true;
     startTime = std::chrono::steady_clock::now();
-#ifdef ROADRUNNER_USE_HPG
-    if (init_hpg) ret = hpg::initialize();
-#endif
-    if (!ret)
-      throw(AipsError("LibHPG::initialize() failed"));
-    return ret;
-  };
-  /**
-   * @brief Finalizes the HPG library.
-   */
-  void finalize()
-  {
+
+    hpg_initialized=true;
 #ifdef ROADRUNNER_USE_HPG
     if (init_hpg)
       {
-	cerr << endl << "CALLING hpg::finalize()" << endl;
-	hpg::finalize();
-      }	
+	static bool once = [&]
+			   {
+			     assert(!hpg::is_finalized());
+			     if (!hpg::is_initialized())
+			       {
+				 hpg_initialized=hpg::initialize();
+				 // cerr << "LibHPG:: hpg::initialize() called" << endl;
+				 // Kokkos::push_finalize_hook([]() {
+				 // 	 cerr << "Kokkos::finalize() called" << endl;
+				 // });
+				 std::atexit(hpg::finalize);
+			       }
+			     return hpg_initialized;
+			   }();
+      } 
+   assert(hpg::is_intialized);
 #endif
+    return hpg_initialized;
   };
+
+  //
+  // @brief Return the internal state variable to indicate if the
+  // hpg::initialize() method was called.
+  //
+  bool hpg_intialize_called() {return hpg_initialized;}
+  
+  //
+  // @brief Return the total time elasped since the class constructor was called.
+  //
+  double runtime()
+  {
+    auto endTime = std::chrono::steady_clock::now();
+    std::chrono::duration<double> totalRuntime = endTime - startTime;
+    return totalRuntime.count();
+  }
   /**
    * @brief Destructor for the LibHPG class.
+   *
+   * If an output stream is give, this also prints the total runtime
+   * on that stream.
    */
   ~LibHPG()
   {
-    finalize();
-
-    auto endTime = std::chrono::steady_clock::now();
-    std::chrono::duration<double> runtime = endTime - startTime;
-    std::cerr << "roadrunner runtime: " << runtime.count()
-	      << " sec" << std::endl;
+    if (os_p != nullptr)
+      (*os_p) << "roadrunner runtime: "
+	      << std::fixed << std::setprecision(2)
+	      << runtime() << " sec" << std::endl;
   };
 
   std::chrono::time_point<std::chrono::steady_clock> startTime;
   bool init_hpg;
+  bool hpg_initialized;
+  std::ostream *os_p;
 };
 //
 //-------------------------------------------------------------------------
@@ -226,8 +249,6 @@ void CFServer(libracore::ThreadCoordinator& thcoord,
 	      casacore::Vector<double>& spwRefFreqList,
 	      int& nDataPol);
 
-
-
 //--------------------------------------------------------------------------------------------
 // Place-holder functions to build the appropriate Mueller index
 // matrix.  For production this should be generalized and perhaps
@@ -337,4 +358,3 @@ void UI(Bool restart, int argc, char **argv, bool interactive,
 
 
 #endif
-

apps/src/tests/CMakeLists.txt

@@ -33,24 +33,11 @@ add_executable(LibRATests ${TEST_SOURCES})
 # Link with Google Test
 set(INTERFACE_LIBRARIES libra libracore roadrunner_cl_interface coyote_cl_interface hummbee_cl_interface mssplit_cl_interface subms_cl_interface tableinfo_cl_interface chip_cl_interface dale_cl_interface casacore_asp)
 
-#target_link_libraries(LibRATests GTest::gtest_main stdc++fs ${APP_LINK_LIBRARIES} ${INTERFACE_LIBRARIES})
+target_link_libraries(LibRATests GTest::gtest_main stdc++fs ${APP_LINK_LIBRARIES} ${INTERFACE_LIBRARIES})
 
-add_executable(test_roadrunner_robustMinus2 test_roadrunner_robustMinus2.cc)
-add_executable(test_roadrunner_robustPlus2 test_roadrunner_robustPlus2.cc)
-add_executable(test_roadrunner_snrpsf test_roadrunner_snrpsf.cc)
-add_executable(test_roadrunner_weight test_roadrunner_weight.cc)
-
-# Link all with required libraries
-foreach(test_target LibRATests test_roadrunner_robustMinus2 test_roadrunner_robustPlus2 test_roadrunner_snrpsf test_roadrunner_weight)
-  target_link_libraries(${test_target} GTest::gtest_main stdc++fs ${APP_LINK_LIBRARIES} ${INTERFACE_LIBRARIES})
-endforeach()
 
 # Add the test executable as a test target
 add_test(NAME LibRATests COMMAND LibRATests)
-add_test(NAME test_roadrunner_robustMinus2 COMMAND test_roadrunner_robustMinus2)
-add_test(NAME test_roadrunner_robustPlus2 COMMAND test_roadrunner_robustPlus2)
-add_test(NAME test_roadrunner_snrpsf COMMAND test_roadrunner_snrpsf)
-add_test(NAME test_roadrunner_weight COMMAND test_roadrunner_weight)
 
 if(Apps_BUILD_TESTS_PYTHON)
   find_program(PYTHON_EXECUTABLE python)
@@ -61,6 +48,6 @@ if(Apps_BUILD_TESTS_PYTHON)
 endif()
 
 # Install the test executable
-install(TARGETS LibRATests test_roadrunner_robustMinus2 test_roadrunner_robustPlus2 test_roadrunner_snrpsf test_roadrunner_weight
+install(TARGETS LibRATests
   RUNTIME DESTINATION "${CMAKE_INSTALL_PREFIX}/bin/tests"
 )

apps/src/tests/test_roadrunner.cc

@@ -8,7 +8,7 @@ namespace test{
   const path goldDir = current_path() / "gold_standard";
 
 
-/*TEST(RoadrunnerTest, InitializeTest) {
+TEST(RoadrunnerTest, InitializeTest) {
   // Create a LibHPG object
   LibHPG lib_hpg;
 
@@ -17,10 +17,8 @@ namespace test{
 
   // Check that the return value is true
   EXPECT_TRUE(ret);
-}*/
+}
 
-// uncomment the following when the multiple hpg initialization error is fixed.
-/*
 TEST(RoadrunnerTest, AppLevelSNRPSF_performance) {
   // Get the test name
   string testName = ::testing::UnitTest::GetInstance()->current_test_info()->name();
@@ -185,8 +183,8 @@ TEST(RoadrunnerTest, AppLevelWeight) {
 
   PagedImage<Float> weight("htclean_gpu_newpsf.weight");
   float tol = 0.1;
-  float goldValLoc0 = -277710.1875;
-  float goldValLoc1 = 470668.9375;
+  float goldValLoc0 = 1255553.875;
+  float goldValLoc1 = 580468.3125;
   EXPECT_NEAR(weight(IPosition(4,2000,2053,0,0)), goldValLoc0, tol);
   EXPECT_NEAR(weight(IPosition(4,1379,1969,0,0)), goldValLoc1, tol);
 
@@ -348,7 +346,7 @@ TEST(RoadrunnerTest, Interface_rmode_minus2) {
   std::filesystem::current_path(testDir.parent_path());
 
   remove_all(testDir);
-}*/
+}
 
 TEST(RoadrunnerTest, UIFactory) {
     // The Factory Settings.