Fixed game of life

Author: kc432959

benchmarks/gbench/mp/CMakeLists.txt

@@ -107,10 +107,6 @@ if(ENABLE_SYCL)
   endforeach()
 endif()
 
-add_executable(game_of_life game_of_life.cpp)
-target_link_libraries(game_of_life cxxopts DR::mpi)
-target_compile_definitions(game_of_life PRIVATE STANDALONE_BENCHMARK)
-
 add_executable(shallow_water shallow_water.cpp)
 target_link_libraries(shallow_water cxxopts DR::mpi)
 target_compile_definitions(shallow_water PRIVATE STANDALONE_BENCHMARK)

test/gtest/mp/CMakeLists.txt

@@ -81,6 +81,13 @@ foreach(test-exec IN ITEMS mp-tests mp-tests-3 mp-quick-test mp-quick-test-3-onl
           "${CMAKE_COMMAND} -E time")
 endforeach()
 
+# Game of life
+add_executable(game_of_life game_of_life.cpp)
+if(ENABLE_ISHMEM)
+  target_link_ishmem(game_of_life)
+endif()
+target_link_libraries(game_of_life cxxopts DR::mpi)
+
 # tests without --sycl flag will fail on IshmemBackend TODO: make them be
 # running somehow if ENABLE_ISHMEM will be default CI config
 if(NOT ENABLE_ISHMEM)

benchmarks/gbench/mp/game_of_life.cpp renamed to test/gtest/mp/game_of_life.cpp

@@ -5,29 +5,48 @@
 #include "cxxopts.hpp"
 #include "dr/mp.hpp"
 #include "mpi.h"
+
+inline void barrier() { dr::mp::barrier(); }
+inline void fence() { dr::mp::fence(); }
+inline void fence_on(auto &&obj) { obj.fence(); }
+
 #include <chrono>
 #include <memory>
 #include <iomanip>
 
-#ifdef STANDALONE_BENCHMARK
+//
+
+struct MPI_data {
+  MPI_Comm comm;
+  int rank;
+  int size;
 
-MPI_Comm comm;
-int comm_rank;
-int comm_size;
+  bool host() {
+    return rank == 0;
+  }
+};
 
-#else
+static MPI_data mpi_data;
 
-#include "../common/dr_bench.hpp"
+struct Options {
+  std::size_t size;
+  std::size_t steps;
+  std::size_t redundancy;
+  bool debug;
 
-#endif
+  std::unique_ptr<std::ofstream> logfile;
+
+  bool sycl;
+  bool device_memory;
+};
 
 namespace GameOfLife {
 
 using T = int;
 using Array = dr::mp::distributed_mdarray<T, 2>;
 
 void init(std::size_t n, Array& out) {
-  std::vector<std::vector<int>> in(n, std::vector<int>(n, 0));
+  std::vector<std::vector<int>> in(4, std::vector<int>(4, 0));
   /*
     1 0 0
     0 1 1
@@ -39,16 +58,16 @@ void init(std::size_t n, Array& out) {
   in[3][1] = 1; in[3][2] = 1; in[3][3] = 0;
   // clang-format on
   std::vector<int> local(n * n);
-  for (int i = 0; i < n; i++) {
-    for (int j = 0; j < n; j++) {
+  for (int i = 0; i < 4; i++) {
+    for (int j = 0; j < 4; j++) {
       local[i * n + j] = in[i][j];
     }
   }
   dr::mp::copy(local.begin(), local.end(), out.begin());
 }
 
 void run(std::size_t n, std::size_t redundancy, std::size_t steps, bool debug) {
-  if (comm_rank == 0) {
+  if (mpi_data.host()) {
     std::cout << "Using backend: dr" << std::endl;
     std::cout << "Grid size: " << n << " x " << n << std::endl;
     std::cout << "Time steps:" << steps << std::endl;
@@ -61,7 +80,6 @@ void run(std::size_t n, std::size_t redundancy, std::size_t steps, bool debug) {
   Array array({n, n}, dist);
   Array array_out({n, n}, dist);
   dr::mp::fill(array, 0);
-  dr::mp::fill(array_out, 0);
 
   init(n, array);
 
@@ -99,80 +117,93 @@ void run(std::size_t n, std::size_t redundancy, std::size_t steps, bool debug) {
       x(0, 0) = x_out(0, 0);
     };
 
-  auto tic = std::chrono::steady_clock::now();
-
   auto print = [n](const auto &v) {
       std::vector<int> local(n * n);
-      copy(v, local.begin());
-      if (comm_rank == 0) {
+      dr::mp::copy(0, v, local.begin());
+      if (mpi_data.host()) {
         for (int i = 0; i < n; i++) {
           for (int j = 0; j < n; j++) {
-            std::cout << local[i * n + j] << " ";
+            fmt::print("{}", local[i * n + j] == 1 ? '#' : '.');
           }
-          std::cout << "\n";
+          fmt::print("\n");
         }
       }
     };
 
-  for (std::size_t i = 0; i < steps; i++) {
-    if (comm_rank == 0) {
-      std::cout << "Step " << i << "\n";
+  auto tic = std::chrono::steady_clock::now();
+  for (std::size_t i = 0, next_treshold = 0; i < steps; i++) {
+    if (i >= next_treshold && mpi_data.host()) {
+      next_treshold += round(static_cast<double>(steps / 100));
+      double percent = round(static_cast<double>(i) * 100 / static_cast<double>(steps));
+      fmt::print("Steps done {}% ({} of {} steps)\n", percent, i, steps);
     }
+
     // step
     stencil_for_each_extended<2>(calculate, {1, 1}, {1, 1}, array, array_out);
     stencil_for_each_extended<2>(assign, {0, 0}, {0, 0}, array, array_out);
+
     // phase with communication - once after (redundancy - 1) steps without communication
     if ((i + 1) % redundancy == 0) {
-      if (comm_rank == 0) {
-        std::cout << "Exchange\n";
+      if (debug && mpi_data.host()) {
+        fmt::print("Exchange at step {}\n", i);
       }
       array.halo().exchange();
       // Array_out is a temporary, no need to exchange it
     }
+
+    // debug print
     if (debug) {
-      if (comm_rank == 0) {
-        std::cout << "Array " << i << ":\n";
+      if (mpi_data.host()) {
+        fmt::print("Array {}:\n", i);
       }
+      // print needs a synchronication accros MPI boundary (dr::mp::copy), each node has to execute it
       print(array);
-      if (comm_rank == 0) {
-        std::cout << "\n";
-      }
     }
   }
-
   auto toc = std::chrono::steady_clock::now();
+
   std::chrono::duration<double> duration = toc - tic;
-  if (comm_rank == 0) {
+
+  if (mpi_data.host()) {
     double t_cpu = duration.count();
     double t_step = t_cpu / static_cast<double>(steps);
-    std::cout << "Duration: " << std::setprecision(3) << t_cpu << " s" << std::endl;
-    std::cout << "Time per step: " << std::setprecision(2) << t_step * 1000 << " ms" << std::endl;
+
+    fmt::print("Steps done 100% ({} of {} steps)\n", steps, steps);
+    fmt::print("Duration {} s\n", t_cpu);
+    fmt::print("Time per step {} ms\n", t_step * 1000);
   }
 }
 
 } // namespace GameOfLife
 
-#ifdef STANDALONE_BENCHMARK
-
-int main(int argc, char *argv[]) {
+// Initialization functions
 
+void init_MPI(int argc, char *argv[]) {
   MPI_Init(&argc, &argv);
-  comm = MPI_COMM_WORLD;
-  MPI_Comm_rank(comm, &comm_rank);
-  MPI_Comm_size(comm, &comm_size);
+  mpi_data.comm = MPI_COMM_WORLD;
+  MPI_Comm_rank(mpi_data.comm, &mpi_data.rank);
+  MPI_Comm_size(mpi_data.comm, &mpi_data.size);
+
+  dr::drlog.debug("MPI: rank = {}, size = {}\n", mpi_data.rank, mpi_data.size);
+}
+
+Options parse_options(int argc, char *argv[]) {
+  Options out;
 
   cxxopts::Options options_spec(argv[0], "game of life");
+
   // clang-format off
   options_spec.add_options()
-    ("n,size", "Grid size", cxxopts::value<std::size_t>()->default_value("128"))
-    ("t,steps", "Run a fixed number of time steps.", cxxopts::value<std::size_t>()->default_value("100"))
-    ("r,redundancy", "Set outer-grid redundancy parameter.", cxxopts::value<std::size_t>()->default_value("2"))
-    ("sycl", "Execute on SYCL device")
-    ("l,log", "enable logging")
-    ("d,debug", "enable debug logging")
+    ("drhelp", "Print help")
+    ("log", "Enable logging")
     ("logprefix", "appended .RANK.log", cxxopts::value<std::string>()->default_value("dr"))
+    ("log-filter", "Filter the log", cxxopts::value<std::vector<std::string>>())
     ("device-memory", "Use device memory")
-    ("h,help", "Print help");
+    ("sycl", "Execute on SYCL device")
+    ("d,debug", "enable debug logging")
+    ("n,size", "Grid size", cxxopts::value<std::size_t>()->default_value("128"))
+    ("t,steps", "Run a fixed number of time steps.", cxxopts::value<std::size_t>()->default_value("100"))
+    ("r,redundancy", "Set outer-grid redundancy parameter.", cxxopts::value<std::size_t>()->default_value("2"));
   // clang-format on
 
   cxxopts::ParseResult options;
@@ -183,51 +214,57 @@ int main(int argc, char *argv[]) {
     exit(1);
   }
 
-  std::unique_ptr<std::ofstream> logfile;
-  if (options.count("log")) {
-    logfile =
-        std::make_unique<std::ofstream>(options["logprefix"].as<std::string>() +
-                                        fmt::format(".{}.log", comm_rank));
-    dr::drlog.set_file(*logfile);
+  out.sycl = options.count("sycl") != 0;
+  out.device_memory = options.count("debug") != 0;
+
+  if (options.count("drhelp")) {
+    std::cout << options_spec.help() << "\n";
+    exit(0);
   }
 
-  if (options.count("sycl")) {
-#ifdef SYCL_LANGUAGE_VERSION
-    sycl::queue q = dr::mp::select_queue();
-    std::cout << "Run on: "
-              << q.get_device().get_info<sycl::info::device::name>() << "\n";
-    dr::mp::init(q, options.count("device-memory") ? sycl::usm::alloc::device
-                                                   : sycl::usm::alloc::shared);
-#else
-    std::cout << "Sycl support requires icpx\n";
-    exit(1);
-#endif
-  } else {
-    if (comm_rank == 0) {
-      std::cout << "Run on: CPU\n";
+  if (options.count("log")) {
+    out.logfile.reset(new std::ofstream(options["logprefix"].as<std::string>() +
+                                    fmt::format(".{}.log", mpi_data.rank)));
+    dr::drlog.set_file(*out.logfile);
+    if (options.count("log-filter")) {
+      dr::drlog.filter(options["log-filter"].as<std::vector<std::string>>());
     }
-    dr::mp::init();
   }
 
-  std::size_t n = options["n"].as<std::size_t>();
-  std::size_t redundancy = options["r"].as<std::size_t>();
-  std::size_t steps = options["t"].as<std::size_t>();
+  out.size = options["n"].as<std::size_t>();
+  out.redundancy = options["r"].as<std::size_t>();
+  out.steps = options["t"].as<std::size_t>();
 
-  bool debug = false;
-  if (options.count("debug")) {
-    debug = true;
-  }
+  out.debug = options.count("debug") != 0;
 
-  GameOfLife::run(n, redundancy, steps, debug);
-  dr::mp::finalize();
-  MPI_Finalize();
-  return 0;
+  return out;
+}
+
+void dr_init(const Options& options) {
+#ifdef SYCL_LANGUAGE_VERSION
+  if (options.sycl) {
+    sycl::queue q;
+    fmt::print("Running on sycl device: {}, memory: {}\n", q.get_device().get_info<sycl::info::device::name>(), options.device_memory ? "devive" : "shared");
+    dr::mp::init(q, options.device_memory ? sycl::usm::alloc::device
+                                          : sycl::usm::alloc::shared);
+    return;
+  }
+#endif
+  fmt::print("Running on CPU\n");
+  dr::mp::init();
 }
 
-#else
+// Main loop
 
-static void GameOfLife_DR(benchmark::State &state) {}
+int main(int argc, char *argv[]) {
+  init_MPI(argc, argv);
+  Options options = parse_options(argc, argv);
+  dr_init(options);
 
-DR_BENCHMARK(GameOfLife_DR);
+  GameOfLife::run(options.size, options.redundancy, options.steps, options.debug);
 
-#endif
+  dr::mp::finalize();
+  MPI_Finalize();
+
+  return 0;
+}