Merge #1882 Use event to handle distributed row gatherer

This PR allows to use event to handle distributed row gatherer.
Event-based way can allow us to submit the next kernel without waiting for the row gatherer preparation, which leads better performance usually.

Related PR: #1882

Author: yhmtsai

benchmark/test/reference/distributed_solver.matrix.stdout

@@ -27,6 +27,7 @@
                         "cg::initialize": 1.0,
                         "advanced_apply(<typename>)": 1.0,
                         "dense::row_gather": 1.0,
+                        "event::record_event": 1.0,
                         "csr::advanced_spmv": 1.0,
                         "dense::compute_squared_norm2": 1.0,
                         "dense::compute_sqrt": 1.0,

benchmark/test/reference/distributed_solver.profile.stderr

@@ -141,6 +141,8 @@ DEBUG: end   cg::initialize
 DEBUG: begin advanced_apply(<typename>)
 DEBUG: begin dense::row_gather
 DEBUG: end   dense::row_gather
+DEBUG: begin event::record_event
+DEBUG: end   event::record_event
 DEBUG: begin advanced_apply(<typename>)
 DEBUG: begin csr::advanced_spmv
 DEBUG: end   csr::advanced_spmv
@@ -181,6 +183,8 @@ DEBUG: end   cg::step_1
 DEBUG: begin apply(<typename>)
 DEBUG: begin dense::row_gather
 DEBUG: end   dense::row_gather
+DEBUG: begin event::record_event
+DEBUG: end   event::record_event
 DEBUG: begin apply(<typename>)
 DEBUG: begin csr::spmv
 DEBUG: end   csr::spmv
@@ -221,6 +225,8 @@ DEBUG: end   cg::step_1
 DEBUG: begin apply(<typename>)
 DEBUG: begin dense::row_gather
 DEBUG: end   dense::row_gather
+DEBUG: begin event::record_event
+DEBUG: end   event::record_event
 DEBUG: begin apply(<typename>)
 DEBUG: begin csr::spmv
 DEBUG: end   csr::spmv
@@ -261,6 +267,8 @@ DEBUG: end   cg::step_1
 DEBUG: begin apply(<typename>)
 DEBUG: begin dense::row_gather
 DEBUG: end   dense::row_gather
+DEBUG: begin event::record_event
+DEBUG: end   event::record_event
 DEBUG: begin apply(<typename>)
 DEBUG: begin csr::spmv
 DEBUG: end   csr::spmv
@@ -301,6 +309,8 @@ DEBUG: end   cg::step_1
 DEBUG: begin apply(<typename>)
 DEBUG: begin dense::row_gather
 DEBUG: end   dense::row_gather
+DEBUG: begin event::record_event
+DEBUG: end   event::record_event
 DEBUG: begin apply(<typename>)
 DEBUG: begin csr::spmv
 DEBUG: end   csr::spmv
@@ -341,6 +351,8 @@ DEBUG: end   cg::step_1
 DEBUG: begin apply(<typename>)
 DEBUG: begin dense::row_gather
 DEBUG: end   dense::row_gather
+DEBUG: begin event::record_event
+DEBUG: end   event::record_event
 DEBUG: begin apply(<typename>)
 DEBUG: begin csr::spmv
 DEBUG: end   csr::spmv
@@ -381,6 +393,8 @@ DEBUG: end   cg::step_1
 DEBUG: begin apply(<typename>)
 DEBUG: begin dense::row_gather
 DEBUG: end   dense::row_gather
+DEBUG: begin event::record_event
+DEBUG: end   event::record_event
 DEBUG: begin apply(<typename>)
 DEBUG: begin csr::spmv
 DEBUG: end   csr::spmv
@@ -421,6 +435,8 @@ DEBUG: end   cg::step_1
 DEBUG: begin apply(<typename>)
 DEBUG: begin dense::row_gather
 DEBUG: end   dense::row_gather
+DEBUG: begin event::record_event
+DEBUG: end   event::record_event
 DEBUG: begin apply(<typename>)
 DEBUG: begin csr::spmv
 DEBUG: end   csr::spmv
@@ -470,6 +486,8 @@ DEBUG: end   cg::initialize
 DEBUG: begin advanced_apply(<typename>)
 DEBUG: begin dense::row_gather
 DEBUG: end   dense::row_gather
+DEBUG: begin event::record_event
+DEBUG: end   event::record_event
 DEBUG: begin advanced_apply(<typename>)
 DEBUG: begin csr::advanced_spmv
 DEBUG: end   csr::advanced_spmv
@@ -508,6 +526,8 @@ DEBUG: end   cg::step_1
 DEBUG: begin apply(<typename>)
 DEBUG: begin dense::row_gather
 DEBUG: end   dense::row_gather
+DEBUG: begin event::record_event
+DEBUG: end   event::record_event
 DEBUG: begin apply(<typename>)
 DEBUG: begin csr::spmv
 DEBUG: end   csr::spmv
@@ -548,6 +568,8 @@ DEBUG: end   cg::step_1
 DEBUG: begin apply(<typename>)
 DEBUG: begin dense::row_gather
 DEBUG: end   dense::row_gather
+DEBUG: begin event::record_event
+DEBUG: end   event::record_event
 DEBUG: begin apply(<typename>)
 DEBUG: begin csr::spmv
 DEBUG: end   csr::spmv
@@ -588,6 +610,8 @@ DEBUG: end   cg::step_1
 DEBUG: begin apply(<typename>)
 DEBUG: begin dense::row_gather
 DEBUG: end   dense::row_gather
+DEBUG: begin event::record_event
+DEBUG: end   event::record_event
 DEBUG: begin apply(<typename>)
 DEBUG: begin csr::spmv
 DEBUG: end   csr::spmv
@@ -628,6 +652,8 @@ DEBUG: end   cg::step_1
 DEBUG: begin apply(<typename>)
 DEBUG: begin dense::row_gather
 DEBUG: end   dense::row_gather
+DEBUG: begin event::record_event
+DEBUG: end   event::record_event
 DEBUG: begin apply(<typename>)
 DEBUG: begin csr::spmv
 DEBUG: end   csr::spmv
@@ -668,6 +694,8 @@ DEBUG: end   cg::step_1
 DEBUG: begin apply(<typename>)
 DEBUG: begin dense::row_gather
 DEBUG: end   dense::row_gather
+DEBUG: begin event::record_event
+DEBUG: end   event::record_event
 DEBUG: begin apply(<typename>)
 DEBUG: begin csr::spmv
 DEBUG: end   csr::spmv
@@ -708,6 +736,8 @@ DEBUG: end   cg::step_1
 DEBUG: begin apply(<typename>)
 DEBUG: begin dense::row_gather
 DEBUG: end   dense::row_gather
+DEBUG: begin event::record_event
+DEBUG: end   event::record_event
 DEBUG: begin apply(<typename>)
 DEBUG: begin csr::spmv
 DEBUG: end   csr::spmv
@@ -748,6 +778,8 @@ DEBUG: end   cg::step_1
 DEBUG: begin apply(<typename>)
 DEBUG: begin dense::row_gather
 DEBUG: end   dense::row_gather
+DEBUG: begin event::record_event
+DEBUG: end   event::record_event
 DEBUG: begin apply(<typename>)
 DEBUG: begin csr::spmv
 DEBUG: end   csr::spmv
@@ -789,6 +821,8 @@ DEBUG: end   copy(<typename>)
 DEBUG: begin advanced_apply(<typename>)
 DEBUG: begin dense::row_gather
 DEBUG: end   dense::row_gather
+DEBUG: begin event::record_event
+DEBUG: end   event::record_event
 DEBUG: begin advanced_apply(<typename>)
 DEBUG: begin csr::advanced_spmv
 DEBUG: end   csr::advanced_spmv

benchmark/test/reference/distributed_solver.simple.stdout

@@ -29,6 +29,7 @@
                         "cg::initialize": 1.0,
                         "advanced_apply(<typename>)": 1.0,
                         "dense::row_gather": 1.0,
+                        "event::record_event": 1.0,
                         "csr::advanced_spmv": 1.0,
                         "dense::compute_squared_norm2": 1.0,
                         "dense::compute_sqrt": 1.0,

benchmark/test/reference/distributed_solver_dcomplex.simple.stdout

@@ -29,6 +29,7 @@
                         "cg::initialize": 1.0,
                         "advanced_apply(<typename>)": 1.0,
                         "dense::row_gather": 1.0,
+                        "event::record_event": 1.0,
                         "csr::advanced_spmv": 1.0,
                         "dense::compute_squared_norm2": 1.0,
                         "dense::compute_sqrt": 1.0,

benchmark/test/reference/spmv_distributed.profile.stderr

@@ -126,6 +126,8 @@ DEBUG: begin repetition
 DEBUG: begin apply(<typename>)
 DEBUG: begin dense::row_gather
 DEBUG: end   dense::row_gather
+DEBUG: begin event::record_event
+DEBUG: end   event::record_event
 DEBUG: begin apply(<typename>)
 DEBUG: begin csr::spmv
 DEBUG: end   csr::spmv

common/unified/components/fill_array_kernels.cpp

@@ -27,6 +27,7 @@ void fill_array(std::shared_ptr<const DefaultExecutor> exec, ValueType* array,
 
 GKO_INSTANTIATE_FOR_EACH_TEMPLATE_TYPE(GKO_DECLARE_FILL_ARRAY_KERNEL);
 template GKO_DECLARE_FILL_ARRAY_KERNEL(bool);
+template GKO_DECLARE_FILL_ARRAY_KERNEL(char);
 template GKO_DECLARE_FILL_ARRAY_KERNEL(uint16);
 template GKO_DECLARE_FILL_ARRAY_KERNEL(uint32);
 #ifndef GKO_SIZE_T_IS_UINT64_T

core/base/array.cpp

@@ -92,6 +92,7 @@ ValueType reduce_add(const array<ValueType>& input_arr,
 
 GKO_INSTANTIATE_FOR_EACH_TEMPLATE_TYPE(GKO_DECLARE_ARRAY_FILL);
 template GKO_DECLARE_ARRAY_FILL(bool);
+template GKO_DECLARE_ARRAY_FILL(char);
 template GKO_DECLARE_ARRAY_FILL(uint16);
 template GKO_DECLARE_ARRAY_FILL(uint32);
 #ifndef GKO_SIZE_T_IS_UINT64_T

core/base/event.hpp

@@ -0,0 +1,37 @@
+// SPDX-FileCopyrightText: 2025 The Ginkgo authors
+//
+// SPDX-License-Identifier: BSD-3-Clause
+
+#ifndef GKO_CORE_BASE_EVENT_HPP_
+#define GKO_CORE_BASE_EVENT_HPP_
+
+#include <memory>
+
+#include <ginkgo/core/base/event.hpp>
+#include <ginkgo/core/base/executor.hpp>
+
+
+namespace gko {
+namespace detail {
+
+/**
+ * NotAsyncEvent is to provide an Event implementation on unsupported executor
+ * like reference. It will ensure the kernels are finished when recording this
+ * event.
+ */
+class NotAsyncEvent : public Event {
+public:
+    NotAsyncEvent(std::shared_ptr<const Executor> exec) { exec->synchronize(); }
+
+    void synchronize() const override
+    {
+        // we have sync in the recording phase
+    }
+};
+
+
+}  // namespace detail
+}  // namespace gko
+
+
+#endif  // #ifndef GKO_CORE_BASE_EVENT_HPP_

core/base/event_kernels.hpp

@@ -0,0 +1,38 @@
+// SPDX-FileCopyrightText: 2025 The Ginkgo authors
+//
+// SPDX-License-Identifier: BSD-3-Clause
+
+#ifndef GKO_CORE_BASE_EVENT_KERNELS_HPP_
+#define GKO_CORE_BASE_EVENT_KERNELS_HPP_
+
+
+#include <memory>
+
+#include <ginkgo/core/base/event.hpp>
+#include <ginkgo/core/base/executor.hpp>
+
+#include "core/base/kernel_declaration.hpp"
+
+
+namespace gko {
+namespace kernels {
+
+
+#define GKO_DECLARE_EVENT_RECORD_EVENT                             \
+    void record_event(std::shared_ptr<const DefaultExecutor> exec, \
+                      std::shared_ptr<const detail::Event>& event)
+
+
+#define GKO_DECLARE_ALL_AS_TEMPLATES GKO_DECLARE_EVENT_RECORD_EVENT
+
+
+GKO_DECLARE_FOR_ALL_EXECUTOR_NAMESPACES(event, GKO_DECLARE_ALL_AS_TEMPLATES);
+
+
+#undef GKO_DECLARE_ALL_AS_TEMPLATES
+
+
+}  // namespace kernels
+}  // namespace gko
+
+#endif  // GKO_CORE_BASE_EVENT_KERNELS_HPP_

core/device_hooks/common_kernels.inc.cpp

@@ -10,6 +10,7 @@
 #include "core/base/batch_instantiation.hpp"
 #include "core/base/batch_multi_vector_kernels.hpp"
 #include "core/base/device_matrix_data_kernels.hpp"
+#include "core/base/event_kernels.hpp"
 #include "core/base/index_set_kernels.hpp"
 #include "core/base/mixed_precision_types.hpp"
 #include "core/components/absolute_array_kernels.hpp"
@@ -253,6 +254,7 @@ template GKO_DECLARE_PREFIX_SUM_NONNEGATIVE_KERNEL(size_type);
 
 GKO_STUB_TEMPLATE_TYPE(GKO_DECLARE_FILL_ARRAY_KERNEL);
 template GKO_DECLARE_FILL_ARRAY_KERNEL(bool);
+template GKO_DECLARE_FILL_ARRAY_KERNEL(char);
 template GKO_DECLARE_FILL_ARRAY_KERNEL(uint16);
 template GKO_DECLARE_FILL_ARRAY_KERNEL(uint32);
 #ifndef GKO_SIZE_T_IS_UINT64_T
@@ -316,6 +318,15 @@ GKO_STUB_INDEX_TYPE(GKO_DECLARE_INDEX_SET_LOCAL_TO_GLOBAL_KERNEL);
 }  // namespace idx_set
 
 
+namespace event {
+
+
+GKO_STUB(GKO_DECLARE_EVENT_RECORD_EVENT);
+
+
+}
+
+
 namespace partition {