FPGA: Reorder submit order for DB11 and DB12 to get more stable timing/Throughput numbers (#2339)

To minimize the variation on the timing, it is important to start the 'first' kernel of a chain of kernels last. This ensures that the timing doesn't start until that kernel is ready to go.

For DB12, there are 2 pipelines that are joined. If PRECISE_TIMING is defined, add a dummy start kernel that will trigger the two pipelines to start.

Author: mendell27

DirectProgramming/C++SYCL_FPGA/ReferenceDesigns/db/CMakeLists.txt

@@ -125,6 +125,12 @@ else()
     set(SF_SMALL_ARG )
 endif()
 
+if (PRECISE_TIMING)
+    set(PRECISE_TIMING_ARG -DPRECISE_TIMING)
+else()
+    set(PRECISE_TIMING_ARG )
+endif()
+
 # setting source file based on query version
 if(${QUERY} EQUAL 1)
     set(SOURCE_FILES ${SOURCE_FILES};src/query1/query1_kernel.cpp)
@@ -144,7 +150,7 @@ endif()
 set(USER_FPGA_FLAGS ${USER_FPGA_FLAGS};${SEED};${CLOCK_TARGET})
 
 # Use cmake -DUSER_FLAGS=<flags> to set extra flags for general compilation.
-set(USER_FLAGS ${USER_FLAGS};-DQUERY=${QUERY};${SF_SMALL_ARG})
+set(USER_FLAGS ${USER_FLAGS};-DQUERY=${QUERY};${SF_SMALL_ARG};${PRECISE_TIMING_ARG})
 
 # Use cmake -DUSER_INCLUDE_PATHS=<paths> to set extra paths for general
 # compilation.

DirectProgramming/C++SYCL_FPGA/ReferenceDesigns/db/src/query11/query11_kernel.cpp

@@ -76,46 +76,6 @@ bool SubmitQuery11(queue& q, Database& dbinfo, std::string& nation,
   // start timer
   high_resolution_clock::time_point host_start = high_resolution_clock::now();
 
-  ///////////////////////////////////////////////////////////////////////////
-  //// ProducePartSupplier Kernel
-  auto produce_ps_event = q.submit([&](handler& h) {
-    // PARTSUPPLIER table accessors
-    accessor ps_partkey_accessor(ps_partkey_buf, h, read_only);
-    accessor ps_suppkey_accessor(ps_suppkey_buf, h, read_only);
-    accessor ps_availqty_accessor(ps_availqty_buf, h, read_only);
-    accessor ps_supplycost_accessor(ps_supplycost_buf, h, read_only);
-
-    // kernel to produce the PARTSUPPLIER table
-    h.single_task<ProducePartSupplier>([=]() [[intel::kernel_args_restrict]] {
-      [[intel::initiation_interval(1)]]
-      for (size_t i = 0; i < ps_iters; i++) {
-        // bulk read of data from global memory
-        NTuple<kJoinWinSize, PartSupplierRow> data;
-
-        UnrolledLoop<0, kJoinWinSize>([&](auto j) {
-          size_t idx = i * kJoinWinSize + j;
-          bool in_range = idx < ps_rows;
-
-          DBIdentifier partkey = ps_partkey_accessor[idx];
-          DBIdentifier suppkey = ps_suppkey_accessor[idx];
-          int availqty = ps_availqty_accessor[idx];
-          DBDecimal supplycost = ps_supplycost_accessor[idx];
-
-          data.get<j>() =
-              PartSupplierRow(in_range, partkey, suppkey, availqty, supplycost);
-        });
-
-        // write to pipe
-        ProducePartSupplierPipe::write(
-            PartSupplierRowPipeData(false, true, data));
-      }
-
-      // tell the downstream kernel we are done producing data
-      ProducePartSupplierPipe::write(PartSupplierRowPipeData(true, false));
-    });
-  });
-  ///////////////////////////////////////////////////////////////////////////
-
   ///////////////////////////////////////////////////////////////////////////
   //// JoinPartSupplierParts Kernel
   auto join_event = q.submit([&](handler& h) {
@@ -250,6 +210,47 @@ bool SubmitQuery11(queue& q, Database& dbinfo, std::string& nation,
   });
   ///////////////////////////////////////////////////////////////////////////
 
+  // Must be last to ensure reliable timings
+  ///////////////////////////////////////////////////////////////////////////
+  //// ProducePartSupplier Kernel
+  auto produce_ps_event = q.submit([&](handler& h) {
+    // PARTSUPPLIER table accessors
+    accessor ps_partkey_accessor(ps_partkey_buf, h, read_only);
+    accessor ps_suppkey_accessor(ps_suppkey_buf, h, read_only);
+    accessor ps_availqty_accessor(ps_availqty_buf, h, read_only);
+    accessor ps_supplycost_accessor(ps_supplycost_buf, h, read_only);
+
+    // kernel to produce the PARTSUPPLIER table
+    h.single_task<ProducePartSupplier>([=]() [[intel::kernel_args_restrict]] {
+      [[intel::initiation_interval(1)]]
+      for (size_t i = 0; i < ps_iters; i++) {
+        // bulk read of data from global memory
+        NTuple<kJoinWinSize, PartSupplierRow> data;
+
+        UnrolledLoop<0, kJoinWinSize>([&](auto j) {
+          size_t idx = i * kJoinWinSize + j;
+          bool in_range = idx < ps_rows;
+
+          DBIdentifier partkey = ps_partkey_accessor[idx];
+          DBIdentifier suppkey = ps_suppkey_accessor[idx];
+          int availqty = ps_availqty_accessor[idx];
+          DBDecimal supplycost = ps_supplycost_accessor[idx];
+
+          data.get<j>() =
+              PartSupplierRow(in_range, partkey, suppkey, availqty, supplycost);
+        });
+
+        // write to pipe
+        ProducePartSupplierPipe::write(
+            PartSupplierRowPipeData(false, true, data));
+      }
+
+      // tell the downstream kernel we are done producing data
+      ProducePartSupplierPipe::write(PartSupplierRowPipeData(true, false));
+    });
+  });
+  ///////////////////////////////////////////////////////////////////////////
+
   // wait for kernels to finish
   produce_ps_event.wait();
   join_event.wait();
@@ -260,9 +261,9 @@ bool SubmitQuery11(queue& q, Database& dbinfo, std::string& nation,
   high_resolution_clock::time_point host_end = high_resolution_clock::now();
   duration<double, std::milli> diff = host_end - host_start;
 
-  // gather profiling info
+  // gather profiling info from start of pipeline to end
   auto start_time =
-      consume_sort_event
+      produce_ps_event
           .get_profiling_info<info::event_profiling::command_start>();
   auto end_time = consume_sort_event
           .get_profiling_info<info::event_profiling::command_end>();

DirectProgramming/C++SYCL_FPGA/ReferenceDesigns/db/src/query12/pipe_types.hpp

@@ -117,4 +117,13 @@ using LineItemProducerPipe =
 using JoinedProducerPipe =
   pipe<class JoinedProducerPipeClass, JoinedRowPipeData>;
 
+#ifdef PRECISE_TIMING
+using OrdersProducerStartPipe =
+  pipe<class OrdersProducerPipeStartClass, bool>;
+
+using LineItemProducerStartPipe =
+  pipe<class LineItemProducerPipeStartClass, bool>;
+
+#endif
+
 #endif /* __PIPE_TYPES_H__ */

DirectProgramming/C++SYCL_FPGA/ReferenceDesigns/db/src/query12/query12_kernel.cpp

@@ -16,6 +16,7 @@ class LineItemProducer;
 class OrdersProducer;
 class Join;
 class Compute;
+class StartProduction;
 
 bool SubmitQuery12(queue& q, Database& dbinfo, DBDate low_date,
                     DBDate high_date, int shipmode1, int shipmode2,
@@ -60,6 +61,9 @@ bool SubmitQuery12(queue& q, Database& dbinfo, DBDate low_date,
     accessor l_receiptdate_accessor(l_receiptdate_buf, h, read_only);
 
     h.single_task<LineItemProducer>([=]() [[intel::kernel_args_restrict]] {
+#ifdef PRECISE_TIMING
+      (void) LineItemProducerStartPipe::read();
+#endif
       [[intel::initiation_interval(1)]]
       for (size_t i = 0; i < l_iters + 1; i++) {
         bool done = (i == l_iters);
@@ -99,6 +103,9 @@ bool SubmitQuery12(queue& q, Database& dbinfo, DBDate low_date,
     accessor o_orderpriority_accessor(o_orderpriority_buf, h, read_only);
 
     h.single_task<OrdersProducer>([=]() [[intel::kernel_args_restrict]] {
+#ifdef PRECISE_TIMING
+      (void) OrdersProducerStartPipe::read();
+#endif
       [[intel::initiation_interval(1)]]
       for (size_t i = 0; i < o_iters + 1; i++) {
         bool done = (i == o_iters);
@@ -237,7 +244,20 @@ bool SubmitQuery12(queue& q, Database& dbinfo, DBDate low_date,
   });
   /////////////////////////////////////////////////////////////////////////////
 
-  // wait for the Compute kernel to finish
+#ifdef PRECISE_TIMING
+  // Started last to get more reliable timings
+  /////////////////////////////////////////////////////////////////////////////
+  //// Start Production - Ensure accurate timings
+  auto start_production_event = q.submit([&](handler& h) {
+    h.single_task<StartProduction>([=]() [[intel::kernel_args_restrict]] {
+      OrdersProducerStartPipe::write(true);
+      LineItemProducerStartPipe::write(true);
+    });
+  });
+  /////////////////////////////////////////////////////////////////////////////
+
+  start_production_event.wait();
+#endif
   produce_orders_event.wait();
   produce_lineitem_event.wait();
   join_event.wait();
@@ -248,8 +268,15 @@ bool SubmitQuery12(queue& q, Database& dbinfo, DBDate low_date,
   duration<double, std::milli> diff = host_end - host_start;
 
   //// gather profiling info
+#ifdef PRECISE_TIMING
+  // Measure complete timing from start of pipeline to the end.
+  auto start_time =
+      start_production_event.get_profiling_info<info::event_profiling::command_start>();
+#else
+  // Just measure computation
   auto start_time =
       compute_event.get_profiling_info<info::event_profiling::command_start>();
+#endif // PRECISE_TIMING
   auto end_time =
       compute_event.get_profiling_info<info::event_profiling::command_end>();