Merge pull request #1 from lanhin/dev_pim_multithread

Dev pim multithread

Author: lanhin

dpu/dpu_main.c

@@ -91,25 +91,35 @@ int wram2mram(__mram_ptr void *pmram,void *pwram,uint32_t size)
 }
 
 
-// main
-int main() {
-    unsigned int tasklet_id = me();
+// set psumf to global value for each thread access
+static float *psumf = NULL;
+
+void init(unsigned int tasklet_id) {
 #if PRINT
-    // printf("tasklet_id = %u\n", tasklet_id);
+    printf("tasklet_id = %u\n", tasklet_id);
 #endif
     if (tasklet_id == 0){ // Initialize once the cycle counter
         mem_reset(); // Reset the heap
+        // first thread set fp32->fp16 table
+        ptable_f32_f16 = (__mram_ptr float *)DPU_MRAM_HEAP_POINTER;
     }
     // Barrier
     barrier_wait(&my_barrier);
+}
 
-    //fp32->fp16 table
-    ptable_f32_f16 = (__mram_ptr float *)DPU_MRAM_HEAP_POINTER;
+// main
+int main() {
+
+    unsigned int tasklet_id = me();
+    
+    init(tasklet_id);
+    
+    //set fp32->fp16 table configure
     uint32_t table_f32_f16_len = (1 << 16)*sizeof(float);
     uint32_t offset = table_f32_f16_len;
     int input_row_size = 0;
     int input_cols = 0;
-    float *psumf = NULL;
+    
 
 #if PRINT
     printf("table_f32_f16_len=%d\n",table_f32_f16_len);
@@ -129,6 +139,11 @@ int main() {
         cache_meta->layer_num,cache_meta->weight_type,cache_meta->rows_per_dpu,cache_meta->rest_rows,cache_meta->input_offset);
 #endif
 
+    // set sart line, end line and line number in each thread
+    uint16_t weight_rows_per_thread = cache_meta->rows_per_dpu / NR_TASKLETS;
+    uint16_t weight_start_row = tasklet_id * weight_rows_per_thread;
+    uint16_t weight_end_row = weight_start_row + weight_rows_per_thread;
+
     // todo:rest row is existed, first thread in every dpu can one more row
     uint16_t weight_rows_cur_thread;
     if (cache_meta->rest_rows) {
@@ -142,14 +157,17 @@ int main() {
 
     //input metadata
     offset += (cache_meta->layer_len * cache_meta->layer_num);
+
 #if PRINT
     printf("layer_len=%d, input metadata offset=%d\n",cache_meta->layer_len,offset);
 #endif
+
     uint32_t inputmetadatabase = weightmetadatabase + sizeof(struct pim_meta) + cache_meta->layer_len * cache_meta->layer_num;
     pim_matrix_des *pinputcache = (pim_matrix_des *) mem_alloc(sizeof(pim_matrix_des));
     mram_read((__mram_ptr void const*) (inputmetadatabase), pinputcache, sizeof(pim_matrix_des));
     input_cols = pinputcache->ne[1];
     assert(input_cols == 1 && "Only support vector as input.");
+
 #if PRINT
     printf("input_type=%d, layerID=%d\n",pinputcache->type,pinputcache->layerid);
     for(int nn=0;nn<GGML_MAX_DIMS;nn++) {
@@ -158,6 +176,7 @@ int main() {
 #endif
 
     assert(cache_meta->weight_type == ((uint16_t)GGML_TYPE_Q4_0) && "Only support Q4_0 weight.");
+
     //weight info: GGML_TYPE_Q4_0 default
     if (cache_meta->weight_type == ((uint16_t)GGML_TYPE_Q4_0)) {
         if (pinputcache->type != GGML_TYPE_Q8_0) {
@@ -169,8 +188,15 @@ int main() {
         input_row_size = nb*sizeof(block_q8_0);
         __mram_ptr void *pweight_base = (__mram_ptr void *)(weightmetadatabase + sizeof(struct pim_meta));
         __mram_ptr void *pinput_base = DPU_MRAM_HEAP_POINTER + cache_meta->input_offset + sizeof(pim_matrix_des);
-        psumf = (float *)mem_alloc(sizeof(float)*input_cols*weight_rows_cur_thread);
+
+        if (tasklet_id == 0) {
+            psumf = (float *)mem_alloc(sizeof(float)*input_cols*weight_rows_cur_thread);
+        }
+        barrier_wait(&my_barrier);
+
+        // psumf = (float *)mem_alloc(sizeof(float)*input_cols*weight_rows_cur_thread);
         memset(psumf, 0 ,sizeof(float)*input_cols*weight_rows_cur_thread);
+        
 #if PRINT
         printf("input_cols=%d, rows_cur_thread=%d, nb=%d, input_row_size=%d\n",input_cols,weight_rows_cur_thread,nb,input_row_size);
 #endif
@@ -179,7 +205,7 @@ int main() {
 
         // weight_rows_cur_thread = 16;
         for(int l = 0;l < input_cols;l++) {
-          __mram_ptr block_q8_0 *pinput = pinput_base + l*nb*sizeof(block_q8_0);
+          __mram_ptr block_q8_0 *pinput = pinput_base + l * nb * sizeof(block_q8_0);
             mram2wram(pinput, pinput_cache, sizeof(block_q8_0)*nb);
 #if PRINT
             printf("input:\n");
@@ -192,8 +218,9 @@ int main() {
             }
             printf("pweight_base: %p\n", pweight_base);
 #endif
-            for(int k = 0;k < weight_rows_cur_thread;k++) {
-              __mram_ptr block_q4_0 *pweight = pweight_base + pinputcache->layerid*cache_meta->layer_len + k*nb*sizeof(block_q4_0);
+            // for(int k = 0;k < weight_rows_cur_thread;k++) {
+            for (int k = weight_start_row; k < weight_end_row; ++k) {
+              __mram_ptr block_q4_0 *pweight = pweight_base + pinputcache->layerid * cache_meta->layer_len + k * nb * sizeof(block_q4_0);
                 mram2wram(pweight, pweight_cache, sizeof(block_q4_0)*nb);
 #if PRINT
                 if (k % 64 == 0) {

dpu/pim_build.sh

@@ -1,2 +1,2 @@
 #!/bin/bash
-dpu-upmem-dpurte-clang -Wall -Wextra -O2 -DNR_TASKLETS=1 -DBL=11 -o gemv_dpu dpu_main.c
+dpu-upmem-dpurte-clang -Wall -Wextra -O2 -DNR_TASKLETS=8 -DBL=11 -o gemv_dpu dpu_main.c

examples/tensor/ts.cpp

@@ -1,6 +1,7 @@
 #include "trace_driver.h"
 #include <iostream>
 #include <iomanip>
+#include <chrono>
 
 #define NR_DPUS 8
 #define NR_LAYER 2
@@ -24,6 +25,8 @@ int gemv_dpu_kernel(struct pim_context *context, struct ggml_tensor * w, struct
   DPU_ASSERT(dpu_broadcast_to(context->dpu_set, DPU_MRAM_HEAP_POINTER_NAME, pim_offset, (void *)(ggml_table_f32_f16), sizeof(ggml_table_f32_f16), DPU_XFER_DEFAULT));
   pim_offset += sizeof(ggml_table_f32_f16);
 
+  std::cout << "ggml_table_f32_f16 len = " << sizeof(ggml_table_f32_f16) << std::endl;
+
   // Transfer pim_metadata into DPUs
   context->pim_metadata.layer_num = NR_LAYER;
   context->pim_metadata.weight_type = (uint16_t)(w->type);
@@ -36,6 +39,8 @@ int gemv_dpu_kernel(struct pim_context *context, struct ggml_tensor * w, struct
   context->pim_metadata.layer_len = w->nb[1] * (context->pim_metadata.rows_per_dpu);
   context->pim_metadata.input_offset = sizeof(ggml_table_f32_f16) + sizeof(struct pim_meta) + context->pim_metadata.layer_len * NR_LAYER;
 
+  std::cout << "layer_num = " << NR_LAYER << ", weight_type = " << (uint16_t)(w->type) << ", rows_per_dpu = " << w->ne[1] / NR_DPUS << ", rest_rows = " << w->ne[1] % NR_DPUS << ", layer_len = " << context->pim_metadata.layer_len << ", input_offset = " << context->pim_metadata.input_offset << std::endl;
+
   //Todo: NR_DPUS contexts are dispatched to different dpus(rest row is different on different dpu)
   DPU_ASSERT(dpu_broadcast_to(context->dpu_set, DPU_MRAM_HEAP_POINTER_NAME, pim_offset, &(context->pim_metadata), sizeof(struct pim_meta), DPU_XFER_DEFAULT));
   pim_offset += sizeof(struct pim_meta);
@@ -50,10 +55,10 @@ int gemv_dpu_kernel(struct pim_context *context, struct ggml_tensor * w, struct
       uint32_t prev_rows_dpu = i * context->pim_metadata.rows_per_dpu;
 
       // every dpu's data
-      DPU_ASSERT(dpu_prepare_xfer(dpu, ((unsigned char *)w->data) + prev_rows_dpu*size_per_row));
+      DPU_ASSERT(dpu_prepare_xfer(dpu, ((unsigned char *)w->data) + prev_rows_dpu * size_per_row));
     }
 
-    DPU_ASSERT(dpu_push_xfer(context->dpu_set, DPU_XFER_TO_DPU, DPU_MRAM_HEAP_POINTER_NAME, pim_offset + layer_len*layeridx, layer_len, DPU_XFER_DEFAULT));
+    DPU_ASSERT(dpu_push_xfer(context->dpu_set, DPU_XFER_TO_DPU, DPU_MRAM_HEAP_POINTER_NAME, pim_offset + layer_len * layeridx, layer_len, DPU_XFER_DEFAULT));
   }
 
   // Transfer input into DPUs
@@ -72,8 +77,14 @@ int gemv_dpu_kernel(struct pim_context *context, struct ggml_tensor * w, struct
   DPU_ASSERT(dpu_broadcast_to(context->dpu_set, DPU_MRAM_HEAP_POINTER_NAME, input_offset, in_q->data, bclen, DPU_XFER_DEFAULT));
   input_offset += bclen;
 
+  std::chrono::high_resolution_clock::time_point ex_tp1 = std::chrono::high_resolution_clock::now();
   // Launch DPU kernel
   DPU_ASSERT(dpu_launch(context->dpu_set, DPU_SYNCHRONOUS));
+  std::chrono::high_resolution_clock::time_point ex_tp2 = std::chrono::high_resolution_clock::now();
+
+  std::chrono::duration<size_t, std::nano> dur = ex_tp2 - ex_tp1;
+
+  std::cout << "执行用时：" << std::chrono::duration_cast<std::chrono::milliseconds>(dur).count() << " ms" << std::endl;
 
   // Check results
   float *mul_mat_res = (float *)res->data;
@@ -105,23 +116,30 @@ int main(int argc, char** argv) {
   struct ggml_tensor * ts_bq = tensor_import(filenamebq);
   struct ggml_tensor * ts_c = tensor_import(filenamec);
   struct ggml_tensor * ts_c_pim = tensor_import(filenamec_p);
-  std::cout<<"ts_a:"<<std::endl;
-  dump_tensor(ts_a, stdout);
-  std::cout<<"ts_b:"<<std::endl;
-  dump_tensor(ts_b, stdout);
-  std::cout<<"ts_bq:"<<std::endl;
-  dump_tensor(ts_bq, stdout);
-  std::cout<<"ts_c:"<<std::endl;
-  dump_tensor(ts_c, stdout);
-  std::cout<<"ts_c_pim:"<<std::endl;
-  dump_tensor(ts_c_pim, stdout);
-
+  // std::cout<<"ts_a:"<<std::endl;
+  // dump_tensor(ts_a, stdout);
+  // std::cout<<"ts_b:"<<std::endl;
+  // dump_tensor(ts_b, stdout);
+  // std::cout<<"ts_bq:"<<std::endl;
+  // dump_tensor(ts_bq, stdout);
+  // std::cout<<"ts_c:"<<std::endl;
+  // dump_tensor(ts_c, stdout);
+  // std::cout<<"ts_c_pim:"<<std::endl;
+  // dump_tensor(ts_c_pim, stdout);
+
+  std::cout << "ts_a: " << std::endl;
+  print_tensor(ts_a, stdout);
+  std::cout << "ts_b: " << std::endl;
+  print_tensor(ts_b, stdout);
 
   gemv_dpu_kernel(pqcontext, ts_a, ts_bq, ts_c_pim);
-  std::cout<<"ts_c_pim calculated by DPUs:"<<std::endl;
-  dump_tensor(ts_c_pim, stdout);
+  // std::cout<<"ts_c_pim calculated by DPUs:"<<std::endl;
+  // dump_tensor(ts_c_pim, stdout);
 
   float first_res = mul_add_q4_0_q8_0(ts_a, ts_bq);
   std::cout<<"first element: "<<std::fixed << std::setprecision(6)<<first_res<<std::endl;
+
+  std::cout << "error between c and c_pim:" << std::endl;
+  compare_tensor(ts_c, ts_c_pim);
   return 0;
 }

include/trace_driver.h

@@ -20,6 +20,9 @@ extern "C" {
   void tensor_export(const struct ggml_tensor * tensor, const char * fname);
   struct ggml_tensor * tensor_import(const char * fname);
   void dump_tensor_first_n(const struct ggml_tensor * tensor, int n, FILE * fout);
+  void compare_tensor(const struct ggml_tensor *a, const struct ggml_tensor *b);
+  void print_q4_tensor(const struct ggml_tensor *a);
+  void print_tensor(const struct ggml_tensor *tensor, FILE *fout);
   void dump_tensor(const struct ggml_tensor * tensor, FILE * fout);
 
   float mul_add_q4_0_q8_0(struct ggml_tensor * a, struct ggml_tensor * b);

src/trace_driver.c

@@ -1,5 +1,6 @@
 #include <stdlib.h>
 #include <stdio.h>
+#include "ggml-quants.h"
 #include "trace_driver.h"
 
 void tensor_export(const struct ggml_tensor * tensor, const char * fname) {
@@ -167,6 +168,91 @@ void dump_tensor_first_n(const struct ggml_tensor * tensor, int n, FILE * fout)
     }
 }
 
+void compare_tensor(const struct ggml_tensor *a, const struct ggml_tensor *b) {
+  assert(a->type == b->type);
+  assert(ggml_nelements(a) == ggml_nelements(b));
+
+  int nelems = ggml_nelements(a);
+
+  float max_err = 0.0f;
+  float min_err = 100000.0f;
+  float total_err = 0.0f;
+  float avg_err = 0.0f;
+  float *a_f32;
+  float *b_f32;
+  switch (a->type) {
+    case GGML_TYPE_F32:
+    case GGML_TYPE_F16:
+    case GGML_TYPE_BF16: {
+      a_f32 = (float *)a->data;
+      b_f32 = (float *)b->data;
+      break;
+    }
+    case GGML_TYPE_Q4_0: {
+      void *a_data = a->data;
+      void *b_data = b->data;
+
+      a_f32 = malloc(nelems * sizeof(float));
+      b_f32 = malloc(nelems * sizeof(float));
+
+      dequantize_row_q4_0(a_data, a_f32, nelems);
+      dequantize_row_q4_0(b_data, b_f32, nelems);
+      break;
+    }
+    case GGML_TYPE_Q8_0: {
+      void *a_data = a->data;
+      void *b_data = b->data;
+
+      a_f32 = malloc(nelems * sizeof(float));
+      b_f32 = malloc(nelems * sizeof(float));
+
+      dequantize_row_q8_0(a_data, a_f32, nelems);
+      dequantize_row_q8_0(b_data, b_f32, nelems);
+      break;
+    }
+    default: {
+      assert(false);
+    }
+  }
+
+  for (int i = 0; i < nelems; ++i) {
+    float err = abs(a_f32[i] - b_f32[i]);
+    total_err += err;
+    if (max_err < err) {
+      max_err = err;
+    }
+    if (min_err > err) {
+      min_err = err;
+    }
+  }
+
+  avg_err = total_err / nelems;
+
+  printf("max_error = %.5f, min_error = %.5f, total_error = %.5f, avg_error = %.5f\n", max_err, min_err, total_err, avg_err);
+}
+
+void print_q4_tensor(const struct ggml_tensor *a) {
+  void *d4_data = a->data;
+  int nelems = ggml_nelements(a);
+  printf("nelems = %d\n", nelems);
+
+  float *fp32_data = malloc(nelems * sizeof(float));
+  dequantize_row_q4_0(d4_data, fp32_data, nelems);
+
+  for (int i = 0; i < 10; ++i) {
+    printf("fp32_data[%d] = %f\n", i, fp32_data[i]);
+  }
+}
+
+void print_tensor(const struct ggml_tensor *tensor, FILE *fout) {
+  const int64_t *ne = tensor->ne;
+  const size_t  *nb = tensor->nb;
+
+  fprintf(fout, "shape = {%d, %d, %d, %d}, stride = {%d, %d, %d, %d}\n",
+    ne[0], ne[1], ne[2], ne[3],
+    nb[0], nb[1], nb[2], nb[3]);
+}
+
 void dump_tensor(const struct ggml_tensor * tensor, FILE * fout) {
     const int64_t * ne = tensor->ne;
     const size_t  * nb = tensor->nb;