Merge pull request #6 from kpouget/remoting

Remoting

Author: kpouget

build.backend.sh

@@ -4,7 +4,22 @@ rm -f READY_backend FAILED_backend
 echo "int isatty(int fd) { return 1; }" | gcc -O2 -fpic -shared -ldl -o /tmp/isatty.so -xc -
 export LD_PRELOAD=/tmp/isatty.so
 
-cmake --build ../build.remoting-backend --parallel 8 --target llama-run "$@"
+if [[ "${PERF_MODE:-}" ]]; then
+    FLAVOR="-prod"
+else
+    FLAVOR=""
+fi
+
+if [[ "$FLAVOR" == "-prod" ]]; then
+    cat <<EOF
+###
+### Building the prod flavor
+###
+EOF
+fi
+
+WHAT="llama-run llama-bench"
+cmake --build ../build.remoting-backend$FLAVOR --parallel 8 --target $WHAT "$@"
 
 if [[ $? == 0 ]]; then
     touch READY_backend

examples/run/run.cpp

@@ -965,6 +965,36 @@ static void print_word_and_concatenate_to_response(const std::string & piece, st
     response += piece;
 }
 
+static long long timer_start = 0;
+static long long timer_total = 0;
+static long long timer_count = 0;
+
+static inline void start_timer(void) {
+  struct timespec ts;
+  clock_gettime(CLOCK_REALTIME, &ts);  // Use CLOCK_MONOTONIC for elapsed time
+  timer_start = (long long)ts.tv_sec * 1000000000LL + ts.tv_nsec;
+}
+
+static inline void stop_timer(void) {
+  struct timespec ts;
+  clock_gettime(CLOCK_REALTIME, &ts);  // Use CLOCK_MONOTONIC for elapsed time
+  long long timer_end = (long long)ts.tv_sec * 1000000000LL + ts.tv_nsec;
+
+  timer_total += (timer_end - timer_start);
+  timer_count += 1;
+}
+
+static void show_timer(void) {
+  //printe("[%15lld] ns\n", timer_total);
+  long long ms = timer_total/1000000;
+  long long itl = ms/timer_count;
+  float speed = 1/((float)itl) * 1000;
+  printe("INFO: generate: [%7lld] ms for %lld invokations | ITL %lldms | throughput = %.2f t/s\n", timer_total/1000000, timer_count, itl, speed);
+
+  printe("INFO: generate: [%7lld] s\n", timer_total/1000000/1000);
+}
+
+
 // helper function to evaluate a prompt and generate a response
 static int generate(LlamaData & llama_data, const std::string & prompt, std::string & response) {
     const llama_vocab * vocab = llama_model_get_vocab(llama_data.model.get());
@@ -974,10 +1004,22 @@ static int generate(LlamaData & llama_data, const std::string & prompt, std::str
         return 1;
     }
 
+      int cr = atexit(show_timer);
+      assert(cr == 0);
+
     // prepare a batch for the prompt
     llama_batch batch = llama_batch_get_one(tokens.data(), tokens.size());
     llama_token new_token_id;
+
+    int count = 0;
     while (true) {
+#if 0
+      if (count > 25) {
+	printe("WARNING: stopping after %d tokens", count);
+	break;
+      }
+#endif
+        start_timer();
         check_context_size(llama_data.context, batch);
         if (llama_decode(llama_data.context.get(), batch)) {
             printe("failed to decode\n");
@@ -999,6 +1041,8 @@ static int generate(LlamaData & llama_data, const std::string & prompt, std::str
 
         // prepare the next batch with the sampled token
         batch = llama_batch_get_one(&new_token_id, 1);
+	stop_timer();
+	count += 1;
     }
 
     printf(LOG_COL_DEFAULT);

ggml/src/ggml-remotingbackend/backend-convert.h

@@ -1,7 +1,15 @@
 #include "shared/apir_backend.h"
 
-static inline apir_buffer_handle_t
+#define BUFFER_TO_HOST_HANDLE(name) ggml_buffer_to_apir_handle(name)
+
+static inline apir_buffer_host_handle_t
 ggml_buffer_to_apir_handle(ggml_backend_buffer_t buffer) {
   // in the backend, the buffer handle is the buffer pointer
-  return (apir_buffer_handle_t) buffer;
+  return (apir_buffer_host_handle_t) buffer;
+}
+
+static inline apir_buffer_type_host_handle_t
+ggml_buffer_type_to_apir_handle(ggml_backend_buffer_type_t buft) {
+  // in the backend, the buffer handle is the buffer pointer
+  return (apir_buffer_type_host_handle_t) buft;
 }

ggml/src/ggml-remotingbackend/backend-dispatched-backend.cpp

@@ -6,11 +6,15 @@
 #include "ggml-backend-impl.h"
 #include "ggml-backend.h"
 
+#include "shared/apir_backend.h"
+
 uint32_t
 backend_graph_compute(struct vn_cs_encoder *enc, struct vn_cs_decoder *dec, struct virgl_apir_context *ctx) {
   UNUSED(ctx);
   UNUSED(enc);
 
+  start_timer();
+
   uint32_t shmem_res_id;
   vn_decode_virtgpu_shmem_res_id(dec, &shmem_res_id);
 
@@ -30,5 +34,7 @@ backend_graph_compute(struct vn_cs_encoder *enc, struct vn_cs_decoder *dec, stru
 
   vn_encode_ggml_status(enc, &status);
 
+  stop_timer();
+
   return 0;
 }

ggml/src/ggml-remotingbackend/backend-dispatched-buffer-type.cpp

@@ -10,7 +10,7 @@ uint32_t
 backend_buffer_type_get_name(struct vn_cs_encoder *enc, struct vn_cs_decoder *dec, struct virgl_apir_context *ctx) {
   UNUSED(ctx);
   ggml_backend_buffer_type_t buft;
-  buft = vn_decode_ggml_buft(dec);
+  buft = vn_decode_ggml_buffer_type(dec);
 
   const char *string = buft->iface.get_name(buft);
 
@@ -25,7 +25,7 @@ uint32_t
 backend_buffer_type_get_alignment(struct vn_cs_encoder *enc, struct vn_cs_decoder *dec, struct virgl_apir_context *ctx) {
   UNUSED(ctx);
   ggml_backend_buffer_type_t buft;
-  buft = vn_decode_ggml_buft(dec);
+  buft = vn_decode_ggml_buffer_type(dec);
 
   size_t value = buft->iface.get_alignment(buft);
   vn_encode_size_t(enc, &value);
@@ -37,7 +37,7 @@ uint32_t
 backend_buffer_type_get_max_size(struct vn_cs_encoder *enc, struct vn_cs_decoder *dec, struct virgl_apir_context *ctx) {
   UNUSED(ctx);
   ggml_backend_buffer_type_t buft;
-  buft = vn_decode_ggml_buft(dec);
+  buft = vn_decode_ggml_buffer_type(dec);
 
   size_t value = buft->iface.get_max_size(buft);
   vn_encode_size_t(enc, &value);
@@ -49,7 +49,7 @@ uint32_t
 backend_buffer_type_is_host(struct vn_cs_encoder *enc, struct vn_cs_decoder *dec, struct virgl_apir_context *ctx) {
   UNUSED(ctx);
   ggml_backend_buffer_type_t buft;
-  buft = vn_decode_ggml_buft(dec);
+  buft = vn_decode_ggml_buffer_type(dec);
 
   bool is_host = buft->iface.is_host(buft);
   vn_encode_bool_t(enc, &is_host);
@@ -60,15 +60,32 @@ backend_buffer_type_is_host(struct vn_cs_encoder *enc, struct vn_cs_decoder *dec
 uint32_t
 backend_buffer_type_alloc_buffer(struct vn_cs_encoder *enc, struct vn_cs_decoder *dec, struct virgl_apir_context *ctx) {
   UNUSED(ctx);
-  ggml_backend_buffer_type_t buft;
-  buft = vn_decode_ggml_buft(dec);
+#if APIR_ALLOC_FROM_HOST_PTR
+  uint32_t shmem_res_id;
+  vn_decode_virtgpu_shmem_res_id(dec, &shmem_res_id);
 
+  void *shmem_data = ctx->iface.get_shmem_ptr(ctx->virgl_ctx, shmem_res_id);
+  if (!shmem_data) {
+    FATAL("Couldn't get the shmem addr from virgl :/");
+  }
+#else
+  ggml_backend_buffer_type_t buft;
+  buft = vn_decode_ggml_buffer_type(dec);
+#endif
   size_t size;
   vn_decode_size_t(dec, &size);
 
-  ggml_backend_buffer_t buffer = buft->iface.alloc_buffer(buft, size);
-  apir_buffer_handle_t *buffer_handle = (apir_buffer_handle_t *) buffer;
-  vn_encode_ggml_buffer_handle(enc, buffer_handle);
+  ggml_backend_buffer_t buffer;
+#if APIR_ALLOC_FROM_HOST_PTR
+  #define MAX_TENSOR_SIZE 323205120
+  buffer = dev->iface.buffer_from_host_ptr(dev, shmem_data, size, MAX_TENSOR_SIZE);
+
+  vn_encode_ggml_buffer_type(enc, buffer->buft);
+#else
+  buffer = buft->iface.alloc_buffer(buft, size);
+#endif
+
+  vn_encode_ggml_buffer(enc, buffer);
 
   if (buffer) {
     track_backend_buffer(buffer);

ggml/src/ggml-remotingbackend/backend-dispatched-buffer.cpp

@@ -15,8 +15,6 @@ backend_buffer_get_base(struct vn_cs_encoder *enc, struct vn_cs_decoder *dec, st
   uintptr_t base = (uintptr_t) buffer->iface.get_base(buffer);
   vn_encode_uintptr_t(enc, &base);
 
-  //INFO("%s: send base %p\n", __func__,  (void *) base);
-
   return 0;
 }
 
@@ -123,6 +121,11 @@ backend_buffer_free_buffer(struct vn_cs_encoder *enc, struct vn_cs_decoder *dec,
   ggml_backend_buffer_t buffer;
   buffer = vn_decode_ggml_buffer(dec);
 
+  if (!untrack_backend_buffer(buffer)) {
+    WARNING("%s: unknown buffer %p", (void *) buffer);
+    return 1;
+  }
+
   buffer->iface.free_buffer(buffer);
 
   return 0;

ggml/src/ggml-remotingbackend/backend-dispatched-device.cpp

@@ -89,8 +89,7 @@ backend_device_get_buffer_type(struct vn_cs_encoder *enc, struct vn_cs_decoder *
 
   ggml_backend_buffer_type_t bufft = dev->iface.get_buffer_type(dev);
 
-  apir_buffer_type_handle_t buft_handle = (apir_buffer_type_handle_t) bufft;
-  vn_encode_apir_buffer_type_handle_t(enc, &buft_handle);
+  vn_encode_ggml_buffer_type(enc, bufft);
 
   return 0;
 }

ggml/src/ggml-remotingbackend/backend-dispatched.cpp

@@ -12,6 +12,10 @@ ggml_backend_reg_t reg = NULL;
 ggml_backend_dev_t dev = NULL;
 ggml_backend_t bck = NULL;
 
+long long timer_start = 0;
+long long timer_total = 0;
+long long timer_count = 0;
+
 uint32_t backend_dispatch_initialize(void *ggml_backend_reg_fct_p, void *ggml_backend_init_fct_p) {
   if (reg != NULL) {
     FATAL("%s: already initialized :/", __func__);
@@ -35,5 +39,9 @@ uint32_t backend_dispatch_initialize(void *ggml_backend_reg_fct_p, void *ggml_ba
     return APIR_BACKEND_INITIALIZE_BACKEND_FAILED;
   }
 
+  size_t free, total;
+  dev->iface.get_memory(dev, &free, &total);
+  WARNING("%s: free memory: %ld MB\n", __func__, (size_t) free/1024/1024);
+
   return APIR_BACKEND_INITIALIZE_SUCCESSS;
 }

ggml/src/ggml-remotingbackend/backend.cpp

@@ -18,6 +18,20 @@ static void *backend_library_handle = NULL;
 
 extern "C" {
   void apir_backend_deinit(void) {
+    auto buffers = get_track_backend_buffers();
+    for (const auto& buffer: buffers) {
+      untrack_backend_buffer(buffer);
+      buffer->iface.free_buffer(buffer);
+    }
+
+    size_t free, total;
+    dev->iface.get_memory(dev, &free, &total);
+    WARNING("%s: free memory: %ld MB\n", __func__, (size_t) free/1024/1024);
+
+    show_timer();
+
+    /* *** */
+
     if (backend_library_handle) {
       INFO("%s: The GGML backend library was loaded. Unloading it.", __func__);
       dlclose(backend_library_handle);
@@ -55,6 +69,14 @@ extern "C" {
       return APIR_BACKEND_INITIALIZE_MISSING_GGML_SYMBOLS;
     }
 
+    INFO("#");
+#if APIR_ALLOC_FROM_HOST_PTR
+    INFO("# USING ALLOC_FROM_HOST_PTR");
+#else
+    INFO("# USING ALLOC_BUFFER");
+#endif
+    INFO("#");
+
     return backend_dispatch_initialize(ggml_backend_reg_fct, ggml_backend_init_fct);
   }
 
@@ -81,6 +103,11 @@ extern "C" {
       return APIR_BACKEND_FORWARD_INDEX_INVALID;
     }
 
+#if 0
+    static long long count = 0;
+    INFO("[%lld] Calling %s", count, backend_dispatch_command_name((ApirBackendCommandType) cmd_type));
+    count += 1;
+#endif
     backend_dispatch_t forward_fct = apir_backend_dispatch_table[cmd_type];
     uint32_t ret = forward_fct(enc, dec, ctx);
 

ggml/src/ggml-remotingbackend/shared/apir_backend.h

@@ -1,6 +1,6 @@
 #pragma once
 
-#define APIR_LIBRARY_PATH "/Users/kevinpouget/remoting/llama_cpp/build.remoting-backend/bin/libggml-remotingbackend.dylib"
+#define APIR_LIBRARY_PATH "/Users/kevinpouget/remoting/llama_cpp/build.remoting-backend-prod/bin/libggml-remotingbackend.dylib"
 #define APIR_INITIALIZE_FCT_NAME "apir_backend_initialize"
 #define APIR_DEINIT_FCT_NAME "apir_backend_deinit"
 #define APIR_DISPATCH_FCT_NAME "apir_backend_dispatcher"
@@ -14,8 +14,18 @@
 
 #define APIR_BACKEND_FORWARD_INDEX_INVALID 6
 
-typedef uintptr_t apir_buffer_type_handle_t;
-typedef uintptr_t apir_buffer_handle_t;
+#define APIR_ALLOC_FROM_HOST_PTR 0
+
+typedef uintptr_t apir_buffer_type_host_handle_t;
+typedef uintptr_t apir_buffer_host_handle_t;
+
+typedef struct {
+  apir_buffer_host_handle_t host_handle;
+#if APIR_ALLOC_FROM_HOST_PTR
+  struct vn_renderer_shmem *shmem;
+  apir_buffer_type_host_handle_t buft_host_handle;
+#endif
+} apir_buffer_context_t;
 
 typedef uint32_t (*apir_backend_initialize_t)(void);
 typedef void (*apir_backend_deinit_t)(void);
@@ -72,7 +82,30 @@ struct virgl_apir_context {
   struct virgl_apir_callbacks iface;
 };
 
-#define TENSOR_MAX_DEPTH_DEVICE_SUPPORTS_OP 2
-#define TENSOR_MAX_DEPTH_BUFFER_GET_TENSOR 2
-#define TENSOR_MAX_DEPTH_BUFFER_SET_TENSOR 2
-#define TENSOR_MAX_DEPTH_CGRAPH_DATA 10
+extern long long timer_start;
+extern long long timer_total;
+extern long long timer_count;
+
+static inline void start_timer(void) {
+  struct timespec ts;
+  clock_gettime(CLOCK_REALTIME, &ts);  // Use CLOCK_MONOTONIC for elapsed time
+  timer_start = (long long)ts.tv_sec * 1000000000LL + ts.tv_nsec;
+}
+
+static inline void stop_timer(void) {
+  struct timespec ts;
+  clock_gettime(CLOCK_REALTIME, &ts);  // Use CLOCK_MONOTONIC for elapsed time
+  long long timer_end = (long long)ts.tv_sec * 1000000000LL + ts.tv_nsec;
+
+  timer_total += (timer_end - timer_start);
+  timer_count += 1;
+}
+
+static inline void show_timer(void) {
+  long long ms = timer_total/1000000;
+  long long itl = ms/timer_count;
+  float speed = 1/((float)itl) * 1000;
+
+  INFO("compute_graph: [%9ld] ms for %ld invokations | ITL %lldms | throughput = %.2f t/s\n", timer_total/1000000, timer_count, itl, speed);
+  INFO("compute_graph: [%9ld] s", (ms)/1000);
+}