Merge pull request #7 from kpouget/remoting

Remoting

Author: kpouget

build.backend.sh

@@ -10,6 +10,8 @@ else
     FLAVOR=""
 fi
 
+export SDKROOT=$(xcrun --sdk macosx --show-sdk-path)
+
 if [[ "$FLAVOR" == "-prod" ]]; then
     cat <<EOF
 ###

examples/run/run.cpp

@@ -985,13 +985,11 @@ static inline void stop_timer(void) {
 }
 
 static void show_timer(void) {
-  //printe("[%15lld] ns\n", timer_total);
-  long long ms = timer_total/1000000;
-  long long itl = ms/timer_count;
-  float speed = 1/((float)itl) * 1000;
-  printe("INFO: generate: [%7lld] ms for %lld invokations | ITL %lldms | throughput = %.2f t/s\n", timer_total/1000000, timer_count, itl, speed);
+  double ms = timer_total/1000000;
+  double itl = ms/timer_count;
+  double speed = 1/itl * 1000;
 
-  printe("INFO: generate: [%7lld] s\n", timer_total/1000000/1000);
+  printe("LLAMA generate [%9.0f] ms for %4lld invocations | ITL %2.2f ms | throughput = %4.2f t/s\n", ms, timer_count, itl, speed);
 }
 
 
@@ -1011,14 +1009,7 @@ static int generate(LlamaData & llama_data, const std::string & prompt, std::str
     llama_batch batch = llama_batch_get_one(tokens.data(), tokens.size());
     llama_token new_token_id;
 
-    int count = 0;
     while (true) {
-#if 0
-      if (count > 25) {
-	printe("WARNING: stopping after %d tokens", count);
-	break;
-      }
-#endif
         start_timer();
         check_context_size(llama_data.context, batch);
         if (llama_decode(llama_data.context.get(), batch)) {
@@ -1042,7 +1033,6 @@ static int generate(LlamaData & llama_data, const std::string & prompt, std::str
         // prepare the next batch with the sampled token
         batch = llama_batch_get_one(&new_token_id, 1);
 	stop_timer();
-	count += 1;
     }
 
     printf(LOG_COL_DEFAULT);

ggml/src/ggml-metal/ggml-metal.m

@@ -4485,9 +4485,53 @@ static void ggml_metal_encode_node(
     }
 }
 
+long long timer_start;
+long long timer_total;
+long long timer_count;
+
+static inline void start_timer(void) {
+  struct timespec ts;
+  clock_gettime(CLOCK_REALTIME, &ts);  // Use CLOCK_MONOTONIC for elapsed time
+  timer_start = (long long)ts.tv_sec * 1000000000LL + ts.tv_nsec;
+}
+
+static inline void stop_timer(void) {
+  struct timespec ts;
+  clock_gettime(CLOCK_REALTIME, &ts);  // Use CLOCK_MONOTONIC for elapsed time
+  long long timer_end = (long long)ts.tv_sec * 1000000000LL + ts.tv_nsec;
+
+  timer_total += (timer_end - timer_start);
+  timer_count += 1;
+}
+
+static void show_timer(void) {
+  double ms = timer_total/1000000;
+  double itl = ms/timer_count;
+  double speed = 1/itl * 1000;
+
+  printf("METAL compute_graph: [%9.0f] ms for %lld invokations | ITL %.2f ms | throughput = %.2f t/s\n",ms, timer_count, itl, speed);
+
+  timer_start = 0;
+  timer_total = 1; // to avoid re-registering
+  timer_count = 0;
+}
+
+static void show_timer_signal(int sig) {
+  GGML_UNUSED(sig);
+  show_timer();
+}
+
 static enum ggml_status ggml_metal_graph_compute(
             ggml_backend_t   backend,
         struct ggml_cgraph * gf) {
+
+  if (timer_total == 0) {
+    signal(SIGUSR1, show_timer_signal); // kill -USR1 $(cat /tmp/krunkit.pid)
+    atexit(show_timer);
+  }
+
+  start_timer();
+
     struct ggml_backend_metal_context        * ctx     = backend->context;
     struct ggml_backend_metal_device_context * ctx_dev = backend->device->context;
 
@@ -4615,6 +4659,8 @@ static enum ggml_status ggml_metal_graph_compute(
         }
     }
 
+  stop_timer();
+
     return GGML_STATUS_SUCCESS;
 }
 

ggml/src/ggml-remotingbackend/backend-dispatched-backend.cpp

@@ -8,12 +8,14 @@
 
 #include "shared/apir_backend.h"
 
+struct timer_data graph_compute_timer = {0, 0, 0, "compute_timer"};
+
 uint32_t
 backend_graph_compute(struct vn_cs_encoder *enc, struct vn_cs_decoder *dec, struct virgl_apir_context *ctx) {
   UNUSED(ctx);
   UNUSED(enc);
 
-  start_timer();
+  start_timer(&graph_compute_timer);
 
   uint32_t shmem_res_id;
   vn_decode_virtgpu_shmem_res_id(dec, &shmem_res_id);
@@ -34,7 +36,7 @@ backend_graph_compute(struct vn_cs_encoder *enc, struct vn_cs_decoder *dec, stru
 
   vn_encode_ggml_status(enc, &status);
 
-  stop_timer();
+  stop_timer(&graph_compute_timer);
 
   return 0;
 }

ggml/src/ggml-remotingbackend/backend-dispatched-buffer-type.cpp

@@ -60,30 +60,16 @@ backend_buffer_type_is_host(struct vn_cs_encoder *enc, struct vn_cs_decoder *dec
 uint32_t
 backend_buffer_type_alloc_buffer(struct vn_cs_encoder *enc, struct vn_cs_decoder *dec, struct virgl_apir_context *ctx) {
   UNUSED(ctx);
-#if APIR_ALLOC_FROM_HOST_PTR
-  uint32_t shmem_res_id;
-  vn_decode_virtgpu_shmem_res_id(dec, &shmem_res_id);
 
-  void *shmem_data = ctx->iface.get_shmem_ptr(ctx->virgl_ctx, shmem_res_id);
-  if (!shmem_data) {
-    FATAL("Couldn't get the shmem addr from virgl :/");
-  }
-#else
   ggml_backend_buffer_type_t buft;
   buft = vn_decode_ggml_buffer_type(dec);
-#endif
+
   size_t size;
   vn_decode_size_t(dec, &size);
 
   ggml_backend_buffer_t buffer;
-#if APIR_ALLOC_FROM_HOST_PTR
-  #define MAX_TENSOR_SIZE 323205120
-  buffer = dev->iface.buffer_from_host_ptr(dev, shmem_data, size, MAX_TENSOR_SIZE);
 
-  vn_encode_ggml_buffer_type(enc, buffer->buft);
-#else
   buffer = buft->iface.alloc_buffer(buft, size);
-#endif
 
   vn_encode_ggml_buffer(enc, buffer);
 

ggml/src/ggml-remotingbackend/backend-dispatched-buffer.cpp

@@ -6,6 +6,9 @@
 #include "ggml-backend-impl.h"
 #include "ggml-backend.h"
 
+struct timer_data get_tensor_timer = {0, 0, 0, "get_tensor"};
+struct timer_data set_tensor_timer = {0, 0, 0, "set_tensor"};
+
 uint32_t
 backend_buffer_get_base(struct vn_cs_encoder *enc, struct vn_cs_decoder *dec, struct virgl_apir_context *ctx) {
   UNUSED(ctx);
@@ -23,6 +26,8 @@ backend_buffer_set_tensor(struct vn_cs_encoder *enc, struct vn_cs_decoder *dec,
   UNUSED(ctx);
   UNUSED(enc);
 
+  start_timer(&set_tensor_timer);
+
   ggml_backend_buffer_t buffer;
   buffer = vn_decode_ggml_buffer(dec);
 
@@ -60,6 +65,8 @@ backend_buffer_set_tensor(struct vn_cs_encoder *enc, struct vn_cs_decoder *dec,
 
   buffer->iface.set_tensor(buffer, tensor, shmem_data, offset, size);
 
+  stop_timer(&set_tensor_timer);
+
   return 0;
 }
 
@@ -68,6 +75,8 @@ backend_buffer_get_tensor(struct vn_cs_encoder *enc, struct vn_cs_decoder *dec,
   UNUSED(ctx);
   UNUSED(enc);
 
+  start_timer(&get_tensor_timer);
+
   ggml_backend_buffer_t buffer;
   buffer = vn_decode_ggml_buffer(dec);
 
@@ -94,6 +103,8 @@ backend_buffer_get_tensor(struct vn_cs_encoder *enc, struct vn_cs_decoder *dec,
   UNUSED(tensor);
   buffer->iface.get_tensor(buffer, tensor, shmem_data, offset, size);
 
+  stop_timer(&get_tensor_timer);
+
   return 0;
 }
 

ggml/src/ggml-remotingbackend/backend-dispatched-device.cpp

@@ -109,3 +109,34 @@ backend_device_get_props(struct vn_cs_encoder *enc, struct vn_cs_decoder *dec, s
 
   return 0;
 }
+
+uint32_t
+backend_device_buffer_from_ptr(struct vn_cs_encoder *enc, struct vn_cs_decoder *dec, struct virgl_apir_context *ctx) {
+  UNUSED(ctx);
+  UNUSED(dec);
+
+  uint32_t shmem_res_id;
+  vn_decode_virtgpu_shmem_res_id(dec, &shmem_res_id);
+
+  void *shmem_ptr = ctx->iface.get_shmem_ptr(ctx->virgl_ctx, shmem_res_id);
+  if (!shmem_ptr) {
+    FATAL("Couldn't get the shmem addr from virgl :/");
+  }
+
+  size_t size;
+  vn_decode_size_t(dec, &size);
+  size_t max_tensor_size;
+  vn_decode_size_t(dec, &max_tensor_size);
+
+  ggml_backend_buffer_t buffer;
+  buffer = dev->iface.buffer_from_host_ptr(dev, shmem_ptr, size, max_tensor_size);
+
+  vn_encode_ggml_buffer(enc, buffer);
+  vn_encode_ggml_buffer_type(enc, buffer->buft);
+
+  if (buffer) {
+    track_backend_buffer(buffer);
+  }
+
+  return 0;
+}

ggml/src/ggml-remotingbackend/backend-dispatched.cpp

@@ -31,9 +31,9 @@ uint32_t backend_dispatch_initialize(void *ggml_backend_reg_fct_p, void *ggml_ba
     dev = reg->iface.get_device(reg, 0);
   }
 
-  ggml_backend_t (* ggml_backend_fct)(void) = (ggml_backend_t (*)()) ggml_backend_init_fct_p;
+  ggml_backend_t (* ggml_backend_fct)(int) = (ggml_backend_t (*)(int)) ggml_backend_init_fct_p;
 
-  bck = ggml_backend_fct();
+  bck = ggml_backend_fct(0);
   if (!bck) {
     ERROR("%s: backend initialization failed :/", __func__);
     return APIR_BACKEND_INITIALIZE_BACKEND_FAILED;

ggml/src/ggml-remotingbackend/backend-dispatched.h

@@ -27,6 +27,7 @@ uint32_t backend_device_get_memory(struct vn_cs_encoder *enc, struct vn_cs_decod
 uint32_t backend_device_supports_op(struct vn_cs_encoder *enc, struct vn_cs_decoder *dec, struct virgl_apir_context *ctx);
 uint32_t backend_device_get_buffer_type(struct vn_cs_encoder *enc, struct vn_cs_decoder *dec, struct virgl_apir_context *ctx);
 uint32_t backend_device_get_props(struct vn_cs_encoder *enc, struct vn_cs_decoder *dec, struct virgl_apir_context *ctx);
+uint32_t backend_device_buffer_from_ptr(struct vn_cs_encoder *enc, struct vn_cs_decoder *dec, struct virgl_apir_context *ctx);
 
 /* buffer-type */
 uint32_t backend_buffer_type_get_name(struct vn_cs_encoder *enc, struct vn_cs_decoder *dec, struct virgl_apir_context *ctx);
@@ -57,6 +58,7 @@ static inline const char *backend_dispatch_command_name(ApirBackendCommandType t
   case APIR_COMMAND_TYPE_DEVICE_SUPPORTS_OP: return "backend_device_supports_op";
   case APIR_COMMAND_TYPE_DEVICE_GET_BUFFER_TYPE: return "backend_get_buffer_type";
   case APIR_COMMAND_TYPE_DEVICE_GET_PROPS: return "backend_get_props";
+  case APIR_COMMAND_TYPE_DEVICE_BUFFER_FROM_PTR: return "backend_buffer_from_ptr";
 
   /* buffer-type */
   case APIR_COMMAND_TYPE_BUFFER_TYPE_GET_NAME: return "backend_buffer_type_get_name";
@@ -88,6 +90,7 @@ static const backend_dispatch_t apir_backend_dispatch_table[APIR_BACKEND_DISPATC
   [APIR_COMMAND_TYPE_DEVICE_SUPPORTS_OP] = backend_device_supports_op,
   [APIR_COMMAND_TYPE_DEVICE_GET_BUFFER_TYPE] = backend_device_get_buffer_type,
   [APIR_COMMAND_TYPE_DEVICE_GET_PROPS] = backend_device_get_props,
+  [APIR_COMMAND_TYPE_DEVICE_BUFFER_FROM_PTR] = backend_device_buffer_from_ptr,
 
   /* buffer-type */
   [APIR_COMMAND_TYPE_BUFFER_TYPE_GET_NAME] = backend_buffer_type_get_name,

ggml/src/ggml-remotingbackend/backend.cpp

@@ -10,9 +10,10 @@
 #include "shared/apir_backend.h"
 #include "shared/venus_cs.h"
 
-#define GGML_BACKEND_LIBRARY_PATH "/Users/kevinpouget/remoting/llama_cpp/build.remoting-backend/bin/libggml-metal.dylib"
-#define GGML_BACKEND_REG_FCT_NAME "ggml_backend_metal_reg"
-#define GGML_BACKEND_INIT_FCT_NAME "ggml_backend_metal_init"
+#define GGML_BACKEND_LIBRARY_PATH_ENV "APIR_LLAMA_CPP_GGML_LIBRARY_PATH"
+#define GGML_BACKEND_LIBRARY_REG_ENV "APIR_LLAMA_CPP_GGML_LIBRARY_REG"
+#define GGML_BACKEND_LIBRARY_INIT_ENV "APIR_LLAMA_CPP_GGML_LIBRARY_INIT"
+
 
 static void *backend_library_handle = NULL;
 
@@ -28,8 +29,9 @@ extern "C" {
     dev->iface.get_memory(dev, &free, &total);
     WARNING("%s: free memory: %ld MB\n", __func__, (size_t) free/1024/1024);
 
-    show_timer();
-
+    show_timer(&graph_compute_timer);
+    show_timer(&set_tensor_timer);
+    show_timer(&get_tensor_timer);
     /* *** */
 
     if (backend_library_handle) {
@@ -43,40 +45,54 @@ extern "C" {
   uint32_t apir_backend_initialize() {
     const char* dlsym_error;
 
-    INFO("%s: hello :wave: \\o/", __func__);
+    const char* library_name = getenv(GGML_BACKEND_LIBRARY_PATH_ENV);
+    const char* library_reg = getenv(GGML_BACKEND_LIBRARY_REG_ENV);
+    const char* library_init = getenv(GGML_BACKEND_LIBRARY_INIT_ENV);
+
+    INFO("%s: loading %s (%s|%s)", __func__, library_name, library_reg, library_init);
 
-    backend_library_handle = dlopen(GGML_BACKEND_LIBRARY_PATH, RTLD_LAZY);
+    if (!library_name) {
+      ERROR("Cannot open library: env var '%s' not defined\n", GGML_BACKEND_LIBRARY_PATH_ENV);
+
+      return APIR_BACKEND_INITIALIZE_CANNOT_OPEN_GGML_LIBRARY;
+    }
+
+    backend_library_handle = dlopen(library_name, RTLD_LAZY);
 
     if (!backend_library_handle) {
       ERROR("Cannot open library: %s\n", dlerror());
 
       return APIR_BACKEND_INITIALIZE_CANNOT_OPEN_GGML_LIBRARY;
     }
 
-    void *ggml_backend_reg_fct = dlsym(backend_library_handle, GGML_BACKEND_REG_FCT_NAME);
+    if (!library_reg) {
+      ERROR("Cannot register library: env var '%s' not defined\n", GGML_BACKEND_LIBRARY_REG_ENV);
+
+      return APIR_BACKEND_INITIALIZE_CANNOT_OPEN_GGML_LIBRARY;
+    }
+
+    void *ggml_backend_reg_fct = dlsym(backend_library_handle, library_reg);
     dlsym_error = dlerror();
     if (dlsym_error) {
       ERROR("Cannot load symbol: %s\n", dlsym_error);
 
       return APIR_BACKEND_INITIALIZE_MISSING_GGML_SYMBOLS;
     }
 
-    void *ggml_backend_init_fct = dlsym(backend_library_handle, GGML_BACKEND_INIT_FCT_NAME);
+    if (!library_init) {
+      ERROR("Cannot initialize library: env var '%s' not defined\n", library_init);
+
+      return APIR_BACKEND_INITIALIZE_CANNOT_OPEN_GGML_LIBRARY;
+    }
+
+    void *ggml_backend_init_fct = dlsym(backend_library_handle, library_init);
     dlsym_error = dlerror();
     if (dlsym_error) {
       ERROR("Cannot load symbol: %s\n", dlsym_error);
 
       return APIR_BACKEND_INITIALIZE_MISSING_GGML_SYMBOLS;
     }
 
-    INFO("#");
-#if APIR_ALLOC_FROM_HOST_PTR
-    INFO("# USING ALLOC_FROM_HOST_PTR");
-#else
-    INFO("# USING ALLOC_BUFFER");
-#endif
-    INFO("#");
-
     return backend_dispatch_initialize(ggml_backend_reg_fct, ggml_backend_init_fct);
   }