Merge pull request #5 from kpouget/remoting

Remoting: keep working

Author: kpouget

build.backend.sh

@@ -4,7 +4,7 @@ rm -f READY_backend FAILED_backend
 echo "int isatty(int fd) { return 1; }" | gcc -O2 -fpic -shared -ldl -o /tmp/isatty.so -xc -
 export LD_PRELOAD=/tmp/isatty.so
 
-cmake --build ../build.remoting-backend --parallel 8 --target llama-cli "$@"
+cmake --build ../build.remoting-backend --parallel 8 --target llama-run "$@"
 
 if [[ $? == 0 ]]; then
     touch READY_backend

ggml/CMakeLists.txt

@@ -271,8 +271,7 @@ set(GGML_PUBLIC_HEADERS
     include/ggml-rpc.h
     include/ggml-sycl.h
     include/ggml-vulkan.h
-    ggml/include/ggml-remoting-frontend.h
-    ggml/include/ggml-remoting-backend.h
+    include/ggml-remoting-frontend.h
     include/gguf.h)
 
 set_target_properties(ggml PROPERTIES PUBLIC_HEADER "${GGML_PUBLIC_HEADERS}")

ggml/include/ggml-remoting-backend.h

@@ -49,10 +49,6 @@
 #include "ggml-remoting-frontend.h"
 #endif
 
-#ifdef GGML_USE_REMOTINGBACKEND
-#include "ggml-remoting-backend.h"
-#endif
-
 #ifdef GGML_USE_OPENCL
 #include "ggml-opencl.h"
 #endif
@@ -183,9 +179,7 @@ struct ggml_backend_registry {
 #ifdef GGML_USE_REMOTINGFRONTEND
         register_backend(ggml_backend_remoting_frontend_reg());
 #endif
-#ifdef GGML_USE_REMOTINGBACKEND
-        register_backend(ggml_backend_remoting_backend_reg());
-#endif
+
 #ifdef GGML_USE_OPENCL
         register_backend(ggml_backend_opencl_reg());
 #endif

ggml/src/ggml-backend-reg.cpp

@@ -8,6 +8,9 @@
 
 #import <Metal/Metal.h>
 
+#undef GGML_LOG_DEBUG
+#define GGML_LOG_DEBUG(...)
+
 #undef MIN
 #undef MAX
 #define MIN(a, b) ((a) < (b) ? (a) : (b))
@@ -776,8 +779,6 @@ @implementation GGMLMetalClass
                 GGML_LOG_ERROR("%s: error: load pipeline error: %s\n", __func__, [[error description] UTF8String]); \
                 return NULL; \
             } \
-        } else { \
-            GGML_LOG_WARN("%s: skipping %-40s (not supported)\n", __func__, "kernel_"#name); \
         }
 
         const bool has_simdgroup_mm        = ctx_dev->has_simdgroup_mm;

ggml/src/ggml-metal/ggml-metal.m

@@ -5,7 +5,16 @@ message(STATUS "Enable API Remoting backend")
 
 ggml_add_backend_library(ggml-remotingbackend
                          backend.cpp
-                         ../../include/ggml-remoting-backend.h
+                         backend-dispatched.cpp
+                         backend-dispatched-backend.cpp
+                         backend-dispatched-device.cpp
+                         backend-dispatched-buffer.cpp
+                         backend-dispatched-buffer-type.cpp
+                         backend-utils.cpp
+                         shared/api_remoting.h
+                         shared/apir_backend.h
+                         shared/venus_cs.h
+                         venus_cs_ggml-rpc-back.cpp
                         )
 
 target_compile_options(ggml-remotingbackend PRIVATE -std=c++20)

ggml/src/ggml-remotingbackend/CMakeLists.txt

@@ -0,0 +1,7 @@
+#include "shared/apir_backend.h"
+
+static inline apir_buffer_handle_t
+ggml_buffer_to_apir_handle(ggml_backend_buffer_t buffer) {
+  // in the backend, the buffer handle is the buffer pointer
+  return (apir_buffer_handle_t) buffer;
+}

ggml/src/ggml-remotingbackend/backend-convert.h

@@ -0,0 +1,34 @@
+#include <cstdint>
+#include "backend-internal.h"
+#include "backend-dispatched.h"
+
+#include "ggml-impl.h"
+#include "ggml-backend-impl.h"
+#include "ggml-backend.h"
+
+uint32_t
+backend_graph_compute(struct vn_cs_encoder *enc, struct vn_cs_decoder *dec, struct virgl_apir_context *ctx) {
+  UNUSED(ctx);
+  UNUSED(enc);
+
+  uint32_t shmem_res_id;
+  vn_decode_virtgpu_shmem_res_id(dec, &shmem_res_id);
+
+  const void *shmem_data = ctx->iface.get_shmem_ptr(ctx->virgl_ctx, shmem_res_id);
+  if (!shmem_data) {
+    FATAL("Couldn't get the shmem addr from virgl :/");
+  }
+  size_t cgraph_size;
+  vn_decode_size_t(dec, &cgraph_size);
+
+  struct vn_cs_decoder secondary_dec = vn_cs_new_decoder((const char *) shmem_data, cgraph_size);
+
+  ggml_cgraph *cgraph = vn_decode_ggml_cgraph(&secondary_dec, cgraph_size);
+
+  ggml_status status;
+  status = bck->iface.graph_compute(bck, cgraph);
+
+  vn_encode_ggml_status(enc, &status);
+
+  return 0;
+}

ggml/src/ggml-remotingbackend/backend-dispatched-backend.cpp

@@ -0,0 +1,78 @@
+#include <cstdint>
+#include "backend-internal.h"
+#include "backend-dispatched.h"
+
+#include "ggml-impl.h"
+#include "ggml-backend-impl.h"
+#include "ggml-backend.h"
+
+uint32_t
+backend_buffer_type_get_name(struct vn_cs_encoder *enc, struct vn_cs_decoder *dec, struct virgl_apir_context *ctx) {
+  UNUSED(ctx);
+  ggml_backend_buffer_type_t buft;
+  buft = vn_decode_ggml_buft(dec);
+
+  const char *string = buft->iface.get_name(buft);
+
+  const size_t string_size = strlen(string) + 1;
+  vn_encode_array_size(enc, string_size);
+  vn_encode_char_array(enc, string, string_size);
+
+  return 0;
+}
+
+uint32_t
+backend_buffer_type_get_alignment(struct vn_cs_encoder *enc, struct vn_cs_decoder *dec, struct virgl_apir_context *ctx) {
+  UNUSED(ctx);
+  ggml_backend_buffer_type_t buft;
+  buft = vn_decode_ggml_buft(dec);
+
+  size_t value = buft->iface.get_alignment(buft);
+  vn_encode_size_t(enc, &value);
+
+  return 0;
+}
+
+uint32_t
+backend_buffer_type_get_max_size(struct vn_cs_encoder *enc, struct vn_cs_decoder *dec, struct virgl_apir_context *ctx) {
+  UNUSED(ctx);
+  ggml_backend_buffer_type_t buft;
+  buft = vn_decode_ggml_buft(dec);
+
+  size_t value = buft->iface.get_max_size(buft);
+  vn_encode_size_t(enc, &value);
+
+  return 0;
+}
+
+uint32_t
+backend_buffer_type_is_host(struct vn_cs_encoder *enc, struct vn_cs_decoder *dec, struct virgl_apir_context *ctx) {
+  UNUSED(ctx);
+  ggml_backend_buffer_type_t buft;
+  buft = vn_decode_ggml_buft(dec);
+
+  bool is_host = buft->iface.is_host(buft);
+  vn_encode_bool_t(enc, &is_host);
+
+  return 0;
+}
+
+uint32_t
+backend_buffer_type_alloc_buffer(struct vn_cs_encoder *enc, struct vn_cs_decoder *dec, struct virgl_apir_context *ctx) {
+  UNUSED(ctx);
+  ggml_backend_buffer_type_t buft;
+  buft = vn_decode_ggml_buft(dec);
+
+  size_t size;
+  vn_decode_size_t(dec, &size);
+
+  ggml_backend_buffer_t buffer = buft->iface.alloc_buffer(buft, size);
+  apir_buffer_handle_t *buffer_handle = (apir_buffer_handle_t *) buffer;
+  vn_encode_ggml_buffer_handle(enc, buffer_handle);
+
+  if (buffer) {
+    track_backend_buffer(buffer);
+  }
+
+  return 0;
+}

ggml/src/ggml-remotingbackend/backend-dispatched-buffer-type.cpp

@@ -0,0 +1,129 @@
+#include <cstdint>
+#include "backend-internal.h"
+#include "backend-dispatched.h"
+
+#include "ggml-impl.h"
+#include "ggml-backend-impl.h"
+#include "ggml-backend.h"
+
+uint32_t
+backend_buffer_get_base(struct vn_cs_encoder *enc, struct vn_cs_decoder *dec, struct virgl_apir_context *ctx) {
+  UNUSED(ctx);
+  ggml_backend_buffer_t buffer;
+  buffer = vn_decode_ggml_buffer(dec);
+
+  uintptr_t base = (uintptr_t) buffer->iface.get_base(buffer);
+  vn_encode_uintptr_t(enc, &base);
+
+  //INFO("%s: send base %p\n", __func__,  (void *) base);
+
+  return 0;
+}
+
+uint32_t
+backend_buffer_set_tensor(struct vn_cs_encoder *enc, struct vn_cs_decoder *dec, struct virgl_apir_context *ctx) {
+  UNUSED(ctx);
+  UNUSED(enc);
+
+  ggml_backend_buffer_t buffer;
+  buffer = vn_decode_ggml_buffer(dec);
+
+  ggml_tensor *tensor;
+  // safe to remove the const qualifier here
+  tensor = (ggml_tensor *) (uintptr_t) vn_decode_ggml_tensor(dec);
+
+  uint32_t shmem_res_id;
+  vn_decode_virtgpu_shmem_res_id(dec, &shmem_res_id);
+
+  size_t offset;
+  vn_decode_size_t(dec, &offset);
+
+  size_t size;
+  vn_decode_size_t(dec, &size);
+
+  void *shmem_data = ctx->iface.get_shmem_ptr(ctx->virgl_ctx, shmem_res_id);
+
+  if (!shmem_data) {
+    FATAL("Couldn't get the shmem addr from virgl :/");
+  }
+
+#if 0
+  INFO("Calling (%p)->set_tensor(tensor=%p, data=%p, offset=%lu, size=%lu",
+       buffer, tensor, shmem_data, offset, size);
+#endif
+#if 0
+  void **addr = (void **)(uintptr_t) shmem_data;
+  for (int i = 0; i <= 10; i++) {
+    INFO("%s: %p | %llx", __func__, addr, *addr);
+    addr++;
+  }
+  INFO("\n");
+#endif
+
+  buffer->iface.set_tensor(buffer, tensor, shmem_data, offset, size);
+
+  return 0;
+}
+
+uint32_t
+backend_buffer_get_tensor(struct vn_cs_encoder *enc, struct vn_cs_decoder *dec, struct virgl_apir_context *ctx) {
+  UNUSED(ctx);
+  UNUSED(enc);
+
+  ggml_backend_buffer_t buffer;
+  buffer = vn_decode_ggml_buffer(dec);
+
+
+  const ggml_tensor *tensor;
+  // safe to remove the const qualifier here
+  tensor = vn_decode_ggml_tensor(dec);
+
+  uint32_t shmem_res_id;
+  vn_decode_virtgpu_shmem_res_id(dec, &shmem_res_id);
+
+  size_t offset;
+  vn_decode_size_t(dec, &offset);
+
+  size_t size;
+  vn_decode_size_t(dec, &size);
+
+  void *shmem_data = ctx->iface.get_shmem_ptr(ctx->virgl_ctx, shmem_res_id);
+    if (!shmem_data) {
+    FATAL("Couldn't get the shmem addr from virgl :/");
+  }
+
+  UNUSED(buffer);
+  UNUSED(tensor);
+  buffer->iface.get_tensor(buffer, tensor, shmem_data, offset, size);
+
+  return 0;
+}
+
+uint32_t
+backend_buffer_clear(struct vn_cs_encoder *enc, struct vn_cs_decoder *dec, struct virgl_apir_context *ctx) {
+  UNUSED(ctx);
+  UNUSED(enc);
+
+  ggml_backend_buffer_t buffer;
+  buffer = vn_decode_ggml_buffer(dec);
+
+  uint8_t value;
+  vn_decode_uint8_t(dec, &value);
+
+  buffer->iface.clear(buffer, value);
+
+  return 0;
+}
+
+uint32_t
+backend_buffer_free_buffer(struct vn_cs_encoder *enc, struct vn_cs_decoder *dec, struct virgl_apir_context *ctx) {
+  UNUSED(ctx);
+  UNUSED(enc);
+
+  ggml_backend_buffer_t buffer;
+  buffer = vn_decode_ggml_buffer(dec);
+
+  buffer->iface.free_buffer(buffer);
+
+  return 0;
+}