Merge pull request #4 from kpouget/remoting

Remoting: add the remoting backend skeleton code

Author: kpouget

CMakePresets.json

@@ -31,6 +31,7 @@
     { "name": "sycl_f16", "hidden": true, "cacheVariables": { "GGML_SYCL_F16":    "ON" } },
     { "name": "vulkan",   "hidden": true, "cacheVariables": { "GGML_VULKAN":      "ON" } },
     { "name": "remoting_frontend",   "hidden": true, "cacheVariables": { "GGML_REMOTING_FRONTEND":      "ON" } },
+    { "name": "remoting_backend",   "hidden": true, "cacheVariables": { "GGML_REMOTING_BACKEND":      "ON" } },
 
     {
         "name": "x64-windows-llvm", "hidden": true,

Makefile

@@ -721,6 +721,11 @@ ifdef GGML_REMOTING_FRONTEND
 	OBJ_GGML_EXT += ggml/src/ggml-remotingfrontend/ggml-remoting-frontend.o
 endif
 
+ifdef GGML_REMOTING_BACKEND
+	MK_CPPFLAGS  += -DGGML_USE_REMOTINGBACKEND
+	OBJ_GGML_EXT += ggml/src/ggml-remotingbackend/ggml-remoting-backend.o
+endif
+
 ifdef GGML_VULKAN
 	MK_CPPFLAGS  += -DGGML_USE_VULKAN
 	MK_LDFLAGS   += $(shell pkg-config --libs vulkan)
@@ -763,6 +768,9 @@ ggml/src/ggml-vulkan.o: ggml/src/ggml-vulkan/ggml-vulkan.cpp ggml/include/ggml-v
 ggml/src/ggml-remotingfrontend/frontend.o: ggml/src/ggml-remotingfrontend/frontend.cpp
 	$(CXX) $(CXXFLAGS) -c $< -o $@
 
+ggml/src/ggml-remotingbackend/backend.o: ggml/src/ggml-remotingbackend/backend.cpp
+	$(CXX) $(CXXFLAGS) -c $< -o $@
+
 $(_ggml_vk_header): $(_ggml_vk_source)
 
 $(_ggml_vk_source): $(_ggml_vk_shader_deps) vulkan-shaders-gen

build.backend.sh

@@ -0,0 +1,13 @@
+# force isatty-->true, so that $0 |& head -50 has colors ...
+rm -f READY_backend FAILED_backend
+
+echo "int isatty(int fd) { return 1; }" | gcc -O2 -fpic -shared -ldl -o /tmp/isatty.so -xc -
+export LD_PRELOAD=/tmp/isatty.so
+
+cmake --build ../build.remoting-backend --parallel 8 --target llama-cli "$@"
+
+if [[ $? == 0 ]]; then
+    touch READY_backend
+else
+    touch FAILED_backend
+fi

ggml/CMakeLists.txt

@@ -180,6 +180,7 @@ option(GGML_VULKAN_PERF                     "ggml: enable Vulkan perf output"
 option(GGML_VULKAN_VALIDATE                 "ggml: enable Vulkan validation"                  OFF)
 option(GGML_VULKAN_RUN_TESTS                "ggml: run Vulkan tests"                          OFF)
 option(GGML_REMOTING_FRONTEND               "ggml: use the API Remoting frontend"             OFF)
+option(GGML_REMOTING_BACKEND                "ggml: use the API Remoting backend"              OFF)
 option(GGML_KOMPUTE                         "ggml: use Kompute"                               OFF)
 option(GGML_METAL                           "ggml: use Metal"                                 ${GGML_METAL_DEFAULT})
 option(GGML_METAL_USE_BF16                  "ggml: use bfloat if available"                   OFF)
@@ -270,7 +271,8 @@ set(GGML_PUBLIC_HEADERS
     include/ggml-rpc.h
     include/ggml-sycl.h
     include/ggml-vulkan.h
-    include/ggml-remoting-frontend.h
+    ggml/include/ggml-remoting-frontend.h
+    ggml/include/ggml-remoting-backend.h
     include/gguf.h)
 
 set_target_properties(ggml PROPERTIES PUBLIC_HEADER "${GGML_PUBLIC_HEADERS}")

ggml/include/ggml-remoting-backend.h

@@ -0,0 +1,16 @@
+#pragma once
+
+#include "ggml.h"
+#include "ggml-backend.h"
+
+#ifdef  __cplusplus
+extern "C" {
+#endif
+
+#define GGML_REMOTING_BACKEND_NAME "RemotingBackend"
+
+GGML_BACKEND_API ggml_backend_reg_t ggml_backend_remoting_backend_reg();
+
+#ifdef  __cplusplus
+}
+#endif

ggml/include/ggml-remoting-frontend.h

@@ -7,9 +7,9 @@
 extern "C" {
 #endif
 
-#define GGML_REMOTING_NAME "RemotingFrontend"
+#define GGML_REMOTING_FRONTEND_NAME "RemotingFrontend"
 
-GGML_BACKEND_API ggml_backend_reg_t ggml_backend_remoting_reg();
+GGML_BACKEND_API ggml_backend_reg_t ggml_backend_remoting_frontend_reg();
 
 #ifdef  __cplusplus
 }

ggml/src/CMakeLists.txt

@@ -310,6 +310,7 @@ ggml_add_backend(RPC)
 ggml_add_backend(SYCL)
 ggml_add_backend(Vulkan)
 ggml_add_backend(RemotingFrontend)
+ggml_add_backend(RemotingBackend)
 ggml_add_backend(OpenCL)
 
 foreach (target ggml-base ggml)

ggml/src/ggml-backend-reg.cpp

@@ -49,6 +49,10 @@
 #include "ggml-remoting-frontend.h"
 #endif
 
+#ifdef GGML_USE_REMOTINGBACKEND
+#include "ggml-remoting-backend.h"
+#endif
+
 #ifdef GGML_USE_OPENCL
 #include "ggml-opencl.h"
 #endif
@@ -177,7 +181,10 @@ struct ggml_backend_registry {
         register_backend(ggml_backend_vk_reg());
 #endif
 #ifdef GGML_USE_REMOTINGFRONTEND
-        register_backend(ggml_backend_remoting_reg());
+        register_backend(ggml_backend_remoting_frontend_reg());
+#endif
+#ifdef GGML_USE_REMOTINGBACKEND
+        register_backend(ggml_backend_remoting_backend_reg());
 #endif
 #ifdef GGML_USE_OPENCL
         register_backend(ggml_backend_opencl_reg());

ggml/src/ggml-remotingbackend/CMakeLists.txt

@@ -0,0 +1,11 @@
+cmake_minimum_required(VERSION 3.19)
+cmake_policy(SET CMP0114 NEW)
+
+message(STATUS "Enable API Remoting backend")
+
+ggml_add_backend_library(ggml-remotingbackend
+                         backend.cpp
+                         ../../include/ggml-remoting-backend.h
+                        )
+
+target_compile_options(ggml-remotingbackend PRIVATE -std=c++20)

ggml/src/ggml-remotingbackend/backend-internal.h

@@ -0,0 +1,30 @@
+#include <cstdio>
+#include <cstdarg>
+
+static inline void LOG(const char* fmt, ...) {
+    va_list args;
+    va_start(args, fmt);
+    vprintf(fmt, args);
+    va_end(args);
+
+    printf("\n");
+}
+
+static inline void FATAL(const char* fmt, ...) {
+  printf("FATAL: ");
+  va_list args;
+  va_start(args, fmt);
+  vprintf(fmt, args);
+  va_end(args);
+
+  printf("\n");
+
+  if (!fmt)
+    return; // avoid the noreturn attribute
+
+  exit(1);
+}
+
+extern "C" {
+  void ggml_backend_remoting_backend_say_hello();
+}