virtgpu: allocate a shared page with the host

Author: kpouget

ggml/src/ggml-remotingfrontend/CMakeLists.txt

@@ -19,6 +19,8 @@ ggml_add_backend_library(ggml-remotingfrontend
                          ggml-buffer-type.cpp
                          ggml-host-buffer-type.cpp
                          virtgpu.cpp
+                         virtgpu-shm.cpp
+                         virtgpu-utils.cpp
                          ../../include/ggml-remoting-frontend.h
                         )
 

ggml/src/ggml-remotingfrontend/virtgpu-shm.cpp

@@ -0,0 +1,107 @@
+#include <assert.h>
+
+#include "virtgpu-shm.h"
+
+static uint32_t
+virtgpu_ioctl_resource_create_blob(struct virtgpu *gpu,
+                                   uint32_t blob_mem,
+                                   uint32_t blob_flags,
+                                   size_t blob_size,
+                                   uint64_t blob_id,
+                                   uint32_t *res_id)
+{
+#ifdef SIMULATE_BO_SIZE_FIX
+   blob_size = align64(blob_size, 4096);
+#endif
+
+   struct drm_virtgpu_resource_create_blob args = {
+      .blob_mem = blob_mem,
+      .blob_flags = blob_flags,
+      .bo_handle = 0,
+      .res_handle = 0,
+      .size = blob_size,
+      .pad = 0,
+      .cmd_size = 0,
+      .cmd = 0,
+      .blob_id = blob_id,
+   };
+
+   if (virtgpu_ioctl(gpu, DRM_IOCTL_VIRTGPU_RESOURCE_CREATE_BLOB, &args))
+      return 0;
+
+   *res_id = args.res_handle;
+   return args.bo_handle;
+}
+
+static void
+virtgpu_ioctl_gem_close(struct virtgpu *gpu, uint32_t gem_handle)
+{
+   struct drm_gem_close args = {
+      .handle = gem_handle,
+      .pad = 0,
+   };
+
+   const int ret = virtgpu_ioctl(gpu, DRM_IOCTL_GEM_CLOSE, &args);
+   assert(!ret);
+}
+
+static void *
+virtgpu_ioctl_map(struct virtgpu *gpu, uint32_t gem_handle, size_t size)
+{
+   struct drm_virtgpu_map args = {
+      .offset = 0,
+      .handle = gem_handle,
+      .pad = 0,
+   };
+   printf("Say hello world\n");
+   if (virtgpu_ioctl(gpu, DRM_IOCTL_VIRTGPU_MAP, &args))
+      return NULL;
+
+   void *ptr = mmap(NULL, size, PROT_READ | PROT_WRITE, MAP_SHARED, gpu->fd,
+                    args.offset);
+   if (ptr == MAP_FAILED)
+      return NULL;
+
+   return ptr;
+}
+
+void
+virtgpu_shmem_destroy(struct virtgpu *gpu,
+                      struct virtgpu_shmem *shmem)
+{
+   munmap(shmem->base.mmap_ptr, shmem->base.mmap_size);
+   virtgpu_ioctl_gem_close(gpu, shmem->gem_handle);
+}
+
+struct vn_renderer_shmem *
+virtgpu_shmem_create(struct virtgpu *gpu, size_t size)
+{
+   size = align64(size, 16384);
+
+   uint32_t res_id;
+   uint32_t gem_handle = virtgpu_ioctl_resource_create_blob(
+      gpu, gpu->shmem_blob_mem, VIRTGPU_BLOB_FLAG_USE_MAPPABLE, size, 0,
+      &res_id);
+   if (!gem_handle)
+      return NULL;
+
+   void *ptr = virtgpu_ioctl_map(gpu, gem_handle, size);
+   if (!ptr) {
+      virtgpu_ioctl_gem_close(gpu, gem_handle);
+      return NULL;
+   }
+   if (gpu->shmem_array.elem_size == 0) {
+     INFO("gpu->shmem_array.elem_size == 0 | Not working :/\n");
+     assert(false);
+   }
+   struct virtgpu_shmem *shmem = (struct virtgpu_shmem *) util_sparse_array_get(&gpu->shmem_array, gem_handle);
+
+   shmem->gem_handle = gem_handle;
+   shmem->base.res_id = res_id;
+   shmem->base.mmap_size = size;
+   shmem->base.mmap_ptr = ptr;
+   shmem->base.refcount.count = 1;
+   shmem->base.gem_handle = gem_handle;
+
+   return &shmem->base;
+}

ggml/src/ggml-remotingfrontend/virtgpu-shm.h

@@ -0,0 +1,37 @@
+#pragma once
+
+#include <cassert>
+#include <cstdint>
+#include <cstddef>
+#include <stdatomic.h>
+#include <sys/mman.h>
+
+#include "virtgpu.h"
+#include "virtgpu-utils.h"
+
+struct vn_refcount {
+   int count; //atomic_int
+};
+
+
+struct vn_renderer_shmem {
+   struct vn_refcount refcount;
+
+   uint32_t res_id;
+   size_t mmap_size; /* for internal use only (i.e., munmap) */
+   void *mmap_ptr;
+
+   struct list_head cache_head;
+   int64_t cache_timestamp;
+
+   uint32_t gem_handle;
+};
+
+struct vn_renderer_shmem *virtgpu_shmem_create(struct virtgpu *gpu, size_t size);
+void virtgpu_shmem_destroy(struct virtgpu *gpu, struct virtgpu_shmem *shmem);
+
+
+struct virtgpu_shmem {
+   struct vn_renderer_shmem base;
+   uint32_t gem_handle;
+};

ggml/src/ggml-remotingfrontend/virtgpu-utils.cpp

@@ -0,0 +1,186 @@
+#include "virtgpu-utils.h"
+#include <malloc.h>
+#include <cstring>
+#include <stdlib.h>
+
+#define NODE_ALLOC_ALIGN 64
+#define NODE_PTR_MASK (~((uintptr_t)NODE_ALLOC_ALIGN - 1))
+#define NODE_LEVEL_MASK ((uintptr_t)NODE_ALLOC_ALIGN - 1)
+#define NULL_NODE 0
+
+#define os_malloc_aligned(_size, _align) _aligned_malloc(_size, _align)
+#define os_free_aligned(_ptr) free(_ptr)
+#define p_atomic_cmpxchg(v, old, _new) \
+   __sync_val_compare_and_swap((v), (old), (_new))
+
+static inline uint64_t
+util_logbase2_64(uint64_t n)
+{
+#if defined(HAVE___BUILTIN_CLZLL)
+   return ((sizeof(uint64_t) * 8 - 1) - __builtin_clzll(n | 1));
+#else
+   uint64_t pos = 0ull;
+   if (n >= 1ull<<32) { n >>= 32; pos += 32; }
+   if (n >= 1ull<<16) { n >>= 16; pos += 16; }
+   if (n >= 1ull<< 8) { n >>=  8; pos +=  8; }
+   if (n >= 1ull<< 4) { n >>=  4; pos +=  4; }
+   if (n >= 1ull<< 2) { n >>=  2; pos +=  2; }
+   if (n >= 1ull<< 1) {           pos +=  1; }
+   return pos;
+#endif
+}
+
+void
+util_sparse_array_init(struct util_sparse_array *arr,
+                       size_t elem_size, size_t node_size)
+{
+   memset(arr, 0, sizeof(*arr));
+   arr->elem_size = elem_size;
+   arr->node_size_log2 = util_logbase2_64(node_size);
+   assert(node_size >= 2 && node_size == (1ull << arr->node_size_log2));
+}
+
+static inline void *
+os_malloc_aligned(size_t size, size_t alignment)
+{
+   void *ptr;
+   alignment = (alignment + sizeof(void*) - 1) & ~(sizeof(void*) - 1);
+   if(posix_memalign(&ptr, alignment, size) != 0)
+      return NULL;
+   return ptr;
+}
+
+static inline void *
+_util_sparse_array_node_data(uintptr_t handle)
+{
+   return (void *)(handle & NODE_PTR_MASK);
+}
+
+static inline unsigned
+_util_sparse_array_node_level(uintptr_t handle)
+{
+   return handle & NODE_LEVEL_MASK;
+}
+
+static inline void
+_util_sparse_array_node_finish(struct util_sparse_array *arr,
+                               uintptr_t node)
+{
+   if (_util_sparse_array_node_level(node) > 0) {
+      uintptr_t *children = (uintptr_t *) _util_sparse_array_node_data(node);
+      size_t node_size = 1ull << arr->node_size_log2;
+      for (size_t i = 0; i < node_size; i++) {
+         if (children[i])
+            _util_sparse_array_node_finish(arr, children[i]);
+      }
+   }
+
+   os_free_aligned(_util_sparse_array_node_data(node));
+}
+
+static inline uintptr_t
+_util_sparse_array_node(void *data, unsigned level)
+{
+   assert(data != NULL);
+   assert(((uintptr_t)data & NODE_LEVEL_MASK) == 0);
+   assert((level & NODE_PTR_MASK) == 0);
+   return (uintptr_t)data | level;
+}
+
+inline uintptr_t
+_util_sparse_array_node_alloc(struct util_sparse_array *arr,
+                              unsigned level)
+{
+   size_t size;
+   if (level == 0) {
+      size = arr->elem_size << arr->node_size_log2;
+   } else {
+      size = sizeof(uintptr_t) << arr->node_size_log2;
+   }
+
+   void *data = os_malloc_aligned(size, NODE_ALLOC_ALIGN);
+   memset(data, 0, size);
+
+   return _util_sparse_array_node(data, level);
+}
+
+static inline uintptr_t
+_util_sparse_array_set_or_free_node(uintptr_t *node_ptr,
+                                    uintptr_t cmp_node,
+                                    uintptr_t node)
+{
+   uintptr_t prev_node = p_atomic_cmpxchg(node_ptr, cmp_node, node);
+
+   if (prev_node != cmp_node) {
+      /* We lost the race.  Free this one and return the one that was already
+       * allocated.
+       */
+      os_free_aligned(_util_sparse_array_node_data(node));
+      return prev_node;
+   } else {
+      return node;
+   }
+}
+
+void *
+util_sparse_array_get(struct util_sparse_array *arr, uint64_t idx)
+{
+   const unsigned node_size_log2 = arr->node_size_log2;
+   uintptr_t root = p_atomic_read(&arr->root);
+   if (unlikely(!root)) {
+      unsigned root_level = 0;
+      uint64_t idx_iter = idx >> node_size_log2;
+      while (idx_iter) {
+         idx_iter >>= node_size_log2;
+         root_level++;
+      }
+      uintptr_t new_root = _util_sparse_array_node_alloc(arr, root_level);
+      root = _util_sparse_array_set_or_free_node(&arr->root,
+                                                 NULL_NODE, new_root);
+   }
+
+   while (1) {
+      unsigned root_level = _util_sparse_array_node_level(root);
+      uint64_t root_idx = idx >> (root_level * node_size_log2);
+      if (likely(root_idx < (1ull << node_size_log2)))
+         break;
+
+      /* In this case, we have a root but its level is low enough that the
+       * requested index is out-of-bounds.
+       */
+      uintptr_t new_root = _util_sparse_array_node_alloc(arr, root_level + 1);
+
+      uintptr_t *new_root_children = (uintptr_t *) _util_sparse_array_node_data(new_root);
+      new_root_children[0] = root;
+
+      /* We only add one at a time instead of the whole tree because it's
+       * easier to ensure correctness of both the tree building and the
+       * clean-up path.  Because we're only adding one node we never have to
+       * worry about trying to free multiple things without freeing the old
+       * things.
+       */
+      root = _util_sparse_array_set_or_free_node(&arr->root, root, new_root);
+   }
+
+   void *node_data = _util_sparse_array_node_data(root);
+   unsigned node_level = _util_sparse_array_node_level(root);
+   while (node_level > 0) {
+      uint64_t child_idx = (idx >> (node_level * node_size_log2)) &
+                           ((1ull << node_size_log2) - 1);
+
+      uintptr_t *children = (uintptr_t *) node_data;
+      uintptr_t child = p_atomic_read(&children[child_idx]);
+
+      if (unlikely(!child)) {
+         child = _util_sparse_array_node_alloc(arr, node_level - 1);
+         child = _util_sparse_array_set_or_free_node(&children[child_idx],
+                                                     NULL_NODE, child);
+      }
+
+      node_data = _util_sparse_array_node_data(child);
+      node_level = _util_sparse_array_node_level(child);
+   }
+
+   uint64_t elem_idx = idx & ((1ull << node_size_log2) - 1);
+   return (void *)((char *)node_data + (elem_idx * arr->elem_size));
+}

ggml/src/ggml-remotingfrontend/virtgpu-utils.h

@@ -0,0 +1,50 @@
+#pragma once
+
+#include <cstdint>
+#include <cassert>
+#include <cstddef>
+
+#define unlikely(x) __builtin_expect(!!(x), 0)
+#define likely(x) __builtin_expect(!!(x), 1)
+
+/** Checks is a value is a power of two. Does not handle zero. */
+#define IS_POT(v) (((v) & ((v) - 1)) == 0)
+
+/** Checks is a value is a power of two. Zero handled. */
+#define IS_POT_NONZERO(v) ((v) != 0 && IS_POT(v))
+
+/** Align a value to a power of two */
+#define ALIGN_POT(x, pot_align) (((x) + (pot_align) - 1) & ~((pot_align) - 1))
+
+#define p_atomic_read(_v) __atomic_load_n((_v), __ATOMIC_ACQUIRE)
+
+
+static inline bool
+util_is_power_of_two_nonzero64(uint64_t v)
+{
+   return IS_POT_NONZERO(v);
+}
+
+static inline uint64_t
+align64(uint64_t value, uint64_t alignment)
+{
+   assert(util_is_power_of_two_nonzero64(alignment));
+   return ALIGN_POT(value, alignment);
+}
+
+struct list_head
+{
+    struct list_head *prev;
+    struct list_head *next;
+};
+
+struct util_sparse_array {
+   size_t elem_size;
+   unsigned node_size_log2;
+
+   uintptr_t root;
+};
+
+void *util_sparse_array_get(struct util_sparse_array *arr, uint64_t idx);
+void util_sparse_array_init(struct util_sparse_array *arr,
+			    size_t elem_size, size_t node_size);