refactor: remove deps of userdma lib

Author: chraac

ggml/src/ggml-qnn/npu/CMakeLists.txt

@@ -188,14 +188,12 @@ else()
     file(GLOB common_srcs "${CMAKE_CURRENT_LIST_DIR}/common/*.cpp")
     file(GLOB device_srcs "${CMAKE_CURRENT_LIST_DIR}/device/*.cpp")
     file(GLOB device_op_srcs "${CMAKE_CURRENT_LIST_DIR}/device/op/*.cpp")
-    file(GLOB dma_srcs "${HEXAGON_SDK_ROOT}/addons/compute/libs/userdma/utils_lib/src/*.c")
     set(skel_srcs "${CMAKE_CURRENT_BINARY_DIR}/npu_device_skel.c")
     add_library(hexagon_npu_skel_OBJS OBJECT
         ${common_srcs}
         ${device_srcs}
         ${device_op_srcs}
         ${skel_srcs}
-        ${dma_srcs}
     )
 
     if(CMAKE_BUILD_TYPE MATCHES "Debug|Dbg")
@@ -243,8 +241,6 @@ else()
         ${HEXAGON_SDK_ROOT}/libs/qprintf/inc/
 
         # TODO: find a better way to include these
-        ${HEXAGON_SDK_ROOT}/addons/compute/libs/userdma/utils_lib/api/
-        ${HEXAGON_SDK_ROOT}/addons/compute/libs/userdma/utils_lib/inc/
         ${CMAKE_CURRENT_LIST_DIR}/device/
         ${CMAKE_CURRENT_LIST_DIR}/device/op/
     )

ggml/src/ggml-qnn/npu/device/dma_transfer.cpp

@@ -1,11 +1,228 @@
 #include "dma_transfer.hpp"
 
-#include <dma_desc.h>
 #include <qurt.h>
 
 #include <array>
 #include <cstdlib>
 
+namespace {
+
+// From addons/compute/libs/userdma/utils_lib/
+
+#define DM0_STATUS_MASK  0x00000003
+#define DM0_STATUS_SHIFT 0
+#define DM0_STATUS_IDLE  0
+#define DM0_STATUS_RUN   1
+#define DM0_STATUS_ERROR 2
+
+#define DM0_DESC_ADDR_MASK  0xFFFFFFF0
+#define DM0_DESC_ADDR_SHIFT 4
+
+#define DMA_COMPLETE   1
+#define DMA_INCOMPLETE 0
+
+#define DMA_SUCCESS 0
+#define DMA_FAIL    -1
+
+#define DMA_DESC_TYPE_1D 0
+#define DMA_DESC_TYPE_2D 1
+
+#define DESC_NEXT_MASK  0xFFFFFFFF
+#define DESC_NEXT_SHIFT 0
+
+#define DESC_DSTATE_MASK       0x80000000
+#define DESC_DSTATE_SHIFT      31
+#define DESC_DSTATE_INCOMPLETE 0
+#define DESC_DSTATE_COMPLETE   1
+
+#define DESC_ORDER_MASK    0x40000000
+#define DESC_ORDER_SHIFT   30
+#define DESC_ORDER_NOORDER 0
+#define DESC_ORDER_ORDER   1
+
+#define DESC_BYPASSSRC_MASK  0x20000000
+#define DESC_BYPASSSRC_SHIFT 29
+#define DESC_BYPASSDST_MASK  0x10000000
+#define DESC_BYPASSDST_SHIFT 28
+#define DESC_BYPASS_OFF      0
+#define DESC_BYPASS_ON       1
+
+#define DESC_DESCTYPE_MASK  0x03000000
+#define DESC_DESCTYPE_SHIFT 24
+#define DESC_DESCTYPE_1D    0
+#define DESC_DESCTYPE_2D    1
+
+#define DESC_LENGTH_MASK  0x00FFFFFF
+#define DESC_LENGTH_SHIFT 0
+#define DESC_SRC_MASK     0xFFFFFFFF
+#define DESC_SRC_SHIFT    0
+#define DESC_DST_MASK     0xFFFFFFFF
+#define DESC_DST_SHIFT    0
+
+#define DESC_CACHEALLOC_MASK      0x03000000
+#define DESC_CACHEALLOC_SHIFT     24
+#define DESC_CACHEALLOC_NONE      0
+#define DESC_CACHEALLOC_WRITEONLY 1
+#define DESC_CACHEALLOC_READONLY  2
+#define DESC_CACHEALLOC_READWRITE 3
+
+#define DESC_ROIWIDTH_MASK   0x0000FFFF
+#define DESC_ROIWIDTH_SHIFT  0
+#define DESC_ROIHEIGHT_MASK  0xFFFF0000
+#define DESC_ROIHEIGHT_SHIFT 16
+
+#define DESC_SRCSTRIDE_MASK  0x0000FFFF
+#define DESC_SRCSTRIDE_SHIFT 0
+#define DESC_DSTSTRIDE_MASK  0xFFFF0000
+#define DESC_DSTSTRIDE_SHIFT 16
+
+#define DESC_SRCWIDTHOFFSET_MASK  0x0000FFFF
+#define DESC_SRCWIDTHOFFSET_SHIFT 0
+#define DESC_DSTWIDTHOFFSET_MASK  0xFFFF0000
+#define DESC_DSTWIDTHOFFSET_SHIFT 16
+
+/**************************/
+/* 1D (linear) descriptor */
+/**************************/
+typedef struct _dma_desc_1d_t {
+    uint32_t next;
+    uint32_t dstate_order_bypass_desctype_length;
+    uint32_t src;
+    uint32_t dst;
+} dma_desc_1d_t;
+
+static_assert(sizeof(dma_desc_1d_t) == hexagon::dma::kDmaDescSize1D, "kDmaDescSize1D size incorrect");
+
+/***********************/
+/* 2D (box) descriptor */
+/***********************/
+typedef struct _dma_desc_2d_t {
+    uint32_t next;
+    uint32_t dstate_order_bypass_desctype_length;
+    uint32_t src;
+    uint32_t dst;
+    uint32_t allocation;
+    uint32_t roiheight_roiwidth;
+    uint32_t dststride_srcstride;
+    uint32_t dstwidthoffset_srcwidthoffset;
+} dma_desc_2d_t;
+
+static_assert(sizeof(dma_desc_2d_t) == hexagon::dma::kDmaDescSize2D, "kDmaDescSize2D size incorrect");
+
+inline void dmstart(void * next) {
+    asm volatile(" release(%0):at" : : "r"(next));
+    asm volatile(" dmstart(%0)" : : "r"(next));
+}
+
+inline void dmlink(void * cur, void * next) {
+    asm volatile(" release(%0):at" : : "r"(next));
+    asm volatile(" dmlink(%0, %1)" : : "r"(cur), "r"(next));
+}
+
+inline unsigned int dmpoll(void) {
+    unsigned int ret = 0;
+    asm volatile(" %0 = dmpoll" : "=r"(ret) : : "memory");
+    return ret;
+}
+
+inline unsigned int dmwait(void) {
+    unsigned int ret = 0;
+    asm volatile(" %0 = dmwait" : "=r"(ret) : : "memory");
+    return ret;
+}
+
+inline void dma_desc_set_next(void * d, uint32_t v) {
+    (((dma_desc_1d_t *) d)->next) &= ~DESC_NEXT_MASK;
+    (((dma_desc_1d_t *) d)->next) |= ((v << DESC_NEXT_SHIFT) & DESC_NEXT_MASK);
+}
+
+inline uint32_t dma_desc_get_dstate(void * d) {
+    return (((((dma_desc_1d_t *) d)->dstate_order_bypass_desctype_length) & DESC_DSTATE_MASK) >> DESC_DSTATE_SHIFT);
+}
+
+inline void dma_desc_set_dstate(void * d, uint32_t v) {
+    (((dma_desc_1d_t *) d)->dstate_order_bypass_desctype_length) &= ~DESC_DSTATE_MASK;
+    (((dma_desc_1d_t *) d)->dstate_order_bypass_desctype_length) |= ((v << DESC_DSTATE_SHIFT) & DESC_DSTATE_MASK);
+}
+
+inline void dma_desc_set_desctype(void * d, uint32_t v) {
+    (((dma_desc_1d_t *) d)->dstate_order_bypass_desctype_length) &= ~DESC_DESCTYPE_MASK;
+    (((dma_desc_1d_t *) d)->dstate_order_bypass_desctype_length) |= ((v << DESC_DESCTYPE_SHIFT) & DESC_DESCTYPE_MASK);
+}
+
+inline void dma_desc_set_order(void * d, uint32_t v) {
+    (((dma_desc_1d_t *) d)->dstate_order_bypass_desctype_length) &= ~DESC_ORDER_MASK;
+    (((dma_desc_1d_t *) d)->dstate_order_bypass_desctype_length) |= ((v << DESC_ORDER_SHIFT) & DESC_ORDER_MASK);
+}
+
+inline void dma_desc_set_bypasssrc(void * d, uint32_t v) {
+    (((dma_desc_1d_t *) d)->dstate_order_bypass_desctype_length) &= ~DESC_BYPASSSRC_MASK;
+    (((dma_desc_1d_t *) d)->dstate_order_bypass_desctype_length) |= ((v << DESC_BYPASSSRC_SHIFT) & DESC_BYPASSSRC_MASK);
+}
+
+inline void dma_desc_set_bypassdst(void * d, uint32_t v) {
+    (((dma_desc_1d_t *) d)->dstate_order_bypass_desctype_length) &= ~DESC_BYPASSDST_MASK;
+    (((dma_desc_1d_t *) d)->dstate_order_bypass_desctype_length) |= ((v << DESC_BYPASSDST_SHIFT) & DESC_BYPASSDST_MASK);
+}
+
+inline void dma_desc_set_length(void * d, uint32_t v) {
+    (((dma_desc_1d_t *) d)->dstate_order_bypass_desctype_length) &= ~DESC_LENGTH_MASK;
+    (((dma_desc_1d_t *) d)->dstate_order_bypass_desctype_length) |= ((v << DESC_LENGTH_SHIFT) & DESC_LENGTH_MASK);
+}
+
+inline uint32_t dma_desc_get_src(void * d) {
+    return (((((dma_desc_1d_t *) d)->src) & DESC_SRC_MASK) >> DESC_SRC_SHIFT);
+}
+
+inline void dma_desc_set_src(void * d, uint32_t v) {
+    (((dma_desc_1d_t *) d)->src) &= ~DESC_SRC_MASK;
+    (((dma_desc_1d_t *) d)->src) |= ((v << DESC_SRC_SHIFT) & DESC_SRC_MASK);
+}
+
+inline void dma_desc_set_dst(void * d, uint32_t v) {
+    (((dma_desc_1d_t *) d)->dst) &= ~DESC_DST_MASK;
+    (((dma_desc_1d_t *) d)->dst) |= ((v << DESC_DST_SHIFT) & DESC_DST_MASK);
+}
+
+inline void dma_desc_set_roiwidth(void * d, uint32_t v) {
+    (((dma_desc_2d_t *) d)->roiheight_roiwidth) &= ~DESC_ROIWIDTH_MASK;
+    (((dma_desc_2d_t *) d)->roiheight_roiwidth) |= ((v << DESC_ROIWIDTH_SHIFT) & DESC_ROIWIDTH_MASK);
+}
+
+inline void dma_desc_set_roiheight(void * d, uint32_t v) {
+    (((dma_desc_2d_t *) d)->roiheight_roiwidth) &= ~DESC_ROIHEIGHT_MASK;
+    (((dma_desc_2d_t *) d)->roiheight_roiwidth) |= ((v << DESC_ROIHEIGHT_SHIFT) & DESC_ROIHEIGHT_MASK);
+}
+
+inline void dma_desc_set_srcstride(void * d, uint32_t v) {
+    (((dma_desc_2d_t *) d)->dststride_srcstride) &= ~DESC_SRCSTRIDE_MASK;
+    (((dma_desc_2d_t *) d)->dststride_srcstride) |= ((v << DESC_SRCSTRIDE_SHIFT) & DESC_SRCSTRIDE_MASK);
+}
+
+inline void dma_desc_set_dststride(void * d, uint32_t v) {
+    (((dma_desc_2d_t *) d)->dststride_srcstride) &= ~DESC_DSTSTRIDE_MASK;
+    (((dma_desc_2d_t *) d)->dststride_srcstride) |= ((v << DESC_DSTSTRIDE_SHIFT) & DESC_DSTSTRIDE_MASK);
+}
+
+inline void dma_desc_set_srcwidthoffset(void * d, uint32_t v) {
+    (((dma_desc_2d_t *) d)->dstwidthoffset_srcwidthoffset) &= ~DESC_SRCWIDTHOFFSET_MASK;
+    (((dma_desc_2d_t *) d)->dstwidthoffset_srcwidthoffset) |=
+        ((v << DESC_SRCWIDTHOFFSET_SHIFT) & DESC_SRCWIDTHOFFSET_MASK);
+}
+
+inline void dma_desc_set_dstwidthoffset(void * d, uint32_t v) {
+    (((dma_desc_2d_t *) d)->dstwidthoffset_srcwidthoffset) &= ~DESC_DSTWIDTHOFFSET_MASK;
+    (((dma_desc_2d_t *) d)->dstwidthoffset_srcwidthoffset) |=
+        ((v << DESC_DSTWIDTHOFFSET_SHIFT) & DESC_DSTWIDTHOFFSET_MASK);
+}
+
+inline void dma_desc_set_cachealloc(void * d, uint32_t v) {
+    (((dma_desc_2d_t *) d)->allocation) &= ~DESC_CACHEALLOC_MASK;
+    (((dma_desc_2d_t *) d)->allocation) |= ((v << DESC_CACHEALLOC_SHIFT) & DESC_CACHEALLOC_MASK);
+}
+
+}  // namespace
+
 namespace hexagon::dma {
 
 dma_transfer::dma_transfer() {
@@ -162,23 +379,58 @@ bool dma_transfer::submit2d(const uint8_t * src,
 }
 
 void dma_transfer::wait() {
-    auto ret = dma_wait_for_idle();
-    if (ret != DMA_SUCCESS) {
-        DEVICE_LOG_ERROR("dma_transfer: failed to wait for DMA idle: %d\n", ret);
+    uint32_t dm0_status = dmwait() & DM0_STATUS_MASK;
+    if (dm0_status != DM0_STATUS_IDLE) {
+        DEVICE_LOG_ERROR("dma_transfer: failed to wait for DMA idle, dm0_status: %d\n", (int) dm0_status);
     }
 }
 
 bool dma_transfer::is_desc_done(uint8_t * desc) {
-    return !dma_desc_get_src(desc) || dma_desc_is_done(desc) == DMA_COMPLETE;
+    if (!dma_desc_get_src(desc)) {
+        return true;
+    }
+
+    if (dma_desc_get_dstate(desc) == DESC_DSTATE_COMPLETE) {
+        return true;
+    }
+
+    dmpoll();
+    return false;
 }
 
-bool dma_transfer::submit_impl(void ** desc_batch, int batch_len) {
+bool dma_transfer::submit_impl(void ** desc_batch, size_t batch_len) {
     _dma_desc_mutex.lock();
-    const bool succ = dma_desc_submit(desc_batch, batch_len) == DMA_SUCCESS;
+    for (size_t i = 0; i < batch_len - 1; i++) {
+        dma_desc_set_next(desc_batch[i], (uint32_t) desc_batch[i + 1]);
+    }
+
+    dma_desc_set_next(desc_batch[batch_len - 1], (uint32_t) nullptr);
+    uint32_t dm0_status = dmpoll() & DM0_STATUS_MASK;
+    if (dm0_status == DM0_STATUS_IDLE) {
+        dmstart(desc_batch[0]);
+    } else if (dm0_status == DM0_STATUS_RUN) {
+        if (_dma_last_desc == nullptr) {
+            _dma_desc_mutex.unlock();
+            DEVICE_LOG_ERROR("dma_transfer: last descriptor not found for linking. Submission failed\n");
+            return false;
+        } else {
+            dmlink(_dma_last_desc, desc_batch[0]);
+        }
+    } else {
+        _dma_desc_mutex.unlock();
+        DEVICE_LOG_ERROR("dma_transfer: DMA not idle or running. Submission failed\n");
+        return false;
+    }
+
+    dmpoll();
+
+    _dma_last_desc = (void *) desc_batch[batch_len - 1];
+
     _dma_desc_mutex.unlock();
-    return succ;
+    return true;
 }
 
 qurt_mutex dma_transfer::_dma_desc_mutex;
+void *     dma_transfer::_dma_last_desc = nullptr;
 
 }  // namespace hexagon::dma

ggml/src/ggml-qnn/npu/device/dma_transfer.hpp

@@ -2,10 +2,11 @@
 
 #include "util.hpp"
 
-#include <dma_utils.h>
-
 namespace hexagon::dma {
 
+constexpr const size_t kDmaDescSize1D = 16;
+constexpr const size_t kDmaDescSize2D = 32;
+
 class dma_transfer {
   public:
     dma_transfer();
@@ -32,12 +33,14 @@ class dma_transfer {
   private:
     static bool       is_desc_done(uint8_t * desc);  // TODO: should we use void * here?
     static qurt_mutex _dma_desc_mutex;
+    static void *     _dma_last_desc;
 
-    bool submit_impl(void ** desc_batch, int batch_len);
+    // TODO: can we avoid the void ** here?
+    bool submit_impl(void ** desc_batch, size_t batch_len);
 
-    alignas(DMA_DESC_SIZE_1D) uint8_t _dma_1d_desc0[DMA_DESC_SIZE_1D] = {};
-    alignas(DMA_DESC_SIZE_1D) uint8_t _dma_1d_desc1[DMA_DESC_SIZE_1D] = {};
-    alignas(DMA_DESC_SIZE_2D) uint8_t _dma_2d_desc0[DMA_DESC_SIZE_2D] = {};
+    alignas(kDmaDescSize1D) uint8_t _dma_1d_desc0[kDmaDescSize1D] = {};
+    alignas(kDmaDescSize1D) uint8_t _dma_1d_desc1[kDmaDescSize1D] = {};
+    alignas(kDmaDescSize2D) uint8_t _dma_2d_desc0[kDmaDescSize2D] = {};
 
     DISABLE_COPY_AND_MOVE(dma_transfer);
 };