Skip to content

Commit 6802c82

Browse files
cqderekcqderek
committed
ggml-hexagon: add HTP unary ops for ABS and LOG
Add HVX-accelerated implementations for GGML_OP_LOG and GGML_UNARY_OP_ABS on the HTP backend. - Register HTP_OP_UNARY_ABS and HTP_OP_UNARY_LOG in op_remap_to_htp() - Add ABS and LOG to ggml_backend_hexagon_device_supports_op() - Implement hvx_abs_f32_aa() in hvx-arith.h using hvx_vec_abs_f32() - Implement hvx_log_f32_aa() in hvx-log.h using hvx_vec_log_f32() - Add abs_f32() and log_f32() row-wise dispatch in unary-ops.c - Define tiled and non-tiled task functions via DEFINE_UNARY_TASK and DEFINE_UNARY_TILED_TASK macros - Route HTP_OP_UNARY_ABS and HTP_OP_UNARY_LOG through execute_op() in main.c
1 parent a130532 commit 6802c82

7 files changed

Lines changed: 100 additions & 0 deletions

File tree

‎ggml/src/ggml-hexagon/ggml-hexagon.cpp‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3453,6 +3453,7 @@ static htp_op_code op_remap_to_htp(const ggml_tensor * t) {
34533453
case GGML_OP_CLAMP: return HTP_OP_CLAMP;
34543454
case GGML_OP_SQR: return HTP_OP_SQR;
34553455
case GGML_OP_SQRT: return HTP_OP_SQRT;
3456+
case GGML_OP_LOG: return HTP_OP_UNARY_LOG;
34563457
case GGML_OP_SOFT_MAX: return HTP_OP_SOFTMAX;
34573458
case GGML_OP_SSM_CONV: return HTP_OP_SSM_CONV;
34583459
case GGML_OP_GATED_DELTA_NET: return HTP_OP_GATED_DELTA_NET;
@@ -3476,6 +3477,7 @@ static htp_op_code op_remap_to_htp(const ggml_tensor * t) {
34763477
case GGML_UNARY_OP_EXP: return HTP_OP_UNARY_EXP;
34773478
case GGML_UNARY_OP_SOFTPLUS: return HTP_OP_UNARY_SOFTPLUS;
34783479
case GGML_UNARY_OP_TANH: return HTP_OP_UNARY_TANH;
3480+
case GGML_UNARY_OP_ABS: return HTP_OP_UNARY_ABS;
34793481
default:
34803482
break;
34813483
}
@@ -4112,6 +4114,7 @@ static bool ggml_backend_hexagon_device_supports_op(ggml_backend_dev_t dev, cons
41124114

41134115
case GGML_OP_SQR:
41144116
case GGML_OP_SQRT:
4117+
case GGML_OP_LOG:
41154118
supp = ggml_hexagon_supported_unary(sess, op);
41164119
break;
41174120

@@ -4130,6 +4133,7 @@ static bool ggml_backend_hexagon_device_supports_op(ggml_backend_dev_t dev, cons
41304133
case GGML_UNARY_OP_SIGMOID:
41314134
case GGML_UNARY_OP_SOFTPLUS:
41324135
case GGML_UNARY_OP_TANH:
4136+
case GGML_UNARY_OP_ABS:
41334137
case GGML_UNARY_OP_SILU:
41344138
case GGML_UNARY_OP_GELU:
41354139
case GGML_UNARY_OP_GELU_QUICK:

‎ggml/src/ggml-hexagon/htp/htp-ops.h‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -70,6 +70,8 @@ enum htp_op_code {
7070
HTP_OP_UNARY_NEG,
7171
HTP_OP_UNARY_SOFTPLUS,
7272
HTP_OP_UNARY_TANH,
73+
HTP_OP_UNARY_ABS,
74+
HTP_OP_UNARY_LOG,
7375
HTP_OP_GLU_SWIGLU,
7476
HTP_OP_GLU_SWIGLU_OAI,
7577
HTP_OP_GLU_GEGLU,

‎ggml/src/ggml-hexagon/htp/hvx-arith.h‎

Lines changed: 28 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -358,6 +358,34 @@ static inline void hvx_clamp_scalar_f32(uint8_t * restrict dst, const uint8_t *
358358
}
359359
}
360360

361+
//
362+
// Abs
363+
//
364+
365+
static inline void hvx_abs_f32_aa(uint8_t * restrict dst, const uint8_t * restrict src, uint32_t n) {
366+
assert((unsigned long) dst % 128 == 0);
367+
assert((unsigned long) src % 128 == 0);
368+
369+
HVX_Vector * restrict vdst = (HVX_Vector *) dst;
370+
HVX_Vector * restrict vsrc = (HVX_Vector *) src;
371+
372+
const uint32_t elem_size = sizeof(float);
373+
const uint32_t epv = 128 / elem_size;
374+
const uint32_t nvec = n / epv;
375+
const uint32_t nloe = n % epv;
376+
377+
uint32_t i = 0;
378+
379+
_Pragma("unroll(4)")
380+
for (; i < nvec; i++) {
381+
vdst[i] = hvx_vec_abs_f32(vsrc[i]);
382+
}
383+
if (nloe) {
384+
HVX_Vector v = hvx_vec_abs_f32(vsrc[i]);
385+
hvx_vec_store_a((void *) &vdst[i], nloe * elem_size, v);
386+
}
387+
}
388+
361389
//
362390
// Square
363391
//

‎ggml/src/ggml-hexagon/htp/hvx-log.h‎

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -62,4 +62,28 @@ static inline HVX_Vector hvx_vec_log_f32(HVX_Vector x) {
6262
return hvx_vec_add_f32_f32(term_e, res);
6363
}
6464

65+
static inline void hvx_log_f32_aa(uint8_t * restrict dst, const uint8_t * restrict src, uint32_t n) {
66+
assert((unsigned long) dst % 128 == 0);
67+
assert((unsigned long) src % 128 == 0);
68+
69+
HVX_Vector * restrict vdst = (HVX_Vector *) dst;
70+
HVX_Vector * restrict vsrc = (HVX_Vector *) src;
71+
72+
const uint32_t elem_size = sizeof(float);
73+
const uint32_t epv = 128 / elem_size;
74+
const uint32_t nvec = n / epv;
75+
const uint32_t nloe = n % epv;
76+
77+
uint32_t i = 0;
78+
79+
_Pragma("unroll(4)")
80+
for (; i < nvec; i++) {
81+
vdst[i] = hvx_vec_log_f32(vsrc[i]);
82+
}
83+
if (nloe) {
84+
HVX_Vector v = hvx_vec_log_f32(vsrc[i]);
85+
hvx_vec_store_a((void *) &vdst[i], nloe * elem_size, v);
86+
}
87+
}
88+
6589
#endif /* HVX_LOG_H */

‎ggml/src/ggml-hexagon/htp/main.c‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -728,6 +728,8 @@ static int execute_op(struct htp_ops_context * octx) {
728728
case HTP_OP_UNARY_NEG:
729729
case HTP_OP_UNARY_EXP:
730730
case HTP_OP_UNARY_TANH:
731+
case HTP_OP_UNARY_ABS:
732+
case HTP_OP_UNARY_LOG:
731733
case HTP_OP_L2_NORM:
732734
return op_unary(octx);
733735

‎ggml/src/ggml-hexagon/htp/unary-ops.c‎

Lines changed: 38 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -443,6 +443,34 @@ static void tanh_f32(const float * restrict src,
443443
}
444444
}
445445

446+
static void abs_f32(const float * restrict src,
447+
float * restrict dst,
448+
const uint32_t num_rows,
449+
const struct htp_unary_context * uctx) {
450+
htp_unary_op_preamble;
451+
452+
for (uint32_t ir = 0; ir < num_rows; ir++) {
453+
const uint8_t * restrict src_local = (const uint8_t *)src + (ir * src0_row_size_aligned);
454+
uint8_t * restrict dst_local = (uint8_t *)dst + (ir * dst_row_size_aligned);
455+
456+
hvx_abs_f32_aa(dst_local, src_local, ne0);
457+
}
458+
}
459+
460+
static void log_f32(const float * restrict src,
461+
float * restrict dst,
462+
const uint32_t num_rows,
463+
const struct htp_unary_context * uctx) {
464+
htp_unary_op_preamble;
465+
466+
for (uint32_t ir = 0; ir < num_rows; ir++) {
467+
const uint8_t * restrict src_local = (const uint8_t *)src + (ir * src0_row_size_aligned);
468+
uint8_t * restrict dst_local = (uint8_t *)dst + (ir * dst_row_size_aligned);
469+
470+
hvx_log_f32_aa(dst_local, src_local, ne0);
471+
}
472+
}
473+
446474
#define DEFINE_UNARY_TASK(NAME, IS_RMS_NORM_MUL, IS_TRI, CORE_EXPR) \
447475
static void unary_task_f32_##NAME(unsigned int nth, unsigned int ith, void * data) { \
448476
const struct htp_unary_context * uctx = (const struct htp_unary_context *) data; \
@@ -603,6 +631,8 @@ DEFINE_UNARY_TASK(unary_silu, false, false, silu_f32(src0_vtcm, dst_vtcm, bl
603631
DEFINE_UNARY_TASK(unary_gelu, false, false, gelu_f32(src0_vtcm, dst_vtcm, block_size, uctx))
604632
DEFINE_UNARY_TASK(unary_softplus, false, false, softplus_f32(src0_vtcm, dst_vtcm, block_size, uctx))
605633
DEFINE_UNARY_TASK(unary_tanh, false, false, tanh_f32(src0_vtcm, dst_vtcm, block_size, uctx))
634+
DEFINE_UNARY_TASK(unary_abs, false, false, abs_f32(src0_vtcm, dst_vtcm, block_size, uctx))
635+
DEFINE_UNARY_TASK(unary_log, false, false, log_f32(src0_vtcm, dst_vtcm, block_size, uctx))
606636
DEFINE_UNARY_TASK(l2_norm, false, false, l2_norm_f32(src0_vtcm, dst_vtcm, block_size, uctx))
607637
DEFINE_UNARY_TASK(tri, false, true, tri_f32(src0_vtcm, dst_vtcm, block_size, ir, uctx))
608638

@@ -850,6 +880,8 @@ DEFINE_UNARY_TILED_TASK(unary_silu, false, tile_silu_f32(dst_vtcm, src_vtcm,
850880
DEFINE_UNARY_TILED_TASK(unary_gelu, false, tile_gelu_f32(dst_vtcm, src_vtcm, tw))
851881
DEFINE_UNARY_TILED_TASK(unary_softplus, false, tile_unary_softplus_f32(dst_vtcm, src_vtcm, tw))
852882
DEFINE_UNARY_TILED_TASK(unary_tanh, false, hvx_tanh_f32_aa(dst_vtcm, src_vtcm, tw))
883+
DEFINE_UNARY_TILED_TASK(unary_abs, false, hvx_abs_f32_aa(dst_vtcm, src_vtcm, tw))
884+
DEFINE_UNARY_TILED_TASK(unary_log, false, hvx_log_f32_aa(dst_vtcm, src_vtcm, tw))
853885
DEFINE_UNARY_TILED_TASK(tri, true, tri_apply_tile_f32(src_vtcm, dst_vtcm, tw, col, i01, ne0, tri_ttype))
854886

855887
static int execute_op_unary_f32(struct htp_ops_context * octx) {
@@ -875,6 +907,8 @@ static int execute_op_unary_f32(struct htp_ops_context * octx) {
875907
case HTP_OP_UNARY_GELU: op_type = "gelu-f32"; break;
876908
case HTP_OP_UNARY_SOFTPLUS: op_type = "softplus-f32"; break;
877909
case HTP_OP_UNARY_TANH: op_type = "tanh-f32"; break;
910+
case HTP_OP_UNARY_ABS: op_type = "abs-f32"; break;
911+
case HTP_OP_UNARY_LOG: op_type = "log-f32"; break;
878912
case HTP_OP_L2_NORM: op_type = "l2norm-f32"; break;
879913
case HTP_OP_TRI: op_type = "tri-f32"; break;
880914

@@ -973,6 +1007,8 @@ static int execute_op_unary_f32(struct htp_ops_context * octx) {
9731007
case HTP_OP_UNARY_GELU: task_func = unary_task_f32_tiled_unary_gelu; break;
9741008
case HTP_OP_UNARY_SOFTPLUS: task_func = unary_task_f32_tiled_unary_softplus; break;
9751009
case HTP_OP_UNARY_TANH: task_func = unary_task_f32_tiled_unary_tanh; break;
1010+
case HTP_OP_UNARY_ABS: task_func = unary_task_f32_tiled_unary_abs; break;
1011+
case HTP_OP_UNARY_LOG: task_func = unary_task_f32_tiled_unary_log; break;
9761012
case HTP_OP_TRI: task_func = unary_task_f32_tiled_tri; break;
9771013
default: break;
9781014
}
@@ -992,6 +1028,8 @@ static int execute_op_unary_f32(struct htp_ops_context * octx) {
9921028
case HTP_OP_UNARY_GELU: task_func = unary_task_f32_unary_gelu; break;
9931029
case HTP_OP_UNARY_SOFTPLUS: task_func = unary_task_f32_unary_softplus; break;
9941030
case HTP_OP_UNARY_TANH: task_func = unary_task_f32_unary_tanh; break;
1031+
case HTP_OP_UNARY_ABS: task_func = unary_task_f32_unary_abs; break;
1032+
case HTP_OP_UNARY_LOG: task_func = unary_task_f32_unary_log; break;
9951033
case HTP_OP_L2_NORM: task_func = unary_task_f32_l2_norm; break;
9961034
case HTP_OP_TRI: task_func = unary_task_f32_tri; break;
9971035
default: break;

‎ggml/src/ggml-hexagon/htp/unary-ops.h‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -55,6 +55,8 @@ static inline bool htp_op_is_unary(uint32_t opcode) {
5555
case HTP_OP_UNARY_GELU:
5656
case HTP_OP_UNARY_SOFTPLUS:
5757
case HTP_OP_UNARY_TANH:
58+
case HTP_OP_UNARY_ABS:
59+
case HTP_OP_UNARY_LOG:
5860
case HTP_OP_L2_NORM:
5961
case HTP_OP_TRI:
6062
return true;

0 commit comments

Comments
 (0)